From 4f01e1fdcf59927a1f4e87ea947a52e9ef6240bb Mon Sep 17 00:00:00 2001 From: Ben Visness Date: Tue, 17 Aug 2021 21:09:42 -0500 Subject: [PATCH 01/10] Add a title to the user profile page --- src/website/user.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/website/user.go b/src/website/user.go index 19f2cc2..b6e7111 100644 --- a/src/website/user.go +++ b/src/website/user.go @@ -215,11 +215,15 @@ func UserProfile(c *RequestContext) ResponseData { c.Perf.EndBlock() + templateUser := templates.UserToTemplate(profileUser, c.Theme) + baseData := getBaseData(c) + baseData.Title = templateUser.Name + var res ResponseData res.MustWriteTemplate("user_profile.html", UserProfileTemplateData{ BaseData: baseData, - ProfileUser: templates.UserToTemplate(profileUser, c.Theme), + ProfileUser: templateUser, ProfileUserLinks: profileUserLinks, ProfileUserProjects: templateProjects, TimelineItems: timelineItems, From 76f9256e97ff090b7e482a96548adc3fc94490bc Mon Sep 17 00:00:00 2001 From: Ben Visness Date: Sat, 21 Aug 2021 11:15:27 -0500 Subject: [PATCH 02/10] Save Discord messages and attachments --- go.mod | 5 + go.sum | 30 ++++ src/assets/assets.go | 155 +++++++++++++++++++ src/assets/assets_test.go | 13 ++ src/db/db.go | 2 + src/discord/gateway.go | 2 +- src/discord/library.go | 45 ++++++ src/discord/payloads.go | 63 ++++++-- src/discord/showcase.go | 311 ++++++++++++++++++++++++++++++++------ src/discord/todo.txt | 43 ++++++ src/main.go | 1 + 11 files changed, 617 insertions(+), 53 deletions(-) create mode 100644 src/assets/assets.go create mode 100644 src/assets/assets_test.go create mode 100644 src/discord/library.go create mode 100644 src/discord/todo.txt diff --git a/go.mod b/go.mod index 16fef9d..8c56138 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,11 @@ require ( github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible github.com/alecthomas/chroma v0.9.2 + github.com/aws/aws-sdk-go-v2 v1.8.1 + github.com/aws/aws-sdk-go-v2/config v1.6.1 + github.com/aws/aws-sdk-go-v2/credentials v1.3.3 + github.com/aws/aws-sdk-go-v2/service/s3 v1.13.0 + github.com/aws/smithy-go v1.7.0 github.com/frustra/bbcode v0.0.0-20201127003707-6ef347fbe1c8 github.com/go-stack/stack v1.8.0 github.com/google/uuid v1.2.0 diff --git a/go.sum b/go.sum index 474fba9..6e6029e 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,30 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go-v2 v1.8.1 h1:GcFgQl7MsBygmeeqXyV1ivrTEmsVz/rdFJaTcltG9ag= +github.com/aws/aws-sdk-go-v2 v1.8.1/go.mod h1:xEFuWz+3TYdlPRuo+CqATbeDWIWyaT5uAPwPaWtgse0= +github.com/aws/aws-sdk-go-v2/config v1.6.1 h1:qrZINaORyr78syO1zfD4l7r4tZjy0Z1l0sy4jiysyOM= +github.com/aws/aws-sdk-go-v2/config v1.6.1/go.mod h1:t/y3UPu0XEDy0cEw6mvygaBQaPzWiYAxfP2SzgtvclA= +github.com/aws/aws-sdk-go-v2/credentials v1.3.3 h1:A13QPatmUl41SqUfnuT3V0E3XiNGL6qNTOINbE8cZL4= +github.com/aws/aws-sdk-go-v2/credentials v1.3.3/go.mod h1:oVieKMT3m9BSfqhOfuQ+E0j/yN84ZAJ7Qv8Sfume/ak= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.1 h1:rc+fRGvlKbeSd9IFhFS1KWBs0XjTkq0CfK5xqyLgIp0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.1/go.mod h1:+GTydg3uHmVlQdkRoetz6VHKbOMEYof70m19IpMLifc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.1 h1:IkqRRUZTKaS16P2vpX+FNc2jq3JWa3c478gykQp4ow4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.1/go.mod h1:Pv3WenDjI0v2Jl7UaMFIIbPOBbhn33RmmAmGgkXDoqY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.2 h1:YcGVEqLQGHDa81776C3daai6ZkkRGf/8RAQ07hV0QcU= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.2/go.mod h1:EASdTcM1lGhUe1/p4gkojHwlGJkeoRjjr1sRCzup3Is= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.3 h1:VxFCgxsqWe7OThOwJ5IpFX3xrObtuIH9Hg/NW7oot1Y= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.3/go.mod h1:7gcsONBmFoCcKrAqrm95trrMd2+C/ReYKP7Vfu8yHHA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.3 h1:7tPSbUWzuoMJ2woUKgOfIPuZS88hMdFHJBBB2vR0bHI= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.3/go.mod h1:/ugW3qFkJe/h7sNtI6/zJnwRbvavs6GyOid69uI9eek= +github.com/aws/aws-sdk-go-v2/service/s3 v1.13.0 h1:2oMLrNpOSpkDTocIVv3Fut1XrmlbKPlgnnYMGYqFp0Y= +github.com/aws/aws-sdk-go-v2/service/s3 v1.13.0/go.mod h1:Tzxhu3GnCpj45WJqXyxcLF2gUHzTcmY7CzpQ9x9KVls= +github.com/aws/aws-sdk-go-v2/service/sso v1.3.3 h1:K2gCnGvAASpz+jqP9iyr+F/KNjmTYf8aWOtTQzhmZ5w= +github.com/aws/aws-sdk-go-v2/service/sso v1.3.3/go.mod h1:Jgw5O+SK7MZ2Yi9Yvzb4PggAPYaFSliiQuWR0hNjexk= +github.com/aws/aws-sdk-go-v2/service/sts v1.6.2 h1:l504GWCoQi1Pk68vSUFGLmDIEMzRfVGNgLakDK+Uj58= +github.com/aws/aws-sdk-go-v2/service/sts v1.6.2/go.mod h1:RBhoMJB8yFToaCnbe0jNq5Dcdy0jp6LhHqg55rjClkM= +github.com/aws/smithy-go v1.7.0 h1:+cLHMRrDZvQ4wk+KuQ9yH6eEg6KZEJ9RI2IkDqnygCg= +github.com/aws/smithy-go v1.7.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -92,6 +116,9 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -185,6 +212,8 @@ github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -484,6 +513,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/src/assets/assets.go b/src/assets/assets.go new file mode 100644 index 0000000..cf2bc83 --- /dev/null +++ b/src/assets/assets.go @@ -0,0 +1,155 @@ +package assets + +import ( + "bytes" + "context" + "crypto/sha1" + "errors" + "fmt" + "regexp" + + "git.handmade.network/hmn/hmn/src/config" + "git.handmade.network/hmn/hmn/src/db" + "git.handmade.network/hmn/hmn/src/models" + "git.handmade.network/hmn/hmn/src/oops" + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/google/uuid" +) + +var client *s3.Client + +func init() { + cfg, err := awsconfig.LoadDefaultConfig(context.Background(), + awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider( + config.Config.DigitalOcean.AssetsSpacesKey, + config.Config.DigitalOcean.AssetsSpacesSecret, + "", + ), + ), + awsconfig.WithRegion(config.Config.DigitalOcean.AssetsSpacesRegion), + awsconfig.WithEndpointResolver(aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: config.Config.DigitalOcean.AssetsSpacesEndpoint, + }, nil + })), + ) + if err != nil { + panic(err) + } + client = s3.NewFromConfig(cfg, func(o *s3.Options) { + o.UsePathStyle = true + }) +} + +type CreateInput struct { + Content []byte + Filename string + MimeType string + + // Optional params + UploaderID *int // HMN user id + Width, Height int +} + +var REIllegalFilenameChars = regexp.MustCompile(`[^\w \-.]`) + +func SanitizeFilename(filename string) string { + if filename == "" { + return "unnamed" + } + return REIllegalFilenameChars.ReplaceAllString(filename, "") +} + +func AssetKey(id, filename string) string { + return fmt.Sprintf("%s%s/%s", config.Config.DigitalOcean.AssetsPathPrefix, id, filename) +} + +type InvalidAssetError error + +func Create(ctx context.Context, dbConn db.ConnOrTx, in CreateInput) (*models.Asset, error) { + filename := SanitizeFilename(in.Filename) + + if len(in.Content) == 0 { + return nil, InvalidAssetError(fmt.Errorf("could not upload asset '%s': no bytes of data were provided", filename)) + } + if in.MimeType == "" { + return nil, InvalidAssetError(fmt.Errorf("could not upload asset '%s': no mime type provided", filename)) + } + + // Upload the asset to the DO space + id := uuid.New() + key := AssetKey(id.String(), filename) + checksum := fmt.Sprintf("%x", sha1.Sum(in.Content)) + + upload := func() error { + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &config.Config.DigitalOcean.AssetsSpacesBucket, + Key: &key, + Body: bytes.NewReader(in.Content), + ACL: types.ObjectCannedACLPublicRead, + }) + return err + } + + err := upload() + if err != nil { + var apiError smithy.APIError + if errors.As(err, &apiError) && apiError.ErrorCode() == "NoSuchBucket" { + _, err := client.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: &config.Config.DigitalOcean.AssetsSpacesBucket, + }) + if err != nil { + return nil, oops.New(err, "failed to create assets bucket") + } + + err = upload() + if err != nil { + return nil, oops.New(err, "failed to upload asset") + } + } else { + return nil, oops.New(err, "failed to upload asset") + } + } + + // Save a record in our database + // TODO(db): Would be convient to use RETURNING here... + _, err = dbConn.Exec(ctx, + ` + INSERT INTO handmade_asset (id, s3_key, filename, size, mime_type, sha1sum, width, height, uploader_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + `, + id, + key, + filename, + len(in.Content), + in.MimeType, + checksum, + in.Width, + in.Height, + in.UploaderID, + ) + if err != nil { + return nil, oops.New(err, "failed to save asset record") + } + + // Fetch and return the new record + iasset, err := db.QueryOne(ctx, dbConn, models.Asset{}, + ` + SELECT $columns + FROM handmade_asset + WHERE id = $1 + `, + id, + ) + if err != nil { + return nil, oops.New(err, "failed to fetch newly-created asset") + } + + return iasset.(*models.Asset), nil +} diff --git a/src/assets/assets_test.go b/src/assets/assets_test.go new file mode 100644 index 0000000..29a0f08 --- /dev/null +++ b/src/assets/assets_test.go @@ -0,0 +1,13 @@ +package assets + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSanitizeFilename(t *testing.T) { + assert.Equal(t, "cool filename.txt.wow", SanitizeFilename("cool filename.txt.wow")) + assert.Equal(t, " hi doggy ", SanitizeFilename("😎 hi doggy 🐶")) + assert.Equal(t, "newlinesaretotallylegal", SanitizeFilename("newlines\naretotallylegal")) +} diff --git a/src/db/db.go b/src/db/db.go index 6359f0e..3962082 100644 --- a/src/db/db.go +++ b/src/db/db.go @@ -11,6 +11,7 @@ import ( "git.handmade.network/hmn/hmn/src/logging" "git.handmade.network/hmn/hmn/src/oops" "github.com/google/uuid" + "github.com/jackc/pgconn" "github.com/jackc/pgtype" "github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4/log/zerologadapter" @@ -57,6 +58,7 @@ func typeIsQueryable(t reflect.Type) bool { // This interface should match both a direct pgx connection or a pgx transaction. type ConnOrTx interface { Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error) + Exec(ctx context.Context, sql string, args ...interface{}) (pgconn.CommandTag, error) } var connInfo = pgtype.NewConnInfo() diff --git a/src/discord/gateway.go b/src/discord/gateway.go index 7cdfdac..5962086 100644 --- a/src/discord/gateway.go +++ b/src/discord/gateway.go @@ -563,7 +563,7 @@ func (bot *discordBotInstance) processEventMsg(ctx context.Context, msg *Gateway } func (bot *discordBotInstance) messageCreateOrUpdate(ctx context.Context, msg *Message) error { - if msg.Author != nil && msg.Author.ID == config.Config.Discord.BotUserID { + if msg.OriginalHasFields("author") && msg.Author.ID == config.Config.Discord.BotUserID { // Don't process your own messages return nil } diff --git a/src/discord/library.go b/src/discord/library.go new file mode 100644 index 0000000..3eaa30c --- /dev/null +++ b/src/discord/library.go @@ -0,0 +1,45 @@ +package discord + +import ( + "context" + + "git.handmade.network/hmn/hmn/src/oops" +) + +func (bot *discordBotInstance) processLibraryMsg(ctx context.Context, msg *Message) error { + switch msg.Type { + case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: + default: + return nil + } + + if !msg.OriginalHasFields("content") { + return nil + } + + if !messageHasLinks(msg.Content) { + err := DeleteMessage(ctx, msg.ChannelID, msg.ID) + if err != nil { + return oops.New(err, "failed to delete message") + } + + if !msg.Author.IsBot { + channel, err := CreateDM(ctx, msg.Author.ID) + if err != nil { + return oops.New(err, "failed to create DM channel") + } + + err = SendMessages(ctx, bot.dbConn, MessageToSend{ + ChannelID: channel.ID, + Req: CreateMessageRequest{ + Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.", + }, + }) + if err != nil { + return oops.New(err, "failed to send showcase warning message") + } + } + } + + return nil +} diff --git a/src/discord/payloads.go b/src/discord/payloads.go index edb061a..18b1b4c 100644 --- a/src/discord/payloads.go +++ b/src/discord/payloads.go @@ -2,6 +2,8 @@ package discord import ( "encoding/json" + "fmt" + "time" ) type Opcode int @@ -170,19 +172,50 @@ const ( // https://discord.com/developers/docs/resources/channel#message-object type Message struct { - ID string `json:"id"` - ChannelID string `json:"channel_id"` - Content string `json:"content"` - Author *User `json:"author"` // note that this may not be an actual valid user (see the docs) - // TODO: Author info - // TODO: Timestamp parsing, yay - Type MessageType `json:"type"` + ID string `json:"id"` + ChannelID string `json:"channel_id"` + GuildID *string `json:"guild_id"` + Content string `json:"content"` + Author User `json:"author"` // note that this may not be an actual valid user (see the docs) + Timestamp string `json:"timestamp"` + Type MessageType `json:"type"` Attachments []Attachment `json:"attachments"` + // TODO: Embeds originalMap map[string]interface{} } +func (m *Message) JumpURL() string { + guildStr := "@me" + if m.GuildID != nil { + guildStr = *m.GuildID + } + return fmt.Sprintf("https://discord.com/channels/%s/%s/%s", guildStr, m.ChannelID, m.ID) +} + +func (m *Message) Time() time.Time { + t, err := time.Parse(time.RFC3339Nano, m.Timestamp) + if err != nil { + panic(err) + } + return t +} + +func (m *Message) OriginalHasFields(fields ...string) bool { + if m.originalMap == nil { + return false + } + + for _, field := range fields { + _, ok := m.originalMap[field] + if !ok { + return false + } + } + return true +} + func MessageFromMap(m interface{}) Message { /* Some gateway events, like MESSAGE_UPDATE, do not contain the @@ -194,15 +227,16 @@ func MessageFromMap(m interface{}) Message { msg := Message{ ID: mmap["id"].(string), ChannelID: mmap["channel_id"].(string), + GuildID: maybeStringP(mmap, "guild_id"), Content: maybeString(mmap, "content"), + Timestamp: maybeString(mmap, "timestamp"), Type: MessageType(maybeInt(mmap, "type")), originalMap: mmap, } if author, ok := mmap["author"]; ok { - u := UserFromMap(author) - msg.Author = &u + msg.Author = UserFromMap(author) } if iattachments, ok := mmap["attachments"]; ok { @@ -212,6 +246,8 @@ func MessageFromMap(m interface{}) Message { } } + // TODO: Embeds + return msg } @@ -276,6 +312,15 @@ func maybeString(m map[string]interface{}, k string) string { return val.(string) } +func maybeStringP(m map[string]interface{}, k string) *string { + val, ok := m[k] + if !ok { + return nil + } + strval := val.(string) + return &strval +} + func maybeInt(m map[string]interface{}, k string) int { val, ok := m[k] if !ok { diff --git a/src/discord/showcase.go b/src/discord/showcase.go index 6dacb16..65abbdc 100644 --- a/src/discord/showcase.go +++ b/src/discord/showcase.go @@ -2,15 +2,25 @@ package discord import ( "context" + "errors" + "io" + "net/http" "net/url" "regexp" "strings" + "git.handmade.network/hmn/hmn/src/assets" + "git.handmade.network/hmn/hmn/src/db" + "git.handmade.network/hmn/hmn/src/logging" + "git.handmade.network/hmn/hmn/src/models" "git.handmade.network/hmn/hmn/src/oops" + "github.com/jackc/pgx/v4" ) var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`) +var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this") + func (bot *discordBotInstance) processShowcaseMsg(ctx context.Context, msg *Message) error { switch msg.Type { case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: @@ -18,26 +28,70 @@ func (bot *discordBotInstance) processShowcaseMsg(ctx context.Context, msg *Mess return nil } + didDelete, err := bot.maybeDeleteShowcaseMsg(ctx, msg) + if err != nil { + return err + } + if didDelete { + return nil + } + + tx, err := bot.dbConn.Begin(ctx) + if err != nil { + panic(err) + } + defer tx.Rollback(ctx) + + // save the message, maybe save its contents, and maybe make a snippet too + _, err = bot.saveMessageAndContents(ctx, tx, msg) + if errors.Is(err, errNotEnoughInfo) { + logging.ExtractLogger(ctx).Warn(). + Interface("msg", msg). + Msg("didn't have enough info to process Discord message") + return nil + } else if err != nil { + return err + } + if doSnippet, err := bot.allowedToCreateMessageSnippet(ctx, msg); doSnippet && err == nil { + _, err := bot.createMessageSnippet(ctx, msg) + if err != nil { + return oops.New(err, "failed to create snippet in gateway") + } + } else if err != nil { + return oops.New(err, "failed to check snippet permissions in gateway") + } + + err = tx.Commit(ctx) + if err != nil { + return oops.New(err, "failed to commit Discord message updates") + } + + return nil +} + +func (bot *discordBotInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message) (didDelete bool, err error) { hasGoodContent := true - if originalMessageHasField(msg, "content") && !messageHasLinks(msg.Content) { + if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) { hasGoodContent = false } hasGoodAttachments := true - if originalMessageHasField(msg, "attachments") && len(msg.Attachments) == 0 { + if msg.OriginalHasFields("attachments") && len(msg.Attachments) == 0 { hasGoodAttachments = false } + didDelete = false if !hasGoodContent && !hasGoodAttachments { + didDelete = true err := DeleteMessage(ctx, msg.ChannelID, msg.ID) if err != nil { - return oops.New(err, "failed to delete message") + return false, oops.New(err, "failed to delete message") } - if msg.Author != nil && !msg.Author.IsBot { + if !msg.Author.IsBot { channel, err := CreateDM(ctx, msg.Author.ID) if err != nil { - return oops.New(err, "failed to create DM channel") + return false, oops.New(err, "failed to create DM channel") } err = SendMessages(ctx, bot.dbConn, MessageToSend{ @@ -47,50 +101,230 @@ func (bot *discordBotInstance) processShowcaseMsg(ctx context.Context, msg *Mess }, }) if err != nil { - return oops.New(err, "failed to send showcase warning message") + return false, oops.New(err, "failed to send showcase warning message") } } } - return nil + return didDelete, nil } -func (bot *discordBotInstance) processLibraryMsg(ctx context.Context, msg *Message) error { - switch msg.Type { - case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: - default: - return nil - } +/* +Ensures that a Discord message is stored in the database. This function is +idempotent and can be called regardless of whether the item already exists in +the database. - if !originalMessageHasField(msg, "content") { - return nil - } +This does not create snippets or do anything besides save the message itself. +*/ +func (bot *discordBotInstance) saveMessage( + ctx context.Context, + tx pgx.Tx, + msg *Message, +) (*models.DiscordMessage, error) { + iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{}, + ` + SELECT $columns + FROM handmade_discordmessage + WHERE id = $1 + `, + msg.ID, + ) + if errors.Is(err, db.ErrNoMatchingRows) { + if !msg.OriginalHasFields("author", "timestamp") { + return nil, errNotEnoughInfo + } - if !messageHasLinks(msg.Content) { - err := DeleteMessage(ctx, msg.ChannelID, msg.ID) + _, err = tx.Exec(ctx, + ` + INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created) + VALUES ($1, $2, $3, $4, $5, $6, $7) + `, + msg.ID, + msg.ChannelID, + *msg.GuildID, + msg.JumpURL(), + msg.Author.ID, + msg.Time(), + false, + ) if err != nil { - return oops.New(err, "failed to delete message") + return nil, oops.New(err, "failed to save new discord message") } - if msg.Author != nil && !msg.Author.IsBot { - channel, err := CreateDM(ctx, msg.Author.ID) - if err != nil { - return oops.New(err, "failed to create DM channel") - } + /* + TODO(db): This is a spot where it would be really nice to be able + to use RETURNING, and avoid this second query. + */ + iDiscordMessage, err = db.QueryOne(ctx, tx, models.DiscordMessage{}, + ` + SELECT $columns + FROM handmade_discordmessage + WHERE id = $1 + `, + msg.ID, + ) + if err != nil { + panic(err) + } + } else if err != nil { + return nil, oops.New(err, "failed to check for existing Discord message") + } - err = SendMessages(ctx, bot.dbConn, MessageToSend{ - ChannelID: channel.ID, - Req: CreateMessageRequest{ - Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.", - }, - }) - if err != nil { - return oops.New(err, "failed to send showcase warning message") - } + return iDiscordMessage.(*models.DiscordMessage), nil +} + +/* +Processes a single Discord message, saving as much of the message's content +and attachments as allowed by our rules and user settings. Does NOT create +snippets. + +Idempotent; can be called any time whether the message exists or not. +*/ +func (bot *discordBotInstance) saveMessageAndContents( + ctx context.Context, + tx pgx.Tx, + msg *Message, +) (*models.DiscordMessage, error) { + newMsg, err := bot.saveMessage(ctx, tx, msg) + if err != nil { + return nil, err + } + + // Check for linked Discord user + iDiscordUser, err := db.QueryOne(ctx, tx, models.DiscordUser{}, + ` + SELECT $columns + FROM handmade_discorduser + WHERE userid = $1 + `, + msg.Author.ID, + ) + if errors.Is(err, db.ErrNoMatchingRows) { + return newMsg, nil + } else if err != nil { + return nil, oops.New(err, "failed to look up linked Discord user") + } + discordUser := iDiscordUser.(*models.DiscordUser) + + // We have a linked Discord account, so save the message contents (regardless of + // whether we create a snippet or not). + + _, err = tx.Exec(ctx, + ` + INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content) + VALUES ($1, $2, $3) + ON CONFLICT (message_id) DO UPDATE SET + discord_id = EXCLUDED.discord_id, + last_content = EXCLUDED.last_content + `, + msg.ID, + discordUser.ID, + msg.Content, // TODO: Add a method that can fill in mentions and stuff (https://discord.com/developers/docs/reference#message-formatting) + ) + + // Save attachments + for _, attachment := range msg.Attachments { + _, err := bot.saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID) + if err != nil { + return nil, oops.New(err, "failed to save attachment") } } - return nil + // TODO: Save embeds + + return newMsg, nil +} + +func (bot *discordBotInstance) saveAttachment(ctx context.Context, tx pgx.Tx, attachment *Attachment, hmnUserID int, discordMessageID string) (*models.DiscordMessageAttachment, error) { + // TODO: Return an existing attachment if it exists + + width := 0 + height := 0 + if attachment.Width != nil { + width = *attachment.Width + } + if attachment.Height != nil { + height = *attachment.Height + } + + // TODO: Timeouts and stuff, context cancellation + res, err := http.Get(attachment.Url) + if err != nil { + return nil, oops.New(err, "failed to fetch attachment data") + } + defer res.Body.Close() + + content, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + } + + asset, err := assets.Create(ctx, tx, assets.CreateInput{ + Content: content, + Filename: attachment.Filename, + MimeType: attachment.ContentType, + + UploaderID: &hmnUserID, + Width: width, + Height: height, + }) + if err != nil { + return nil, oops.New(err, "failed to save asset for Discord attachment") + } + + // TODO(db): RETURNING plz thanks + _, err = tx.Exec(ctx, + ` + INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id) + VALUES ($1, $2, $3) + `, + attachment.ID, + asset.ID, + discordMessageID, + ) + if err != nil { + return nil, oops.New(err, "failed to save Discord attachment data") + } + + iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{}, + ` + SELECT $columns + FROM handmade_discordmessageattachment + WHERE id = $1 + `, + attachment.ID, + ) + if err != nil { + return nil, oops.New(err, "failed to fetch new Discord attachment data") + } + + return iDiscordAttachment.(*models.DiscordMessageAttachment), nil +} + +func (bot *discordBotInstance) allowedToCreateMessageSnippet(ctx context.Context, msg *Message) (bool, error) { + canSave, err := db.QueryBool(ctx, bot.dbConn, + ` + SELECT u.discord_save_showcase + FROM + handmade_discorduser AS duser + JOIN auth_user AS u ON duser.hmn_user_id = u.id + WHERE + duser.userid = $1 + `, + msg.Author.ID, + ) + if errors.Is(err, db.ErrNoMatchingRows) { + return false, nil + } else if err != nil { + return false, oops.New(err, "failed to check if we can save Discord message") + } + + return canSave, nil +} + +func (bot *discordBotInstance) createMessageSnippet(ctx context.Context, msg *Message) (*models.Snippet, error) { + // TODO: Actually do this + return nil, nil } func messageHasLinks(content string) bool { @@ -104,12 +338,3 @@ func messageHasLinks(content string) bool { return false } - -func originalMessageHasField(msg *Message, field string) bool { - if msg.originalMap == nil { - return false - } - - _, ok := msg.originalMap[field] - return ok -} diff --git a/src/discord/todo.txt b/src/discord/todo.txt new file mode 100644 index 0000000..2d683ac --- /dev/null +++ b/src/discord/todo.txt @@ -0,0 +1,43 @@ +the goal: port the old discord showcase bot + +what it does: save #project-showcase posts to your HMN user profile if you have your account linked + +stuff we need to worry about: +- old posts from before you linked your account +- posts that come in while the bot is down +- what to do with posts if you unlink your account +- what to do with posts if you re-link your account +- what to do if you edit the original discord message +- what to do if you delete the original discord message +- the user's preferences re: saving content + - we don't want to save content without the user's consent, especially since it may persist after they disable the integration +- manually adding content for various reasons + - maybe a bug prevented something from saving + - ryan used to post everything in #projects for some reason + + +real-time stuff: +- on new showcase message + - always save the lightweight record + - if we have permission, create a snippet +- on edit + - re-save the lightweight record and content as if it was new + - create snippet, unconditionally???? (bug??) + - update snippet contents if the edit makes sense +- on delete + - delete snippet if the user so desires + - delete the message records + + +background stuff: +- watch mode + - every five seconds + - fetch all HMN users with Discord accounts + - check if we have message records without content + - if so, run a full scrape (no snippets) + - every hour + - run a full scrape, creating snippets +- scrape behavior + - look at every message ever in the channel + - do exactly what the real-time bot does on new messages (although maybe don't do snippets depending on context) + diff --git a/src/main.go b/src/main.go index 6bfb901..96bcdab 100644 --- a/src/main.go +++ b/src/main.go @@ -2,6 +2,7 @@ package main import ( _ "git.handmade.network/hmn/hmn/src/admintools" + _ "git.handmade.network/hmn/hmn/src/assets" _ "git.handmade.network/hmn/hmn/src/buildscss" _ "git.handmade.network/hmn/hmn/src/initimage" _ "git.handmade.network/hmn/hmn/src/migration" From 72ae9383026a94d9e6d1b22c727269dc9c8e3cdc Mon Sep 17 00:00:00 2001 From: Ben Visness Date: Mon, 23 Aug 2021 16:52:57 -0500 Subject: [PATCH 03/10] Save Discord attachments and embeds --- src/assets/assets.go | 12 +- src/discord/gateway.go | 20 +-- src/discord/library.go | 2 +- src/discord/payloads.go | 268 ++++++++++++++++++++++++++++++++++++++-- src/discord/showcase.go | 181 ++++++++++++++++++++++++--- 5 files changed, 436 insertions(+), 47 deletions(-) diff --git a/src/assets/assets.go b/src/assets/assets.go index cf2bc83..bf02a50 100644 --- a/src/assets/assets.go +++ b/src/assets/assets.go @@ -48,9 +48,9 @@ func init() { } type CreateInput struct { - Content []byte - Filename string - MimeType string + Content []byte + Filename string + ContentType string // Optional params UploaderID *int // HMN user id @@ -78,8 +78,8 @@ func Create(ctx context.Context, dbConn db.ConnOrTx, in CreateInput) (*models.As if len(in.Content) == 0 { return nil, InvalidAssetError(fmt.Errorf("could not upload asset '%s': no bytes of data were provided", filename)) } - if in.MimeType == "" { - return nil, InvalidAssetError(fmt.Errorf("could not upload asset '%s': no mime type provided", filename)) + if in.ContentType == "" { + return nil, InvalidAssetError(fmt.Errorf("could not upload asset '%s': no content type provided", filename)) } // Upload the asset to the DO space @@ -128,7 +128,7 @@ func Create(ctx context.Context, dbConn db.ConnOrTx, in CreateInput) (*models.As key, filename, len(in.Content), - in.MimeType, + in.ContentType, checksum, in.Width, in.Height, diff --git a/src/discord/gateway.go b/src/discord/gateway.go index 5962086..abf9185 100644 --- a/src/discord/gateway.go +++ b/src/discord/gateway.go @@ -93,7 +93,7 @@ func RunDiscordBot(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{} { var outgoingMessagesReady = make(chan struct{}, 1) -type discordBotInstance struct { +type botInstance struct { conn *websocket.Conn dbConn *pgxpool.Pool @@ -116,8 +116,8 @@ type discordBotInstance struct { wg sync.WaitGroup } -func newBotInstance(dbConn *pgxpool.Pool) *discordBotInstance { - return &discordBotInstance{ +func newBotInstance(dbConn *pgxpool.Pool) *botInstance { + return &botInstance{ dbConn: dbConn, forceHeartbeat: make(chan struct{}), didAckHeartbeat: true, @@ -129,7 +129,7 @@ Runs a bot instance to completion. It will start up a gateway connection and ret connection is closed. It only returns an error when something unexpected occurs; if so, you should do exponential backoff before reconnecting. Otherwise you can reconnect right away. */ -func (bot *discordBotInstance) Run(ctx context.Context) (err error) { +func (bot *botInstance) Run(ctx context.Context) (err error) { defer utils.RecoverPanicAsError(&err) ctx, bot.cancel = context.WithCancel(ctx) @@ -223,7 +223,7 @@ and RESUMED messages in our main message receiving loop instead of here. That way, we could receive exactly one message after sending Resume, either a Resume ACK or an Invalid Session, and from there it would be crystal clear what to do. Alas!) */ -func (bot *discordBotInstance) connect(ctx context.Context) error { +func (bot *botInstance) connect(ctx context.Context) error { res, err := GetGatewayBot(ctx) if err != nil { return oops.New(err, "failed to get gateway URL") @@ -328,7 +328,7 @@ func (bot *discordBotInstance) connect(ctx context.Context) error { Sends outgoing gateway messages and channel messages. Handles heartbeats. This function should be run as its own goroutine. */ -func (bot *discordBotInstance) doSender(ctx context.Context) { +func (bot *botInstance) doSender(ctx context.Context) { defer bot.wg.Done() defer bot.cancel() @@ -507,7 +507,7 @@ func (bot *discordBotInstance) doSender(ctx context.Context) { } } -func (bot *discordBotInstance) receiveGatewayMessage(ctx context.Context) (*GatewayMessage, error) { +func (bot *botInstance) receiveGatewayMessage(ctx context.Context) (*GatewayMessage, error) { _, msgBytes, err := bot.conn.ReadMessage() if err != nil { return nil, err @@ -524,7 +524,7 @@ func (bot *discordBotInstance) receiveGatewayMessage(ctx context.Context) (*Gate return &msg, nil } -func (bot *discordBotInstance) sendGatewayMessage(ctx context.Context, msg GatewayMessage) error { +func (bot *botInstance) sendGatewayMessage(ctx context.Context, msg GatewayMessage) error { logging.ExtractLogger(ctx).Debug().Interface("msg", msg).Msg("sending gateway message") return bot.conn.WriteMessage(websocket.TextMessage, msg.ToJSON()) } @@ -534,7 +534,7 @@ Processes a single event message from Discord. If this returns an error, it mean really gone wrong, bad enough that the connection should be shut down. Otherwise it will just log any errors that occur. */ -func (bot *discordBotInstance) processEventMsg(ctx context.Context, msg *GatewayMessage) error { +func (bot *botInstance) processEventMsg(ctx context.Context, msg *GatewayMessage) error { if msg.Opcode != OpcodeDispatch { panic(fmt.Sprintf("processEventMsg must only be used on Dispatch messages (opcode %d). Validate this before you call this function.", OpcodeDispatch)) } @@ -562,7 +562,7 @@ func (bot *discordBotInstance) processEventMsg(ctx context.Context, msg *Gateway return nil } -func (bot *discordBotInstance) messageCreateOrUpdate(ctx context.Context, msg *Message) error { +func (bot *botInstance) messageCreateOrUpdate(ctx context.Context, msg *Message) error { if msg.OriginalHasFields("author") && msg.Author.ID == config.Config.Discord.BotUserID { // Don't process your own messages return nil diff --git a/src/discord/library.go b/src/discord/library.go index 3eaa30c..bff896c 100644 --- a/src/discord/library.go +++ b/src/discord/library.go @@ -6,7 +6,7 @@ import ( "git.handmade.network/hmn/hmn/src/oops" ) -func (bot *discordBotInstance) processLibraryMsg(ctx context.Context, msg *Message) error { +func (bot *botInstance) processLibraryMsg(ctx context.Context, msg *Message) error { switch msg.Type { case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: default: diff --git a/src/discord/payloads.go b/src/discord/payloads.go index 18b1b4c..765608e 100644 --- a/src/discord/payloads.go +++ b/src/discord/payloads.go @@ -112,8 +112,9 @@ type Resume struct { type ChannelType int +// https://discord.com/developers/docs/resources/channel#channel-object-channel-types const ( - ChannelTypeGuildext ChannelType = 0 + ChannelTypeGuildText ChannelType = 0 ChannelTypeDM ChannelType = 1 ChannelTypeGuildVoice ChannelType = 2 ChannelTypeGroupDM ChannelType = 3 @@ -126,6 +127,7 @@ const ( ChannelTypeGuildStageVoice ChannelType = 13 ) +// https://discord.com/developers/docs/resources/channel#channel-object type Channel struct { ID string `json:"id"` Type ChannelType `json:"type"` @@ -138,6 +140,7 @@ type Channel struct { type MessageType int +// https://discord.com/developers/docs/resources/channel#message-object-message-types const ( MessageTypeDefault MessageType = 0 @@ -181,7 +184,7 @@ type Message struct { Type MessageType `json:"type"` Attachments []Attachment `json:"attachments"` - // TODO: Embeds + Embeds []Embed `json:"embeds"` originalMap map[string]interface{} } @@ -246,7 +249,12 @@ func MessageFromMap(m interface{}) Message { } } - // TODO: Embeds + if iembeds, ok := mmap["embeds"]; ok { + embeds := iembeds.([]interface{}) + for _, iembed := range embeds { + msg.Embeds = append(msg.Embeds, EmbedFromMap(iembed)) + } + } return msg } @@ -277,15 +285,16 @@ func UserFromMap(m interface{}) User { return u } +// https://discord.com/developers/docs/resources/channel#attachment-object type Attachment struct { - ID string `json:"id"` - Filename string `json:"filename"` - ContentType string `json:"content_type"` - Size int `json:"size"` - Url string `json:"url"` - ProxyUrl string `json:"proxy_url"` - Height *int `json:"height"` - Width *int `json:"width"` + ID string `json:"id"` + Filename string `json:"filename"` + ContentType *string `json:"content_type"` + Size int `json:"size"` + Url string `json:"url"` + ProxyUrl string `json:"proxy_url"` + Height *int `json:"height"` + Width *int `json:"width"` } func AttachmentFromMap(m interface{}) Attachment { @@ -293,7 +302,7 @@ func AttachmentFromMap(m interface{}) Attachment { a := Attachment{ ID: mmap["id"].(string), Filename: mmap["filename"].(string), - ContentType: maybeString(mmap, "content_type"), + ContentType: maybeStringP(mmap, "content_type"), Size: int(mmap["size"].(float64)), Url: mmap["url"].(string), ProxyUrl: mmap["proxy_url"].(string), @@ -304,6 +313,224 @@ func AttachmentFromMap(m interface{}) Attachment { return a } +// https://discord.com/developers/docs/resources/channel#embed-object +type Embed struct { + Title *string `json:"title"` + Type *string `json:"type"` + Description *string `json:"description"` + Url *string `json:"url"` + Timestamp *string `json:"timestamp"` + Color *int `json:"color"` + Footer *EmbedFooter `json:"footer"` + Image *EmbedImage `json:"image"` + Thumbnail *EmbedThumbnail `json:"thumbnail"` + Video *EmbedVideo `json:"video"` + Provider *EmbedProvider `json:"provider"` + Author *EmbedAuthor `json:"author"` + Fields []EmbedField `json:"fields"` +} + +type EmbedFooter struct { + Text string `json:"text"` + IconUrl *string `json:"icon_url"` + ProxyIconUrl *string `json:"proxy_icon_url"` +} + +type EmbedImageish struct { + Url *string `json:"url"` + ProxyUrl *string `json:"proxy_url"` + Height *int `json:"height"` + Width *int `json:"width"` +} + +type EmbedImage struct { + EmbedImageish +} + +type EmbedThumbnail struct { + EmbedImageish +} + +type EmbedVideo struct { + EmbedImageish +} + +type EmbedProvider struct { + Name *string `json:"name"` + Url *string `json:"url"` +} + +type EmbedAuthor struct { + Name *string `json:"name"` + Url *string `json:"url"` + IconUrl *string `json:"icon_url"` + ProxyIconUrl *string `json:"proxy_icon_url"` +} + +type EmbedField struct { + Name string `json:"name"` + Value string `json:"value"` + Inline *bool `json:"inline"` +} + +func EmbedFromMap(m interface{}) Embed { + mmap := m.(map[string]interface{}) + + e := Embed{ + Title: maybeStringP(mmap, "title"), + Type: maybeStringP(mmap, "type"), + Description: maybeStringP(mmap, "description"), + Url: maybeStringP(mmap, "url"), + Timestamp: maybeStringP(mmap, "timestamp"), + Color: maybeIntP(mmap, "color"), + Footer: EmbedFooterFromMap(mmap, "footer"), + Image: EmbedImageFromMap(mmap, "image"), + Thumbnail: EmbedThumbnailFromMap(mmap, "thumbnail"), + Video: EmbedVideoFromMap(mmap, "video"), + Provider: EmbedProviderFromMap(mmap, "provider"), + Author: EmbedAuthorFromMap(mmap, "author"), + Fields: EmbedFieldsFromMap(mmap, "fields"), + } + + return e +} + +func EmbedFooterFromMap(m map[string]interface{}, k string) *EmbedFooter { + f, ok := m[k] + if !ok { + return nil + } + fMap, ok := f.(map[string]interface{}) + if !ok { + return nil + } + + return &EmbedFooter{ + Text: maybeString(fMap, "text"), + IconUrl: maybeStringP(fMap, "icon_url"), + ProxyIconUrl: maybeStringP(fMap, "proxy_icon_url"), + } +} + +func EmbedImageFromMap(m map[string]interface{}, k string) *EmbedImage { + val, ok := m[k] + if !ok { + return nil + } + valMap, ok := val.(map[string]interface{}) + if !ok { + return nil + } + + return &EmbedImage{ + EmbedImageish: EmbedImageish{ + Url: maybeStringP(valMap, "url"), + ProxyUrl: maybeStringP(valMap, "proxy_url"), + Height: maybeIntP(valMap, "height"), + Width: maybeIntP(valMap, "width"), + }, + } +} + +func EmbedThumbnailFromMap(m map[string]interface{}, k string) *EmbedThumbnail { + val, ok := m[k] + if !ok { + return nil + } + valMap, ok := val.(map[string]interface{}) + if !ok { + return nil + } + + return &EmbedThumbnail{ + EmbedImageish: EmbedImageish{ + Url: maybeStringP(valMap, "url"), + ProxyUrl: maybeStringP(valMap, "proxy_url"), + Height: maybeIntP(valMap, "height"), + Width: maybeIntP(valMap, "width"), + }, + } +} + +func EmbedVideoFromMap(m map[string]interface{}, k string) *EmbedVideo { + val, ok := m[k] + if !ok { + return nil + } + valMap, ok := val.(map[string]interface{}) + if !ok { + return nil + } + + return &EmbedVideo{ + EmbedImageish: EmbedImageish{ + Url: maybeStringP(valMap, "url"), + ProxyUrl: maybeStringP(valMap, "proxy_url"), + Height: maybeIntP(valMap, "height"), + Width: maybeIntP(valMap, "width"), + }, + } +} + +func EmbedProviderFromMap(m map[string]interface{}, k string) *EmbedProvider { + val, ok := m[k] + if !ok { + return nil + } + valMap, ok := val.(map[string]interface{}) + if !ok { + return nil + } + + return &EmbedProvider{ + Name: maybeStringP(valMap, "name"), + Url: maybeStringP(valMap, "url"), + } +} + +func EmbedAuthorFromMap(m map[string]interface{}, k string) *EmbedAuthor { + val, ok := m[k] + if !ok { + return nil + } + valMap, ok := val.(map[string]interface{}) + if !ok { + return nil + } + + return &EmbedAuthor{ + Name: maybeStringP(valMap, "name"), + Url: maybeStringP(valMap, "url"), + } +} + +func EmbedFieldsFromMap(m map[string]interface{}, k string) []EmbedField { + val, ok := m[k] + if !ok { + return nil + } + valSlice, ok := val.([]interface{}) + if !ok { + return nil + } + + var result []EmbedField + for _, innerVal := range valSlice { + valMap, ok := innerVal.(map[string]interface{}) + if !ok { + continue + } + + result = append(result, EmbedField{ + Name: maybeString(valMap, "name"), + Value: maybeString(valMap, "value"), + Inline: maybeBoolP(valMap, "inline"), + }) + } + + return result +} + func maybeString(m map[string]interface{}, k string) string { val, ok := m[k] if !ok { @@ -337,3 +564,20 @@ func maybeIntP(m map[string]interface{}, k string) *int { intval := int(val.(float64)) return &intval } + +func maybeBool(m map[string]interface{}, k string) bool { + val, ok := m[k] + if !ok { + return false + } + return val.(bool) +} + +func maybeBoolP(m map[string]interface{}, k string) *bool { + val, ok := m[k] + if !ok { + return nil + } + boolval := val.(bool) + return &boolval +} diff --git a/src/discord/showcase.go b/src/discord/showcase.go index 65abbdc..a0de713 100644 --- a/src/discord/showcase.go +++ b/src/discord/showcase.go @@ -3,17 +3,20 @@ package discord import ( "context" "errors" + "fmt" "io" "net/http" "net/url" "regexp" "strings" + "time" "git.handmade.network/hmn/hmn/src/assets" "git.handmade.network/hmn/hmn/src/db" "git.handmade.network/hmn/hmn/src/logging" "git.handmade.network/hmn/hmn/src/models" "git.handmade.network/hmn/hmn/src/oops" + "github.com/google/uuid" "github.com/jackc/pgx/v4" ) @@ -21,7 +24,8 @@ var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`) var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this") -func (bot *discordBotInstance) processShowcaseMsg(ctx context.Context, msg *Message) error { +// TODO: Can this function be called asynchronously? +func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) error { switch msg.Type { case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: default: @@ -69,7 +73,7 @@ func (bot *discordBotInstance) processShowcaseMsg(ctx context.Context, msg *Mess return nil } -func (bot *discordBotInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message) (didDelete bool, err error) { +func (bot *botInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message) (didDelete bool, err error) { hasGoodContent := true if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) { hasGoodContent = false @@ -116,7 +120,7 @@ the database. This does not create snippets or do anything besides save the message itself. */ -func (bot *discordBotInstance) saveMessage( +func (bot *botInstance) saveMessage( ctx context.Context, tx pgx.Tx, msg *Message, @@ -180,7 +184,7 @@ snippets. Idempotent; can be called any time whether the message exists or not. */ -func (bot *discordBotInstance) saveMessageAndContents( +func (bot *botInstance) saveMessageAndContents( ctx context.Context, tx pgx.Tx, msg *Message, @@ -230,12 +234,53 @@ func (bot *discordBotInstance) saveMessageAndContents( } } - // TODO: Save embeds + // Save embeds + for _, embed := range msg.Embeds { + _, err := bot.saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID) + if err != nil { + return nil, oops.New(err, "failed to save embed") + } + } return newMsg, nil } -func (bot *discordBotInstance) saveAttachment(ctx context.Context, tx pgx.Tx, attachment *Attachment, hmnUserID int, discordMessageID string) (*models.DiscordMessageAttachment, error) { +var discordDownloadClient = &http.Client{ + Timeout: 10 * time.Second, +} + +type DiscordResourceBadStatusCode error + +func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, "", oops.New(err, "failed to make Discord download request") + } + res, err := discordDownloadClient.Do(req) + if err != nil { + return nil, "", oops.New(err, "failed to fetch Discord resource data") + } + defer res.Body.Close() + + if res.StatusCode < 200 || 299 < res.StatusCode { + return nil, "", DiscordResourceBadStatusCode(fmt.Errorf("status code %d from Discord resource: %s", res.StatusCode, url)) + } + + content, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + } + + return content, res.Header.Get("Content-Type"), nil +} + +func (bot *botInstance) saveAttachment( + ctx context.Context, + tx pgx.Tx, + attachment *Attachment, + hmnUserID int, + discordMessageID string, +) (*models.DiscordMessageAttachment, error) { // TODO: Return an existing attachment if it exists width := 0 @@ -247,22 +292,20 @@ func (bot *discordBotInstance) saveAttachment(ctx context.Context, tx pgx.Tx, at height = *attachment.Height } - // TODO: Timeouts and stuff, context cancellation - res, err := http.Get(attachment.Url) + content, _, err := downloadDiscordResource(ctx, attachment.Url) if err != nil { - return nil, oops.New(err, "failed to fetch attachment data") + return nil, oops.New(err, "failed to download Discord attachment") } - defer res.Body.Close() - content, err := io.ReadAll(res.Body) - if err != nil { - panic(err) + contentType := "application/octet-stream" + if attachment.ContentType != nil { + contentType = *attachment.ContentType } asset, err := assets.Create(ctx, tx, assets.CreateInput{ - Content: content, - Filename: attachment.Filename, - MimeType: attachment.ContentType, + Content: content, + Filename: attachment.Filename, + ContentType: contentType, UploaderID: &hmnUserID, Width: width, @@ -301,7 +344,109 @@ func (bot *discordBotInstance) saveAttachment(ctx context.Context, tx pgx.Tx, at return iDiscordAttachment.(*models.DiscordMessageAttachment), nil } -func (bot *discordBotInstance) allowedToCreateMessageSnippet(ctx context.Context, msg *Message) (bool, error) { +func (bot *botInstance) saveEmbed( + ctx context.Context, + tx pgx.Tx, + embed *Embed, + hmnUserID int, + discordMessageID string, +) (*models.DiscordMessageEmbed, error) { + // TODO: Does this need to be idempotent + + isOkImageType := func(contentType string) bool { + return strings.HasPrefix(contentType, "image/") + } + + isOkVideoType := func(contentType string) bool { + return strings.HasPrefix(contentType, "video/") + } + + maybeSaveImageish := func(i EmbedImageish, contentTypeCheck func(string) bool) (*uuid.UUID, error) { + content, contentType, err := downloadDiscordResource(ctx, *i.Url) + if err != nil { + var statusError DiscordResourceBadStatusCode + if errors.As(err, &statusError) { + return nil, nil + } else { + return nil, oops.New(err, "failed to save Discord embed") + } + } + if contentTypeCheck(contentType) { + in := assets.CreateInput{ + Content: content, + Filename: "embed", + ContentType: contentType, + UploaderID: &hmnUserID, + } + + if i.Width != nil { + in.Width = *i.Width + } + if i.Height != nil { + in.Height = *i.Height + } + + asset, err := assets.Create(ctx, tx, in) + if err != nil { + return nil, oops.New(err, "failed to create asset from embed") + } + return &asset.ID, nil + } + + return nil, nil + } + + var imageAssetId *uuid.UUID + var videoAssetId *uuid.UUID + var err error + + if embed.Video != nil && embed.Video.Url != nil { + videoAssetId, err = maybeSaveImageish(embed.Video.EmbedImageish, isOkVideoType) + } else if embed.Image != nil && embed.Image.Url != nil { + imageAssetId, err = maybeSaveImageish(embed.Image.EmbedImageish, isOkImageType) + } else if embed.Thumbnail != nil && embed.Thumbnail.Url != nil { + imageAssetId, err = maybeSaveImageish(embed.Thumbnail.EmbedImageish, isOkImageType) + } + if err != nil { + return nil, err + } + + // Save the embed into the db + // TODO(db): Insert, RETURNING + var savedEmbedId int + err = tx.QueryRow(ctx, + ` + INSERT INTO handmade_discordmessageembed (title, description, url, message_id, image_id, video_id) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id + `, + embed.Title, + embed.Description, + embed.Url, + discordMessageID, + imageAssetId, + videoAssetId, + ).Scan(&savedEmbedId) + if err != nil { + return nil, oops.New(err, "failed to insert new embed") + } + + iDiscordEmbed, err := db.QueryOne(ctx, tx, models.DiscordMessageEmbed{}, + ` + SELECT $columns + FROM handmade_discordmessageembed + WHERE id = $1 + `, + savedEmbedId, + ) + if err != nil { + return nil, oops.New(err, "failed to fetch new Discord embed data") + } + + return iDiscordEmbed.(*models.DiscordMessageEmbed), nil +} + +func (bot *botInstance) allowedToCreateMessageSnippet(ctx context.Context, msg *Message) (bool, error) { canSave, err := db.QueryBool(ctx, bot.dbConn, ` SELECT u.discord_save_showcase @@ -322,7 +467,7 @@ func (bot *discordBotInstance) allowedToCreateMessageSnippet(ctx context.Context return canSave, nil } -func (bot *discordBotInstance) createMessageSnippet(ctx context.Context, msg *Message) (*models.Snippet, error) { +func (bot *botInstance) createMessageSnippet(ctx context.Context, msg *Message) (*models.Snippet, error) { // TODO: Actually do this return nil, nil } From ec64babdd6ef3d4677d088ef60815f41ff36c55f Mon Sep 17 00:00:00 2001 From: Ben Visness Date: Mon, 23 Aug 2021 19:49:39 -0500 Subject: [PATCH 04/10] Create snippets! --- src/discord/showcase.go | 198 ++++++++++++++++-- .../2021-08-23T230559Z_DiscordDefaults.go | 55 +++++ src/models/snippet.go | 2 +- 3 files changed, 231 insertions(+), 24 deletions(-) create mode 100644 src/migration/migrations/2021-08-23T230559Z_DiscordDefaults.go diff --git a/src/discord/showcase.go b/src/discord/showcase.go index a0de713..ba3a1cd 100644 --- a/src/discord/showcase.go +++ b/src/discord/showcase.go @@ -47,7 +47,7 @@ func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) er defer tx.Rollback(ctx) // save the message, maybe save its contents, and maybe make a snippet too - _, err = bot.saveMessageAndContents(ctx, tx, msg) + newMsg, err := bot.saveMessageAndContents(ctx, tx, msg) if errors.Is(err, errNotEnoughInfo) { logging.ExtractLogger(ctx).Warn(). Interface("msg", msg). @@ -56,8 +56,8 @@ func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) er } else if err != nil { return err } - if doSnippet, err := bot.allowedToCreateMessageSnippet(ctx, msg); doSnippet && err == nil { - _, err := bot.createMessageSnippet(ctx, msg) + if doSnippet, err := bot.allowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil { + _, err := bot.createMessageSnippet(ctx, tx, msg) if err != nil { return oops.New(err, "failed to create snippet in gateway") } @@ -201,7 +201,7 @@ func (bot *botInstance) saveMessageAndContents( FROM handmade_discorduser WHERE userid = $1 `, - msg.Author.ID, + newMsg.UserID, ) if errors.Is(err, db.ErrNoMatchingRows) { return newMsg, nil @@ -213,18 +213,20 @@ func (bot *botInstance) saveMessageAndContents( // We have a linked Discord account, so save the message contents (regardless of // whether we create a snippet or not). - _, err = tx.Exec(ctx, - ` - INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content) - VALUES ($1, $2, $3) - ON CONFLICT (message_id) DO UPDATE SET - discord_id = EXCLUDED.discord_id, - last_content = EXCLUDED.last_content - `, - msg.ID, - discordUser.ID, - msg.Content, // TODO: Add a method that can fill in mentions and stuff (https://discord.com/developers/docs/reference#message-formatting) - ) + if msg.OriginalHasFields("content") { + _, err = tx.Exec(ctx, + ` + INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content) + VALUES ($1, $2, $3) + ON CONFLICT (message_id) DO UPDATE SET + discord_id = EXCLUDED.discord_id, + last_content = EXCLUDED.last_content + `, + newMsg.ID, + discordUser.ID, + msg.Content, // TODO: Add a method that can fill in mentions and stuff (https://discord.com/developers/docs/reference#message-formatting) + ) + } // Save attachments for _, attachment := range msg.Attachments { @@ -274,6 +276,10 @@ func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, e return content, res.Header.Get("Content-Type"), nil } +/* +Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment +that already exists +*/ func (bot *botInstance) saveAttachment( ctx context.Context, tx pgx.Tx, @@ -281,7 +287,21 @@ func (bot *botInstance) saveAttachment( hmnUserID int, discordMessageID string, ) (*models.DiscordMessageAttachment, error) { - // TODO: Return an existing attachment if it exists + iexisting, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{}, + ` + SELECT $columns + FROM handmade_discordmessageattachment + WHERE id = $1 + `, + attachment.ID, + ) + if err == nil { + return iexisting.(*models.DiscordMessageAttachment), nil + } else if errors.Is(err, db.ErrNoMatchingRows) { + // this is fine, just create it + } else { + return nil, oops.New(err, "failed to check for existing attachment") + } width := 0 height := 0 @@ -351,7 +371,8 @@ func (bot *botInstance) saveEmbed( hmnUserID int, discordMessageID string, ) (*models.DiscordMessageEmbed, error) { - // TODO: Does this need to be idempotent + // TODO: Does this need to be idempotent? Embeds don't have IDs... + // Maybe Discord will never actually send us the same embed twice? isOkImageType := func(contentType string) bool { return strings.HasPrefix(contentType, "image/") @@ -446,7 +467,7 @@ func (bot *botInstance) saveEmbed( return iDiscordEmbed.(*models.DiscordMessageEmbed), nil } -func (bot *botInstance) allowedToCreateMessageSnippet(ctx context.Context, msg *Message) (bool, error) { +func (bot *botInstance) allowedToCreateMessageSnippet(ctx context.Context, tx pgx.Tx, discordUserId string) (bool, error) { canSave, err := db.QueryBool(ctx, bot.dbConn, ` SELECT u.discord_save_showcase @@ -456,7 +477,7 @@ func (bot *botInstance) allowedToCreateMessageSnippet(ctx context.Context, msg * WHERE duser.userid = $1 `, - msg.Author.ID, + discordUserId, ) if errors.Is(err, db.ErrNoMatchingRows) { return false, nil @@ -467,9 +488,140 @@ func (bot *botInstance) allowedToCreateMessageSnippet(ctx context.Context, msg * return canSave, nil } -func (bot *botInstance) createMessageSnippet(ctx context.Context, msg *Message) (*models.Snippet, error) { - // TODO: Actually do this - return nil, nil +func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg *Message) (*models.Snippet, error) { + // Check for existing snippet, maybe return it + type existingSnippetResult struct { + Message models.DiscordMessage `db:"msg"` + MessageContent *models.DiscordMessageContent `db:"c"` + Snippet *models.Snippet `db:"snippet"` + DiscordUser *models.DiscordUser `db:"duser"` + } + iexisting, err := db.QueryOne(ctx, tx, existingSnippetResult{}, + ` + SELECT $columns + FROM + handmade_discordmessage AS msg + LEFT JOIN handmade_discordmessagecontent AS c ON c.message_id = msg.id + LEFT JOIN handmade_snippet AS snippet ON snippet.discord_message_id = msg.id + LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid + WHERE + msg.id = $1 + `, + msg.ID, + ) + if err != nil { + return nil, oops.New(err, "failed to check for existing snippet") + } + existing := iexisting.(*existingSnippetResult) + + if existing.Snippet != nil { + // A snippet already exists + return existing.Snippet, nil + } + + if existing.Message.SnippetCreated { + // A snippet once existed but no longer does + // (we do not create another one in this case) + return nil, nil + } + + if existing.MessageContent == nil || existing.DiscordUser == nil { + return nil, nil + } + + // Get an asset ID or URL to make a snippet from + assetId, url, err := bot.getSnippetAssetOrUrl(ctx, tx, &existing.Message) + if assetId == nil && url == "" { + // Nothing to make a snippet from! + return nil, nil + } + + contentMarkdown := existing.MessageContent.LastContent + contentHTML := contentMarkdown // TODO: Actually parse Discord's Markdown + + // TODO(db): Insert + isnippet, err := db.QueryOne(ctx, tx, models.Snippet{}, + ` + INSERT INTO handmade_snippet (url, "when", description, _description_html, asset_id, discord_message_id, owner_id) + VALUES ($1, $2, $3, $4, $5, $6, $7) + RETURNING $columns + `, + nil, + existing.Message.SentAt, + contentMarkdown, + contentHTML, + assetId, + msg.ID, + existing.DiscordUser.HMNUserId, + ) + if err != nil { + return nil, oops.New(err, "failed to create snippet from attachment") + } + _, err = tx.Exec(ctx, + ` + UPDATE handmade_discordmessage + SET snippet_created = TRUE + WHERE id = $1 + `, + msg.ID, + ) + if err != nil { + return nil, oops.New(err, "failed to mark message as having snippet") + } + + return isnippet.(*models.Snippet), nil +} + +// NOTE(ben): This is maybe redundant with the regexes we use for markdown. But +// do we actually want to reuse those, or should we keep them separate? +var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`) + +func (bot *botInstance) getSnippetAssetOrUrl(ctx context.Context, tx pgx.Tx, msg *models.DiscordMessage) (*uuid.UUID, string, error) { + // Check attachments + itAttachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{}, + ` + SELECT $columns + FROM handmade_discordmessageattachment + WHERE message_id = $1 + `, + msg.ID, + ) + if err != nil { + return nil, "", oops.New(err, "failed to fetch message attachments") + } + attachments := itAttachments.ToSlice() + for _, iattachment := range attachments { + attachment := iattachment.(*models.DiscordMessageAttachment) + return &attachment.AssetID, "", nil + } + + // Check embeds + itEmbeds, err := db.Query(ctx, tx, models.DiscordMessageEmbed{}, + ` + SELECT $columns + FROM handmade_discordmessageembed + WHERE message_id = $1 + `, + msg.ID, + ) + if err != nil { + return nil, "", oops.New(err, "failed to fetch discord embeds") + } + embeds := itEmbeds.ToSlice() + for _, iembed := range embeds { + embed := iembed.(*models.DiscordMessageEmbed) + if embed.VideoID != nil { + return embed.VideoID, "", nil + } else if embed.ImageID != nil { + return embed.ImageID, "", nil + } else if embed.URL != nil { + if RESnippetableUrl.MatchString(*embed.URL) { + return nil, *embed.URL, nil + } + } + } + + return nil, "", nil } func messageHasLinks(content string) bool { diff --git a/src/migration/migrations/2021-08-23T230559Z_DiscordDefaults.go b/src/migration/migrations/2021-08-23T230559Z_DiscordDefaults.go new file mode 100644 index 0000000..b06e3b2 --- /dev/null +++ b/src/migration/migrations/2021-08-23T230559Z_DiscordDefaults.go @@ -0,0 +1,55 @@ +package migrations + +import ( + "context" + "time" + + "git.handmade.network/hmn/hmn/src/migration/types" + "git.handmade.network/hmn/hmn/src/oops" + "github.com/jackc/pgx/v4" +) + +func init() { + registerMigration(DiscordDefaults{}) +} + +type DiscordDefaults struct{} + +func (m DiscordDefaults) Version() types.MigrationVersion { + return types.MigrationVersion(time.Date(2021, 8, 23, 23, 5, 59, 0, time.UTC)) +} + +func (m DiscordDefaults) Name() string { + return "DiscordDefaults" +} + +func (m DiscordDefaults) Description() string { + return "Add some default values to Discord fields" +} + +func (m DiscordDefaults) Up(ctx context.Context, tx pgx.Tx) error { + var err error + + _, err = tx.Exec(ctx, ` + ALTER TABLE handmade_discordmessage + ALTER snippet_created SET DEFAULT FALSE; + `) + if err != nil { + return oops.New(err, "failed to set message defaults") + } + + _, err = tx.Exec(ctx, ` + ALTER TABLE handmade_snippet + ALTER "when" SET DEFAULT NOW(), + ALTER edited_on_website SET DEFAULT FALSE; + `) + if err != nil { + return oops.New(err, "failed to set snippet defaults") + } + + return nil +} + +func (m DiscordDefaults) Down(ctx context.Context, tx pgx.Tx) error { + panic("Implement me") +} diff --git a/src/models/snippet.go b/src/models/snippet.go index 3721e22..9310e6e 100644 --- a/src/models/snippet.go +++ b/src/models/snippet.go @@ -10,7 +10,7 @@ type Snippet struct { ID int `db:"id"` OwnerID int `db:"owner_id"` - When time.Time `db:"when"` + When time.Time `db:"\"when\""` Description string `db:"description"` DescriptionHtml string `db:"_description_html"` From 69ead1f34776bfd18845fe3997b1845aca6b7d8b Mon Sep 17 00:00:00 2001 From: Ben Visness Date: Mon, 23 Aug 2021 20:06:32 -0500 Subject: [PATCH 05/10] Handle message updates better --- src/discord/showcase.go | 68 +++++++++++++++++++++++++++++++++++------ 1 file changed, 59 insertions(+), 9 deletions(-) diff --git a/src/discord/showcase.go b/src/discord/showcase.go index ba3a1cd..d43359f 100644 --- a/src/discord/showcase.go +++ b/src/discord/showcase.go @@ -229,18 +229,48 @@ func (bot *botInstance) saveMessageAndContents( } // Save attachments - for _, attachment := range msg.Attachments { - _, err := bot.saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID) - if err != nil { - return nil, oops.New(err, "failed to save attachment") + if msg.OriginalHasFields("attachments") { + for _, attachment := range msg.Attachments { + _, err := bot.saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID) + if err != nil { + return nil, oops.New(err, "failed to save attachment") + } } } - // Save embeds - for _, embed := range msg.Embeds { - _, err := bot.saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID) + // Save / delete embeds + if msg.OriginalHasFields("embeds") { + numSavedEmbeds, err := db.QueryInt(ctx, tx, + ` + SELECT COUNT(*) + FROM handmade_discordmessageembed + WHERE message_id = $1 + `, + msg.ID, + ) if err != nil { - return nil, oops.New(err, "failed to save embed") + return nil, oops.New(err, "failed to count existing embeds") + } + if numSavedEmbeds == 0 { + // No embeds yet, so save new ones + for _, embed := range msg.Embeds { + _, err := bot.saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID) + if err != nil { + return nil, oops.New(err, "failed to save embed") + } + } + } else if len(msg.Embeds) > 0 { + // Embeds were removed from the message + _, err := tx.Exec(ctx, + ` + DELETE FROM handmade_discordmessageembed + WHERE message_id = $1 + `, + msg.ID, + ) + if err != nil { + return nil, oops.New(err, "failed to delete embeds") + } } } @@ -515,7 +545,27 @@ func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg existing := iexisting.(*existingSnippetResult) if existing.Snippet != nil { - // A snippet already exists + // A snippet already exists - maybe update its content, then return it + if msg.OriginalHasFields("content") && !existing.Snippet.EditedOnWebsite { + contentMarkdown := msg.Content + contentHTML := contentMarkdown // TODO: Parse Markdown's HTML + + _, err := tx.Exec(ctx, + ` + UPDATE handmade_snippet + SET + description = $1, + _description_html = $2 + WHERE id = $3 + `, + contentMarkdown, + contentHTML, + existing.Snippet.ID, + ) + if err != nil { + logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to update content of snippet on message edit") + } + } return existing.Snippet, nil } From 7d5590ee10be509db5a0287b20228dcab9c340ff Mon Sep 17 00:00:00 2001 From: Ben Visness Date: Mon, 23 Aug 2021 22:26:27 -0500 Subject: [PATCH 06/10] Process Discord markdown for snippets --- src/discord/markdown.go | 147 +++++++++++++++++++++++++++++++++++ src/discord/markdown_test.go | 38 +++++++++ src/discord/payloads.go | 14 ++++ src/discord/rest.go | 98 +++++++++++++++++++++++ src/discord/showcase.go | 10 +-- src/discord/todo.txt | 4 + src/models/discord.go | 2 +- 7 files changed, 307 insertions(+), 6 deletions(-) create mode 100644 src/discord/markdown.go create mode 100644 src/discord/markdown_test.go diff --git a/src/discord/markdown.go b/src/discord/markdown.go new file mode 100644 index 0000000..313bc7d --- /dev/null +++ b/src/discord/markdown.go @@ -0,0 +1,147 @@ +package discord + +import ( + "context" + "errors" + "fmt" + "regexp" + "strings" + "sync" + + "git.handmade.network/hmn/hmn/src/config" + "git.handmade.network/hmn/hmn/src/logging" +) + +var ( + REMarkdownUser = regexp.MustCompile(`<@([0-9]+)>`) + REMarkdownUserNickname = regexp.MustCompile(`<@!([0-9]+)>`) + REMarkdownChannel = regexp.MustCompile(`<#([0-9]+)>`) + REMarkdownRole = regexp.MustCompile(`<@&([0-9]+)>`) + REMarkdownCustomEmoji = regexp.MustCompile(``) // includes animated + REMarkdownTimestamp = regexp.MustCompile(``) +) + +func CleanUpMarkdown(ctx context.Context, original string) string { + userMatches := REMarkdownUser.FindAllStringSubmatch(original, -1) + userNicknameMatches := REMarkdownUserNickname.FindAllStringSubmatch(original, -1) + channelMatches := REMarkdownChannel.FindAllStringSubmatch(original, -1) + roleMatches := REMarkdownRole.FindAllStringSubmatch(original, -1) + customEmojiMatches := REMarkdownCustomEmoji.FindAllStringSubmatch(original, -1) + timestampMatches := REMarkdownTimestamp.FindAllStringSubmatch(original, -1) + + userIdsToFetch := map[string]struct{}{} + + for _, m := range userMatches { + userIdsToFetch[m[1]] = struct{}{} + } + for _, m := range userNicknameMatches { + userIdsToFetch[m[1]] = struct{}{} + } + + // do the requests, gathering the resulting data + userNames := map[string]string{} + userNicknames := map[string]string{} + channelNames := map[string]string{} + roleNames := map[string]string{} + var wg sync.WaitGroup + var mutex sync.Mutex + + for userId := range userIdsToFetch { + wg.Add(1) + go func(ctx context.Context, userId string) { + defer wg.Done() + member, err := GetGuildMember(ctx, config.Config.Discord.GuildID, userId) + if err != nil { + if errors.Is(err, NotFound) { + // not a problem + } else if err != nil { + logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to fetch guild member for markdown") + } + return + } + func() { + mutex.Lock() + defer mutex.Unlock() + if member.User != nil { + userNames[userId] = member.User.Username + } + if member.Nick != nil { + userNicknames[userId] = *member.Nick + } + }() + }(ctx, userId) + } + + if len(channelMatches) > 0 { + wg.Add(1) + go func(ctx context.Context) { + defer wg.Done() + channels, err := GetGuildChannels(ctx, config.Config.Discord.GuildID) + if err != nil { + logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to fetch channels for markdown") + return + } + for _, channel := range channels { + channelNames[channel.ID] = channel.Name + } + }(ctx) + } + + if len(roleMatches) > 0 { + wg.Add(1) + go func(ctx context.Context) { + defer wg.Done() + roles, err := GetGuildRoles(ctx, config.Config.Discord.GuildID) + if err != nil { + logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to fetch roles for markdown") + return + } + for _, role := range roles { + roleNames[role.ID] = role.Name + } + }(ctx) + } + + wg.Wait() + + // Replace all the everything + res := original + for _, m := range userMatches { + resultName := "Unknown User" + if name, ok := userNames[m[1]]; ok { + resultName = name + } + res = strings.Replace(res, m[0], fmt.Sprintf("@%s", resultName), 1) + } + for _, m := range userNicknameMatches { + resultName := "Unknown User" + if name, ok := userNicknames[m[1]]; ok { + resultName = name + } else if name, ok := userNames[m[1]]; ok { + resultName = name + } + res = strings.Replace(res, m[0], fmt.Sprintf("@%s", resultName), 1) + } + for _, m := range channelMatches { + resultName := "Unknown Channel" + if name, ok := channelNames[m[1]]; ok { + resultName = name + } + res = strings.Replace(res, m[0], fmt.Sprintf("#%s", resultName), 1) + } + for _, m := range roleMatches { + resultName := "Unknown Role" + if name, ok := roleNames[m[1]]; ok { + resultName = name + } + res = strings.Replace(res, m[0], fmt.Sprintf("@%s", resultName), 1) + } + for _, m := range customEmojiMatches { + res = strings.Replace(res, m[0], fmt.Sprintf(":%s:", m[1]), 1) + } + for _, m := range timestampMatches { + res = strings.Replace(res, m[0], "", 1) // TODO: Actual timestamp stuff? Is it worth it? + } + + return res +} diff --git a/src/discord/markdown_test.go b/src/discord/markdown_test.go new file mode 100644 index 0000000..6c55373 --- /dev/null +++ b/src/discord/markdown_test.go @@ -0,0 +1,38 @@ +package discord + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestCleanUpMarkdown(t *testing.T) { + t.Skip("Skipping these tests because they are server-specific and make network requests. Feel free to re-enable, but don't commit :)") + + const userBen = "<@!132715550571888640>" + const channelShowcaseTest = "<#759497527883202582>" + const roleHmnMember = "<@&876685379770646538>" + + t.Run("normal behavior", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + assert.Equal(t, "@Frogbot some stuff", CleanUpMarkdown(ctx, "<@!745051593728196732> some stuff")) + assert.Equal(t, + "users: @Unknown User @bvisness @bvisness, channels: #Unknown Channel #showcase-test #showcase-test, roles: @Unknown Role @HMN Member @HMN Member, :shakefist: also normal text", + CleanUpMarkdown(ctx, fmt.Sprintf("users: <@!000000> %s %s, channels: <#000000> %s %s, roles: <@&000000> %s %s, also normal text", userBen, userBen, channelShowcaseTest, channelShowcaseTest, roleHmnMember, roleHmnMember)), + ) + }) + t.Run("context cancellation", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // immediately cancel + + assert.Equal(t, + "@Unknown User #Unknown Channel @Unknown Role", + CleanUpMarkdown(ctx, fmt.Sprintf("%s %s %s", userBen, channelShowcaseTest, roleHmnMember)), + ) + }) +} diff --git a/src/discord/payloads.go b/src/discord/payloads.go index 765608e..0b0f786 100644 --- a/src/discord/payloads.go +++ b/src/discord/payloads.go @@ -127,6 +127,13 @@ const ( ChannelTypeGuildStageVoice ChannelType = 13 ) +// https://discord.com/developers/docs/topics/permissions#role-object +type Role struct { + ID string `json:"id"` + Name string `json:"name"` + // more fields not yet present +} + // https://discord.com/developers/docs/resources/channel#channel-object type Channel struct { ID string `json:"id"` @@ -285,6 +292,13 @@ func UserFromMap(m interface{}) User { return u } +// https://discord.com/developers/docs/resources/guild#guild-member-object +type GuildMember struct { + User *User `json:"user"` + Nick *string `json:"nick"` + // more fields not yet handled here +} + // https://discord.com/developers/docs/resources/channel#attachment-object type Attachment struct { ID string `json:"id"` diff --git a/src/discord/rest.go b/src/discord/rest.go index e11dae5..8387e11 100644 --- a/src/discord/rest.go +++ b/src/discord/rest.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -26,6 +27,8 @@ const ( var UserAgent = fmt.Sprintf("%s (%s, %s)", BotName, UserAgentURL, UserAgentVersion) +var NotFound = errors.New("not found") + var httpClient = &http.Client{} func buildUrl(path string) string { @@ -83,6 +86,101 @@ func GetGatewayBot(ctx context.Context) (*GetGatewayBotResponse, error) { return &result, nil } +func GetGuildRoles(ctx context.Context, guildID string) ([]Role, error) { + const name = "Get Guild Roles" + + path := fmt.Sprintf("/guilds/%s/roles", guildID) + res, err := doWithRateLimiting(ctx, name, func(ctx context.Context) *http.Request { + return makeRequest(ctx, http.MethodGet, path, nil) + }) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode >= 400 { + logErrorResponse(ctx, name, res, "") + return nil, oops.New(nil, "received error from Discord") + } + + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + } + + var roles []Role + err = json.Unmarshal(bodyBytes, &roles) + if err != nil { + return nil, oops.New(err, "failed to unmarshal Discord message") + } + + return roles, nil +} + +func GetGuildChannels(ctx context.Context, guildID string) ([]Channel, error) { + const name = "Get Guild Channels" + + path := fmt.Sprintf("/guilds/%s/channels", guildID) + res, err := doWithRateLimiting(ctx, name, func(ctx context.Context) *http.Request { + return makeRequest(ctx, http.MethodGet, path, nil) + }) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode >= 400 { + logErrorResponse(ctx, name, res, "") + return nil, oops.New(nil, "received error from Discord") + } + + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + } + + var channels []Channel + err = json.Unmarshal(bodyBytes, &channels) + if err != nil { + return nil, oops.New(err, "failed to unmarshal Discord message") + } + + return channels, nil +} + +func GetGuildMember(ctx context.Context, guildID, userID string) (*GuildMember, error) { + const name = "Get Guild Member" + + path := fmt.Sprintf("/guilds/%s/members/%s", guildID, userID) + res, err := doWithRateLimiting(ctx, name, func(ctx context.Context) *http.Request { + return makeRequest(ctx, http.MethodGet, path, nil) + }) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode == http.StatusNotFound { + return nil, NotFound + } else if res.StatusCode >= 400 { + logErrorResponse(ctx, name, res, "") + return nil, oops.New(nil, "received error from Discord") + } + + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + } + + var msg GuildMember + err = json.Unmarshal(bodyBytes, &msg) + if err != nil { + return nil, oops.New(err, "failed to unmarshal Discord message") + } + + return &msg, nil +} + type CreateMessageRequest struct { Content string `json:"content"` } diff --git a/src/discord/showcase.go b/src/discord/showcase.go index d43359f..3f1f68a 100644 --- a/src/discord/showcase.go +++ b/src/discord/showcase.go @@ -16,6 +16,7 @@ import ( "git.handmade.network/hmn/hmn/src/logging" "git.handmade.network/hmn/hmn/src/models" "git.handmade.network/hmn/hmn/src/oops" + "git.handmade.network/hmn/hmn/src/parsing" "github.com/google/uuid" "github.com/jackc/pgx/v4" ) @@ -24,7 +25,6 @@ var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`) var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this") -// TODO: Can this function be called asynchronously? func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) error { switch msg.Type { case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: @@ -224,7 +224,7 @@ func (bot *botInstance) saveMessageAndContents( `, newMsg.ID, discordUser.ID, - msg.Content, // TODO: Add a method that can fill in mentions and stuff (https://discord.com/developers/docs/reference#message-formatting) + CleanUpMarkdown(ctx, msg.Content), ) } @@ -547,8 +547,8 @@ func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg if existing.Snippet != nil { // A snippet already exists - maybe update its content, then return it if msg.OriginalHasFields("content") && !existing.Snippet.EditedOnWebsite { - contentMarkdown := msg.Content - contentHTML := contentMarkdown // TODO: Parse Markdown's HTML + contentMarkdown := existing.MessageContent.LastContent + contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.RealMarkdown) _, err := tx.Exec(ctx, ` @@ -587,7 +587,7 @@ func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg } contentMarkdown := existing.MessageContent.LastContent - contentHTML := contentMarkdown // TODO: Actually parse Discord's Markdown + contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.RealMarkdown) // TODO(db): Insert isnippet, err := db.QueryOne(ctx, tx, models.Snippet{}, diff --git a/src/discord/todo.txt b/src/discord/todo.txt index 2d683ac..ba71d42 100644 --- a/src/discord/todo.txt +++ b/src/discord/todo.txt @@ -41,3 +41,7 @@ background stuff: - look at every message ever in the channel - do exactly what the real-time bot does on new messages (although maybe don't do snippets depending on context) + +what the heck do we do with discord's markdown +- when we save message contents, we should save both the raw discord markdown and a version with their custom stuff replaced. We do _not_ (yet) need a full markdown parse with HTML tags and stuff. (That arguably doesn't make sense for the handmade_discordmessagecontent record anyway.) +- when we create a snippet, we should store both markdown that makes sense to a user and the rendered version of that HTML. THIS MEANS: The markdown we save is the "clean" version of the Discord markdown. diff --git a/src/models/discord.go b/src/models/discord.go index 4b2eeb7..33eda4a 100644 --- a/src/models/discord.go +++ b/src/models/discord.go @@ -41,7 +41,7 @@ account, regardless of whether we create snippets or not. */ type DiscordMessageContent struct { MessageID string `db:"message_id"` - LastContent string `db:"last_content"` + LastContent string `db:"last_content"` // This should always be cleaned up with nice user IDs and stuff DiscordID int `db:"discord_id"` } From 719c0d230c985cde6e22f8c587805b0c3ef19179 Mon Sep 17 00:00:00 2001 From: Ben Visness Date: Thu, 26 Aug 2021 18:33:39 -0500 Subject: [PATCH 07/10] Delete stuff on message delete still need to do bulk delete --- src/discord/gateway.go | 75 +++++++++++++++++++ src/discord/payloads.go | 17 +++++ src/discord/todo.txt | 8 +- ...021-08-26T005607Z_FixSnippetConstraints.go | 49 ++++++++++++ 4 files changed, 145 insertions(+), 4 deletions(-) create mode 100644 src/migration/migrations/2021-08-26T005607Z_FixSnippetConstraints.go diff --git a/src/discord/gateway.go b/src/discord/gateway.go index abf9185..a98f2d6 100644 --- a/src/discord/gateway.go +++ b/src/discord/gateway.go @@ -557,11 +557,14 @@ func (bot *botInstance) processEventMsg(ctx context.Context, msg *GatewayMessage if err != nil { return oops.New(err, "error on updated message") } + case "MESSAGE_DELETE": + bot.messageDelete(ctx, MessageDeleteFromMap(msg.Data)) } return nil } +// TODO: Should this return an error? Or just log errors? func (bot *botInstance) messageCreateOrUpdate(ctx context.Context, msg *Message) error { if msg.OriginalHasFields("author") && msg.Author.ID == config.Config.Discord.BotUserID { // Don't process your own messages @@ -587,6 +590,78 @@ func (bot *botInstance) messageCreateOrUpdate(ctx context.Context, msg *Message) return nil } +func (bot *botInstance) messageDelete(ctx context.Context, msgDelete MessageDelete) { + log := logging.ExtractLogger(ctx) + + tx, err := bot.dbConn.Begin(ctx) + if err != nil { + log.Error().Err(err).Msg("failed to start transaction") + return + } + defer tx.Rollback(ctx) + + type deleteMessageQuery struct { + Message models.DiscordMessage `db:"msg"` + DiscordUser *models.DiscordUser `db:"duser"` + HMNUser *models.User `db:"hmnuser"` + SnippetID *int `db:"snippet.id"` + } + iresult, err := db.QueryOne(ctx, tx, deleteMessageQuery{}, + ` + SELECT $columns + FROM + handmade_discordmessage AS msg + LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid + LEFT JOIN auth_user AS hmnuser ON duser.hmn_user_id = hmnuser.id + LEFT JOIN handmade_snippet AS snippet ON snippet.discord_message_id = msg.id + WHERE msg.id = $1 AND msg.channel_id = $2 + `, + msgDelete.ID, msgDelete.ChannelID, + ) + if errors.Is(err, db.ErrNoMatchingRows) { + return + } else if err != nil { + log.Error().Err(err).Msg("failed to check for message to delete") + return + } + result := iresult.(*deleteMessageQuery) + + log.Debug().Msg("deleting Discord message") + _, err = tx.Exec(ctx, + ` + DELETE FROM handmade_discordmessage + WHERE id = $1 AND channel_id = $2 + `, + msgDelete.ID, + msgDelete.ChannelID, + ) + + shouldDeleteSnippet := result.HMNUser != nil && result.HMNUser.DiscordDeleteSnippetOnMessageDelete + if result.SnippetID != nil && shouldDeleteSnippet { + log.Debug(). + Int("snippet_id", *result.SnippetID). + Int("user_id", result.HMNUser.ID). + Msg("deleting snippet from Discord message") + _, err = tx.Exec(ctx, + ` + DELETE FROM handmade_snippet + WHERE id = $1 + `, + result.SnippetID, + ) + if err != nil { + log.Error().Err(err).Msg("failed to delete snippet") + return + } + } + + err = tx.Commit(ctx) + if err != nil { + log.Error().Err(err).Msg("failed to delete Discord message") + return + } +} + type MessageToSend struct { ChannelID string Req CreateMessageRequest diff --git a/src/discord/payloads.go b/src/discord/payloads.go index 0b0f786..0ab88aa 100644 --- a/src/discord/payloads.go +++ b/src/discord/payloads.go @@ -110,6 +110,23 @@ type Resume struct { SequenceNumber int `json:"seq"` } +// https://discord.com/developers/docs/topics/gateway#message-delete +type MessageDelete struct { + ID string `json:"id"` + ChannelID string `json:"channel_id"` + GuildID string `json:"guild_id"` +} + +func MessageDeleteFromMap(m interface{}) MessageDelete { + mmap := m.(map[string]interface{}) + + return MessageDelete{ + ID: mmap["id"].(string), + ChannelID: mmap["channel_id"].(string), + GuildID: maybeString(mmap, "guild_id"), + } +} + type ChannelType int // https://discord.com/developers/docs/resources/channel#channel-object-channel-types diff --git a/src/discord/todo.txt b/src/discord/todo.txt index ba71d42..0a17cd0 100644 --- a/src/discord/todo.txt +++ b/src/discord/todo.txt @@ -7,9 +7,9 @@ stuff we need to worry about: - posts that come in while the bot is down - what to do with posts if you unlink your account - what to do with posts if you re-link your account -- what to do if you edit the original discord message +✔ - what to do if you edit the original discord message - what to do if you delete the original discord message -- the user's preferences re: saving content +✔ - the user's preferences re: saving content - we don't want to save content without the user's consent, especially since it may persist after they disable the integration - manually adding content for various reasons - maybe a bug prevented something from saving @@ -17,10 +17,10 @@ stuff we need to worry about: real-time stuff: -- on new showcase message +✔ - on new showcase message - always save the lightweight record - if we have permission, create a snippet -- on edit +✔ - on edit - re-save the lightweight record and content as if it was new - create snippet, unconditionally???? (bug??) - update snippet contents if the edit makes sense diff --git a/src/migration/migrations/2021-08-26T005607Z_FixSnippetConstraints.go b/src/migration/migrations/2021-08-26T005607Z_FixSnippetConstraints.go new file mode 100644 index 0000000..ff5ec55 --- /dev/null +++ b/src/migration/migrations/2021-08-26T005607Z_FixSnippetConstraints.go @@ -0,0 +1,49 @@ +package migrations + +import ( + "context" + "time" + + "git.handmade.network/hmn/hmn/src/migration/types" + "git.handmade.network/hmn/hmn/src/oops" + "github.com/jackc/pgx/v4" +) + +func init() { + registerMigration(FixSnippetConstraints{}) +} + +type FixSnippetConstraints struct{} + +func (m FixSnippetConstraints) Version() types.MigrationVersion { + return types.MigrationVersion(time.Date(2021, 8, 26, 0, 56, 7, 0, time.UTC)) +} + +func (m FixSnippetConstraints) Name() string { + return "FixSnippetConstraints" +} + +func (m FixSnippetConstraints) Description() string { + return "Fix the ON DELETE behaviors of snippets" +} + +func (m FixSnippetConstraints) Up(ctx context.Context, tx pgx.Tx) error { + _, err := tx.Exec(ctx, ` + ALTER TABLE handmade_snippet + DROP CONSTRAINT handmade_snippet_asset_id_c786de4f_fk_handmade_asset_id, + DROP CONSTRAINT handmade_snippet_discord_message_id_d16f1f4e_fk_handmade_, + DROP CONSTRAINT handmade_snippet_owner_id_fcca1783_fk_auth_user_id, + ADD FOREIGN KEY (asset_id) REFERENCES handmade_asset (id) ON DELETE SET NULL, + ADD FOREIGN KEY (discord_message_id) REFERENCES handmade_discordmessage (id) ON DELETE SET NULL, + ADD FOREIGN KEY (owner_id) REFERENCES auth_user (id) ON DELETE CASCADE; + `) + if err != nil { + return oops.New(err, "failed to fix constraints") + } + + return nil +} + +func (m FixSnippetConstraints) Down(ctx context.Context, tx pgx.Tx) error { + panic("Implement me") +} From 042e9166fdc73dd92dd72901c0b0a356c8fb5471 Mon Sep 17 00:00:00 2001 From: Ben Visness Date: Thu, 26 Aug 2021 19:58:41 -0500 Subject: [PATCH 08/10] Add bulk delete --- src/discord/gateway.go | 9 +++++++++ src/discord/payloads.go | 23 +++++++++++++++++++++++ src/discord/todo.txt | 7 ++++--- 3 files changed, 36 insertions(+), 3 deletions(-) diff --git a/src/discord/gateway.go b/src/discord/gateway.go index a98f2d6..9e61204 100644 --- a/src/discord/gateway.go +++ b/src/discord/gateway.go @@ -559,6 +559,15 @@ func (bot *botInstance) processEventMsg(ctx context.Context, msg *GatewayMessage } case "MESSAGE_DELETE": bot.messageDelete(ctx, MessageDeleteFromMap(msg.Data)) + case "MESSAGE_BULK_DELETE": + bulkDelete := MessageBulkDeleteFromMap(msg.Data) + for _, id := range bulkDelete.IDs { + bot.messageDelete(ctx, MessageDelete{ + ID: id, + ChannelID: bulkDelete.ChannelID, + GuildID: bulkDelete.GuildID, + }) + } } return nil diff --git a/src/discord/payloads.go b/src/discord/payloads.go index 0ab88aa..1256cd3 100644 --- a/src/discord/payloads.go +++ b/src/discord/payloads.go @@ -127,6 +127,29 @@ func MessageDeleteFromMap(m interface{}) MessageDelete { } } +// https://discord.com/developers/docs/topics/gateway#message-delete +type MessageBulkDelete struct { + IDs []string `json:"ids"` + ChannelID string `json:"channel_id"` + GuildID string `json:"guild_id"` +} + +func MessageBulkDeleteFromMap(m interface{}) MessageBulkDelete { + mmap := m.(map[string]interface{}) + + iids := mmap["ids"].([]interface{}) + ids := make([]string, len(iids)) + for i, iid := range iids { + ids[i] = iid.(string) + } + + return MessageBulkDelete{ + IDs: ids, + ChannelID: mmap["channel_id"].(string), + GuildID: maybeString(mmap, "guild_id"), + } +} + type ChannelType int // https://discord.com/developers/docs/resources/channel#channel-object-channel-types diff --git a/src/discord/todo.txt b/src/discord/todo.txt index 0a17cd0..41c3d9b 100644 --- a/src/discord/todo.txt +++ b/src/discord/todo.txt @@ -16,7 +16,7 @@ stuff we need to worry about: - ryan used to post everything in #projects for some reason -real-time stuff: +✔ real-time stuff: ✔ - on new showcase message - always save the lightweight record - if we have permission, create a snippet @@ -24,10 +24,11 @@ real-time stuff: - re-save the lightweight record and content as if it was new - create snippet, unconditionally???? (bug??) - update snippet contents if the edit makes sense -- on delete +✔ - on delete - delete snippet if the user so desires - delete the message records - +✔ - on bulk delete + - same stuff background stuff: - watch mode From 16ae2188d1b0513f2a77f92cf2f6577220d7508f Mon Sep 17 00:00:00 2001 From: Ben Visness Date: Thu, 26 Aug 2021 22:59:12 -0500 Subject: [PATCH 09/10] Add background features to the Discord bot --- src/db/db.go | 1 + src/discord/cmd/cmd.go | 30 ++++ src/discord/history.go | 210 ++++++++++++++++++++++++ src/discord/message_test.go | 16 ++ src/discord/payloads.go | 9 +- src/discord/rest.go | 88 ++++++++++ src/discord/showcase.go | 74 +++++---- src/discord/todo.txt | 48 ------ src/main.go | 1 + src/parsing/parsing.go | 57 +++++-- src/parsing/parsing_test.go | 14 +- src/website/podcast.go | 2 +- src/website/threads_and_posts_helper.go | 2 +- src/website/website.go | 1 + 14 files changed, 451 insertions(+), 102 deletions(-) create mode 100644 src/discord/cmd/cmd.go create mode 100644 src/discord/history.go create mode 100644 src/discord/message_test.go delete mode 100644 src/discord/todo.txt diff --git a/src/db/db.go b/src/db/db.go index 3962082..e6713b2 100644 --- a/src/db/db.go +++ b/src/db/db.go @@ -58,6 +58,7 @@ func typeIsQueryable(t reflect.Type) bool { // This interface should match both a direct pgx connection or a pgx transaction. type ConnOrTx interface { Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error) + QueryRow(ctx context.Context, sql string, args ...interface{}) pgx.Row Exec(ctx context.Context, sql string, args ...interface{}) (pgconn.CommandTag, error) } diff --git a/src/discord/cmd/cmd.go b/src/discord/cmd/cmd.go new file mode 100644 index 0000000..c9f52e6 --- /dev/null +++ b/src/discord/cmd/cmd.go @@ -0,0 +1,30 @@ +package cmd + +import ( + "context" + "time" + + "git.handmade.network/hmn/hmn/src/db" + "git.handmade.network/hmn/hmn/src/discord" + "git.handmade.network/hmn/hmn/src/website" + "github.com/spf13/cobra" +) + +func init() { + scrapeCommand := &cobra.Command{ + Use: "discordscrapechannel [...]", + Short: "Scrape the entire history of Discord channels", + Long: "Scrape the entire history of Discord channels, saving message content (but not creating snippets)", + Run: func(cmd *cobra.Command, args []string) { + ctx := context.Background() + conn := db.NewConnPool(1, 1) + defer conn.Close() + + for _, channelID := range args { + discord.Scrape(ctx, conn, channelID, time.Time{}, false) + } + }, + } + + website.WebsiteCommand.AddCommand(scrapeCommand) +} diff --git a/src/discord/history.go b/src/discord/history.go new file mode 100644 index 0000000..a042f0e --- /dev/null +++ b/src/discord/history.go @@ -0,0 +1,210 @@ +package discord + +import ( + "context" + "errors" + "time" + + "git.handmade.network/hmn/hmn/src/config" + "git.handmade.network/hmn/hmn/src/db" + "git.handmade.network/hmn/hmn/src/logging" + "git.handmade.network/hmn/hmn/src/models" + "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/pgxpool" +) + +func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{} { + log := logging.ExtractLogger(ctx).With().Str("discord goroutine", "history watcher").Logger() + ctx = logging.AttachLoggerToContext(&log, ctx) + + done := make(chan struct{}) + + go func() { + defer func() { + log.Debug().Msg("shut down Discord history watcher") + done <- struct{}{} + }() + + backfillInterval := 1 * time.Hour + + newUserTicker := time.NewTicker(5 * time.Second) + backfillTicker := time.NewTicker(backfillInterval) + + lastBackfillTime := time.Now().Add(-backfillInterval) + for { + select { + case <-ctx.Done(): + return + case <-newUserTicker.C: + // Get content for messages when a user links their account (but do not create snippets) + fetchMissingContent(ctx, dbConn) + case <-backfillTicker.C: + // Run a backfill to patch up places where the Discord bot missed (does create snippets) + Scrape(ctx, dbConn, + config.Config.Discord.ShowcaseChannelID, + lastBackfillTime, + true, + ) + } + } + }() + + return done +} + +func fetchMissingContent(ctx context.Context, dbConn *pgxpool.Pool) { + log := logging.ExtractLogger(ctx) + + type query struct { + Message models.DiscordMessage `db:"msg"` + } + result, err := db.Query(ctx, dbConn, query{}, + ` + SELECT $columns + FROM + handmade_discordmessage AS msg + JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid -- only fetch messages for linked discord users + LEFT JOIN handmade_discordmessagecontent AS c ON c.message_id = msg.id + WHERE + c.last_content IS NULL + AND msg.guild_id = $1 + `, + config.Config.Discord.GuildID, + ) + if err != nil { + log.Error().Err(err).Msg("failed to check for messages without content") + return + } + imessagesWithoutContent := result.ToSlice() + + if len(imessagesWithoutContent) > 0 { + log.Info().Msgf("There are %d Discord messages without content, fetching their content now...", len(imessagesWithoutContent)) + msgloop: + for _, imsg := range imessagesWithoutContent { + select { + case <-ctx.Done(): + log.Info().Msg("Scrape was canceled") + break msgloop + default: + } + + msg := imsg.(*query).Message + + discordMsg, err := GetChannelMessage(ctx, msg.ChannelID, msg.ID) + if errors.Is(err, NotFound) { + // This message has apparently been deleted; delete it from our database + _, err = dbConn.Exec(ctx, + ` + DELETE FROM handmade_discordmessage + WHERE id = $1 + `, + msg.ID, + ) + if err != nil { + log.Error().Err(err).Msg("failed to delete missing message") + continue + } + log.Info().Str("msg id", msg.ID).Msg("deleted missing Discord message") + continue + } else if err != nil { + log.Error().Err(err).Msg("failed to get message") + continue + } + + log.Info().Str("msg", discordMsg.ShortString()).Msg("fetched message for content") + + err = handleHistoryMessage(ctx, dbConn, discordMsg, false) + if err != nil { + log.Error().Err(err).Msg("failed to save content for message") + continue + } + } + log.Info().Msgf("Done fetching missing content") + } +} + +func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earliestMessageTime time.Time, createSnippets bool) { + log := logging.ExtractLogger(ctx) + + log.Info().Msg("Starting scrape") + defer log.Info().Msg("Done with scrape!") + + before := "" + for { + msgs, err := GetChannelMessages(ctx, channelID, GetChannelMessagesInput{ + Limit: 100, + Before: before, + }) + if err != nil { + panic(err) // TODO + } + + if len(msgs) == 0 { + logging.Debug().Msg("out of messages, stopping scrape") + return + } + + for _, msg := range msgs { + select { + case <-ctx.Done(): + log.Info().Msg("Scrape was canceled") + return + default: + } + + log.Info().Str("msg", msg.ShortString()).Msg("") + + if !earliestMessageTime.IsZero() && msg.Time().Before(earliestMessageTime) { + logging.ExtractLogger(ctx).Info().Time("earliest", earliestMessageTime).Msg("Saw a message before the specified earliest time; exiting") + return + } + + err := handleHistoryMessage(ctx, dbConn, &msg, true) + if err != nil { + errLog := logging.ExtractLogger(ctx).Error() + if errors.Is(err, errNotEnoughInfo) { + errLog = logging.ExtractLogger(ctx).Warn() + } + errLog.Err(err).Msg("failed to process Discord message") + } + + before = msg.ID + } + } +} + +func handleHistoryMessage(ctx context.Context, dbConn *pgxpool.Pool, msg *Message, createSnippets bool) error { + var tx pgx.Tx + for { + var err error + tx, err = dbConn.Begin(ctx) + if err != nil { + logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to start transaction for message") + time.Sleep(1 * time.Second) + continue + } + break + } + + newMsg, err := saveMessageAndContents(ctx, tx, msg) + if err != nil { + return err + } + if createSnippets { + if doSnippet, err := allowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil { + _, err := createMessageSnippet(ctx, tx, msg) + if err != nil { + return err + } + } else if err != nil { + return err + } + } + + err = tx.Commit(ctx) + if err != nil { + return err + } + + return nil +} diff --git a/src/discord/message_test.go b/src/discord/message_test.go new file mode 100644 index 0000000..abea1c6 --- /dev/null +++ b/src/discord/message_test.go @@ -0,0 +1,16 @@ +package discord + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGetMessage(t *testing.T) { + // t.Skip("this test is only for debugging") + + msg, err := GetChannelMessage(context.Background(), "404399251276169217", "764575065772916790") + assert.Nil(t, err) + t.Logf("%+v", msg) +} diff --git a/src/discord/payloads.go b/src/discord/payloads.go index 1256cd3..4b6e3c5 100644 --- a/src/discord/payloads.go +++ b/src/discord/payloads.go @@ -252,9 +252,16 @@ func (m *Message) Time() time.Time { return t } +func (m *Message) ShortString() string { + return fmt.Sprintf("%s / %s: \"%s\" (%d attachments, %d embeds)", m.Timestamp, m.Author.Username, m.Content, len(m.Attachments), len(m.Embeds)) +} + func (m *Message) OriginalHasFields(fields ...string) bool { if m.originalMap == nil { - return false + // If we don't know, we assume the fields are there. + // Usually this is because it came from their API, where we + // always have all fields. + return true } for _, field := range fields { diff --git a/src/discord/rest.go b/src/discord/rest.go index 8387e11..db91554 100644 --- a/src/discord/rest.go +++ b/src/discord/rest.go @@ -10,6 +10,7 @@ import ( "net/http" "net/http/httputil" "net/url" + "strconv" "strings" "git.handmade.network/hmn/hmn/src/config" @@ -411,6 +412,93 @@ func RemoveGuildMemberRole(ctx context.Context, userID, roleID string) error { return nil } +func GetChannelMessage(ctx context.Context, channelID, messageID string) (*Message, error) { + const name = "Get Channel Message" + + path := fmt.Sprintf("/channels/%s/messages/%s", channelID, messageID) + res, err := doWithRateLimiting(ctx, name, func(ctx context.Context) *http.Request { + return makeRequest(ctx, http.MethodGet, path, nil) + }) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode == http.StatusNotFound { + return nil, NotFound + } else if res.StatusCode >= 400 { + logErrorResponse(ctx, name, res, "") + return nil, oops.New(nil, "received error from Discord") + } + + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + } + + var msg Message + err = json.Unmarshal(bodyBytes, &msg) + if err != nil { + return nil, oops.New(err, "failed to unmarshal Discord message") + } + + return &msg, nil +} + +type GetChannelMessagesInput struct { + Around string + Before string + After string + Limit int +} + +func GetChannelMessages(ctx context.Context, channelID string, in GetChannelMessagesInput) ([]Message, error) { + const name = "Get Channel Messages" + + path := fmt.Sprintf("/channels/%s/messages", channelID) + res, err := doWithRateLimiting(ctx, name, func(ctx context.Context) *http.Request { + req := makeRequest(ctx, http.MethodGet, path, nil) + q := req.URL.Query() + if in.Around != "" { + q.Add("around", in.Around) + } + if in.Before != "" { + q.Add("before", in.Before) + } + if in.After != "" { + q.Add("after", in.After) + } + if in.Limit != 0 { + q.Add("limit", strconv.Itoa(in.Limit)) + } + req.URL.RawQuery = q.Encode() + + return req + }) + if err != nil { + return nil, err + } + defer res.Body.Close() + + if res.StatusCode >= 400 { + logErrorResponse(ctx, name, res, "") + return nil, oops.New(nil, "received error from Discord") + } + + bodyBytes, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + } + + var msgs []Message + err = json.Unmarshal(bodyBytes, &msgs) + if err != nil { + return nil, oops.New(err, "failed to unmarshal Discord message") + } + + return msgs, nil +} + func logErrorResponse(ctx context.Context, name string, res *http.Response, msg string) { dump, err := httputil.DumpResponse(res, true) if err != nil { diff --git a/src/discord/showcase.go b/src/discord/showcase.go index 3f1f68a..df4ad71 100644 --- a/src/discord/showcase.go +++ b/src/discord/showcase.go @@ -12,13 +12,13 @@ import ( "time" "git.handmade.network/hmn/hmn/src/assets" + "git.handmade.network/hmn/hmn/src/config" "git.handmade.network/hmn/hmn/src/db" "git.handmade.network/hmn/hmn/src/logging" "git.handmade.network/hmn/hmn/src/models" "git.handmade.network/hmn/hmn/src/oops" "git.handmade.network/hmn/hmn/src/parsing" "github.com/google/uuid" - "github.com/jackc/pgx/v4" ) var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`) @@ -47,7 +47,7 @@ func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) er defer tx.Rollback(ctx) // save the message, maybe save its contents, and maybe make a snippet too - newMsg, err := bot.saveMessageAndContents(ctx, tx, msg) + newMsg, err := saveMessageAndContents(ctx, tx, msg) if errors.Is(err, errNotEnoughInfo) { logging.ExtractLogger(ctx).Warn(). Interface("msg", msg). @@ -56,8 +56,8 @@ func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) er } else if err != nil { return err } - if doSnippet, err := bot.allowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil { - _, err := bot.createMessageSnippet(ctx, tx, msg) + if doSnippet, err := allowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil { + _, err := createMessageSnippet(ctx, tx, msg) if err != nil { return oops.New(err, "failed to create snippet in gateway") } @@ -120,9 +120,9 @@ the database. This does not create snippets or do anything besides save the message itself. */ -func (bot *botInstance) saveMessage( +func saveMessage( ctx context.Context, - tx pgx.Tx, + tx db.ConnOrTx, msg *Message, ) (*models.DiscordMessage, error) { iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{}, @@ -138,6 +138,16 @@ func (bot *botInstance) saveMessage( return nil, errNotEnoughInfo } + guildID := msg.GuildID + if guildID == nil { + /* + This is weird, but it can happen when we fetch messages from + history instead of receiving it from the gateway. In this case + we just assume it's from the HMN server. + */ + guildID = &config.Config.Discord.GuildID + } + _, err = tx.Exec(ctx, ` INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created) @@ -145,7 +155,7 @@ func (bot *botInstance) saveMessage( `, msg.ID, msg.ChannelID, - *msg.GuildID, + *guildID, msg.JumpURL(), msg.Author.ID, msg.Time(), @@ -184,12 +194,12 @@ snippets. Idempotent; can be called any time whether the message exists or not. */ -func (bot *botInstance) saveMessageAndContents( +func saveMessageAndContents( ctx context.Context, - tx pgx.Tx, + tx db.ConnOrTx, msg *Message, ) (*models.DiscordMessage, error) { - newMsg, err := bot.saveMessage(ctx, tx, msg) + newMsg, err := saveMessage(ctx, tx, msg) if err != nil { return nil, err } @@ -231,7 +241,7 @@ func (bot *botInstance) saveMessageAndContents( // Save attachments if msg.OriginalHasFields("attachments") { for _, attachment := range msg.Attachments { - _, err := bot.saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID) + _, err := saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID) if err != nil { return nil, oops.New(err, "failed to save attachment") } @@ -254,7 +264,7 @@ func (bot *botInstance) saveMessageAndContents( if numSavedEmbeds == 0 { // No embeds yet, so save new ones for _, embed := range msg.Embeds { - _, err := bot.saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID) + _, err := saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID) if err != nil { return nil, oops.New(err, "failed to save embed") } @@ -310,9 +320,9 @@ func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, e Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment that already exists */ -func (bot *botInstance) saveAttachment( +func saveAttachment( ctx context.Context, - tx pgx.Tx, + tx db.ConnOrTx, attachment *Attachment, hmnUserID int, discordMessageID string, @@ -394,9 +404,9 @@ func (bot *botInstance) saveAttachment( return iDiscordAttachment.(*models.DiscordMessageAttachment), nil } -func (bot *botInstance) saveEmbed( +func saveEmbed( ctx context.Context, - tx pgx.Tx, + tx db.ConnOrTx, embed *Embed, hmnUserID int, discordMessageID string, @@ -497,8 +507,8 @@ func (bot *botInstance) saveEmbed( return iDiscordEmbed.(*models.DiscordMessageEmbed), nil } -func (bot *botInstance) allowedToCreateMessageSnippet(ctx context.Context, tx pgx.Tx, discordUserId string) (bool, error) { - canSave, err := db.QueryBool(ctx, bot.dbConn, +func allowedToCreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, discordUserId string) (bool, error) { + canSave, err := db.QueryBool(ctx, tx, ` SELECT u.discord_save_showcase FROM @@ -518,7 +528,7 @@ func (bot *botInstance) allowedToCreateMessageSnippet(ctx context.Context, tx pg return canSave, nil } -func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg *Message) (*models.Snippet, error) { +func createMessageSnippet(ctx context.Context, tx db.ConnOrTx, msg *Message) (*models.Snippet, error) { // Check for existing snippet, maybe return it type existingSnippetResult struct { Message models.DiscordMessage `db:"msg"` @@ -548,7 +558,7 @@ func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg // A snippet already exists - maybe update its content, then return it if msg.OriginalHasFields("content") && !existing.Snippet.EditedOnWebsite { contentMarkdown := existing.MessageContent.LastContent - contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.RealMarkdown) + contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown) _, err := tx.Exec(ctx, ` @@ -580,14 +590,14 @@ func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg } // Get an asset ID or URL to make a snippet from - assetId, url, err := bot.getSnippetAssetOrUrl(ctx, tx, &existing.Message) - if assetId == nil && url == "" { + assetId, url, err := getSnippetAssetOrUrl(ctx, tx, &existing.Message) + if assetId == nil && url == nil { // Nothing to make a snippet from! return nil, nil } contentMarkdown := existing.MessageContent.LastContent - contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.RealMarkdown) + contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown) // TODO(db): Insert isnippet, err := db.QueryOne(ctx, tx, models.Snippet{}, @@ -596,7 +606,7 @@ func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING $columns `, - nil, + url, existing.Message.SentAt, contentMarkdown, contentHTML, @@ -626,7 +636,7 @@ func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg // do we actually want to reuse those, or should we keep them separate? var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`) -func (bot *botInstance) getSnippetAssetOrUrl(ctx context.Context, tx pgx.Tx, msg *models.DiscordMessage) (*uuid.UUID, string, error) { +func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msg *models.DiscordMessage) (*uuid.UUID, *string, error) { // Check attachments itAttachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{}, ` @@ -637,12 +647,12 @@ func (bot *botInstance) getSnippetAssetOrUrl(ctx context.Context, tx pgx.Tx, msg msg.ID, ) if err != nil { - return nil, "", oops.New(err, "failed to fetch message attachments") + return nil, nil, oops.New(err, "failed to fetch message attachments") } attachments := itAttachments.ToSlice() for _, iattachment := range attachments { attachment := iattachment.(*models.DiscordMessageAttachment) - return &attachment.AssetID, "", nil + return &attachment.AssetID, nil, nil } // Check embeds @@ -655,23 +665,23 @@ func (bot *botInstance) getSnippetAssetOrUrl(ctx context.Context, tx pgx.Tx, msg msg.ID, ) if err != nil { - return nil, "", oops.New(err, "failed to fetch discord embeds") + return nil, nil, oops.New(err, "failed to fetch discord embeds") } embeds := itEmbeds.ToSlice() for _, iembed := range embeds { embed := iembed.(*models.DiscordMessageEmbed) if embed.VideoID != nil { - return embed.VideoID, "", nil + return embed.VideoID, nil, nil } else if embed.ImageID != nil { - return embed.ImageID, "", nil + return embed.ImageID, nil, nil } else if embed.URL != nil { if RESnippetableUrl.MatchString(*embed.URL) { - return nil, *embed.URL, nil + return nil, embed.URL, nil } } } - return nil, "", nil + return nil, nil, nil } func messageHasLinks(content string) bool { diff --git a/src/discord/todo.txt b/src/discord/todo.txt deleted file mode 100644 index 41c3d9b..0000000 --- a/src/discord/todo.txt +++ /dev/null @@ -1,48 +0,0 @@ -the goal: port the old discord showcase bot - -what it does: save #project-showcase posts to your HMN user profile if you have your account linked - -stuff we need to worry about: -- old posts from before you linked your account -- posts that come in while the bot is down -- what to do with posts if you unlink your account -- what to do with posts if you re-link your account -✔ - what to do if you edit the original discord message -- what to do if you delete the original discord message -✔ - the user's preferences re: saving content - - we don't want to save content without the user's consent, especially since it may persist after they disable the integration -- manually adding content for various reasons - - maybe a bug prevented something from saving - - ryan used to post everything in #projects for some reason - - -✔ real-time stuff: -✔ - on new showcase message - - always save the lightweight record - - if we have permission, create a snippet -✔ - on edit - - re-save the lightweight record and content as if it was new - - create snippet, unconditionally???? (bug??) - - update snippet contents if the edit makes sense -✔ - on delete - - delete snippet if the user so desires - - delete the message records -✔ - on bulk delete - - same stuff - -background stuff: -- watch mode - - every five seconds - - fetch all HMN users with Discord accounts - - check if we have message records without content - - if so, run a full scrape (no snippets) - - every hour - - run a full scrape, creating snippets -- scrape behavior - - look at every message ever in the channel - - do exactly what the real-time bot does on new messages (although maybe don't do snippets depending on context) - - -what the heck do we do with discord's markdown -- when we save message contents, we should save both the raw discord markdown and a version with their custom stuff replaced. We do _not_ (yet) need a full markdown parse with HTML tags and stuff. (That arguably doesn't make sense for the handmade_discordmessagecontent record anyway.) -- when we create a snippet, we should store both markdown that makes sense to a user and the rendered version of that HTML. THIS MEANS: The markdown we save is the "clean" version of the Discord markdown. diff --git a/src/main.go b/src/main.go index 96bcdab..2377cbc 100644 --- a/src/main.go +++ b/src/main.go @@ -4,6 +4,7 @@ import ( _ "git.handmade.network/hmn/hmn/src/admintools" _ "git.handmade.network/hmn/hmn/src/assets" _ "git.handmade.network/hmn/hmn/src/buildscss" + _ "git.handmade.network/hmn/hmn/src/discord/cmd" _ "git.handmade.network/hmn/hmn/src/initimage" _ "git.handmade.network/hmn/hmn/src/migration" "git.handmade.network/hmn/hmn/src/website" diff --git a/src/parsing/parsing.go b/src/parsing/parsing.go index fbd70da..68bf27b 100644 --- a/src/parsing/parsing.go +++ b/src/parsing/parsing.go @@ -10,21 +10,38 @@ import ( ) // Used for rendering real-time previews of post content. -var PreviewMarkdown = goldmark.New( - goldmark.WithExtensions(makeGoldmarkExtensions(true)...), +var ForumPreviewMarkdown = goldmark.New( + goldmark.WithExtensions(makeGoldmarkExtensions(MarkdownOptions{ + Previews: true, + Embeds: true, + })...), ) // Used for generating the final HTML for a post. -var RealMarkdown = goldmark.New( - goldmark.WithExtensions(makeGoldmarkExtensions(false)...), +var ForumRealMarkdown = goldmark.New( + goldmark.WithExtensions(makeGoldmarkExtensions(MarkdownOptions{ + Previews: false, + Embeds: true, + })...), ) // Used for generating plain-text previews of posts. var PlaintextMarkdown = goldmark.New( - goldmark.WithExtensions(makeGoldmarkExtensions(false)...), + goldmark.WithExtensions(makeGoldmarkExtensions(MarkdownOptions{ + Previews: false, + Embeds: true, + })...), goldmark.WithRenderer(plaintextRenderer{}), ) +// Used for processing Discord messages +var DiscordMarkdown = goldmark.New( + goldmark.WithExtensions(makeGoldmarkExtensions(MarkdownOptions{ + Previews: false, + Embeds: false, + })...), +) + func ParseMarkdown(source string, md goldmark.Markdown) string { var buf bytes.Buffer if err := md.Convert([]byte(source), &buf); err != nil { @@ -34,19 +51,35 @@ func ParseMarkdown(source string, md goldmark.Markdown) string { return buf.String() } -func makeGoldmarkExtensions(preview bool) []goldmark.Extender { - return []goldmark.Extender{ +type MarkdownOptions struct { + Previews bool + Embeds bool +} + +func makeGoldmarkExtensions(opts MarkdownOptions) []goldmark.Extender { + var extenders []goldmark.Extender + extenders = append(extenders, extension.GFM, highlightExtension, SpoilerExtension{}, - EmbedExtension{ - Preview: preview, - }, + ) + + if opts.Embeds { + extenders = append(extenders, + EmbedExtension{ + Preview: opts.Previews, + }, + ) + } + + extenders = append(extenders, MathjaxExtension{}, BBCodeExtension{ - Preview: preview, + Preview: opts.Previews, }, - } + ) + + return extenders } var highlightExtension = highlighting.NewHighlighting( diff --git a/src/parsing/parsing_test.go b/src/parsing/parsing_test.go index 1e68bdf..42ad807 100644 --- a/src/parsing/parsing_test.go +++ b/src/parsing/parsing_test.go @@ -10,14 +10,14 @@ import ( func TestMarkdown(t *testing.T) { t.Run("fenced code blocks", func(t *testing.T) { t.Run("multiple lines", func(t *testing.T) { - html := ParseMarkdown("```\nmultiple lines\n\tof code\n```", RealMarkdown) + html := ParseMarkdown("```\nmultiple lines\n\tof code\n```", ForumRealMarkdown) t.Log(html) assert.Equal(t, 1, strings.Count(html, " Date: Fri, 27 Aug 2021 12:58:52 -0500 Subject: [PATCH 10/10] Add user edit form Add most of the user settings backend still need to do discord lol Add the Discord settings Add avatar uploads --- public/js/tabs.js | 95 ++++++ public/style.css | 177 ++++------- public/themes/dark/theme.css | 3 - public/themes/light/theme.css | 3 - src/admintools/admintools.go | 5 +- src/auth/auth.go | 12 +- src/discord/history.go | 7 +- src/discord/rest.go | 11 + src/discord/showcase.go | 39 ++- src/hmnurl/urls.go | 16 +- .../2021-08-27T190408Z_NewLinkData.go | 60 ++++ src/models/link.go | 13 +- src/rawdata/scss/_core.scss | 24 +- src/rawdata/scss/_editor.scss | 142 +++------ src/rawdata/scss/_forms.scss | 14 +- .../scss/tachyons/scss/_variables.scss | 1 + src/rawdata/scss/tachyons/scss/_widths.scss | 4 + src/rawdata/scss/themes/dark/_variables.scss | 3 - src/rawdata/scss/themes/light/_variables.scss | 3 - src/templates/mapping.go | 112 ++++--- src/templates/src/project_homepage.html | 4 +- src/templates/src/user_profile.html | 4 +- src/templates/src/user_settings.html | 201 +++++++++++++ src/templates/types.go | 9 +- src/website/auth.go | 22 +- src/website/discord.go | 127 ++++---- src/website/imagefile_helper.go | 93 ++++++ src/website/podcast.go | 71 +---- src/website/routes.go | 9 +- src/website/user.go | 274 ++++++++++++++++++ 30 files changed, 1074 insertions(+), 484 deletions(-) create mode 100644 public/js/tabs.js create mode 100644 src/migration/migrations/2021-08-27T190408Z_NewLinkData.go create mode 100644 src/templates/src/user_settings.html create mode 100644 src/website/imagefile_helper.go diff --git a/public/js/tabs.js b/public/js/tabs.js new file mode 100644 index 0000000..2c7b239 --- /dev/null +++ b/public/js/tabs.js @@ -0,0 +1,95 @@ +function TabState(tabbed) { + this.container = tabbed; + this.tabs = tabbed.querySelector(".tab"); + + this.tabbar = document.createElement("div"); + this.tabbar.classList.add("tab-bar"); + this.container.insertBefore(this.tabbar, this.container.firstChild); + + this.current_i = -1; + this.tab_buttons = []; +} + +function switch_tab_old(state, tab_i) { + return function() { + if (state.current_i >= 0) { + state.tabs[state.current_i].classList.add("hidden"); + state.tab_buttons[state.current_i].classList.remove("current"); + } + + state.tabs[tab_i].classList.remove("hidden"); + state.tab_buttons[tab_i].classList.add("current"); + + var hash = ""; + if (state.tabs[tab_i].hasAttribute("data-url-hash")) { + hash = state.tabs[tab_i].getAttribute("data-url-hash"); + } + window.location.hash = hash; + + state.current_i = tab_i; + }; +} + +document.addEventListener("DOMContentLoaded", function() { + const tabContainers = document.getElementsByClassName("tabbed"); + for (const container of tabContainers) { + const tabBar = document.createElement("div"); + tabBar.classList.add("tab-bar"); + container.insertAdjacentElement('afterbegin', tabBar); + + const tabs = container.querySelectorAll(".tab"); + for (let i = 0; i < tabs.length; i++) { + const tab = tabs[i]; + tab.classList.toggle('dn', i > 0); + + const slug = tab.getAttribute("data-slug"); + + // TODO: Should this element be a link? + const tabButton = document.createElement("div"); + tabButton.classList.add("tab-button"); + tabButton.classList.toggle("current", i === 0); + tabButton.innerText = tab.getAttribute("data-name"); + tabButton.setAttribute("data-slug", slug); + + tabButton.addEventListener("click", () => { + switchTab(container, slug); + }); + + tabBar.appendChild(tabButton); + } + + const initialSlug = window.location.hash; + if (initialSlug) { + switchTab(container, initialSlug.substring(1)); + } + } +}); + +function switchTab(container, slug) { + const tabs = container.querySelectorAll('.tab'); + + let didMatch = false; + for (const tab of tabs) { + const slugMatches = tab.getAttribute("data-slug") === slug; + tab.classList.toggle('dn', !slugMatches); + // TODO: Also update the tab button styles + + if (slugMatches) { + didMatch = true; + } + } + + const tabButtons = document.querySelectorAll(".tab-button"); + for (const tabButton of tabButtons) { + const buttonSlug = tabButton.getAttribute("data-slug"); + tabButton.classList.toggle('current', slug === buttonSlug); + } + + if (!didMatch) { + // switch to first tab as a fallback + tabs[0].classList.remove('dn'); + tabButtons[0].classList.add('current'); + } + + window.location.hash = slug; +} \ No newline at end of file diff --git a/public/style.css b/public/style.css index 8f1fde0..62f609a 100644 --- a/public/style.css +++ b/public/style.css @@ -1994,7 +1994,7 @@ img, video { -l = large */ -.flex { +.flex, .tab-bar, .edit-form .edit-form-row { display: flex; } .inline-flex { @@ -2012,10 +2012,10 @@ img, video { .flex-none { flex: none; } -.flex-column { +.flex-column, .edit-form .edit-form-row { flex-direction: column; } -.flex-row { +.flex-row, .tab-bar { flex-direction: row; } .flex-wrap { @@ -2126,13 +2126,13 @@ img, video { .order-last { order: 99999; } -.flex-grow-0 { +.flex-grow-0, .edit-form .edit-form-row > :first-child { flex-grow: 0; } -.flex-grow-1 { +.flex-grow-1, .edit-form .edit-form-row > :nth-child(2) { flex-grow: 1; } -.flex-shrink-0 { +.flex-shrink-0, .edit-form .edit-form-row > :first-child { flex-shrink: 0; } .flex-shrink-1 { @@ -2153,7 +2153,7 @@ img, video { flex: none; } .flex-column-ns { flex-direction: column; } - .flex-row-ns { + .flex-row-ns, .edit-form .edit-form-row { flex-direction: row; } .flex-wrap-ns { flex-wrap: wrap; } @@ -2771,7 +2771,7 @@ code, .code { .h2 { height: 2rem; } -.h3 { +.h3, .edit-form textarea { height: 4rem; } .h4 { @@ -3079,7 +3079,7 @@ code, .code { */ /* Max Width Percentages */ -.mw-100 { +.mw-100, .edit-form textarea { max-width: 100%; } /* Max Width Scale */ @@ -3125,7 +3125,7 @@ code, .code { max-width: 4rem; } .mw4-ns { max-width: 8rem; } - .mw5-ns { + .mw5-ns, .edit-form input[type=text] { max-width: 16rem; } .mw6-ns { max-width: 32rem; } @@ -3243,6 +3243,9 @@ code, .code { .w5 { width: 16rem; } +.w6 { + width: 32rem; } + .w-10 { width: 10%; } @@ -3282,7 +3285,7 @@ code, .code { .w-90 { width: 90%; } -.w-100 { +.w-100, .edit-form .edit-form-row > :first-child, .edit-form input[type=text], .edit-form textarea { width: 100%; } .w-third { @@ -3301,10 +3304,12 @@ code, .code { width: 2rem; } .w3-ns { width: 4rem; } - .w4-ns { + .w4-ns, .edit-form .edit-form-row > :first-child { width: 8rem; } .w5-ns { width: 16rem; } + .w6-ns, .edit-form textarea { + width: 32rem; } .w-10-ns { width: 10%; } .w-20-ns { @@ -3351,6 +3356,8 @@ code, .code { width: 8rem; } .w5-m { width: 16rem; } + .w6-m { + width: 32rem; } .w-10-m { width: 10%; } .w-20-m { @@ -3397,6 +3404,8 @@ code, .code { width: 8rem; } .w5-l { width: 16rem; } + .w6-l { + width: 32rem; } .w-10-l { width: 10%; } .w-20-l { @@ -3445,7 +3454,7 @@ code, .code { .overflow-visible { overflow: visible; } -.overflow-hidden { +.overflow-hidden, .edit-form .edit-form-row > :nth-child(2) { overflow: hidden; } .overflow-scroll { @@ -4614,7 +4623,7 @@ code, .code { .pl7 { padding-left: 16rem; } -.pr0 { +.pr0, .edit-form .edit-form-row > :first-child { padding-right: 0; } .pr1 { @@ -4641,7 +4650,7 @@ code, .code { .pb0 { padding-bottom: 0; } -.pb1 { +.pb1, .edit-form .edit-form-row > :first-child { padding-bottom: 0.25rem; } .pb2 { @@ -4698,7 +4707,7 @@ code, .code { padding-top: 0.25rem; padding-bottom: 0.25rem; } -.pv2, header .menu-bar .items a.project-logo, +.pv2, header .menu-bar .items a.project-logo, .tab-bar .tab-button, button, .button, input[type=button], @@ -4742,7 +4751,7 @@ input[type=submit] { padding-left: 0.5rem; padding-right: 0.5rem; } -.ph3, +.ph3, .tab-bar .tab-button, button, .button, input[type=button], @@ -4898,7 +4907,7 @@ input[type=submit] { margin-top: 0.5rem; margin-bottom: 0.5rem; } -.mv3, hr { +.mv3, hr, .edit-form .edit-form-row { margin-top: 1rem; margin-bottom: 1rem; } @@ -4987,7 +4996,7 @@ input[type=submit] { padding-right: 0; } .pr1-ns { padding-right: 0.25rem; } - .pr2-ns { + .pr2-ns, .edit-form .edit-form-row > :first-child { padding-right: 0.5rem; } .pr3-ns { padding-right: 1rem; } @@ -4999,7 +5008,7 @@ input[type=submit] { padding-right: 8rem; } .pr7-ns { padding-right: 16rem; } - .pb0-ns { + .pb0-ns, .edit-form .edit-form-row > :first-child { padding-bottom: 0; } .pb1-ns { padding-bottom: 0.25rem; } @@ -6169,7 +6178,7 @@ input[type=submit] { -l = large */ -.tl { +.tl, .edit-form .edit-form-row > :first-child { text-align: left; } .tr { @@ -6184,7 +6193,7 @@ input[type=submit] { @media screen and (min-width: 30em) { .tl-ns { text-align: left; } - .tr-ns { + .tr-ns, .edit-form .edit-form-row > :first-child { text-align: right; } .tc-ns { text-align: center; } @@ -7204,7 +7213,7 @@ body { min-height: 100vh; box-sizing: border-box; font-size: 0.875rem; - line-height: 1.5em; + line-height: 1.2em; font-weight: 400; } a { @@ -7321,10 +7330,10 @@ article code { margin-left: auto; margin-right: auto; } -.flex-shrink-0 { +.flex-shrink-0, .edit-form .edit-form-row > :first-child { flex-shrink: 0; } -.flex-grow-1 { +.flex-grow-1, .edit-form .edit-form-row > :nth-child(2) { flex-grow: 1; } .flex-fair { @@ -7780,32 +7789,20 @@ header { .tab-bar { border-color: #d8d8d8; border-color: var(--tab-border-color); - width: 100%; - border-bottom-width: 1px; - border-bottom-style: solid; - box-sizing: border-box; } + width: 100%; } .tab-bar .tab-button { background-color: #dfdfdf; background-color: var(--tab-button-background); border-color: #d8d8d8; border-color: var(--tab-border-color); - height: 100%; - display: inline-block; - padding: 10px 15px; - line-height: 100%; - cursor: pointer; - border-width: 1px; - border-style: solid; - box-sizing: border-box; } + cursor: pointer; } .tab-bar .tab-button:hover { background-color: #efefef; background-color: var(--tab-button-background-hover); } .tab-bar .tab-button.current { background-color: #fff; background-color: var(--tab-button-background-current); - border-bottom-color: transparent; - font-weight: bold; - height: 105%; } + font-weight: 500; } .pagination .page.current { cursor: default; @@ -8016,81 +8013,12 @@ pre { max-height: calc(100vh - 20rem); overflow: auto; } } -.edit-form .error { - margin-left: 5em; - padding: 10px; - color: red; } +.edit-form .edit-form-row > :first-child { + font-weight: 500; } -.edit-form input[type=text] { - min-width: 20em; } - -.edit-form textarea { - font-size: 13pt; } - -.edit-form .note { - margin-bottom: 5px; - font-style: italic; - font-size: 90%; } - -.edit-form .links { - width: 80%; - min-height: 200px; - height: 15vh; } - -.edit-form .half { - padding: 10px; - text-align: center; } - -.edit-form table { - width: 95%; - margin: auto; - border-collapse: separate; - border-spacing: 0px 10px; } - .edit-form table td { - padding-bottom: 15px; - width: 90%; } - .edit-form table td.half { - width: 50%; } - .edit-form table td table { - width: 100%; } - -.edit-form th { - text-align: right; - font-weight: bold; - padding-right: 10px; - padding-bottom: 15px; - vertical-align: top; - max-width: 5em; } - -.edit-form td table th { - text-align: left; } - -.edit-form .page-options label { - font-weight: bold; - margin-right: 20px; } - -.edit-form.profile-edit .longbio { - width: 100%; - min-height: 400px; - height: 30vh; } - -.edit-form.profile-edit .avatar-preview { - border: 1px solid transparent; - margin: 10px; - margin-bottom: 0px; } - -.edit-form.profile-edit textarea.shortbio, -.edit-form.profile-edit textarea.signature { - min-width: 300px; - width: 50%; - min-height: 100px; - height: 4em; } - -.edit-form.profile-edit .logo-preview { - border-color: #999; - border-color: var(--project-edit-logo-previw-border-color); - width: 200px; - border-width: 1px; } +@media screen and (min-width: 30em) { + .edit-form .edit-form-row .pt-input-ns { + padding-top: 0.3rem; } } .edit-form.project-edit .project_description { width: 100%; @@ -8103,14 +8031,10 @@ pre { width: 50%; } .edit-form.project-edit .quota-bar { - border-color: #999; - border-color: var(--project-edit-quota-bar-border-color); width: 500px; border-width: 1px; margin-bottom: 10px; } .edit-form.project-edit .quota-bar .quota-filled { - background-color: #444; - background-color: var(--project-edit-quota-bar-filled-background); height: 100%; } .episode-list .description p { @@ -8361,6 +8285,7 @@ nav.timecodes { input[type=text], input[type=password], +input[type=email], textarea, select { color: black; @@ -8375,6 +8300,7 @@ select { outline: none; } input[type=text].lite, input[type=password].lite, + input[type=email].lite, textarea.lite, select.lite { background-color: transparent; @@ -8386,6 +8312,8 @@ select { input[type=text].lite:focus, input[type=text].lite:active, input[type=password].lite:focus, input[type=password].lite:active, + input[type=email].lite:focus, + input[type=email].lite:active, textarea.lite:focus, textarea.lite:active, select.lite:focus, @@ -8396,6 +8324,8 @@ select { input[type=text]:active, input[type=text]:focus, input[type=password]:active, input[type=password]:focus, + input[type=email]:active, + input[type=email]:focus, textarea:active, textarea:focus, select:active, @@ -8405,14 +8335,19 @@ select { border-color: #4c9ed9; border-color: var(--form-text-border-color-active); } -input[type=text]:not(.lite), input[type=password]:not(.lite) { - padding: 5px; } +input[type=text]:not(.lite), +input[type=password]:not(.lite), +input[type=email]:not(.lite) { + padding: 0.3rem; } + +textarea { + padding: 0.3rem; } form .note { font-style: italic; } select { - padding: 5px 10px; } + padding: 0.3rem 0.6rem; } option[selected] { font-weight: bold; } diff --git a/public/themes/dark/theme.css b/public/themes/dark/theme.css index 190a1dc..f0c0a78 100644 --- a/public/themes/dark/theme.css +++ b/public/themes/dark/theme.css @@ -237,9 +237,6 @@ will throw an error. --project-card-border-color: #333; --project-user-suggestions-background: #222; --project-user-suggestions-border-color: #444; - --project-edit-logo-previw-border-color: #444; - --project-edit-quota-bar-border-color: #444; - --project-edit-quota-bar-filled-background: #888; --notice-text-color: #eee; --notice-unapproved-color: #7a2020; --notice-hidden-color: #494949; diff --git a/public/themes/light/theme.css b/public/themes/light/theme.css index d506bde..0f76c4a 100644 --- a/public/themes/light/theme.css +++ b/public/themes/light/theme.css @@ -255,9 +255,6 @@ will throw an error. --project-card-border-color: #aaa; --project-user-suggestions-background: #fff; --project-user-suggestions-border-color: #ddd; - --project-edit-logo-previw-border-color: #999; - --project-edit-quota-bar-border-color: #999; - --project-edit-quota-bar-filled-background: #444; --notice-text-color: #fff; --notice-unapproved-color: #b42222; --notice-hidden-color: #b6b6b6; diff --git a/src/admintools/admintools.go b/src/admintools/admintools.go index 952c47d..97c532d 100644 --- a/src/admintools/admintools.go +++ b/src/admintools/admintools.go @@ -51,10 +51,7 @@ func init() { } } - hashedPassword, err := auth.HashPassword(password) - if err != nil { - panic(err) - } + hashedPassword := auth.HashPassword(password) err = auth.UpdatePassword(ctx, conn, canonicalUsername, hashedPassword) if err != nil { diff --git a/src/auth/auth.go b/src/auth/auth.go index ba8f669..1fdee8e 100644 --- a/src/auth/auth.go +++ b/src/auth/auth.go @@ -13,6 +13,7 @@ import ( "strings" "time" + "git.handmade.network/hmn/hmn/src/db" "git.handmade.network/hmn/hmn/src/logging" "git.handmade.network/hmn/hmn/src/models" "git.handmade.network/hmn/hmn/src/oops" @@ -150,15 +151,12 @@ func CheckPassword(password string, hashedPassword HashedPassword) (bool, error) } } -func HashPassword(password string) (HashedPassword, error) { +func HashPassword(password string) HashedPassword { // Follows the OWASP recommendations as of March 2021. // https://cheatsheetseries.owasp.org/cheatsheets/Password_Storage_Cheat_Sheet.html salt := make([]byte, saltLength) - _, err := io.ReadFull(rand.Reader, salt) - if err != nil { - return HashedPassword{}, oops.New(err, "failed to generate salt") - } + io.ReadFull(rand.Reader, salt) saltEnc := base64.StdEncoding.EncodeToString(salt) cfg := Argon2idConfig{ @@ -176,12 +174,12 @@ func HashPassword(password string) (HashedPassword, error) { AlgoConfig: cfg.String(), Salt: saltEnc, Hash: keyEnc, - }, nil + } } var ErrUserDoesNotExist = errors.New("user does not exist") -func UpdatePassword(ctx context.Context, conn *pgxpool.Pool, username string, hp HashedPassword) error { +func UpdatePassword(ctx context.Context, conn db.ConnOrTx, username string, hp HashedPassword) error { tag, err := conn.Exec(ctx, "UPDATE auth_user SET password = $1 WHERE username = $2", hp.String(), username) if err != nil { return oops.New(err, "failed to update password") diff --git a/src/discord/history.go b/src/discord/history.go index a042f0e..80c820b 100644 --- a/src/discord/history.go +++ b/src/discord/history.go @@ -68,6 +68,7 @@ func fetchMissingContent(ctx context.Context, dbConn *pgxpool.Pool) { WHERE c.last_content IS NULL AND msg.guild_id = $1 + ORDER BY msg.sent_at DESC `, config.Config.Discord.GuildID, ) @@ -186,13 +187,13 @@ func handleHistoryMessage(ctx context.Context, dbConn *pgxpool.Pool, msg *Messag break } - newMsg, err := saveMessageAndContents(ctx, tx, msg) + newMsg, err := SaveMessageAndContents(ctx, tx, msg) if err != nil { return err } if createSnippets { - if doSnippet, err := allowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil { - _, err := createMessageSnippet(ctx, tx, msg) + if doSnippet, err := AllowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil { + _, err := CreateMessageSnippet(ctx, tx, msg.ID) if err != nil { return err } diff --git a/src/discord/rest.go b/src/discord/rest.go index db91554..925383d 100644 --- a/src/discord/rest.go +++ b/src/discord/rest.go @@ -14,6 +14,7 @@ import ( "strings" "git.handmade.network/hmn/hmn/src/config" + "git.handmade.network/hmn/hmn/src/hmnurl" "git.handmade.network/hmn/hmn/src/logging" "git.handmade.network/hmn/hmn/src/oops" ) @@ -499,6 +500,16 @@ func GetChannelMessages(ctx context.Context, channelID string, in GetChannelMess return msgs, nil } +func GetAuthorizeUrl(state string) string { + params := make(url.Values) + params.Set("response_type", "code") + params.Set("client_id", config.Config.Discord.OAuthClientID) + params.Set("scope", "identify") + params.Set("state", state) + params.Set("redirect_uri", hmnurl.BuildDiscordOAuthCallback()) + return fmt.Sprintf("https://discord.com/api/oauth2/authorize?%s", params.Encode()) +} + func logErrorResponse(ctx context.Context, name string, res *http.Response, msg string) { dump, err := httputil.DumpResponse(res, true) if err != nil { diff --git a/src/discord/showcase.go b/src/discord/showcase.go index df4ad71..8fdc96c 100644 --- a/src/discord/showcase.go +++ b/src/discord/showcase.go @@ -47,7 +47,7 @@ func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) er defer tx.Rollback(ctx) // save the message, maybe save its contents, and maybe make a snippet too - newMsg, err := saveMessageAndContents(ctx, tx, msg) + newMsg, err := SaveMessageAndContents(ctx, tx, msg) if errors.Is(err, errNotEnoughInfo) { logging.ExtractLogger(ctx).Warn(). Interface("msg", msg). @@ -56,8 +56,8 @@ func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) er } else if err != nil { return err } - if doSnippet, err := allowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil { - _, err := createMessageSnippet(ctx, tx, msg) + if doSnippet, err := AllowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil { + _, err := CreateMessageSnippet(ctx, tx, msg.ID) if err != nil { return oops.New(err, "failed to create snippet in gateway") } @@ -120,7 +120,7 @@ the database. This does not create snippets or do anything besides save the message itself. */ -func saveMessage( +func SaveMessage( ctx context.Context, tx db.ConnOrTx, msg *Message, @@ -194,12 +194,12 @@ snippets. Idempotent; can be called any time whether the message exists or not. */ -func saveMessageAndContents( +func SaveMessageAndContents( ctx context.Context, tx db.ConnOrTx, msg *Message, ) (*models.DiscordMessage, error) { - newMsg, err := saveMessage(ctx, tx, msg) + newMsg, err := SaveMessage(ctx, tx, msg) if err != nil { return nil, err } @@ -507,7 +507,11 @@ func saveEmbed( return iDiscordEmbed.(*models.DiscordMessageEmbed), nil } -func allowedToCreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, discordUserId string) (bool, error) { +/* +Checks settings and permissions to decide whether we are allowed to create +snippets for a user. +*/ +func AllowedToCreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, discordUserId string) (bool, error) { canSave, err := db.QueryBool(ctx, tx, ` SELECT u.discord_save_showcase @@ -528,7 +532,16 @@ func allowedToCreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, discordU return canSave, nil } -func createMessageSnippet(ctx context.Context, tx db.ConnOrTx, msg *Message) (*models.Snippet, error) { +/* +Attempts to create a snippet from a Discord message. If a snippet already +exists, it will be returned and no new snippets will be created. + +It uses the content saved in the database to do this. If we do not have +any content saved, nothing will happen. + +Does not check user preferences around snippets. +*/ +func CreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, msgID string) (*models.Snippet, error) { // Check for existing snippet, maybe return it type existingSnippetResult struct { Message models.DiscordMessage `db:"msg"` @@ -547,16 +560,16 @@ func createMessageSnippet(ctx context.Context, tx db.ConnOrTx, msg *Message) (*m WHERE msg.id = $1 `, - msg.ID, + msgID, ) if err != nil { - return nil, oops.New(err, "failed to check for existing snippet") + return nil, oops.New(err, "failed to check for existing snippet for message %s", msgID) } existing := iexisting.(*existingSnippetResult) if existing.Snippet != nil { // A snippet already exists - maybe update its content, then return it - if msg.OriginalHasFields("content") && !existing.Snippet.EditedOnWebsite { + if existing.MessageContent != nil && !existing.Snippet.EditedOnWebsite { contentMarkdown := existing.MessageContent.LastContent contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown) @@ -611,7 +624,7 @@ func createMessageSnippet(ctx context.Context, tx db.ConnOrTx, msg *Message) (*m contentMarkdown, contentHTML, assetId, - msg.ID, + msgID, existing.DiscordUser.HMNUserId, ) if err != nil { @@ -623,7 +636,7 @@ func createMessageSnippet(ctx context.Context, tx db.ConnOrTx, msg *Message) (*m SET snippet_created = TRUE WHERE id = $1 `, - msg.ID, + msgID, ) if err != nil { return nil, oops.New(err, "failed to mark message as having snippet") diff --git a/src/hmnurl/urls.go b/src/hmnurl/urls.go index f71960c..b5b16bc 100644 --- a/src/hmnurl/urls.go +++ b/src/hmnurl/urls.go @@ -194,10 +194,10 @@ func BuildUserProfile(username string) string { return Url("/m/"+url.PathEscape(username), nil) } -var RegexUserSettings = regexp.MustCompile(`^/_settings$`) +var RegexUserSettings = regexp.MustCompile(`^/settings$`) func BuildUserSettings(section string) string { - return ProjectUrlWithFragment("/_settings", nil, "", section) + return ProjectUrlWithFragment("/settings", nil, "", section) } /* @@ -558,12 +558,6 @@ func BuildLibraryResource(projectSlug string, resourceId int) string { * Discord OAuth */ -var RegexDiscordTest = regexp.MustCompile("^/discord$") - -func BuildDiscordTest() string { - return Url("/discord", nil) -} - var RegexDiscordOAuthCallback = regexp.MustCompile("^/_discord_callback$") func BuildDiscordOAuthCallback() string { @@ -576,6 +570,12 @@ func BuildDiscordUnlink() string { return Url("/_discord_unlink", nil) } +var RegexDiscordShowcaseBacklog = regexp.MustCompile("^/discord_showcase_backlog$") + +func BuildDiscordShowcaseBacklog() string { + return Url("/discord_showcase_backlog", nil) +} + /* * Assets */ diff --git a/src/migration/migrations/2021-08-27T190408Z_NewLinkData.go b/src/migration/migrations/2021-08-27T190408Z_NewLinkData.go new file mode 100644 index 0000000..f35e071 --- /dev/null +++ b/src/migration/migrations/2021-08-27T190408Z_NewLinkData.go @@ -0,0 +1,60 @@ +package migrations + +import ( + "context" + "time" + + "git.handmade.network/hmn/hmn/src/migration/types" + "git.handmade.network/hmn/hmn/src/oops" + "github.com/jackc/pgx/v4" +) + +func init() { + registerMigration(NewLinkData{}) +} + +type NewLinkData struct{} + +func (m NewLinkData) Version() types.MigrationVersion { + return types.MigrationVersion(time.Date(2021, 8, 27, 19, 4, 8, 0, time.UTC)) +} + +func (m NewLinkData) Name() string { + return "NewLinkData" +} + +func (m NewLinkData) Description() string { + return "Rework link data to be less completely weird" +} + +func (m NewLinkData) Up(ctx context.Context, tx pgx.Tx) error { + /* + Broadly the goal is to: + - drop `key` + - make `name` not null + - rename `value` to `url` + */ + + _, err := tx.Exec(ctx, `UPDATE handmade_links SET name = '' WHERE name IS NULL`) + if err != nil { + return oops.New(err, "failed to fill in null names") + } + + _, err = tx.Exec(ctx, ` + ALTER TABLE handmade_links + DROP key, + ALTER name SET NOT NULL; + + ALTER TABLE handmade_links + RENAME value TO url; + `) + if err != nil { + return oops.New(err, "failed to alter links table") + } + + return nil +} + +func (m NewLinkData) Down(ctx context.Context, tx pgx.Tx) error { + panic("Implement me") +} diff --git a/src/models/link.go b/src/models/link.go index 319339a..5f9ea61 100644 --- a/src/models/link.go +++ b/src/models/link.go @@ -1,11 +1,10 @@ package models type Link struct { - ID int `db:"id"` - Key string `db:"key"` - Name *string `db:"name"` - Value string `db:"value"` - Ordering int `db:"ordering"` - UserID *int `db:"user_id"` - ProjectID *int `db:"project_id"` + ID int `db:"id"` + Name string `db:"name"` + URL string `db:"url"` + Ordering int `db:"ordering"` + UserID *int `db:"user_id"` + ProjectID *int `db:"project_id"` } diff --git a/src/rawdata/scss/_core.scss b/src/rawdata/scss/_core.scss index ab25006..4e9e095 100644 --- a/src/rawdata/scss/_core.scss +++ b/src/rawdata/scss/_core.scss @@ -1,3 +1,6 @@ +// Global variables +$input-padding: 0.3rem; + .noselect { -webkit-touch-callout: none; /* iOS Safari */ -webkit-user-select: none; /* Chrome/Safari/Opera */ @@ -24,7 +27,7 @@ body { min-height: 100vh; box-sizing: border-box; font-size: px2rem(14px); - line-height: 1.5em; + line-height: 1.2em; font-weight: 400; } @@ -735,24 +738,16 @@ footer { .tab-bar { @include usevar(border-color, tab-border-color); + @extend .flex, .flex-row; width: 100%; - border-bottom-width: 1px; - border-bottom-style: solid; - box-sizing: border-box; .tab-button { @include usevar(background-color, tab-button-background); @include usevar(border-color, tab-border-color); + @extend .ph3, .pv2; - height:100%; - display:inline-block; - padding:10px 15px; - line-height:100%; - cursor:pointer; - border-width: 1px; - border-style: solid; - box-sizing:border-box; + cursor: pointer; // TODO: Should this be a link? &:hover { @include usevar(background-color, tab-button-background-hover); @@ -760,10 +755,7 @@ footer { &.current { @include usevar(background-color, tab-button-background-current); - - border-bottom-color: transparent; - font-weight:bold; - height:105%; + font-weight: 500; } } } diff --git a/src/rawdata/scss/_editor.scss b/src/rawdata/scss/_editor.scss index b6ab5bc..b2e787c 100644 --- a/src/rawdata/scss/_editor.scss +++ b/src/rawdata/scss/_editor.scss @@ -44,103 +44,49 @@ } .edit-form { - .error { - margin-left:5em; - padding:10px; - color:red; + .edit-form-row { + @extend .flex; + @extend .flex-column; + @extend .flex-row-ns; + @extend .mv3; + + > :first-child { + @extend .w-100; + @extend .w4-ns; + @extend .flex-grow-0; + @extend .flex-shrink-0; + @extend .tl; + @extend .tr-ns; + @extend .pr0; + @extend .pr2-ns; + @extend .pb1; + @extend .pb0-ns; + font-weight: 500; + } + + > :nth-child(2) { + @extend .flex-grow-1; + @extend .overflow-hidden; + } + + .pt-input-ns { + // NOTE(ben): This could maybe be more general someday? + @media #{$breakpoint-not-small} { + padding-top: $input-padding; + } + } } input[type=text] { - min-width:20em; + @extend .w-100; + @extend .mw5-ns; } textarea { - font-size:13pt; - } - - .note { - margin-bottom:5px; - font-style:italic; - font-size:90%; - } - - .links { - width: 80%; - min-height: 200px; - height: 15vh; - } - - .half { - padding:10px; - text-align:center; - } - - table { - width:95%; - margin:auto; - border-collapse:separate; - border-spacing: 0px 10px; - - td { - padding-bottom:15px; - width:90%; - - &.half { - width:50%; - } - - table { - width:100%; - } - } - } - - th { - text-align:right; - font-weight:bold; - padding-right:10px; - padding-bottom:15px; - vertical-align:top; - max-width:5em; - } - - td table th { - text-align:left; - } - - .page-options label { - font-weight:bold; - margin-right:20px; - } - - &.profile-edit { - .longbio { - width: 100%; - min-height: 400px; - height: 30vh; - } - - .avatar-preview { - border:1px solid transparent; - margin:10px; - margin-bottom:0px; - } - - textarea.shortbio, - textarea.signature, - { - min-width:300px; - width:50%; - min-height: 100px; - height:4em; - } - - .logo-preview { - @include usevar(border-color, 'project-edit-logo-previw-border-color'); - - width:200px; - border-width: 1px; - } + @extend .w-100; + @extend .w6-ns; + @extend .mw-100; + @extend .h3; } &.project-edit { @@ -153,21 +99,21 @@ input.project_blurb, input.project_name, { - min-width:300px; - width:50%; + min-width: 300px; + width: 50%; } .quota-bar { - @include usevar(border-color, 'project-edit-quota-bar-border-color'); + // @include usevar(border-color, 'project-edit-quota-bar-border-color'); - width:500px; + width: 500px; border-width: 1px; - margin-bottom:10px; + margin-bottom: 10px; .quota-filled { - @include usevar(background-color, 'project-edit-quota-bar-filled-background'); + // @include usevar(background-color, 'project-edit-quota-bar-filled-background'); - height:100%; + height: 100%; } } } diff --git a/src/rawdata/scss/_forms.scss b/src/rawdata/scss/_forms.scss index 906725c..2b794da 100644 --- a/src/rawdata/scss/_forms.scss +++ b/src/rawdata/scss/_forms.scss @@ -68,6 +68,7 @@ input[type=text], input[type=password], +input[type=email], textarea, select, { @@ -102,18 +103,25 @@ select, } } -input[type=text], input[type=password] { +input[type=text], +input[type=password], +input[type=email], +{ &:not(.lite) { - padding:5px; + padding: $input-padding; } } +textarea { + padding: $input-padding; +} + form .note { font-style:italic; } select { - padding: 5px 10px; + padding: $input-padding 2*$input-padding; } option[selected] { diff --git a/src/rawdata/scss/tachyons/scss/_variables.scss b/src/rawdata/scss/tachyons/scss/_variables.scss index d4ee6c5..7ef94a1 100644 --- a/src/rawdata/scss/tachyons/scss/_variables.scss +++ b/src/rawdata/scss/tachyons/scss/_variables.scss @@ -42,6 +42,7 @@ $width-2: 2rem !default; $width-3: 4rem !default; $width-4: 8rem !default; $width-5: 16rem !default; +$width-6: 32rem !default; $max-width-1: 1rem !default; $max-width-2: 2rem !default; $max-width-3: 4rem !default; diff --git a/src/rawdata/scss/tachyons/scss/_widths.scss b/src/rawdata/scss/tachyons/scss/_widths.scss index abc4fc7..2b24968 100644 --- a/src/rawdata/scss/tachyons/scss/_widths.scss +++ b/src/rawdata/scss/tachyons/scss/_widths.scss @@ -54,6 +54,7 @@ .w3 { width: $width-3; } .w4 { width: $width-4; } .w5 { width: $width-5; } +.w6 { width: $width-6; } .w-10 { width: 10%; } .w-20 { width: 20%; } @@ -80,6 +81,7 @@ .w3-ns { width: $width-3; } .w4-ns { width: $width-4; } .w5-ns { width: $width-5; } + .w6-ns { width: $width-6; } .w-10-ns { width: 10%; } .w-20-ns { width: 20%; } .w-25-ns { width: 25%; } @@ -105,6 +107,7 @@ .w3-m { width: $width-3; } .w4-m { width: $width-4; } .w5-m { width: $width-5; } + .w6-m { width: $width-6; } .w-10-m { width: 10%; } .w-20-m { width: 20%; } .w-25-m { width: 25%; } @@ -130,6 +133,7 @@ .w3-l { width: $width-3; } .w4-l { width: $width-4; } .w5-l { width: $width-5; } + .w6-l { width: $width-6; } .w-10-l { width: 10%; } .w-20-l { width: 20%; } .w-25-l { width: 25%; } diff --git a/src/rawdata/scss/themes/dark/_variables.scss b/src/rawdata/scss/themes/dark/_variables.scss index e02f3f8..30a8663 100644 --- a/src/rawdata/scss/themes/dark/_variables.scss +++ b/src/rawdata/scss/themes/dark/_variables.scss @@ -38,9 +38,6 @@ $vars: ( project-card-border-color: #333, project-user-suggestions-background: #222, project-user-suggestions-border-color: #444, - project-edit-logo-previw-border-color: #444, - project-edit-quota-bar-border-color: #444, - project-edit-quota-bar-filled-background: #888, notice-text-color: $fg-font-color, notice-unapproved-color: #7a2020, diff --git a/src/rawdata/scss/themes/light/_variables.scss b/src/rawdata/scss/themes/light/_variables.scss index df360d2..9c35f22 100644 --- a/src/rawdata/scss/themes/light/_variables.scss +++ b/src/rawdata/scss/themes/light/_variables.scss @@ -38,9 +38,6 @@ $vars: ( project-card-border-color: #aaa, project-user-suggestions-background: #fff, project-user-suggestions-border-color: #ddd, - project-edit-logo-previw-border-color: #999, - project-edit-quota-bar-border-color: #999, - project-edit-quota-bar-filled-background: #444, notice-text-color: #fff, notice-unapproved-color: #b42222, diff --git a/src/templates/mapping.go b/src/templates/mapping.go index 0d795ce..de1633c 100644 --- a/src/templates/mapping.go +++ b/src/templates/mapping.go @@ -147,6 +147,7 @@ func UserToTemplate(u *models.User, currentTheme string) User { IsStaff: u.IsStaff, Name: u.BestName(), + Bio: u.Bio, Blurb: u.Blurb, Signature: u.Signature, DateJoined: u.DateJoined, @@ -162,60 +163,85 @@ func UserToTemplate(u *models.User, currentTheme string) User { } } -var RegexServiceYoutube = regexp.MustCompile(`youtube\.com/(c/)?(?P[\w/-]+)$`) -var RegexServiceTwitter = regexp.MustCompile(`twitter\.com/(?P\w+)$`) -var RegexServiceGithub = regexp.MustCompile(`github\.com/(?P[\w/-]+)$`) -var RegexServiceTwitch = regexp.MustCompile(`twitch\.tv/(?P[\w/-]+)$`) -var RegexServiceHitbox = regexp.MustCompile(`hitbox\.tv/(?P[\w/-]+)$`) -var RegexServicePatreon = regexp.MustCompile(`patreon\.com/(?P[\w/-]+)$`) -var RegexServiceSoundcloud = regexp.MustCompile(`soundcloud\.com/(?P[\w/-]+)$`) -var RegexServiceItch = regexp.MustCompile(`(?P[\w/-]+)\.itch\.io/?$`) - -var LinkServiceMap = map[string]*regexp.Regexp{ - "youtube": RegexServiceYoutube, - "twitter": RegexServiceTwitter, - "github": RegexServiceGithub, - "twitch": RegexServiceTwitch, - "hitbox": RegexServiceHitbox, - "patreon": RegexServicePatreon, - "soundcloud": RegexServiceSoundcloud, - "itch": RegexServiceItch, +// An online site/service for which we recognize the link +type LinkService struct { + Name string + IconName string + Regex *regexp.Regexp } -func ParseKnownServicesForLink(link *models.Link) (serviceName string, userData string) { - for name, re := range LinkServiceMap { - match := re.FindStringSubmatch(link.Value) +var LinkServices = []LinkService{ + { + Name: "YouTube", + IconName: "youtube", + Regex: regexp.MustCompile(`youtube\.com/(c/)?(?P[\w/-]+)$`), + }, + { + Name: "Twitter", + IconName: "twitter", + Regex: regexp.MustCompile(`twitter\.com/(?P\w+)$`), + }, + { + Name: "GitHub", + IconName: "github", + Regex: regexp.MustCompile(`github\.com/(?P[\w/-]+)$`), + }, + { + Name: "Twitch", + IconName: "twitch", + Regex: regexp.MustCompile(`twitch\.tv/(?P[\w/-]+)$`), + }, + { + Name: "Hitbox", + IconName: "hitbox", + Regex: regexp.MustCompile(`hitbox\.tv/(?P[\w/-]+)$`), + }, + { + Name: "Patreon", + IconName: "patreon", + Regex: regexp.MustCompile(`patreon\.com/(?P[\w/-]+)$`), + }, + { + Name: "SoundCloud", + IconName: "soundcloud", + Regex: regexp.MustCompile(`soundcloud\.com/(?P[\w/-]+)$`), + }, + { + Name: "itch.io", + IconName: "itch", + Regex: regexp.MustCompile(`(?P[\w/-]+)\.itch\.io/?$`), + }, +} + +func ParseKnownServicesForLink(link *models.Link) (service LinkService, userData string) { + for _, svc := range LinkServices { + match := svc.Regex.FindStringSubmatch(link.URL) if match != nil { - serviceName = name - userData = match[re.SubexpIndex("userdata")] - return + return svc, match[svc.Regex.SubexpIndex("userdata")] } } - return "", "" + return LinkService{}, "" } func LinkToTemplate(link *models.Link) Link { - name := "" - /* - // NOTE(asaf): While Name and Key are separate things, Name is almost always the same as Key in the db, which looks weird. - // So we're just going to ignore Name until we decide it's worth reusing. - if link.Name != nil { - name = *link.Name - } - */ - serviceName, serviceUserData := ParseKnownServicesForLink(link) - if serviceUserData != "" { - name = serviceUserData + tlink := Link{ + Name: link.Name, + Url: link.URL, + LinkText: link.URL, } - if name == "" { - name = link.Value + + service, userData := ParseKnownServicesForLink(link) + if tlink.Name == "" && service.Name != "" { + tlink.Name = service.Name } - return Link{ - Key: link.Key, - Name: name, - Icon: serviceName, - Url: link.Value, + if service.IconName != "" { + tlink.Icon = service.IconName } + if userData != "" { + tlink.LinkText = userData + } + + return tlink } func TimelineItemsToJSON(items []TimelineItem) string { diff --git a/src/templates/src/project_homepage.html b/src/templates/src/project_homepage.html index 0503db6..eb3a9aa 100644 --- a/src/templates/src/project_homepage.html +++ b/src/templates/src/project_homepage.html @@ -63,8 +63,8 @@ {{ range .ProjectLinks }}
-
{{ .Key }}
-
{{ .Name }}
+
{{ .Name }}
+
{{ .LinkText }}
{{ end }} diff --git a/src/templates/src/user_profile.html b/src/templates/src/user_profile.html index f37dfa8..d23793a 100644 --- a/src/templates/src/user_profile.html +++ b/src/templates/src/user_profile.html @@ -67,8 +67,8 @@ {{ range .ProfileUserLinks }}
-
{{ .Key }}
-
{{ .Name }}
+
{{ .Name }}
+
{{ .LinkText }}
{{ end }} diff --git a/src/templates/src/user_settings.html b/src/templates/src/user_settings.html new file mode 100644 index 0000000..3ddaaae --- /dev/null +++ b/src/templates/src/user_settings.html @@ -0,0 +1,201 @@ +{{ template "base.html" . }} + +{{ define "extrahead" }} + +{{ end }} + +{{ define "content" }} +
+ {{ csrftoken .Session }} +
+
+
Username:
+
+
{{ .User.Username }}
+
If you would like to change your username, please contact us.
+
+
+
+
Real name:
+
+ +
(optional)
+
+
+
+
Email:
+
+ +
+ + +
+
+
+
+
Theme:
+
+ + +
+
+
+
Avatar:
+
+ +
+
+
(Allowed image types: PNG, JPEG and GIF. Avatars may weigh up to 1MB and will be resized if larger than 400x400 pixels)
+
+
+
+
Short bio:
+
+ +
+
+
+
Forum signature:
+
+ +
+
+
+
+
+ +
+
+
+ +
+
+
Old password:
+
+ +
+
+
+
New password:
+
+ +
+ Your password must be 8 or more characters, and must differ from your username and current password. + Other than that, please follow best practices. +
+
+
+
+
New password confirmation:
+
+ +
+
+
+
+
+ +
+
+
+ +
+
+
Links:
+
+ +
+
Relevant links to put on your profile.
+
Format: url [Title] (e.g. http://example.com/ Example Site)
+
(1 per line, 10 max)
+
+
+
+
+
Description:
+
+ +
+
Include some information about yourself, such as your background, interests, occupation, etc.
+
+
+
+
+
+
+ +
+
+
+ +
+
+ {{ if .DiscordUser }} + Linked account: + {{ .DiscordUser.Username }}#{{ .DiscordUser.Discriminator }} + + Unlink account + + {{ else }} + You haven't linked your Discord account. + Link account + {{ end }} +
+ +
+ + +
Snippets will only be created while this setting is on.
+
+ +
+ + +
+ + {{ if .DiscordUser }} +
+ + Create snippets from all of my #project-showcase posts + +
+ Use this if you have a backlog of content in #project-showcase that you want on your profile. +
+ {{ if gt .DiscordNumUnsavedMessages 0 }} +
+ WARNING: {{ .DiscordNumUnsavedMessages }} of your messages are currently waiting to be processed. If you run this command now, some snippets may still be missing. +
+ {{ end }} +
+ {{ end }} + + +
+
+ + + +
+ {{ csrftoken .Session }} + +
+{{ end }} \ No newline at end of file diff --git a/src/templates/types.go b/src/templates/types.go index 3759720..e86e7d7 100644 --- a/src/templates/types.go +++ b/src/templates/types.go @@ -137,6 +137,7 @@ type User struct { ProfileUrl string DarkTheme bool + ShowEmail bool Timezone string CanEditLibrary bool @@ -145,10 +146,10 @@ type User struct { } type Link struct { - Key string - Name string - Url string - Icon string + Name string + Url string + LinkText string + Icon string } type Podcast struct { diff --git a/src/website/auth.go b/src/website/auth.go index 8eb2ad9..ce32597 100644 --- a/src/website/auth.go +++ b/src/website/auth.go @@ -210,10 +210,7 @@ func RegisterNewUserSubmit(c *RequestContext) ResponseData { return c.Redirect(hmnurl.BuildRegistrationSuccess(), http.StatusSeeOther) } - hashed, err := auth.HashPassword(password) - if err != nil { - return ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to encrypt password")) - } + hashed := auth.HashPassword(password) c.Perf.StartBlock("SQL", "Create user and one time token") tx, err := c.Conn.Begin(c.Context()) @@ -622,10 +619,7 @@ func DoPasswordResetSubmit(c *RequestContext) ResponseData { return RejectRequest(c, "Password confirmation doesn't match password") } - hashed, err := auth.HashPassword(password) - if err != nil { - return ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to encrypt password")) - } + hashed := auth.HashPassword(password) c.Perf.StartBlock("SQL", "Update user's password and delete reset token") tx, err := c.Conn.Begin(c.Context()) @@ -707,14 +701,10 @@ func tryLogin(c *RequestContext, user *models.User, password string) (bool, erro // re-hash and save the user's password if necessary if hashed.IsOutdated() { - newHashed, err := auth.HashPassword(password) - if err == nil { - err := auth.UpdatePassword(c.Context(), c.Conn, user.Username, newHashed) - if err != nil { - c.Logger.Error().Err(err).Msg("failed to update user's password") - } - } else { - c.Logger.Error().Err(err).Msg("failed to re-hash password") + newHashed := auth.HashPassword(password) + err := auth.UpdatePassword(c.Context(), c.Conn, user.Username, newHashed) + if err != nil { + c.Logger.Error().Err(err).Msg("failed to update user's password") } // If errors happen here, we can still continue with logging them in } diff --git a/src/website/discord.go b/src/website/discord.go index 57611fb..2504084 100644 --- a/src/website/discord.go +++ b/src/website/discord.go @@ -2,9 +2,7 @@ package website import ( "errors" - "fmt" "net/http" - "net/url" "time" "git.handmade.network/hmn/hmn/src/auth" @@ -14,62 +12,8 @@ import ( "git.handmade.network/hmn/hmn/src/hmnurl" "git.handmade.network/hmn/hmn/src/models" "git.handmade.network/hmn/hmn/src/oops" - "git.handmade.network/hmn/hmn/src/templates" ) -func DiscordTest(c *RequestContext) ResponseData { - var userDiscord *models.DiscordUser - iUserDiscord, err := db.QueryOne(c.Context(), c.Conn, models.DiscordUser{}, - ` - SELECT $columns - FROM handmade_discorduser - WHERE hmn_user_id = $1 - `, - c.CurrentUser.ID, - ) - if err != nil { - if errors.Is(err, db.ErrNoMatchingRows) { - // we're ok, just no user - } else { - return ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to fetch current user's Discord account")) - } - } else { - userDiscord = iUserDiscord.(*models.DiscordUser) - } - - type templateData struct { - templates.BaseData - DiscordUser *templates.DiscordUser - AuthorizeURL string - UnlinkURL string - } - - baseData := getBaseData(c) - baseData.Title = "Discord Test" - - params := make(url.Values) - params.Set("response_type", "code") - params.Set("client_id", config.Config.Discord.OAuthClientID) - params.Set("scope", "identify") - params.Set("state", c.CurrentSession.CSRFToken) - params.Set("redirect_uri", hmnurl.BuildDiscordOAuthCallback()) - - td := templateData{ - BaseData: baseData, - AuthorizeURL: fmt.Sprintf("https://discord.com/api/oauth2/authorize?%s", params.Encode()), - UnlinkURL: hmnurl.BuildDiscordUnlink(), - } - - if userDiscord != nil { - u := templates.DiscordUserToTemplate(userDiscord) - td.DiscordUser = &u - } - - var res ResponseData - res.MustWriteTemplate("discordtest.html", td, c.Perf) - return res -} - func DiscordOAuthCallback(c *RequestContext) ResponseData { query := c.Req.URL.Query() @@ -95,14 +39,19 @@ func DiscordOAuthCallback(c *RequestContext) ResponseData { } // Check for error values and redirect back to ???? - if query.Get("error") != "" { + if errCode := query.Get("error"); errCode != "" { // TODO: actually handle these errors - return ErrorResponse(http.StatusBadRequest, errors.New(query.Get("error"))) + if errCode == "access_denied" { + // This occurs when the user cancels. Just go back to the profile page. + return c.Redirect(hmnurl.BuildUserSettings("discord"), http.StatusSeeOther) + } else { + return RejectRequest(c, "Failed to authenticate with Discord.") + } } // Do the actual token exchange and redirect back to ???? code := query.Get("code") - res, err := discord.ExchangeOAuthCode(c.Context(), code, hmnurl.BuildDiscordOAuthCallback()) // TODO: Redirect to the right place + res, err := discord.ExchangeOAuthCode(c.Context(), code, hmnurl.BuildDiscordOAuthCallback()) if err != nil { return ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to exchange Discord authorization code")) } @@ -139,7 +88,7 @@ func DiscordOAuthCallback(c *RequestContext) ResponseData { return ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to save new Discord user info")) } - return c.Redirect(hmnurl.BuildDiscordTest(), http.StatusSeeOther) + return c.Redirect(hmnurl.BuildUserSettings("discord"), http.StatusSeeOther) } func DiscordUnlink(c *RequestContext) ResponseData { @@ -159,7 +108,7 @@ func DiscordUnlink(c *RequestContext) ResponseData { ) if err != nil { if errors.Is(err, db.ErrNoMatchingRows) { - return c.Redirect(hmnurl.BuildDiscordTest(), http.StatusSeeOther) + return c.Redirect(hmnurl.BuildUserSettings("discord"), http.StatusSeeOther) } else { return ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to get Discord user for unlink")) } @@ -187,5 +136,59 @@ func DiscordUnlink(c *RequestContext) ResponseData { c.Logger.Warn().Err(err).Msg("failed to remove member role on unlink") } - return c.Redirect(hmnurl.BuildDiscordTest(), http.StatusSeeOther) + return c.Redirect(hmnurl.BuildUserSettings("discord"), http.StatusSeeOther) +} + +func DiscordShowcaseBacklog(c *RequestContext) ResponseData { + iduser, err := db.QueryOne(c.Context(), c.Conn, models.DiscordUser{}, + `SELECT $columns FROM handmade_discorduser WHERE hmn_user_id = $1`, + c.CurrentUser.ID, + ) + if errors.Is(err, db.ErrNoMatchingRows) { + // Nothing to do + c.Logger.Warn().Msg("could not do showcase backlog because no discord user exists") + return c.Redirect(hmnurl.BuildUserProfile(c.CurrentUser.Username), http.StatusSeeOther) + } else if err != nil { + return ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to get discord user")) + } + duser := iduser.(*models.DiscordUser) + + ok, err := discord.AllowedToCreateMessageSnippet(c.Context(), c.Conn, duser.UserID) + if err != nil { + return ErrorResponse(http.StatusInternalServerError, err) + } + + if !ok { + // Not allowed to do this, bail out + c.Logger.Warn().Msg("was not allowed to save user snippets") + return c.Redirect(hmnurl.BuildUserProfile(c.CurrentUser.Username), http.StatusSeeOther) + } + + type messageIdQuery struct { + MessageID string `db:"msg.id"` + } + imsgIds, err := db.Query(c.Context(), c.Conn, messageIdQuery{}, + ` + SELECT $columns + FROM + handmade_discordmessage AS msg + WHERE + msg.user_id = $1 + AND msg.channel_id = $2 + `, + duser.UserID, + config.Config.Discord.ShowcaseChannelID, + ) + msgIds := imsgIds.ToSlice() + + for _, imsgId := range msgIds { + msgId := imsgId.(*messageIdQuery) + _, err := discord.CreateMessageSnippet(c.Context(), c.Conn, msgId.MessageID) + if err != nil { + c.Logger.Warn().Err(err).Msg("failed to create snippet from showcase backlog") + continue + } + } + + return c.Redirect(hmnurl.BuildUserProfile(c.CurrentUser.Username), http.StatusSeeOther) } diff --git a/src/website/imagefile_helper.go b/src/website/imagefile_helper.go new file mode 100644 index 0000000..44b9bd6 --- /dev/null +++ b/src/website/imagefile_helper.go @@ -0,0 +1,93 @@ +package website + +import ( + "crypto/sha1" + "encoding/hex" + "errors" + "fmt" + "image" + "io" + "net/http" + "os" + + "git.handmade.network/hmn/hmn/src/db" + "git.handmade.network/hmn/hmn/src/oops" +) + +// If a helper method returns this, you should call RejectRequest with the value. +type RejectRequestError error + +/* +Reads an image file from form data and saves it to the filesystem and the database. +If the file doesn't exist, this does nothing. + +NOTE(ben): Someday we should replace this with the asset system. +*/ +func SaveImageFile(c *RequestContext, dbConn db.ConnOrTx, fileFieldName string, maxSize int64, filepath string) (imageFileId int, err error) { + img, header, err := c.Req.FormFile(fileFieldName) + filename := "" + width := 0 + height := 0 + if err != nil && !errors.Is(err, http.ErrMissingFile) { + return 0, oops.New(err, "failed to read uploaded file") + } + + if header != nil { + if header.Size > maxSize { + return 0, RejectRequestError(fmt.Errorf("Image filesize too big. Max size: %d bytes", maxSize)) + } else { + c.Perf.StartBlock("IMAGE", "Decoding image") + config, format, err := image.DecodeConfig(img) + c.Perf.EndBlock() + if err != nil { + return 0, RejectRequestError(errors.New("Image type not supported")) + } + width = config.Width + height = config.Height + if width == 0 || height == 0 { + return 0, RejectRequestError(errors.New("Image has zero size")) + } + + filename = fmt.Sprintf("%s.%s", filepath, format) + storageFilename := fmt.Sprintf("public/media/%s", filename) + c.Perf.StartBlock("IMAGE", "Writing image file") + file, err := os.Create(storageFilename) + if err != nil { + return 0, oops.New(err, "Failed to create local image file") + } + img.Seek(0, io.SeekStart) + _, err = io.Copy(file, img) + if err != nil { + return 0, oops.New(err, "Failed to write image to file") + } + file.Close() + img.Close() + c.Perf.EndBlock() + } + } + c.Perf.EndBlock() + + c.Perf.StartBlock("SQL", "Saving image file") + if filename != "" { + hasher := sha1.New() + img.Seek(0, io.SeekStart) + io.Copy(hasher, img) // NOTE(asaf): Writing to hash.Hash never returns an error according to the docs + sha1sum := hasher.Sum(nil) + var imageId int + err = dbConn.QueryRow(c.Context(), + ` + INSERT INTO handmade_imagefile (file, size, sha1sum, protected, width, height) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id + `, + filename, header.Size, hex.EncodeToString(sha1sum), false, width, height, + ).Scan(&imageId) + if err != nil { + return 0, oops.New(err, "Failed to insert image file row") + } + + return imageId, nil + } + + return 0, nil +} diff --git a/src/website/podcast.go b/src/website/podcast.go index 7af26f7..536f3f2 100644 --- a/src/website/podcast.go +++ b/src/website/podcast.go @@ -1,11 +1,8 @@ package website import ( - "crypto/sha1" - "encoding/hex" "errors" "fmt" - "image" "io" "io/fs" "net/http" @@ -142,47 +139,6 @@ func PodcastEditSubmit(c *RequestContext) ResponseData { if len(strings.TrimSpace(description)) == 0 { return RejectRequest(c, "Podcast description is empty") } - podcastImage, header, err := c.Req.FormFile("podcast_image") - imageFilename := "" - imageWidth := 0 - imageHeight := 0 - if err != nil && !errors.Is(err, http.ErrMissingFile) { - return ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to read uploaded file")) - } - if header != nil { - if header.Size > maxFileSize { - return RejectRequest(c, fmt.Sprintf("Image filesize too big. Max size: %d bytes", maxFileSize)) - } else { - c.Perf.StartBlock("PODCAST", "Decoding image") - config, format, err := image.DecodeConfig(podcastImage) - c.Perf.EndBlock() - if err != nil { - return RejectRequest(c, "Image type not supported") - } - imageWidth = config.Width - imageHeight = config.Height - if imageWidth == 0 || imageHeight == 0 { - return RejectRequest(c, "Image has zero size") - } - - imageFilename = fmt.Sprintf("podcast/%s/logo%d.%s", c.CurrentProject.Slug, time.Now().UTC().Unix(), format) - storageFilename := fmt.Sprintf("public/media/%s", imageFilename) - c.Perf.StartBlock("PODCAST", "Writing image file") - file, err := os.Create(storageFilename) - if err != nil { - return ErrorResponse(http.StatusInternalServerError, oops.New(err, "Failed to create local image file")) - } - podcastImage.Seek(0, io.SeekStart) - _, err = io.Copy(file, podcastImage) - if err != nil { - return ErrorResponse(http.StatusInternalServerError, oops.New(err, "Failed to write image to file")) - } - file.Close() - podcastImage.Close() - c.Perf.EndBlock() - } - } - c.Perf.EndBlock() c.Perf.StartBlock("SQL", "Updating podcast") tx, err := c.Conn.Begin(c.Context()) @@ -190,23 +146,18 @@ func PodcastEditSubmit(c *RequestContext) ResponseData { return ErrorResponse(http.StatusInternalServerError, oops.New(err, "Failed to start db transaction")) } defer tx.Rollback(c.Context()) - if imageFilename != "" { - hasher := sha1.New() - podcastImage.Seek(0, io.SeekStart) - io.Copy(hasher, podcastImage) // NOTE(asaf): Writing to hash.Hash never returns an error according to the docs - sha1sum := hasher.Sum(nil) - var imageId int - err = tx.QueryRow(c.Context(), - ` - INSERT INTO handmade_imagefile (file, size, sha1sum, protected, width, height) - VALUES ($1, $2, $3, $4, $5, $6) - RETURNING id - `, - imageFilename, header.Size, hex.EncodeToString(sha1sum), false, imageWidth, imageHeight, - ).Scan(&imageId) - if err != nil { - return ErrorResponse(http.StatusInternalServerError, oops.New(err, "Failed to insert image file row")) + + imageId, err := SaveImageFile(c, tx, "podcast_image", maxFileSize, fmt.Sprintf("podcast/%s/logo%d", c.CurrentProject.Slug, time.Now().UTC().Unix())) + if err != nil { + var rejectErr RejectRequestError + if errors.As(err, &rejectErr) { + return RejectRequest(c, rejectErr.Error()) + } else { + return ErrorResponse(http.StatusInternalServerError, oops.New(err, "Failed to save podcast image")) } + } + + if imageId != 0 { _, err = tx.Exec(c.Context(), ` UPDATE handmade_podcast diff --git a/src/website/routes.go b/src/website/routes.go index 1f4d385..efeff59 100644 --- a/src/website/routes.go +++ b/src/website/routes.go @@ -104,7 +104,7 @@ func NewWebsiteRoutes(longRequestContext context.Context, conn *pgxpool.Pool, pe // CSRF mitigation actions per the OWASP cheat sheet: // https://cheatsheetseries.owasp.org/cheatsheets/Cross-Site_Request_Forgery_Prevention_Cheat_Sheet.html return func(c *RequestContext) ResponseData { - c.Req.ParseForm() + c.Req.ParseMultipartForm(100 * 1024 * 1024) csrfToken := c.Req.Form.Get(auth.CSRFFieldName) if csrfToken != c.CurrentSession.CSRFToken { c.Logger.Warn().Str("userId", c.CurrentUser.Username).Msg("user failed CSRF validation - potential attack?") @@ -228,9 +228,12 @@ func NewWebsiteRoutes(longRequestContext context.Context, conn *pgxpool.Pool, pe mainRoutes.GET(hmnurl.RegexPodcastEpisode, PodcastEpisode) mainRoutes.GET(hmnurl.RegexPodcastRSS, PodcastRSS) - mainRoutes.GET(hmnurl.RegexDiscordTest, authMiddleware(DiscordTest)) // TODO: Delete this route mainRoutes.GET(hmnurl.RegexDiscordOAuthCallback, authMiddleware(DiscordOAuthCallback)) - mainRoutes.POST(hmnurl.RegexDiscordUnlink, authMiddleware(DiscordUnlink)) + mainRoutes.POST(hmnurl.RegexDiscordUnlink, authMiddleware(csrfMiddleware(DiscordUnlink))) + mainRoutes.POST(hmnurl.RegexDiscordShowcaseBacklog, authMiddleware(csrfMiddleware(DiscordShowcaseBacklog))) + + mainRoutes.GET(hmnurl.RegexUserSettings, authMiddleware(UserSettings)) + mainRoutes.POST(hmnurl.RegexUserSettings, authMiddleware(csrfMiddleware(UserSettingsSave))) mainRoutes.GET(hmnurl.RegexProjectCSS, ProjectCSS) mainRoutes.GET(hmnurl.RegexEditorPreviewsJS, func(c *RequestContext) ResponseData { diff --git a/src/website/user.go b/src/website/user.go index b6e7111..dc35a39 100644 --- a/src/website/user.go +++ b/src/website/user.go @@ -2,14 +2,21 @@ package website import ( "errors" + "fmt" "net/http" "sort" "strings" + "git.handmade.network/hmn/hmn/src/auth" + "git.handmade.network/hmn/hmn/src/config" "git.handmade.network/hmn/hmn/src/db" + "git.handmade.network/hmn/hmn/src/discord" + hmnemail "git.handmade.network/hmn/hmn/src/email" + "git.handmade.network/hmn/hmn/src/hmnurl" "git.handmade.network/hmn/hmn/src/models" "git.handmade.network/hmn/hmn/src/oops" "git.handmade.network/hmn/hmn/src/templates" + "github.com/jackc/pgx/v4" ) type UserProfileTemplateData struct { @@ -233,3 +240,270 @@ func UserProfile(c *RequestContext) ResponseData { }, c.Perf) return res } + +func UserSettings(c *RequestContext) ResponseData { + var res ResponseData + + type UserSettingsTemplateData struct { + templates.BaseData + + User templates.User + Email string // these fields are handled specially on templates.User + ShowEmail bool + LinksText string + + SubmitUrl string + ContactUrl string + + DiscordUser *templates.DiscordUser + DiscordNumUnsavedMessages int + DiscordAuthorizeUrl string + DiscordUnlinkUrl string + DiscordShowcaseBacklogUrl string + } + + ilinks, err := db.Query(c.Context(), c.Conn, models.Link{}, + ` + SELECT $columns + FROM handmade_links + WHERE user_id = $1 + ORDER BY ordering + `, + c.CurrentUser.ID, + ) + if err != nil { + return ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to fetch user links")) + } + links := ilinks.ToSlice() + + linksText := "" + for _, ilink := range links { + link := ilink.(*models.Link) + linksText += fmt.Sprintf("%s %s\n", link.URL, link.Name) + } + + var tduser *templates.DiscordUser + var numUnsavedMessages int + iduser, err := db.QueryOne(c.Context(), c.Conn, models.DiscordUser{}, + ` + SELECT $columns + FROM handmade_discorduser + WHERE hmn_user_id = $1 + `, + c.CurrentUser.ID, + ) + if errors.Is(err, db.ErrNoMatchingRows) { + // this is fine, but don't fetch any more messages + } else if err != nil { + return ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to fetch user's Discord account")) + } else { + duser := iduser.(*models.DiscordUser) + tmp := templates.DiscordUserToTemplate(duser) + tduser = &tmp + + numUnsavedMessages, err = db.QueryInt(c.Context(), c.Conn, + ` + SELECT COUNT(*) + FROM + handmade_discordmessage AS msg + LEFT JOIN handmade_discordmessagecontent AS c ON c.message_id = msg.id + WHERE + msg.user_id = $1 + AND msg.channel_id = $2 + AND c.last_content IS NULL + `, + duser.UserID, + config.Config.Discord.ShowcaseChannelID, + ) + if err != nil { + return ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to check for unsaved user messages")) + } + } + + templateUser := templates.UserToTemplate(c.CurrentUser, c.Theme) + + baseData := getBaseData(c) + baseData.Title = templateUser.Name + + res.MustWriteTemplate("user_settings.html", UserSettingsTemplateData{ + BaseData: baseData, + User: templateUser, + Email: c.CurrentUser.Email, + ShowEmail: c.CurrentUser.ShowEmail, + LinksText: linksText, + + SubmitUrl: hmnurl.BuildUserSettings(""), + ContactUrl: hmnurl.BuildContactPage(), + + DiscordUser: tduser, + DiscordNumUnsavedMessages: numUnsavedMessages, + DiscordAuthorizeUrl: discord.GetAuthorizeUrl(c.CurrentSession.CSRFToken), + DiscordUnlinkUrl: hmnurl.BuildDiscordUnlink(), + DiscordShowcaseBacklogUrl: hmnurl.BuildDiscordShowcaseBacklog(), + }, c.Perf) + return res +} + +func UserSettingsSave(c *RequestContext) ResponseData { + tx, err := c.Conn.Begin(c.Context()) + if err != nil { + panic(err) + } + defer tx.Rollback(c.Context()) + + form, err := c.GetFormValues() + if err != nil { + c.Logger.Warn().Err(err).Msg("failed to parse form on user update") + return c.Redirect(hmnurl.BuildUserSettings(""), http.StatusSeeOther) + } + + name := form.Get("realname") + + email := form.Get("email") + if !hmnemail.IsEmail(email) { + return RejectRequest(c, "Your email was not valid.") + } + + showEmail := form.Get("showemail") != "" + darkTheme := form.Get("darktheme") != "" + + blurb := form.Get("shortbio") + signature := form.Get("signature") + bio := form.Get("longbio") + + discordShowcaseAuto := form.Get("discord-showcase-auto") != "" + discordDeleteSnippetOnMessageDelete := form.Get("discord-snippet-keep") == "" + + _, err = tx.Exec(c.Context(), + ` + UPDATE auth_user + SET + name = $2, + email = $3, + showemail = $4, + darktheme = $5, + blurb = $6, + signature = $7, + bio = $8, + discord_save_showcase = $9, + discord_delete_snippet_on_message_delete = $10 + WHERE + id = $1 + `, + c.CurrentUser.ID, + name, + email, + showEmail, + darkTheme, + blurb, + signature, + bio, + discordShowcaseAuto, + discordDeleteSnippetOnMessageDelete, + ) + if err != nil { + return ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to update user")) + } + + // Process links + linksText := form.Get("links") + links := strings.Split(linksText, "\n") + _, err = tx.Exec(c.Context(), `DELETE FROM handmade_links WHERE user_id = $1`, c.CurrentUser.ID) + if err != nil { + c.Logger.Warn().Err(err).Msg("failed to delete old links") + } else { + for i, link := range links { + link = strings.TrimSpace(link) + linkParts := strings.SplitN(link, " ", 2) + url := strings.TrimSpace(linkParts[0]) + name := "" + if len(linkParts) > 1 { + name = strings.TrimSpace(linkParts[1]) + } + + if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") { + continue + } + + _, err := tx.Exec(c.Context(), + ` + INSERT INTO handmade_links (name, url, ordering, user_id) + VALUES ($1, $2, $3, $4) + `, + name, + url, + i, + c.CurrentUser.ID, + ) + if err != nil { + c.Logger.Warn().Err(err).Msg("failed to insert new link") + continue + } + } + } + + // Update password + oldPassword := form.Get("old_password") + newPassword := form.Get("new_password1") + newPasswordConfirmation := form.Get("new_password2") + if oldPassword != "" && newPassword != "" { + errorRes := updatePassword(c, tx, oldPassword, newPassword, newPasswordConfirmation) + if errorRes != nil { + return *errorRes + } + } + + // Update avatar + _, err = SaveImageFile(c, tx, "avatar", 1*1024*1024, fmt.Sprintf("members/avatars/%s", c.CurrentUser.Username)) + if err != nil { + var rejectErr RejectRequestError + if errors.As(err, &rejectErr) { + return RejectRequest(c, rejectErr.Error()) + } else { + return ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to save new avatar")) + } + } + + // TODO: Success message + + err = tx.Commit(c.Context()) + if err != nil { + return ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to save user settings")) + } + + return c.Redirect(hmnurl.BuildUserSettings(""), http.StatusSeeOther) +} + +// TODO: Rework this to use that RejectRequestError thing +func updatePassword(c *RequestContext, tx pgx.Tx, old, new, confirm string) *ResponseData { + if new != confirm { + res := RejectRequest(c, "Your password and password confirmation did not match.") + return &res + } + + oldHashedPassword, err := auth.ParsePasswordString(c.CurrentUser.Password) + if err != nil { + c.Logger.Warn().Err(err).Msg("failed to parse user's password string") + return nil + } + + ok, err := auth.CheckPassword(old, oldHashedPassword) + if err != nil { + res := ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to check user's password")) + return &res + } + + if !ok { + res := RejectRequest(c, "The old password you provided was not correct.") + return &res + } + + newHashedPassword := auth.HashPassword(new) + err = auth.UpdatePassword(c.Context(), tx, c.CurrentUser.Username, newHashedPassword) + if err != nil { + res := ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to update password")) + return &res + } + + return nil +}