From 76f9256e97ff090b7e482a96548adc3fc94490bc Mon Sep 17 00:00:00 2001 From: Ben Visness Date: Sat, 21 Aug 2021 11:15:27 -0500 Subject: [PATCH] Save Discord messages and attachments --- go.mod | 5 + go.sum | 30 ++++ src/assets/assets.go | 155 +++++++++++++++++++ src/assets/assets_test.go | 13 ++ src/db/db.go | 2 + src/discord/gateway.go | 2 +- src/discord/library.go | 45 ++++++ src/discord/payloads.go | 63 ++++++-- src/discord/showcase.go | 311 ++++++++++++++++++++++++++++++++------ src/discord/todo.txt | 43 ++++++ src/main.go | 1 + 11 files changed, 617 insertions(+), 53 deletions(-) create mode 100644 src/assets/assets.go create mode 100644 src/assets/assets_test.go create mode 100644 src/discord/library.go create mode 100644 src/discord/todo.txt diff --git a/go.mod b/go.mod index 16fef9d..8c56138 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,11 @@ require ( github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible github.com/alecthomas/chroma v0.9.2 + github.com/aws/aws-sdk-go-v2 v1.8.1 + github.com/aws/aws-sdk-go-v2/config v1.6.1 + github.com/aws/aws-sdk-go-v2/credentials v1.3.3 + github.com/aws/aws-sdk-go-v2/service/s3 v1.13.0 + github.com/aws/smithy-go v1.7.0 github.com/frustra/bbcode v0.0.0-20201127003707-6ef347fbe1c8 github.com/go-stack/stack v1.8.0 github.com/google/uuid v1.2.0 diff --git a/go.sum b/go.sum index 474fba9..6e6029e 100644 --- a/go.sum +++ b/go.sum @@ -40,6 +40,30 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/aws/aws-sdk-go-v2 v1.8.1 h1:GcFgQl7MsBygmeeqXyV1ivrTEmsVz/rdFJaTcltG9ag= +github.com/aws/aws-sdk-go-v2 v1.8.1/go.mod h1:xEFuWz+3TYdlPRuo+CqATbeDWIWyaT5uAPwPaWtgse0= +github.com/aws/aws-sdk-go-v2/config v1.6.1 h1:qrZINaORyr78syO1zfD4l7r4tZjy0Z1l0sy4jiysyOM= +github.com/aws/aws-sdk-go-v2/config v1.6.1/go.mod h1:t/y3UPu0XEDy0cEw6mvygaBQaPzWiYAxfP2SzgtvclA= +github.com/aws/aws-sdk-go-v2/credentials v1.3.3 h1:A13QPatmUl41SqUfnuT3V0E3XiNGL6qNTOINbE8cZL4= +github.com/aws/aws-sdk-go-v2/credentials v1.3.3/go.mod h1:oVieKMT3m9BSfqhOfuQ+E0j/yN84ZAJ7Qv8Sfume/ak= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.1 h1:rc+fRGvlKbeSd9IFhFS1KWBs0XjTkq0CfK5xqyLgIp0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.1/go.mod h1:+GTydg3uHmVlQdkRoetz6VHKbOMEYof70m19IpMLifc= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.1 h1:IkqRRUZTKaS16P2vpX+FNc2jq3JWa3c478gykQp4ow4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.2.1/go.mod h1:Pv3WenDjI0v2Jl7UaMFIIbPOBbhn33RmmAmGgkXDoqY= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.2 h1:YcGVEqLQGHDa81776C3daai6ZkkRGf/8RAQ07hV0QcU= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.2/go.mod h1:EASdTcM1lGhUe1/p4gkojHwlGJkeoRjjr1sRCzup3Is= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.3 h1:VxFCgxsqWe7OThOwJ5IpFX3xrObtuIH9Hg/NW7oot1Y= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.3/go.mod h1:7gcsONBmFoCcKrAqrm95trrMd2+C/ReYKP7Vfu8yHHA= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.3 h1:7tPSbUWzuoMJ2woUKgOfIPuZS88hMdFHJBBB2vR0bHI= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.3/go.mod h1:/ugW3qFkJe/h7sNtI6/zJnwRbvavs6GyOid69uI9eek= +github.com/aws/aws-sdk-go-v2/service/s3 v1.13.0 h1:2oMLrNpOSpkDTocIVv3Fut1XrmlbKPlgnnYMGYqFp0Y= +github.com/aws/aws-sdk-go-v2/service/s3 v1.13.0/go.mod h1:Tzxhu3GnCpj45WJqXyxcLF2gUHzTcmY7CzpQ9x9KVls= +github.com/aws/aws-sdk-go-v2/service/sso v1.3.3 h1:K2gCnGvAASpz+jqP9iyr+F/KNjmTYf8aWOtTQzhmZ5w= +github.com/aws/aws-sdk-go-v2/service/sso v1.3.3/go.mod h1:Jgw5O+SK7MZ2Yi9Yvzb4PggAPYaFSliiQuWR0hNjexk= +github.com/aws/aws-sdk-go-v2/service/sts v1.6.2 h1:l504GWCoQi1Pk68vSUFGLmDIEMzRfVGNgLakDK+Uj58= +github.com/aws/aws-sdk-go-v2/service/sts v1.6.2/go.mod h1:RBhoMJB8yFToaCnbe0jNq5Dcdy0jp6LhHqg55rjClkM= +github.com/aws/smithy-go v1.7.0 h1:+cLHMRrDZvQ4wk+KuQ9yH6eEg6KZEJ9RI2IkDqnygCg= +github.com/aws/smithy-go v1.7.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= @@ -92,6 +116,9 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -185,6 +212,8 @@ github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= @@ -484,6 +513,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/src/assets/assets.go b/src/assets/assets.go new file mode 100644 index 0000000..cf2bc83 --- /dev/null +++ b/src/assets/assets.go @@ -0,0 +1,155 @@ +package assets + +import ( + "bytes" + "context" + "crypto/sha1" + "errors" + "fmt" + "regexp" + + "git.handmade.network/hmn/hmn/src/config" + "git.handmade.network/hmn/hmn/src/db" + "git.handmade.network/hmn/hmn/src/models" + "git.handmade.network/hmn/hmn/src/oops" + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + "github.com/google/uuid" +) + +var client *s3.Client + +func init() { + cfg, err := awsconfig.LoadDefaultConfig(context.Background(), + awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider( + config.Config.DigitalOcean.AssetsSpacesKey, + config.Config.DigitalOcean.AssetsSpacesSecret, + "", + ), + ), + awsconfig.WithRegion(config.Config.DigitalOcean.AssetsSpacesRegion), + awsconfig.WithEndpointResolver(aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: config.Config.DigitalOcean.AssetsSpacesEndpoint, + }, nil + })), + ) + if err != nil { + panic(err) + } + client = s3.NewFromConfig(cfg, func(o *s3.Options) { + o.UsePathStyle = true + }) +} + +type CreateInput struct { + Content []byte + Filename string + MimeType string + + // Optional params + UploaderID *int // HMN user id + Width, Height int +} + +var REIllegalFilenameChars = regexp.MustCompile(`[^\w \-.]`) + +func SanitizeFilename(filename string) string { + if filename == "" { + return "unnamed" + } + return REIllegalFilenameChars.ReplaceAllString(filename, "") +} + +func AssetKey(id, filename string) string { + return fmt.Sprintf("%s%s/%s", config.Config.DigitalOcean.AssetsPathPrefix, id, filename) +} + +type InvalidAssetError error + +func Create(ctx context.Context, dbConn db.ConnOrTx, in CreateInput) (*models.Asset, error) { + filename := SanitizeFilename(in.Filename) + + if len(in.Content) == 0 { + return nil, InvalidAssetError(fmt.Errorf("could not upload asset '%s': no bytes of data were provided", filename)) + } + if in.MimeType == "" { + return nil, InvalidAssetError(fmt.Errorf("could not upload asset '%s': no mime type provided", filename)) + } + + // Upload the asset to the DO space + id := uuid.New() + key := AssetKey(id.String(), filename) + checksum := fmt.Sprintf("%x", sha1.Sum(in.Content)) + + upload := func() error { + _, err := client.PutObject(ctx, &s3.PutObjectInput{ + Bucket: &config.Config.DigitalOcean.AssetsSpacesBucket, + Key: &key, + Body: bytes.NewReader(in.Content), + ACL: types.ObjectCannedACLPublicRead, + }) + return err + } + + err := upload() + if err != nil { + var apiError smithy.APIError + if errors.As(err, &apiError) && apiError.ErrorCode() == "NoSuchBucket" { + _, err := client.CreateBucket(ctx, &s3.CreateBucketInput{ + Bucket: &config.Config.DigitalOcean.AssetsSpacesBucket, + }) + if err != nil { + return nil, oops.New(err, "failed to create assets bucket") + } + + err = upload() + if err != nil { + return nil, oops.New(err, "failed to upload asset") + } + } else { + return nil, oops.New(err, "failed to upload asset") + } + } + + // Save a record in our database + // TODO(db): Would be convient to use RETURNING here... + _, err = dbConn.Exec(ctx, + ` + INSERT INTO handmade_asset (id, s3_key, filename, size, mime_type, sha1sum, width, height, uploader_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + `, + id, + key, + filename, + len(in.Content), + in.MimeType, + checksum, + in.Width, + in.Height, + in.UploaderID, + ) + if err != nil { + return nil, oops.New(err, "failed to save asset record") + } + + // Fetch and return the new record + iasset, err := db.QueryOne(ctx, dbConn, models.Asset{}, + ` + SELECT $columns + FROM handmade_asset + WHERE id = $1 + `, + id, + ) + if err != nil { + return nil, oops.New(err, "failed to fetch newly-created asset") + } + + return iasset.(*models.Asset), nil +} diff --git a/src/assets/assets_test.go b/src/assets/assets_test.go new file mode 100644 index 0000000..29a0f08 --- /dev/null +++ b/src/assets/assets_test.go @@ -0,0 +1,13 @@ +package assets + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestSanitizeFilename(t *testing.T) { + assert.Equal(t, "cool filename.txt.wow", SanitizeFilename("cool filename.txt.wow")) + assert.Equal(t, " hi doggy ", SanitizeFilename("😎 hi doggy 🐶")) + assert.Equal(t, "newlinesaretotallylegal", SanitizeFilename("newlines\naretotallylegal")) +} diff --git a/src/db/db.go b/src/db/db.go index 6359f0e..3962082 100644 --- a/src/db/db.go +++ b/src/db/db.go @@ -11,6 +11,7 @@ import ( "git.handmade.network/hmn/hmn/src/logging" "git.handmade.network/hmn/hmn/src/oops" "github.com/google/uuid" + "github.com/jackc/pgconn" "github.com/jackc/pgtype" "github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4/log/zerologadapter" @@ -57,6 +58,7 @@ func typeIsQueryable(t reflect.Type) bool { // This interface should match both a direct pgx connection or a pgx transaction. type ConnOrTx interface { Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error) + Exec(ctx context.Context, sql string, args ...interface{}) (pgconn.CommandTag, error) } var connInfo = pgtype.NewConnInfo() diff --git a/src/discord/gateway.go b/src/discord/gateway.go index 7cdfdac..5962086 100644 --- a/src/discord/gateway.go +++ b/src/discord/gateway.go @@ -563,7 +563,7 @@ func (bot *discordBotInstance) processEventMsg(ctx context.Context, msg *Gateway } func (bot *discordBotInstance) messageCreateOrUpdate(ctx context.Context, msg *Message) error { - if msg.Author != nil && msg.Author.ID == config.Config.Discord.BotUserID { + if msg.OriginalHasFields("author") && msg.Author.ID == config.Config.Discord.BotUserID { // Don't process your own messages return nil } diff --git a/src/discord/library.go b/src/discord/library.go new file mode 100644 index 0000000..3eaa30c --- /dev/null +++ b/src/discord/library.go @@ -0,0 +1,45 @@ +package discord + +import ( + "context" + + "git.handmade.network/hmn/hmn/src/oops" +) + +func (bot *discordBotInstance) processLibraryMsg(ctx context.Context, msg *Message) error { + switch msg.Type { + case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: + default: + return nil + } + + if !msg.OriginalHasFields("content") { + return nil + } + + if !messageHasLinks(msg.Content) { + err := DeleteMessage(ctx, msg.ChannelID, msg.ID) + if err != nil { + return oops.New(err, "failed to delete message") + } + + if !msg.Author.IsBot { + channel, err := CreateDM(ctx, msg.Author.ID) + if err != nil { + return oops.New(err, "failed to create DM channel") + } + + err = SendMessages(ctx, bot.dbConn, MessageToSend{ + ChannelID: channel.ID, + Req: CreateMessageRequest{ + Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.", + }, + }) + if err != nil { + return oops.New(err, "failed to send showcase warning message") + } + } + } + + return nil +} diff --git a/src/discord/payloads.go b/src/discord/payloads.go index edb061a..18b1b4c 100644 --- a/src/discord/payloads.go +++ b/src/discord/payloads.go @@ -2,6 +2,8 @@ package discord import ( "encoding/json" + "fmt" + "time" ) type Opcode int @@ -170,19 +172,50 @@ const ( // https://discord.com/developers/docs/resources/channel#message-object type Message struct { - ID string `json:"id"` - ChannelID string `json:"channel_id"` - Content string `json:"content"` - Author *User `json:"author"` // note that this may not be an actual valid user (see the docs) - // TODO: Author info - // TODO: Timestamp parsing, yay - Type MessageType `json:"type"` + ID string `json:"id"` + ChannelID string `json:"channel_id"` + GuildID *string `json:"guild_id"` + Content string `json:"content"` + Author User `json:"author"` // note that this may not be an actual valid user (see the docs) + Timestamp string `json:"timestamp"` + Type MessageType `json:"type"` Attachments []Attachment `json:"attachments"` + // TODO: Embeds originalMap map[string]interface{} } +func (m *Message) JumpURL() string { + guildStr := "@me" + if m.GuildID != nil { + guildStr = *m.GuildID + } + return fmt.Sprintf("https://discord.com/channels/%s/%s/%s", guildStr, m.ChannelID, m.ID) +} + +func (m *Message) Time() time.Time { + t, err := time.Parse(time.RFC3339Nano, m.Timestamp) + if err != nil { + panic(err) + } + return t +} + +func (m *Message) OriginalHasFields(fields ...string) bool { + if m.originalMap == nil { + return false + } + + for _, field := range fields { + _, ok := m.originalMap[field] + if !ok { + return false + } + } + return true +} + func MessageFromMap(m interface{}) Message { /* Some gateway events, like MESSAGE_UPDATE, do not contain the @@ -194,15 +227,16 @@ func MessageFromMap(m interface{}) Message { msg := Message{ ID: mmap["id"].(string), ChannelID: mmap["channel_id"].(string), + GuildID: maybeStringP(mmap, "guild_id"), Content: maybeString(mmap, "content"), + Timestamp: maybeString(mmap, "timestamp"), Type: MessageType(maybeInt(mmap, "type")), originalMap: mmap, } if author, ok := mmap["author"]; ok { - u := UserFromMap(author) - msg.Author = &u + msg.Author = UserFromMap(author) } if iattachments, ok := mmap["attachments"]; ok { @@ -212,6 +246,8 @@ func MessageFromMap(m interface{}) Message { } } + // TODO: Embeds + return msg } @@ -276,6 +312,15 @@ func maybeString(m map[string]interface{}, k string) string { return val.(string) } +func maybeStringP(m map[string]interface{}, k string) *string { + val, ok := m[k] + if !ok { + return nil + } + strval := val.(string) + return &strval +} + func maybeInt(m map[string]interface{}, k string) int { val, ok := m[k] if !ok { diff --git a/src/discord/showcase.go b/src/discord/showcase.go index 6dacb16..65abbdc 100644 --- a/src/discord/showcase.go +++ b/src/discord/showcase.go @@ -2,15 +2,25 @@ package discord import ( "context" + "errors" + "io" + "net/http" "net/url" "regexp" "strings" + "git.handmade.network/hmn/hmn/src/assets" + "git.handmade.network/hmn/hmn/src/db" + "git.handmade.network/hmn/hmn/src/logging" + "git.handmade.network/hmn/hmn/src/models" "git.handmade.network/hmn/hmn/src/oops" + "github.com/jackc/pgx/v4" ) var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`) +var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this") + func (bot *discordBotInstance) processShowcaseMsg(ctx context.Context, msg *Message) error { switch msg.Type { case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: @@ -18,26 +28,70 @@ func (bot *discordBotInstance) processShowcaseMsg(ctx context.Context, msg *Mess return nil } + didDelete, err := bot.maybeDeleteShowcaseMsg(ctx, msg) + if err != nil { + return err + } + if didDelete { + return nil + } + + tx, err := bot.dbConn.Begin(ctx) + if err != nil { + panic(err) + } + defer tx.Rollback(ctx) + + // save the message, maybe save its contents, and maybe make a snippet too + _, err = bot.saveMessageAndContents(ctx, tx, msg) + if errors.Is(err, errNotEnoughInfo) { + logging.ExtractLogger(ctx).Warn(). + Interface("msg", msg). + Msg("didn't have enough info to process Discord message") + return nil + } else if err != nil { + return err + } + if doSnippet, err := bot.allowedToCreateMessageSnippet(ctx, msg); doSnippet && err == nil { + _, err := bot.createMessageSnippet(ctx, msg) + if err != nil { + return oops.New(err, "failed to create snippet in gateway") + } + } else if err != nil { + return oops.New(err, "failed to check snippet permissions in gateway") + } + + err = tx.Commit(ctx) + if err != nil { + return oops.New(err, "failed to commit Discord message updates") + } + + return nil +} + +func (bot *discordBotInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message) (didDelete bool, err error) { hasGoodContent := true - if originalMessageHasField(msg, "content") && !messageHasLinks(msg.Content) { + if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) { hasGoodContent = false } hasGoodAttachments := true - if originalMessageHasField(msg, "attachments") && len(msg.Attachments) == 0 { + if msg.OriginalHasFields("attachments") && len(msg.Attachments) == 0 { hasGoodAttachments = false } + didDelete = false if !hasGoodContent && !hasGoodAttachments { + didDelete = true err := DeleteMessage(ctx, msg.ChannelID, msg.ID) if err != nil { - return oops.New(err, "failed to delete message") + return false, oops.New(err, "failed to delete message") } - if msg.Author != nil && !msg.Author.IsBot { + if !msg.Author.IsBot { channel, err := CreateDM(ctx, msg.Author.ID) if err != nil { - return oops.New(err, "failed to create DM channel") + return false, oops.New(err, "failed to create DM channel") } err = SendMessages(ctx, bot.dbConn, MessageToSend{ @@ -47,50 +101,230 @@ func (bot *discordBotInstance) processShowcaseMsg(ctx context.Context, msg *Mess }, }) if err != nil { - return oops.New(err, "failed to send showcase warning message") + return false, oops.New(err, "failed to send showcase warning message") } } } - return nil + return didDelete, nil } -func (bot *discordBotInstance) processLibraryMsg(ctx context.Context, msg *Message) error { - switch msg.Type { - case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: - default: - return nil - } +/* +Ensures that a Discord message is stored in the database. This function is +idempotent and can be called regardless of whether the item already exists in +the database. - if !originalMessageHasField(msg, "content") { - return nil - } +This does not create snippets or do anything besides save the message itself. +*/ +func (bot *discordBotInstance) saveMessage( + ctx context.Context, + tx pgx.Tx, + msg *Message, +) (*models.DiscordMessage, error) { + iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{}, + ` + SELECT $columns + FROM handmade_discordmessage + WHERE id = $1 + `, + msg.ID, + ) + if errors.Is(err, db.ErrNoMatchingRows) { + if !msg.OriginalHasFields("author", "timestamp") { + return nil, errNotEnoughInfo + } - if !messageHasLinks(msg.Content) { - err := DeleteMessage(ctx, msg.ChannelID, msg.ID) + _, err = tx.Exec(ctx, + ` + INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created) + VALUES ($1, $2, $3, $4, $5, $6, $7) + `, + msg.ID, + msg.ChannelID, + *msg.GuildID, + msg.JumpURL(), + msg.Author.ID, + msg.Time(), + false, + ) if err != nil { - return oops.New(err, "failed to delete message") + return nil, oops.New(err, "failed to save new discord message") } - if msg.Author != nil && !msg.Author.IsBot { - channel, err := CreateDM(ctx, msg.Author.ID) - if err != nil { - return oops.New(err, "failed to create DM channel") - } + /* + TODO(db): This is a spot where it would be really nice to be able + to use RETURNING, and avoid this second query. + */ + iDiscordMessage, err = db.QueryOne(ctx, tx, models.DiscordMessage{}, + ` + SELECT $columns + FROM handmade_discordmessage + WHERE id = $1 + `, + msg.ID, + ) + if err != nil { + panic(err) + } + } else if err != nil { + return nil, oops.New(err, "failed to check for existing Discord message") + } - err = SendMessages(ctx, bot.dbConn, MessageToSend{ - ChannelID: channel.ID, - Req: CreateMessageRequest{ - Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.", - }, - }) - if err != nil { - return oops.New(err, "failed to send showcase warning message") - } + return iDiscordMessage.(*models.DiscordMessage), nil +} + +/* +Processes a single Discord message, saving as much of the message's content +and attachments as allowed by our rules and user settings. Does NOT create +snippets. + +Idempotent; can be called any time whether the message exists or not. +*/ +func (bot *discordBotInstance) saveMessageAndContents( + ctx context.Context, + tx pgx.Tx, + msg *Message, +) (*models.DiscordMessage, error) { + newMsg, err := bot.saveMessage(ctx, tx, msg) + if err != nil { + return nil, err + } + + // Check for linked Discord user + iDiscordUser, err := db.QueryOne(ctx, tx, models.DiscordUser{}, + ` + SELECT $columns + FROM handmade_discorduser + WHERE userid = $1 + `, + msg.Author.ID, + ) + if errors.Is(err, db.ErrNoMatchingRows) { + return newMsg, nil + } else if err != nil { + return nil, oops.New(err, "failed to look up linked Discord user") + } + discordUser := iDiscordUser.(*models.DiscordUser) + + // We have a linked Discord account, so save the message contents (regardless of + // whether we create a snippet or not). + + _, err = tx.Exec(ctx, + ` + INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content) + VALUES ($1, $2, $3) + ON CONFLICT (message_id) DO UPDATE SET + discord_id = EXCLUDED.discord_id, + last_content = EXCLUDED.last_content + `, + msg.ID, + discordUser.ID, + msg.Content, // TODO: Add a method that can fill in mentions and stuff (https://discord.com/developers/docs/reference#message-formatting) + ) + + // Save attachments + for _, attachment := range msg.Attachments { + _, err := bot.saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID) + if err != nil { + return nil, oops.New(err, "failed to save attachment") } } - return nil + // TODO: Save embeds + + return newMsg, nil +} + +func (bot *discordBotInstance) saveAttachment(ctx context.Context, tx pgx.Tx, attachment *Attachment, hmnUserID int, discordMessageID string) (*models.DiscordMessageAttachment, error) { + // TODO: Return an existing attachment if it exists + + width := 0 + height := 0 + if attachment.Width != nil { + width = *attachment.Width + } + if attachment.Height != nil { + height = *attachment.Height + } + + // TODO: Timeouts and stuff, context cancellation + res, err := http.Get(attachment.Url) + if err != nil { + return nil, oops.New(err, "failed to fetch attachment data") + } + defer res.Body.Close() + + content, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + } + + asset, err := assets.Create(ctx, tx, assets.CreateInput{ + Content: content, + Filename: attachment.Filename, + MimeType: attachment.ContentType, + + UploaderID: &hmnUserID, + Width: width, + Height: height, + }) + if err != nil { + return nil, oops.New(err, "failed to save asset for Discord attachment") + } + + // TODO(db): RETURNING plz thanks + _, err = tx.Exec(ctx, + ` + INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id) + VALUES ($1, $2, $3) + `, + attachment.ID, + asset.ID, + discordMessageID, + ) + if err != nil { + return nil, oops.New(err, "failed to save Discord attachment data") + } + + iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{}, + ` + SELECT $columns + FROM handmade_discordmessageattachment + WHERE id = $1 + `, + attachment.ID, + ) + if err != nil { + return nil, oops.New(err, "failed to fetch new Discord attachment data") + } + + return iDiscordAttachment.(*models.DiscordMessageAttachment), nil +} + +func (bot *discordBotInstance) allowedToCreateMessageSnippet(ctx context.Context, msg *Message) (bool, error) { + canSave, err := db.QueryBool(ctx, bot.dbConn, + ` + SELECT u.discord_save_showcase + FROM + handmade_discorduser AS duser + JOIN auth_user AS u ON duser.hmn_user_id = u.id + WHERE + duser.userid = $1 + `, + msg.Author.ID, + ) + if errors.Is(err, db.ErrNoMatchingRows) { + return false, nil + } else if err != nil { + return false, oops.New(err, "failed to check if we can save Discord message") + } + + return canSave, nil +} + +func (bot *discordBotInstance) createMessageSnippet(ctx context.Context, msg *Message) (*models.Snippet, error) { + // TODO: Actually do this + return nil, nil } func messageHasLinks(content string) bool { @@ -104,12 +338,3 @@ func messageHasLinks(content string) bool { return false } - -func originalMessageHasField(msg *Message, field string) bool { - if msg.originalMap == nil { - return false - } - - _, ok := msg.originalMap[field] - return ok -} diff --git a/src/discord/todo.txt b/src/discord/todo.txt new file mode 100644 index 0000000..2d683ac --- /dev/null +++ b/src/discord/todo.txt @@ -0,0 +1,43 @@ +the goal: port the old discord showcase bot + +what it does: save #project-showcase posts to your HMN user profile if you have your account linked + +stuff we need to worry about: +- old posts from before you linked your account +- posts that come in while the bot is down +- what to do with posts if you unlink your account +- what to do with posts if you re-link your account +- what to do if you edit the original discord message +- what to do if you delete the original discord message +- the user's preferences re: saving content + - we don't want to save content without the user's consent, especially since it may persist after they disable the integration +- manually adding content for various reasons + - maybe a bug prevented something from saving + - ryan used to post everything in #projects for some reason + + +real-time stuff: +- on new showcase message + - always save the lightweight record + - if we have permission, create a snippet +- on edit + - re-save the lightweight record and content as if it was new + - create snippet, unconditionally???? (bug??) + - update snippet contents if the edit makes sense +- on delete + - delete snippet if the user so desires + - delete the message records + + +background stuff: +- watch mode + - every five seconds + - fetch all HMN users with Discord accounts + - check if we have message records without content + - if so, run a full scrape (no snippets) + - every hour + - run a full scrape, creating snippets +- scrape behavior + - look at every message ever in the channel + - do exactly what the real-time bot does on new messages (although maybe don't do snippets depending on context) + diff --git a/src/main.go b/src/main.go index 6bfb901..96bcdab 100644 --- a/src/main.go +++ b/src/main.go @@ -2,6 +2,7 @@ package main import ( _ "git.handmade.network/hmn/hmn/src/admintools" + _ "git.handmade.network/hmn/hmn/src/assets" _ "git.handmade.network/hmn/hmn/src/buildscss" _ "git.handmade.network/hmn/hmn/src/initimage" _ "git.handmade.network/hmn/hmn/src/migration"