Save Discord messages and attachments

go.mod

@@ -7,6 +7,11 @@ require (
 	github.com/Masterminds/semver v1.5.0 // indirect
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/alecthomas/chroma v0.9.2
+	github.com/aws/aws-sdk-go-v2 v1.8.1
+	github.com/aws/aws-sdk-go-v2/config v1.6.1
+	github.com/aws/aws-sdk-go-v2/credentials v1.3.3
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.13.0
+	github.com/aws/smithy-go v1.7.0
 	github.com/frustra/bbcode v0.0.0-20201127003707-6ef347fbe1c8
 	github.com/go-stack/stack v1.8.0
 	github.com/google/uuid v1.2.0

go.sum

@@ -40,6 +40,30 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/aws/aws-sdk-go-v2 v1.8.1 h1:GcFgQl7MsBygmeeqXyV1ivrTEmsVz/rdFJaTcltG9ag=
+github.com/aws/aws-sdk-go-v2 v1.8.1/go.mod h1:xEFuWz+3TYdlPRuo+CqATbeDWIWyaT5uAPwPaWtgse0=
+github.com/aws/aws-sdk-go-v2/config v1.6.1 h1:qrZINaORyr78syO1zfD4l7r4tZjy0Z1l0sy4jiysyOM=
+github.com/aws/aws-sdk-go-v2/config v1.6.1/go.mod h1:t/y3UPu0XEDy0cEw6mvygaBQaPzWiYAxfP2SzgtvclA=
+github.com/aws/aws-sdk-go-v2/credentials v1.3.3 h1:A13QPatmUl41SqUfnuT3V0E3XiNGL6qNTOINbE8cZL4=
+github.com/aws/aws-sdk-go-v2/credentials v1.3.3/go.mod h1:oVieKMT3m9BSfqhOfuQ+E0j/yN84ZAJ7Qv8Sfume/ak=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.1 h1:rc+fRGvlKbeSd9IFhFS1KWBs0XjTkq0CfK5xqyLgIp0=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.4.1/go.mod h1:+GTydg3uHmVlQdkRoetz6VHKbOMEYof70m19IpMLifc=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.2.1 h1:IkqRRUZTKaS16P2vpX+FNc2jq3JWa3c478gykQp4ow4=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.2.1/go.mod h1:Pv3WenDjI0v2Jl7UaMFIIbPOBbhn33RmmAmGgkXDoqY=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.2 h1:YcGVEqLQGHDa81776C3daai6ZkkRGf/8RAQ07hV0QcU=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.2/go.mod h1:EASdTcM1lGhUe1/p4gkojHwlGJkeoRjjr1sRCzup3Is=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.3 h1:VxFCgxsqWe7OThOwJ5IpFX3xrObtuIH9Hg/NW7oot1Y=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.3/go.mod h1:7gcsONBmFoCcKrAqrm95trrMd2+C/ReYKP7Vfu8yHHA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.3 h1:7tPSbUWzuoMJ2woUKgOfIPuZS88hMdFHJBBB2vR0bHI=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.3/go.mod h1:/ugW3qFkJe/h7sNtI6/zJnwRbvavs6GyOid69uI9eek=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.13.0 h1:2oMLrNpOSpkDTocIVv3Fut1XrmlbKPlgnnYMGYqFp0Y=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.13.0/go.mod h1:Tzxhu3GnCpj45WJqXyxcLF2gUHzTcmY7CzpQ9x9KVls=
+github.com/aws/aws-sdk-go-v2/service/sso v1.3.3 h1:K2gCnGvAASpz+jqP9iyr+F/KNjmTYf8aWOtTQzhmZ5w=
+github.com/aws/aws-sdk-go-v2/service/sso v1.3.3/go.mod h1:Jgw5O+SK7MZ2Yi9Yvzb4PggAPYaFSliiQuWR0hNjexk=
+github.com/aws/aws-sdk-go-v2/service/sts v1.6.2 h1:l504GWCoQi1Pk68vSUFGLmDIEMzRfVGNgLakDK+Uj58=
+github.com/aws/aws-sdk-go-v2/service/sts v1.6.2/go.mod h1:RBhoMJB8yFToaCnbe0jNq5Dcdy0jp6LhHqg55rjClkM=
+github.com/aws/smithy-go v1.7.0 h1:+cLHMRrDZvQ4wk+KuQ9yH6eEg6KZEJ9RI2IkDqnygCg=
+github.com/aws/smithy-go v1.7.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
@@ -92,6 +116,9 @@ github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Z
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -185,6 +212,8 @@ github.com/jackc/puddle v1.1.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv
 github.com/jackc/puddle v1.1.1/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
 github.com/jackc/puddle v1.1.3 h1:JnPg/5Q9xVJGfjsO5CPUOjnJps1JaRUm8I9FXVCFK94=
 github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk=
+github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
 github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA=
 github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
@@ -484,6 +513,7 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

src/assets/assets.go

@@ -0,0 +1,155 @@
+package assets
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha1"
+	"errors"
+	"fmt"
+	"regexp"
+
+	"git.handmade.network/hmn/hmn/src/config"
+	"git.handmade.network/hmn/hmn/src/db"
+	"git.handmade.network/hmn/hmn/src/models"
+	"git.handmade.network/hmn/hmn/src/oops"
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	"github.com/aws/smithy-go"
+	"github.com/google/uuid"
+)
+
+var client *s3.Client
+
+func init() {
+	cfg, err := awsconfig.LoadDefaultConfig(context.Background(),
+		awsconfig.WithCredentialsProvider(
+			credentials.NewStaticCredentialsProvider(
+				config.Config.DigitalOcean.AssetsSpacesKey,
+				config.Config.DigitalOcean.AssetsSpacesSecret,
+				"",
+			),
+		),
+		awsconfig.WithRegion(config.Config.DigitalOcean.AssetsSpacesRegion),
+		awsconfig.WithEndpointResolver(aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
+			return aws.Endpoint{
+				URL: config.Config.DigitalOcean.AssetsSpacesEndpoint,
+			}, nil
+		})),
+	)
+	if err != nil {
+		panic(err)
+	}
+	client = s3.NewFromConfig(cfg, func(o *s3.Options) {
+		o.UsePathStyle = true
+	})
+}
+
+type CreateInput struct {
+	Content  []byte
+	Filename string
+	MimeType string
+
+	// Optional params
+	UploaderID    *int // HMN user id
+	Width, Height int
+}
+
+var REIllegalFilenameChars = regexp.MustCompile(`[^\w \-.]`)
+
+func SanitizeFilename(filename string) string {
+	if filename == "" {
+		return "unnamed"
+	}
+	return REIllegalFilenameChars.ReplaceAllString(filename, "")
+}
+
+func AssetKey(id, filename string) string {
+	return fmt.Sprintf("%s%s/%s", config.Config.DigitalOcean.AssetsPathPrefix, id, filename)
+}
+
+type InvalidAssetError error
+
+func Create(ctx context.Context, dbConn db.ConnOrTx, in CreateInput) (*models.Asset, error) {
+	filename := SanitizeFilename(in.Filename)
+
+	if len(in.Content) == 0 {
+		return nil, InvalidAssetError(fmt.Errorf("could not upload asset '%s': no bytes of data were provided", filename))
+	}
+	if in.MimeType == "" {
+		return nil, InvalidAssetError(fmt.Errorf("could not upload asset '%s': no mime type provided", filename))
+	}
+
+	// Upload the asset to the DO space
+	id := uuid.New()
+	key := AssetKey(id.String(), filename)
+	checksum := fmt.Sprintf("%x", sha1.Sum(in.Content))
+
+	upload := func() error {
+		_, err := client.PutObject(ctx, &s3.PutObjectInput{
+			Bucket: &config.Config.DigitalOcean.AssetsSpacesBucket,
+			Key:    &key,
+			Body:   bytes.NewReader(in.Content),
+			ACL:    types.ObjectCannedACLPublicRead,
+		})
+		return err
+	}
+
+	err := upload()
+	if err != nil {
+		var apiError smithy.APIError
+		if errors.As(err, &apiError) && apiError.ErrorCode() == "NoSuchBucket" {
+			_, err := client.CreateBucket(ctx, &s3.CreateBucketInput{
+				Bucket: &config.Config.DigitalOcean.AssetsSpacesBucket,
+			})
+			if err != nil {
+				return nil, oops.New(err, "failed to create assets bucket")
+			}
+
+			err = upload()
+			if err != nil {
+				return nil, oops.New(err, "failed to upload asset")
+			}
+		} else {
+			return nil, oops.New(err, "failed to upload asset")
+		}
+	}
+
+	// Save a record in our database
+	// TODO(db): Would be convient to use RETURNING here...
+	_, err = dbConn.Exec(ctx,
+		`
+		INSERT INTO handmade_asset (id, s3_key, filename, size, mime_type, sha1sum, width, height, uploader_id)
+		VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
+		`,
+		id,
+		key,
+		filename,
+		len(in.Content),
+		in.MimeType,
+		checksum,
+		in.Width,
+		in.Height,
+		in.UploaderID,
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to save asset record")
+	}
+
+	// Fetch and return the new record
+	iasset, err := db.QueryOne(ctx, dbConn, models.Asset{},
+		`
+		SELECT $columns
+		FROM handmade_asset
+		WHERE id = $1
+		`,
+		id,
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to fetch newly-created asset")
+	}
+
+	return iasset.(*models.Asset), nil
+}

src/assets/assets_test.go

@@ -0,0 +1,13 @@
+package assets
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestSanitizeFilename(t *testing.T) {
+	assert.Equal(t, "cool filename.txt.wow", SanitizeFilename("cool filename.txt.wow"))
+	assert.Equal(t, " hi doggy ", SanitizeFilename("😎 hi doggy 🐶"))
+	assert.Equal(t, "newlinesaretotallylegal", SanitizeFilename("newlines\naretotallylegal"))
+}

src/db/db.go

@@ -11,6 +11,7 @@ import (
 	"git.handmade.network/hmn/hmn/src/logging"
 	"git.handmade.network/hmn/hmn/src/oops"
 	"github.com/google/uuid"
+	"github.com/jackc/pgconn"
 	"github.com/jackc/pgtype"
 	"github.com/jackc/pgx/v4"
 	"github.com/jackc/pgx/v4/log/zerologadapter"
@@ -57,6 +58,7 @@ func typeIsQueryable(t reflect.Type) bool {
 // This interface should match both a direct pgx connection or a pgx transaction.
 type ConnOrTx interface {
 	Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error)
+	Exec(ctx context.Context, sql string, args ...interface{}) (pgconn.CommandTag, error)
 }
 
 var connInfo = pgtype.NewConnInfo()

src/discord/gateway.go

@@ -563,7 +563,7 @@ func (bot *discordBotInstance) processEventMsg(ctx context.Context, msg *Gateway
 }
 
 func (bot *discordBotInstance) messageCreateOrUpdate(ctx context.Context, msg *Message) error {
-	if msg.Author != nil && msg.Author.ID == config.Config.Discord.BotUserID {
+	if msg.OriginalHasFields("author") && msg.Author.ID == config.Config.Discord.BotUserID {
 		// Don't process your own messages
 		return nil
 	}

src/discord/library.go

@@ -0,0 +1,45 @@
+package discord
+
+import (
+	"context"
+
+	"git.handmade.network/hmn/hmn/src/oops"
+)
+
+func (bot *discordBotInstance) processLibraryMsg(ctx context.Context, msg *Message) error {
+	switch msg.Type {
+	case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
+	default:
+		return nil
+	}
+
+	if !msg.OriginalHasFields("content") {
+		return nil
+	}
+
+	if !messageHasLinks(msg.Content) {
+		err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
+		if err != nil {
+			return oops.New(err, "failed to delete message")
+		}
+
+		if !msg.Author.IsBot {
+			channel, err := CreateDM(ctx, msg.Author.ID)
+			if err != nil {
+				return oops.New(err, "failed to create DM channel")
+			}
+
+			err = SendMessages(ctx, bot.dbConn, MessageToSend{
+				ChannelID: channel.ID,
+				Req: CreateMessageRequest{
+					Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.",
+				},
+			})
+			if err != nil {
+				return oops.New(err, "failed to send showcase warning message")
+			}
+		}
+	}
+
+	return nil
+}

src/discord/payloads.go

@@ -2,6 +2,8 @@ package discord
 
 import (
 	"encoding/json"
+	"fmt"
+	"time"
 )
 
 type Opcode int
@@ -170,19 +172,50 @@ const (
 
 // https://discord.com/developers/docs/resources/channel#message-object
 type Message struct {
-	ID        string `json:"id"`
+	ID        string      `json:"id"`
-	ChannelID string `json:"channel_id"`
+	ChannelID string      `json:"channel_id"`
-	Content   string `json:"content"`
+	GuildID   *string     `json:"guild_id"`
-	Author    *User  `json:"author"` // note that this may not be an actual valid user (see the docs)
+	Content   string      `json:"content"`
-	// TODO: Author info
+	Author    User        `json:"author"` // note that this may not be an actual valid user (see the docs)
-	// TODO: Timestamp parsing, yay
+	Timestamp string      `json:"timestamp"`
-	Type MessageType `json:"type"`
+	Type      MessageType `json:"type"`
 
 	Attachments []Attachment `json:"attachments"`
+	// TODO: Embeds
 
 	originalMap map[string]interface{}
 }
 
+func (m *Message) JumpURL() string {
+	guildStr := "@me"
+	if m.GuildID != nil {
+		guildStr = *m.GuildID
+	}
+	return fmt.Sprintf("https://discord.com/channels/%s/%s/%s", guildStr, m.ChannelID, m.ID)
+}
+
+func (m *Message) Time() time.Time {
+	t, err := time.Parse(time.RFC3339Nano, m.Timestamp)
+	if err != nil {
+		panic(err)
+	}
+	return t
+}
+
+func (m *Message) OriginalHasFields(fields ...string) bool {
+	if m.originalMap == nil {
+		return false
+	}
+
+	for _, field := range fields {
+		_, ok := m.originalMap[field]
+		if !ok {
+			return false
+		}
+	}
+	return true
+}
+
 func MessageFromMap(m interface{}) Message {
 	/*
 		Some gateway events, like MESSAGE_UPDATE, do not contain the
@@ -194,15 +227,16 @@ func MessageFromMap(m interface{}) Message {
 	msg := Message{
 		ID:        mmap["id"].(string),
 		ChannelID: mmap["channel_id"].(string),
+		GuildID:   maybeStringP(mmap, "guild_id"),
 		Content:   maybeString(mmap, "content"),
+		Timestamp: maybeString(mmap, "timestamp"),
 		Type:      MessageType(maybeInt(mmap, "type")),
 
 		originalMap: mmap,
 	}
 
 	if author, ok := mmap["author"]; ok {
-		u := UserFromMap(author)
+		msg.Author = UserFromMap(author)
-		msg.Author = &u
 	}
 
 	if iattachments, ok := mmap["attachments"]; ok {
@@ -212,6 +246,8 @@ func MessageFromMap(m interface{}) Message {
 		}
 	}
 
+	// TODO: Embeds
+
 	return msg
 }
 
@@ -276,6 +312,15 @@ func maybeString(m map[string]interface{}, k string) string {
 	return val.(string)
 }
 
+func maybeStringP(m map[string]interface{}, k string) *string {
+	val, ok := m[k]
+	if !ok {
+		return nil
+	}
+	strval := val.(string)
+	return &strval
+}
+
 func maybeInt(m map[string]interface{}, k string) int {
 	val, ok := m[k]
 	if !ok {

src/discord/showcase.go

@@ -2,15 +2,25 @@ package discord
 
 import (
 	"context"
+	"errors"
+	"io"
+	"net/http"
 	"net/url"
 	"regexp"
 	"strings"
 
+	"git.handmade.network/hmn/hmn/src/assets"
+	"git.handmade.network/hmn/hmn/src/db"
+	"git.handmade.network/hmn/hmn/src/logging"
+	"git.handmade.network/hmn/hmn/src/models"
 	"git.handmade.network/hmn/hmn/src/oops"
+	"github.com/jackc/pgx/v4"
 )
 
 var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`)
 
+var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this")
+
 func (bot *discordBotInstance) processShowcaseMsg(ctx context.Context, msg *Message) error {
 	switch msg.Type {
 	case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
@@ -18,26 +28,70 @@ func (bot *discordBotInstance) processShowcaseMsg(ctx context.Context, msg *Mess
 		return nil
 	}
 
+	didDelete, err := bot.maybeDeleteShowcaseMsg(ctx, msg)
+	if err != nil {
+		return err
+	}
+	if didDelete {
+		return nil
+	}
+
+	tx, err := bot.dbConn.Begin(ctx)
+	if err != nil {
+		panic(err)
+	}
+	defer tx.Rollback(ctx)
+
+	// save the message, maybe save its contents, and maybe make a snippet too
+	_, err = bot.saveMessageAndContents(ctx, tx, msg)
+	if errors.Is(err, errNotEnoughInfo) {
+		logging.ExtractLogger(ctx).Warn().
+			Interface("msg", msg).
+			Msg("didn't have enough info to process Discord message")
+		return nil
+	} else if err != nil {
+		return err
+	}
+	if doSnippet, err := bot.allowedToCreateMessageSnippet(ctx, msg); doSnippet && err == nil {
+		_, err := bot.createMessageSnippet(ctx, msg)
+		if err != nil {
+			return oops.New(err, "failed to create snippet in gateway")
+		}
+	} else if err != nil {
+		return oops.New(err, "failed to check snippet permissions in gateway")
+	}
+
+	err = tx.Commit(ctx)
+	if err != nil {
+		return oops.New(err, "failed to commit Discord message updates")
+	}
+
+	return nil
+}
+
+func (bot *discordBotInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message) (didDelete bool, err error) {
 	hasGoodContent := true
-	if originalMessageHasField(msg, "content") && !messageHasLinks(msg.Content) {
+	if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) {
 		hasGoodContent = false
 	}
 
 	hasGoodAttachments := true
-	if originalMessageHasField(msg, "attachments") && len(msg.Attachments) == 0 {
+	if msg.OriginalHasFields("attachments") && len(msg.Attachments) == 0 {
 		hasGoodAttachments = false
 	}
 
+	didDelete = false
 	if !hasGoodContent && !hasGoodAttachments {
+		didDelete = true
 		err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
 		if err != nil {
-			return oops.New(err, "failed to delete message")
+			return false, oops.New(err, "failed to delete message")
 		}
 
-		if msg.Author != nil && !msg.Author.IsBot {
+		if !msg.Author.IsBot {
 			channel, err := CreateDM(ctx, msg.Author.ID)
 			if err != nil {
-				return oops.New(err, "failed to create DM channel")
+				return false, oops.New(err, "failed to create DM channel")
 			}
 
 			err = SendMessages(ctx, bot.dbConn, MessageToSend{
@@ -47,50 +101,230 @@ func (bot *discordBotInstance) processShowcaseMsg(ctx context.Context, msg *Mess
 				},
 			})
 			if err != nil {
-				return oops.New(err, "failed to send showcase warning message")
+				return false, oops.New(err, "failed to send showcase warning message")
 			}
 		}
 	}
 
-	return nil
+	return didDelete, nil
 }
 
-func (bot *discordBotInstance) processLibraryMsg(ctx context.Context, msg *Message) error {
+/*
-	switch msg.Type {
+Ensures that a Discord message is stored in the database. This function is
-	case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
+idempotent and can be called regardless of whether the item already exists in
-	default:
+the database.
-		return nil
-	}
 
-	if !originalMessageHasField(msg, "content") {
+This does not create snippets or do anything besides save the message itself.
-		return nil
+*/
-	}
+func (bot *discordBotInstance) saveMessage(
+	ctx context.Context,
+	tx pgx.Tx,
+	msg *Message,
+) (*models.DiscordMessage, error) {
+	iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessage
+		WHERE id = $1
+		`,
+		msg.ID,
+	)
+	if errors.Is(err, db.ErrNoMatchingRows) {
+		if !msg.OriginalHasFields("author", "timestamp") {
+			return nil, errNotEnoughInfo
+		}
 
-	if !messageHasLinks(msg.Content) {
+		_, err = tx.Exec(ctx,
-		err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
+			`
+			INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created)
+			VALUES ($1, $2, $3, $4, $5, $6, $7)
+			`,
+			msg.ID,
+			msg.ChannelID,
+			*msg.GuildID,
+			msg.JumpURL(),
+			msg.Author.ID,
+			msg.Time(),
+			false,
+		)
 		if err != nil {
-			return oops.New(err, "failed to delete message")
+			return nil, oops.New(err, "failed to save new discord message")
 		}
 
-		if msg.Author != nil && !msg.Author.IsBot {
+		/*
-			channel, err := CreateDM(ctx, msg.Author.ID)
+			TODO(db): This is a spot where it would be really nice to be able
-			if err != nil {
+			to use RETURNING, and avoid this second query.
-				return oops.New(err, "failed to create DM channel")
+		*/
-			}
+		iDiscordMessage, err = db.QueryOne(ctx, tx, models.DiscordMessage{},
+			`
+			SELECT $columns
+			FROM handmade_discordmessage
+			WHERE id = $1
+			`,
+			msg.ID,
+		)
+		if err != nil {
+			panic(err)
+		}
+	} else if err != nil {
+		return nil, oops.New(err, "failed to check for existing Discord message")
+	}
 
-			err = SendMessages(ctx, bot.dbConn, MessageToSend{
+	return iDiscordMessage.(*models.DiscordMessage), nil
-				ChannelID: channel.ID,
+}
-				Req: CreateMessageRequest{
+
-					Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.",
+/*
-				},
+Processes a single Discord message, saving as much of the message's content
-			})
+and attachments as allowed by our rules and user settings. Does NOT create
-			if err != nil {
+snippets.
-				return oops.New(err, "failed to send showcase warning message")
+
-			}
+Idempotent; can be called any time whether the message exists or not.
+*/
+func (bot *discordBotInstance) saveMessageAndContents(
+	ctx context.Context,
+	tx pgx.Tx,
+	msg *Message,
+) (*models.DiscordMessage, error) {
+	newMsg, err := bot.saveMessage(ctx, tx, msg)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check for linked Discord user
+	iDiscordUser, err := db.QueryOne(ctx, tx, models.DiscordUser{},
+		`
+		SELECT $columns
+		FROM handmade_discorduser
+		WHERE userid = $1
+		`,
+		msg.Author.ID,
+	)
+	if errors.Is(err, db.ErrNoMatchingRows) {
+		return newMsg, nil
+	} else if err != nil {
+		return nil, oops.New(err, "failed to look up linked Discord user")
+	}
+	discordUser := iDiscordUser.(*models.DiscordUser)
+
+	// We have a linked Discord account, so save the message contents (regardless of
+	// whether we create a snippet or not).
+
+	_, err = tx.Exec(ctx,
+		`
+		INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content)
+		VALUES ($1, $2, $3)
+		ON CONFLICT (message_id) DO UPDATE SET
+			discord_id = EXCLUDED.discord_id,
+			last_content = EXCLUDED.last_content
+		`,
+		msg.ID,
+		discordUser.ID,
+		msg.Content, // TODO: Add a method that can fill in mentions and stuff (https://discord.com/developers/docs/reference#message-formatting)
+	)
+
+	// Save attachments
+	for _, attachment := range msg.Attachments {
+		_, err := bot.saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID)
+		if err != nil {
+			return nil, oops.New(err, "failed to save attachment")
 		}
 	}
 
-	return nil
+	// TODO: Save embeds
+
+	return newMsg, nil
+}
+
+func (bot *discordBotInstance) saveAttachment(ctx context.Context, tx pgx.Tx, attachment *Attachment, hmnUserID int, discordMessageID string) (*models.DiscordMessageAttachment, error) {
+	// TODO: Return an existing attachment if it exists
+
+	width := 0
+	height := 0
+	if attachment.Width != nil {
+		width = *attachment.Width
+	}
+	if attachment.Height != nil {
+		height = *attachment.Height
+	}
+
+	// TODO: Timeouts and stuff, context cancellation
+	res, err := http.Get(attachment.Url)
+	if err != nil {
+		return nil, oops.New(err, "failed to fetch attachment data")
+	}
+	defer res.Body.Close()
+
+	content, err := io.ReadAll(res.Body)
+	if err != nil {
+		panic(err)
+	}
+
+	asset, err := assets.Create(ctx, tx, assets.CreateInput{
+		Content:  content,
+		Filename: attachment.Filename,
+		MimeType: attachment.ContentType,
+
+		UploaderID: &hmnUserID,
+		Width:      width,
+		Height:     height,
+	})
+	if err != nil {
+		return nil, oops.New(err, "failed to save asset for Discord attachment")
+	}
+
+	// TODO(db): RETURNING plz thanks
+	_, err = tx.Exec(ctx,
+		`
+		INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id)
+		VALUES ($1, $2, $3)
+		`,
+		attachment.ID,
+		asset.ID,
+		discordMessageID,
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to save Discord attachment data")
+	}
+
+	iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessageattachment
+		WHERE id = $1
+		`,
+		attachment.ID,
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to fetch new Discord attachment data")
+	}
+
+	return iDiscordAttachment.(*models.DiscordMessageAttachment), nil
+}
+
+func (bot *discordBotInstance) allowedToCreateMessageSnippet(ctx context.Context, msg *Message) (bool, error) {
+	canSave, err := db.QueryBool(ctx, bot.dbConn,
+		`
+		SELECT u.discord_save_showcase
+		FROM
+			handmade_discorduser AS duser
+			JOIN auth_user AS u ON duser.hmn_user_id = u.id
+		WHERE
+			duser.userid = $1
+		`,
+		msg.Author.ID,
+	)
+	if errors.Is(err, db.ErrNoMatchingRows) {
+		return false, nil
+	} else if err != nil {
+		return false, oops.New(err, "failed to check if we can save Discord message")
+	}
+
+	return canSave, nil
+}
+
+func (bot *discordBotInstance) createMessageSnippet(ctx context.Context, msg *Message) (*models.Snippet, error) {
+	// TODO: Actually do this
+	return nil, nil
 }
 
 func messageHasLinks(content string) bool {
@@ -104,12 +338,3 @@ func messageHasLinks(content string) bool {
 
 	return false
 }
-
-func originalMessageHasField(msg *Message, field string) bool {
-	if msg.originalMap == nil {
-		return false
-	}
-
-	_, ok := msg.originalMap[field]
-	return ok
-}

src/discord/todo.txt

@@ -0,0 +1,43 @@
+the goal: port the old discord showcase bot
+
+what it does: save #project-showcase posts to your HMN user profile if you have your account linked
+
+stuff we need to worry about:
+- old posts from before you linked your account
+- posts that come in while the bot is down
+- what to do with posts if you unlink your account
+- what to do with posts if you re-link your account
+- what to do if you edit the original discord message
+- what to do if you delete the original discord message
+- the user's preferences re: saving content
+    - we don't want to save content without the user's consent, especially since it may persist after they disable the integration
+- manually adding content for various reasons
+    - maybe a bug prevented something from saving
+    - ryan used to post everything in #projects for some reason
+
+
+real-time stuff:
+- on new showcase message
+    - always save the lightweight record
+    - if we have permission, create a snippet
+- on edit
+    - re-save the lightweight record and content as if it was new
+    - create snippet, unconditionally???? (bug??)
+    - update snippet contents if the edit makes sense
+- on delete
+    - delete snippet if the user so desires
+    - delete the message records
+
+
+background stuff:
+- watch mode
+    - every five seconds
+        - fetch all HMN users with Discord accounts
+        - check if we have message records without content
+        - if so, run a full scrape (no snippets)
+    - every hour
+        - run a full scrape, creating snippets
+- scrape behavior
+    - look at every message ever in the channel
+    - do exactly what the real-time bot does on new messages (although maybe don't do snippets depending on context)
+

src/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	_ "git.handmade.network/hmn/hmn/src/admintools"
+	_ "git.handmade.network/hmn/hmn/src/assets"
 	_ "git.handmade.network/hmn/hmn/src/buildscss"
 	_ "git.handmade.network/hmn/hmn/src/initimage"
 	_ "git.handmade.network/hmn/hmn/src/migration"