WIP. Rethinking things. See the README

src/db/db.go

@@ -438,3 +438,35 @@ func QueryBool(ctx context.Context, conn ConnOrTx, query string, args ...interfa
 		return false, oops.New(nil, "QueryBool got a non-bool result: %v", result)
 	}
 }
+
+// Begins a transaction, runs the provided function, and then commits the
+// transaction. If the provided function returns an error, the transaction will
+// be rolled back, and the original error will be returned.
+func DoWithTransaction(ctx context.Context, conn ConnOrTx, f func(tx pgx.Tx) error) error {
+	tx, err := conn.Begin(ctx)
+	if err != nil {
+		return oops.New(err, "failed to begin transaction")
+	}
+	defer func() {
+		err := tx.Rollback(ctx)
+		if err != nil {
+			// TODO: Maybe in the future we could have a "multi-error" so we
+			// can actually return this error instead of having to log it.
+			logging.ExtractLogger(ctx).Error().
+				Err(err).
+				Msg("failed to roll back transaction")
+		}
+	}()
+
+	err = f(tx)
+	if err != nil {
+		return err
+	}
+
+	err = tx.Commit(ctx)
+	if err != nil {
+		return oops.New(err, "failed to commit transaction")
+	}
+
+	return nil
+}

src/discord/README.md

@@ -0,0 +1,168 @@
+# Discord is crazy
+
+This document describes the entire process of persisting Discord messages and handling:
+
+- showcase messages
+- library messages
+- snippets
+- backfills
+- admin commands
+- who even knows what else
+
+If you want to just hop in and make a quick change to the Discord logic, don't. Not without reading this first, and in particular not without going through the user stories.
+
+## Constraints and design guidelines
+
+We must never save message content for anyone without a linked Discord account.
+
+We always have to respect rate limits, heartbeats, other gateway nuances.
+
+Actions on Discord should never affect the website if the Discord account is not linked to an HMN profile.
+
+Nothing should be added to a user's HMN profile without explicit user action. For example, linking your account should not result in snippets being automatically created from old #project-showcase posts.
+
+We shouldn't assume that just because we have a message (or its contents) stored, we want to create a snippet from it. Sometimes messages could be saved for other reasons.
+
+## All the logic
+
+On new/updated message:
+- if in a showcase channel and it doesn't fit the format, delete and return
+- if in a library channel and it doesn't fit the format, delete and return
+- if we care about the content of this message (see below):
+    - persist mirror of message data (incl. content if allowed)
+        - if persisting content, clean up message markdown
+- if user has a linked Discord account:
+    - if this message is snippet-able (see below):
+        - if snippet exists (in any channel):
+            - update snippet
+        - if no snippet exists:
+            - check conditions for a new snippet:
+                - the flag for snippet creation is set
+                - any of the following is true:
+                    - in "showcase" channel, and the user has the setting enabled
+                    - an admin wants a snippet here
+                - we have message content
+                - SnippetCreated is false on the message
+            - if conditions are met, create snippet
+    - if this message is not snippet-able (see below):
+        - delete any existing snippets that are connected to Discord (huzzah for edge cases)
+
+On deleted message:
+- delete snippets, if the following conditions are true:
+    - snippet is connected to Discord
+    - the author has a Discord account linked
+    - the author chooses not to keep captured snippets on Discord deletes (profile setting)
+- delete mirror of message data
+
+On Discord account unlink:
+- DO NOT disconnect snippets from Discord.
+    - There's no reason to.
+
+We care about the content of the message if:
+- we already have content for this message, or
+- it is in a showcase channel or whatever, or
+- an admin said so (CLI command)
+
+A message is snippet-able if:
+- It contains enough information to make a snippet. Namely:
+    - it has media
+- Things that DO NOT affect snippetability:
+    - whether a Discord account is linked
+    - the user's settings for automatic snippet creation
+    - other permissions or whatever
+
+### Things we do
+
+These actions all fall into one of the two flows above - the create/update flow, or the delete flow. Some may require flags to modify the main flow behavior slightly.
+
+Stuff for the create/update flow:
+- receive message creates/updates in real time
+- general backfill (1 hour) (exactly the same behavior as real-time)
+- new user backfill (5 seconds) (message content only, no snippets)
+- make snippets from #project-showcase posts (exactly the same behavior as real-time)
+- admin scrapechannel (fetch messages and content in arbitrary channels, no snippets)
+- admin makesnippet
+
+Stuff for the delete flow:
+- receive message deletes in real time
+- IN THE FUTURE: delete messages in history backfill
+
+## ✨ :D user story time :D ✨
+
+Whenever touching the logic, consider all of these lovely edge cases!
+
+- bad showcase post
+    - expected: post is immediately deleted
+
+- bad library post
+    - expected: post is immediately deleted
+
+- good showcase post from an unlinked user
+    - expected: lightweight message record is created, but no snippet is created
+
+- good showcase post, then the user unlinks their discord account
+    - expected: the snippet remains on the user's profile
+
+- good showcase post, then the user unlinks their discord account, then edits the message
+    - expected: the message edit has no effect on the snippet
+
+- good showcase post, then the user unlinks their discord account, then deletes the message
+    - expected: the message delete has no effect on the snippet
+
+- good showcase post, then the user unlinks and re-links their discord account, then edits the showcase post
+    - expected: the snippet remained connected to the discord message, and is updated
+
+- good showcase post, then the user unlinks their discord account and links a different one
+    - expected: nothing happens to old snippets; new snippets get added as usual
+
+- good showcase post, then the user unlinks their discord account, then their discord account is linked to a different website profile
+    - expected: snippets may be re-created on the new HMN profile
+
+- good showcase post, then the user unlinks their discord account, then we run a backfill on that specific message
+    - expected: nothing happens because there is no discord account linked for that message
+
+- good showcase post gets edited to be bad
+    - expected: the message and associated snippet are deleted
+
+- good library post gets edited to be bad
+    - expected: the message is deleted
+
+- good showcase post gets edited and is still good
+    - expected: the message record and associated snippet are updated
+
+- good library post gets edited and is still good
+    - expected: nothing happens
+
+- Ryan posts like 50 things in #projects, and we want them to be on his profile
+    - expected: an admin can run `admin makesnippet` on all the messages, and they will be added to Ryan's profile
+
+- Ryan edits his old posts in #projects
+    - same as: someone edits their posts in #jam-showcase
+    - expected: any associated snippets are updated, even though the edit was not in a showcase channel
+
+- Ryan edits an old post in #projects to no longer have a link
+    - expected: the associated snippet is deleted, but the message persists.
+
+- Ryan edits an old post in #projects to no longer have a link, then edits it again to have a link again.
+    - expected: the associated snippet is deleted, and not reinstated on the second edit. (oh well!)
+
+- an admin attempts to make a snippet for someone without a linked discord account
+    - expected: nothing happens
+
+- an admin attempts to make a snippet for a message without media
+    - expected: nothing happens
+
+- good showcase post is deleted
+    - expected: any snippets are deleted, if the user wishes.
+
+- good showcase post, snippet is deleted from website, post is edited
+    - expected: no new snippet is created
+
+- good showcase post, snippet is deleted from website, backfill runs
+    - expected: no new snippet is created
+
+- good showcase post while the user has the showcase integration profile setting disabled, then they enable the showcase integration, then a backfill runs
+    - expected: no new snippets are created for old showcase posts
+
+- good showcase post while the user has the integration disabled, then they enable the integration, then they explicitly create snippets from their showcase posts
+    - expected: all user messages in showcase without snippets get snippets (subject to SnippetCreated)

src/discord/gateway.go

@@ -548,14 +548,14 @@ func (bot *botInstance) processEventMsg(ctx context.Context, msg *GatewayMessage
 	case "MESSAGE_CREATE":
 		newMessage := *MessageFromMap(msg.Data, "")
 
-		err := bot.messageCreateOrUpdate(ctx, &newMessage)
+		err := bot.gatewayMessageCreateOrUpdate(ctx, &newMessage)
 		if err != nil {
 			return oops.New(err, "error on new message")
 		}
 	case "MESSAGE_UPDATE":
 		newMessage := *MessageFromMap(msg.Data, "")
 
-		err := bot.messageCreateOrUpdate(ctx, &newMessage)
+		err := bot.gatewayMessageCreateOrUpdate(ctx, &newMessage)
 		if err != nil {
 			return oops.New(err, "error on updated message")
 		}
@@ -585,76 +585,35 @@ func (bot *botInstance) processEventMsg(ctx context.Context, msg *GatewayMessage
 }
 
 // Only return an error if we want to restart the bot.
-func (bot *botInstance) messageCreateOrUpdate(ctx context.Context, msg *Message) error {
+func (bot *botInstance) gatewayMessageCreateOrUpdate(ctx context.Context, msg *Message) error {
-	if msg.OriginalHasFields("author") && msg.Author.ID == config.Config.Discord.BotUserID {
+	log := logging.ExtractLogger(ctx).With().
-		// Don't process your own messages
+		Str("msgID", msg.ID).
-		return nil
+		Str("channelID", msg.ChannelID).
-	}
+		Logger()
+	ctx = logging.AttachLoggerToContext(&log, ctx)
 
-	if msg.ChannelID == config.Config.Discord.ShowcaseChannelID {
+	err := MessageCreatedOrUpdated(ctx, bot.dbConn, msg, 0)
-		err := bot.processShowcaseMsg(ctx, msg)
+	if errors.Is(err, context.Canceled) {
-		if err != nil {
+		log.Warn().Err(err).Msg("context canceled when handling new Discord message")
-			logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to process showcase message")
-			return nil
-		}
 		return nil
-	}
+	} else if err != nil {
-
+		log.Error().Err(err).Msg("error while handling new Discord message")
-	if msg.ChannelID == config.Config.Discord.LibraryChannelID {
-		err := bot.processLibraryMsg(ctx, msg)
-		if err != nil {
-			logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to process library message")
-			return nil
-		}
 		return nil
 	}
 
-	err := UpdateSnippetTagsIfAny(ctx, bot.dbConn, msg)
-	if err != nil {
-		logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to update tags for Discord snippet")
-	}
-
 	return nil
 }
 
+// TODO: Fold this into MessageDeleted and get rid of it
 func (bot *botInstance) messageDelete(ctx context.Context, msgDelete MessageDelete) {
-	log := logging.ExtractLogger(ctx)
+	log := logging.ExtractLogger(ctx).With().
-
+		Str("msgID", msgDelete.ID).
-	tx, err := bot.dbConn.Begin(ctx)
+		Str("channelID", msgDelete.ChannelID).
-	if err != nil {
+		Logger()
-		log.Error().Err(err).Msg("failed to start transaction")
+	ctx = logging.AttachLoggerToContext(&log, ctx)
-		return
-	}
-	defer tx.Rollback(ctx)
-
-	type deleteMessageQuery struct {
-		Message     models.DiscordMessage `db:"msg"`
-		DiscordUser *models.DiscordUser   `db:"duser"`
-		HMNUser     *models.User          `db:"hmnuser"`
-		SnippetID   *int                  `db:"snippet.id"`
-	}
-	iresult, err := db.QueryOne(ctx, tx, deleteMessageQuery{},
-		`
-		SELECT $columns
-		FROM
-			handmade_discordmessage AS msg
-			LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid
-			LEFT JOIN auth_user AS hmnuser ON duser.hmn_user_id = hmnuser.id
-			LEFT JOIN handmade_snippet AS snippet ON snippet.discord_message_id = msg.id
-		WHERE msg.id = $1 AND msg.channel_id = $2
-		`,
-		msgDelete.ID, msgDelete.ChannelID,
-	)
-	if errors.Is(err, db.NotFound) {
-		return
-	} else if err != nil {
-		log.Error().Err(err).Msg("failed to check for message to delete")
-		return
-	}
-	result := iresult.(*deleteMessageQuery)
 
 	log.Debug().Msg("deleting Discord message")
-	_, err = tx.Exec(ctx,
+	_, err := bot.dbConn.Exec(ctx,
 		`
 		DELETE FROM handmade_discordmessage
 		WHERE id = $1 AND channel_id = $2
@@ -662,30 +621,8 @@ func (bot *botInstance) messageDelete(ctx context.Context, msgDelete MessageDele
 		msgDelete.ID,
 		msgDelete.ChannelID,
 	)
-
-	shouldDeleteSnippet := result.HMNUser != nil && result.HMNUser.DiscordDeleteSnippetOnMessageDelete
-	if result.SnippetID != nil && shouldDeleteSnippet {
-		log.Debug().
-			Int("snippet_id", *result.SnippetID).
-			Int("user_id", result.HMNUser.ID).
-			Msg("deleting snippet from Discord message")
-		_, err = tx.Exec(ctx,
-			`
-			DELETE FROM handmade_snippet
-			WHERE id = $1
-			`,
-			result.SnippetID,
-		)
-		if err != nil {
-			log.Error().Err(err).Msg("failed to delete snippet")
-			return
-		}
-	}
-
-	err = tx.Commit(ctx)
 	if err != nil {
-		log.Error().Err(err).Msg("failed to delete Discord message")
+		logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to delete Discord message")
-		return
 	}
 }
 
@@ -697,10 +634,10 @@ type MessageToSend struct {
 
 func SendMessages(
 	ctx context.Context,
-	conn *pgxpool.Pool,
+	dbConn db.ConnOrTx,
 	msgs ...MessageToSend,
 ) error {
-	tx, err := conn.Begin(ctx)
+	tx, err := dbConn.Begin(ctx)
 	if err != nil {
 		return oops.New(err, "failed to start transaction")
 	}

src/discord/history.go

@@ -193,7 +193,7 @@ func handleHistoryMessage(ctx context.Context, dbConn *pgxpool.Pool, msg *Messag
 		break
 	}
 
-	newMsg, err := SaveMessageAndContents(ctx, tx, msg)
+	newMsg, err := PersistMessage(ctx, tx, msg, true)
 	if err != nil {
 		return err
 	}

src/discord/integration.go

@@ -0,0 +1,210 @@
+package discord
+
+import (
+	"context"
+	"errors"
+	"net/url"
+	"strings"
+
+	"git.handmade.network/hmn/hmn/src/config"
+	"git.handmade.network/hmn/hmn/src/db"
+	"git.handmade.network/hmn/hmn/src/models"
+	"git.handmade.network/hmn/hmn/src/oops"
+	"github.com/jackc/pgx/v4"
+)
+
+//
+// READ THE README!
+// (./README.md)
+//
+
+type MessageCreatedOrUpdatedFlags int
+
+const (
+	// If set, snippet creation will generally be enabled. We will attempt to
+	// make snippets for new messages shared in #project-showcase or whatever.
+	//
+	// If not set, snippet updates and deletes will still occur where
+	// appropriate. But new snippets will never be created.
+	CreateSnippetsOnNewMessages MessageCreatedOrUpdatedFlags = 1 << iota
+
+	// Create a snippet even if a message is not in the usual channels. This
+	// will NOT, in fact, create a snippet if the message doesn't meet the
+	// conditions - e.g. the message has media, and the user has a linked
+	// Discord account.
+	//
+	// Still requires CreateSnippetsOnNewMessages to be set as well.
+	ForceSnippet
+)
+
+// Call whenever you have a new or updated Discord message, in any channel.
+// For example, when:
+//   - You receive a message create/update event from the gateway
+//   - You see a new message in a history backfill
+//   - An admin command wants to make a snippet from some arbitrary message
+//
+// This implements the "on new/updated message" flow in the readme.
+func MessageCreatedOrUpdated(
+	ctx context.Context,
+	dbConn db.ConnOrTx,
+	msg *Message,
+	flags MessageCreatedOrUpdatedFlags,
+) error {
+	return db.DoWithTransaction(ctx, dbConn, func(tx pgx.Tx) error {
+		if msg.OriginalHasFields("author") && msg.Author.ID == config.Config.Discord.BotUserID {
+			// Don't process messages from the bot
+			return nil
+		}
+
+		didDelete, err := maybeDeleteShowcaseMsg(ctx, dbConn, msg)
+		if err != nil {
+			return err
+		} else if didDelete {
+			return nil
+		}
+
+		didDelete, err = maybeDeleteLibraryMsg(ctx, dbConn, msg)
+		if err != nil {
+			return err
+		} else if didDelete {
+			return nil
+		}
+
+		weCareAboutThisMessagesContents := false ||
+			flags&ForceSnippet != 0 ||
+			msg.ChannelID == config.Config.Discord.ShowcaseChannelID ||
+			weAlreadyHaveMessageContent(ctx, dbConn, msg)
+		if weCareAboutThisMessagesContents {
+			_, err := PersistMessage(ctx, dbConn, msg)
+			if err != nil {
+				return oops.New(err, "failed to persist new/updated Discord message")
+			}
+		}
+
+		//
+		// It's snippet time.
+		//
+		{
+			// Fetch the data we need to evaluate what to do from here.
+			type savedMessageAndSnippet struct {
+				Message        models.DiscordMessage         `db:"msg"`
+				MessageContent *models.DiscordMessageContent `db:"c"`
+				Snippet        *models.Snippet               `db:"snippet"`
+				DiscordUser    *models.DiscordUser           `db:"duser"`
+				HMNUser        *models.User                  `db:"u"`
+			}
+			isaved, err := db.QueryOne(ctx, tx, savedMessageAndSnippet{},
+				`
+				SELECT $columns
+				FROM
+					handmade_discordmessage AS msg
+					LEFT JOIN handmade_discordmessagecontent AS c ON c.message_id = msg.id
+					LEFT JOIN handmade_snippet AS snippet ON snippet.discord_message_id = msg.id
+					LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid
+					LEFT JOIN auth_user AS u ON duser.hmn_user_id = u.id
+				WHERE
+					msg.id = $1
+				`,
+				msg.ID,
+			)
+			if errors.Is(err, db.NotFound) {
+				goto nosnippet
+			} else if err != nil {
+				return oops.New(err, "failed to fetch snippet data and stuff")
+			}
+			saved := isaved.(*savedMessageAndSnippet)
+
+			if saved.DiscordUser == nil {
+				goto nosnippet
+			}
+
+			snippetableAsset, snippetableUrl, err := getSnippetAssetOrUrl(ctx, tx, msg.ID)
+			if err != nil {
+				return err
+			}
+
+			messageIsSnippetable := snippetableAsset != nil || snippetableUrl != ""
+			if messageIsSnippetable {
+				createSnippet, updateSnippet := false, false
+
+				if saved.Snippet == nil {
+					// No snippet exists. Maybe create one?
+					wantSnippetBecauseShowcase := true &&
+						msg.ChannelID == config.Config.Discord.ShowcaseChannelID &&
+						saved.HMNUser.DiscordSaveShowcase
+					wantNewSnippet := false ||
+						wantSnippetBecauseShowcase ||
+						flags&ForceSnippet != 0
+
+					createSnippet = true &&
+						flags&CreateSnippetsOnNewMessages != 0 &&
+						wantNewSnippet &&
+						saved.MessageContent != nil &&
+						saved.Message.SnippetCreated == false
+				} else {
+					// Update the snippet
+					updateSnippet = true
+				}
+
+				if createSnippet {
+					// TODO
+				} else if updateSnippet {
+					// TODO
+				}
+			} else {
+				// Delete any existing snippets (don't restrict this by channel!)
+				// Message edits can happen anywhere, even in channels where we don't
+				// delete messages like we do in showcase.
+
+				// TODO
+			}
+		}
+	nosnippet:
+
+		return nil
+	})
+}
+
+// Call whenever a Discord message is deleted, in any channel.
+func MessageDeleted(ctx context.Context, dbConn db.ConnOrTx, msg *Message) {
+
+}
+
+func messageHasLinks(content string) bool {
+	links := reDiscordMessageLink.FindAllString(content, -1)
+	for _, link := range links {
+		_, err := url.Parse(strings.TrimSpace(link))
+		if err == nil {
+			return true
+		}
+	}
+
+	return false
+}
+
+func weAlreadyHaveMessageContent(
+	ctx context.Context,
+	dbConn db.ConnOrTx,
+	msg *Message,
+) bool {
+	hasContent, err := db.QueryBool(ctx, dbConn,
+		`
+		SELECT COUNT(*) > 0
+		FROM
+			handmade_discordmessagecontent
+		WHERE
+			message_id = $1
+		`,
+		msg.ID,
+	)
+	if err != nil {
+		panic(err)
+	}
+
+	return hasContent
+}
+
+// Yields HTML and tags for a snippet, but doesn't actually create the snippet.
+func messageToSnippetData() {
+
+}

src/discord/library.go

@@ -3,43 +3,55 @@ package discord
 import (
 	"context"
 
+	"git.handmade.network/hmn/hmn/src/config"
+	"git.handmade.network/hmn/hmn/src/db"
 	"git.handmade.network/hmn/hmn/src/oops"
 )
 
-func (bot *botInstance) processLibraryMsg(ctx context.Context, msg *Message) error {
+func maybeDeleteLibraryMsg(
+	ctx context.Context,
+	dbConn db.ConnOrTx,
+	msg *Message,
+) (didDelete bool, err error) {
+	// Ensure basic info about the message
+	if msg.ChannelID != config.Config.Discord.LibraryChannelID {
+		return false, nil
+	}
 	switch msg.Type {
 	case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
 	default:
-		return nil
+		return false, nil
 	}
-
 	if !msg.OriginalHasFields("content") {
-		return nil
+		return false, nil
 	}
 
+	// Maybe delete it if it's bad
+	didDelete = false
 	if !messageHasLinks(msg.Content) {
 		err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
 		if err != nil {
-			return oops.New(err, "failed to delete message")
+			return false, oops.New(err, "failed to delete message")
 		}
+		didDelete = true
 
 		if !msg.Author.IsBot {
 			channel, err := CreateDM(ctx, msg.Author.ID)
 			if err != nil {
-				return oops.New(err, "failed to create DM channel")
+				return false, oops.New(err, "failed to create DM channel")
 			}
 
-			err = SendMessages(ctx, bot.dbConn, MessageToSend{
+			err = SendMessages(ctx, dbConn, MessageToSend{
 				ChannelID: channel.ID,
 				Req: CreateMessageRequest{
 					Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.",
 				},
 			})
 			if err != nil {
-				return oops.New(err, "failed to send showcase warning message")
+				return false, oops.New(err, "failed to send showcase warning message")
 			}
 		}
 	}
 
-	return nil
+	return didDelete, nil
 }

src/discord/payloads.go

@@ -246,13 +246,14 @@ const (
 
 // https://discord.com/developers/docs/resources/channel#message-object
 type Message struct {
-	ID        string      `json:"id"`
+	ID              string      `json:"id"`
-	ChannelID string      `json:"channel_id"`
+	ChannelID       string      `json:"channel_id"`
-	GuildID   *string     `json:"guild_id"`
+	GuildID         *string     `json:"guild_id"`
-	Content   string      `json:"content"`
+	Content         string      `json:"content"`
-	Author    *User       `json:"author"` // note that this may not be an actual valid user (see the docs)
+	Author          *User       `json:"author"` // note that this may not be an actual valid user (see the docs)
-	Timestamp string      `json:"timestamp"`
+	Timestamp       string      `json:"timestamp"`
-	Type      MessageType `json:"type"`
+	EditedTimestamp *string     `json:"edited_timestamp"`
+	Type            MessageType `json:"type"`
 
 	Attachments []Attachment `json:"attachments"`
 	Embeds      []Embed      `json:"embeds"`

src/discord/persistence.go

@@ -0,0 +1,373 @@
+package discord
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"git.handmade.network/hmn/hmn/src/assets"
+	"git.handmade.network/hmn/hmn/src/config"
+	"git.handmade.network/hmn/hmn/src/db"
+	"git.handmade.network/hmn/hmn/src/models"
+	"git.handmade.network/hmn/hmn/src/oops"
+	"github.com/google/uuid"
+)
+
+/*
+Ensures that a Discord message is stored in the database. It will also store
+the message's contents, if allowed. This function is idempotent and can be
+called regardless of whether the item already exists in the database.
+
+This does not create snippets or do anything besides save the message itself.
+*/
+func PersistMessage(
+	ctx context.Context,
+	dbConn db.ConnOrTx,
+	msg *Message,
+) (*models.DiscordMessage, error) {
+	tx, err := dbConn.Begin(ctx)
+	if err != nil {
+		return nil, oops.New(err, "failed to start transaction")
+	}
+	defer tx.Rollback(ctx)
+
+	// For every single Discord message on the server, we store a lightweight
+	// record of its existence. This allows us to efficiently retrieve message
+	// contents when someone links their Discord account. Naturally, we do this
+	// regardless of whether a user has linked their Discord account.
+	//
+	// Unless a user has linked their Discord account, no message content is
+	// ever saved.
+	iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessage
+		WHERE id = $1
+		`,
+		msg.ID,
+	)
+	if errors.Is(err, db.NotFound) {
+		if !msg.OriginalHasFields("author", "timestamp") {
+			return nil, errNotEnoughInfo
+		}
+
+		guildID := msg.GuildID
+		if guildID == nil {
+			/*
+				This is weird, but it can happen when we fetch messages from
+				history instead of receiving it from the gateway. In this case
+				we just assume it's from the HMN server.
+			*/
+			guildID = &config.Config.Discord.GuildID
+		}
+
+		iDiscordMessage, err = db.QueryOne(ctx, tx, models.DiscordMessage{},
+			`
+			INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created)
+			VALUES ($1, $2, $3, $4, $5, $6, $7)
+			RETURNING $columns
+			`,
+			msg.ID,
+			msg.ChannelID,
+			*guildID,
+			msg.JumpURL(),
+			msg.Author.ID,
+			msg.Time(),
+			false,
+		)
+		if err != nil {
+			return nil, oops.New(err, "failed to save new discord message")
+		}
+	} else if err != nil {
+		return nil, oops.New(err, "failed to check for existing Discord message")
+	}
+	discordMessage := iDiscordMessage.(*models.DiscordMessage)
+
+	//
+	// Time to save content!
+	//
+	{
+		// Check for linked Discord user
+		iDiscordUser, err := db.QueryOne(ctx, tx, models.DiscordUser{},
+			`
+			SELECT $columns
+			FROM handmade_discorduser
+			WHERE userid = $1
+			`,
+			discordMessage.UserID,
+		)
+		if errors.Is(err, db.NotFound) {
+			goto cancelSavingContent // considered harmful??
+		} else if err != nil {
+			return nil, oops.New(err, "failed to look up linked Discord user")
+		}
+		discordUser := iDiscordUser.(*models.DiscordUser)
+
+		// Save message text
+		if msg.OriginalHasFields("content") {
+			_, err = tx.Exec(ctx,
+				`
+				INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content)
+				VALUES ($1, $2, $3)
+				ON CONFLICT (message_id) DO UPDATE SET
+					discord_id = EXCLUDED.discord_id,
+					last_content = EXCLUDED.last_content
+				`,
+				discordMessage.ID,
+				discordUser.ID,
+				CleanUpMarkdown(ctx, msg.Content),
+			)
+		}
+
+		// Save attachments
+		if msg.OriginalHasFields("attachments") {
+			for _, attachment := range msg.Attachments {
+				_, err := saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID)
+				if err != nil {
+					return nil, oops.New(err, "failed to save attachment")
+				}
+			}
+		}
+
+		// Save / delete embeds
+		if msg.OriginalHasFields("embeds") {
+			numSavedEmbeds, err := db.QueryInt(ctx, tx,
+				`
+				SELECT COUNT(*)
+				FROM handmade_discordmessageembed
+				WHERE message_id = $1
+				`,
+				msg.ID,
+			)
+			if err != nil {
+				return nil, oops.New(err, "failed to count existing embeds")
+			}
+			if numSavedEmbeds == 0 {
+				// No embeds yet, so save new ones
+				for _, embed := range msg.Embeds {
+					_, err := saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID)
+					if err != nil {
+						return nil, oops.New(err, "failed to save embed")
+					}
+				}
+			} else if len(msg.Embeds) > 0 {
+				// Embeds were removed from the message
+				_, err := tx.Exec(ctx,
+					`
+					DELETE FROM handmade_discordmessageembed
+					WHERE message_id = $1
+					`,
+					msg.ID,
+				)
+				if err != nil {
+					return nil, oops.New(err, "failed to delete embeds")
+				}
+			}
+		}
+	}
+cancelSavingContent:
+
+	err = tx.Commit(ctx)
+	if err != nil {
+		return nil, oops.New(err, "failed to commit transaction")
+	}
+
+	return discordMessage, nil
+}
+
+/*
+Saves a Discord attachment as an HMN asset. Idempotent; will not create an
+attachment that already exists.
+*/
+func saveAttachment(
+	ctx context.Context,
+	tx db.ConnOrTx,
+	attachment *Attachment,
+	hmnUserID int,
+	discordMessageID string,
+) (*models.DiscordMessageAttachment, error) {
+	iexisting, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessageattachment
+		WHERE id = $1
+		`,
+		attachment.ID,
+	)
+	if err == nil {
+		return iexisting.(*models.DiscordMessageAttachment), nil
+	} else if errors.Is(err, db.NotFound) {
+		// this is fine, just create it
+	} else {
+		return nil, oops.New(err, "failed to check for existing attachment")
+	}
+
+	width := 0
+	height := 0
+	if attachment.Width != nil {
+		width = *attachment.Width
+	}
+	if attachment.Height != nil {
+		height = *attachment.Height
+	}
+
+	content, _, err := downloadDiscordResource(ctx, attachment.Url)
+	if err != nil {
+		return nil, oops.New(err, "failed to download Discord attachment")
+	}
+
+	contentType := "application/octet-stream"
+	if attachment.ContentType != nil {
+		contentType = *attachment.ContentType
+	}
+
+	asset, err := assets.Create(ctx, tx, assets.CreateInput{
+		Content:     content,
+		Filename:    attachment.Filename,
+		ContentType: contentType,
+
+		UploaderID: &hmnUserID,
+		Width:      width,
+		Height:     height,
+	})
+	if err != nil {
+		return nil, oops.New(err, "failed to save asset for Discord attachment")
+	}
+
+	iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
+		`
+		INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id)
+		VALUES ($1, $2, $3)
+		RETURNING $columns
+		`,
+		attachment.ID,
+		asset.ID,
+		discordMessageID,
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to save Discord attachment data")
+	}
+
+	return iDiscordAttachment.(*models.DiscordMessageAttachment), nil
+}
+
+// Saves an embed from Discord. NOTE: This is _not_ idempotent, so only call it
+// if you do not have any embeds saved for this message yet.
+func saveEmbed(
+	ctx context.Context,
+	tx db.ConnOrTx,
+	embed *Embed,
+	hmnUserID int,
+	discordMessageID string,
+) (*models.DiscordMessageEmbed, error) {
+	isOkImageType := func(contentType string) bool {
+		return strings.HasPrefix(contentType, "image/")
+	}
+
+	isOkVideoType := func(contentType string) bool {
+		return strings.HasPrefix(contentType, "video/")
+	}
+
+	maybeSaveImageish := func(i EmbedImageish, contentTypeCheck func(string) bool) (*uuid.UUID, error) {
+		content, contentType, err := downloadDiscordResource(ctx, *i.Url)
+		if err != nil {
+			var statusError DiscordResourceBadStatusCode
+			if errors.As(err, &statusError) {
+				return nil, nil
+			} else {
+				return nil, oops.New(err, "failed to save Discord embed")
+			}
+		}
+		if contentTypeCheck(contentType) {
+			in := assets.CreateInput{
+				Content:     content,
+				Filename:    "embed",
+				ContentType: contentType,
+				UploaderID:  &hmnUserID,
+			}
+
+			if i.Width != nil {
+				in.Width = *i.Width
+			}
+			if i.Height != nil {
+				in.Height = *i.Height
+			}
+
+			asset, err := assets.Create(ctx, tx, in)
+			if err != nil {
+				return nil, oops.New(err, "failed to create asset from embed")
+			}
+			return &asset.ID, nil
+		}
+
+		return nil, nil
+	}
+
+	var imageAssetId *uuid.UUID
+	var videoAssetId *uuid.UUID
+	var err error
+
+	if embed.Video != nil && embed.Video.Url != nil {
+		videoAssetId, err = maybeSaveImageish(embed.Video.EmbedImageish, isOkVideoType)
+	} else if embed.Image != nil && embed.Image.Url != nil {
+		imageAssetId, err = maybeSaveImageish(embed.Image.EmbedImageish, isOkImageType)
+	} else if embed.Thumbnail != nil && embed.Thumbnail.Url != nil {
+		imageAssetId, err = maybeSaveImageish(embed.Thumbnail.EmbedImageish, isOkImageType)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	// Save the embed into the db
+	iDiscordEmbed, err := db.QueryOne(ctx, tx, models.DiscordMessageEmbed{},
+		`
+		INSERT INTO handmade_discordmessageembed (title, description, url, message_id, image_id, video_id)
+		VALUES ($1, $2, $3, $4, $5, $6)
+		RETURNING id
+		`,
+		embed.Title,
+		embed.Description,
+		embed.Url,
+		discordMessageID,
+		imageAssetId,
+		videoAssetId,
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to insert new embed")
+	}
+
+	return iDiscordEmbed.(*models.DiscordMessageEmbed), nil
+}
+
+var discordDownloadClient = &http.Client{
+	Timeout: 10 * time.Second,
+}
+
+type DiscordResourceBadStatusCode error
+
+func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, "", oops.New(err, "failed to make Discord download request")
+	}
+	res, err := discordDownloadClient.Do(req)
+	if err != nil {
+		return nil, "", oops.New(err, "failed to fetch Discord resource data")
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode < 200 || 299 < res.StatusCode {
+		return nil, "", DiscordResourceBadStatusCode(fmt.Errorf("status code %d from Discord resource: %s", res.StatusCode, url))
+	}
+
+	content, err := io.ReadAll(res.Body)
+	if err != nil {
+		panic(err)
+	}
+
+	return content, res.Header.Get("Content-Type"), nil
+}

src/discord/showcase.go

@@ -3,15 +3,9 @@ package discord
 import (
 	"context"
 	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
 	"regexp"
 	"strings"
-	"time"
 
-	"git.handmade.network/hmn/hmn/src/assets"
 	"git.handmade.network/hmn/hmn/src/config"
 	"git.handmade.network/hmn/hmn/src/db"
 	"git.handmade.network/hmn/hmn/src/hmndata"
@@ -19,49 +13,40 @@ import (
 	"git.handmade.network/hmn/hmn/src/models"
 	"git.handmade.network/hmn/hmn/src/oops"
 	"git.handmade.network/hmn/hmn/src/parsing"
-	"github.com/google/uuid"
+	"github.com/rs/zerolog/log"
 )
 
 var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`)
 
 var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this")
 
-func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) error {
+// For any persisted Discord message, attempt to create or update a snippet.
-	switch msg.Type {
+func makeSnippet(
+	ctx context.Context,
+	dbConn db.ConnOrTx,
+	discordMsg *Message,
+	hmnMsg *models.DiscordMessage,
+) error {
+	if discordMsg.ChannelID != config.Config.Discord.ShowcaseChannelID {
+		return nil
+	}
+
+	switch discordMsg.Type {
 	case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
 	default:
 		return nil
 	}
 
-	didDelete, err := bot.maybeDeleteShowcaseMsg(ctx, msg)
+	tx, err := dbConn.Begin(ctx)
-	if err != nil {
-		return err
-	}
-	if didDelete {
-		return nil
-	}
-
-	tx, err := bot.dbConn.Begin(ctx)
 	if err != nil {
 		panic(err)
 	}
 	defer tx.Rollback(ctx)
 
-	// save the message, maybe save its contents
-	newMsg, err := SaveMessageAndContents(ctx, tx, msg)
-	if errors.Is(err, errNotEnoughInfo) {
-		logging.ExtractLogger(ctx).Warn().
-			Interface("msg", msg).
-			Msg("didn't have enough info to process Discord message")
-		return nil
-	} else if err != nil {
-		return err
-	}
-
 	// ...and maybe make a snippet too, if the user wants us to
-	duser, err := FetchDiscordUser(ctx, tx, newMsg.UserID)
+	duser, err := hmndata.FetchDiscordUser(ctx, tx, hmnMsg.UserID)
 	if err == nil && duser.HMNUser.DiscordSaveShowcase {
-		err = CreateMessageSnippets(ctx, tx, newMsg.UserID, msg.ID)
+		err = CreateMessageSnippets(ctx, tx, hmnMsg.UserID, discordMsg.ID)
 		if err != nil {
 			return oops.New(err, "failed to create snippet in gateway")
 		}
@@ -81,7 +66,19 @@ func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) er
 	return nil
 }
 
-func (bot *botInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message) (didDelete bool, err error) {
+func maybeDeleteShowcaseMsg(ctx context.Context, dbConn db.ConnOrTx, msg *Message) (didDelete bool, err error) {
+	// Ensure basics about the message
+	if msg.ChannelID != config.Config.Discord.ShowcaseChannelID {
+		return false, nil
+	}
+	switch msg.Type {
+	case MessageTypeDefault, MessageTypeReply:
+		// proceed
+	default:
+		return false, nil
+	}
+
+	// Check message content and stuff
 	hasGoodContent := true
 	if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) {
 		hasGoodContent = false
@@ -106,7 +103,7 @@ func (bot *botInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message
 				return false, oops.New(err, "failed to create DM channel")
 			}
 
-			err = SendMessages(ctx, bot.dbConn, MessageToSend{
+			err = SendMessages(ctx, dbConn, MessageToSend{
 				ChannelID: channel.ID,
 				Req: CreateMessageRequest{
 					Content: "Posts in #project-showcase are required to have either an image/video or a link. Discuss showcase content in #projects.",
@@ -121,420 +118,34 @@ func (bot *botInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message
 	return didDelete, nil
 }
 
-/*
+// For any Discord message, attempt to delete any snippets.
-Ensures that a Discord message is stored in the database. This function is
+func deleteSnippet(ctx context.Context, dbConn db.ConnOrTx, msgDelete MessageDelete) {
-idempotent and can be called regardless of whether the item already exists in
+	s, err := hmndata.FetchSnippetForDiscordMessage(ctx, dbConn, nil, msgDelete.ID, hmndata.SnippetQuery{})
-the database.
-
-This does not create snippets or do anything besides save the message itself.
-*/
-func SaveMessage(
-	ctx context.Context,
-	tx db.ConnOrTx,
-	msg *Message,
-) (*models.DiscordMessage, error) {
-	iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessage
-		WHERE id = $1
-		`,
-		msg.ID,
-	)
 	if errors.Is(err, db.NotFound) {
-		if !msg.OriginalHasFields("author", "timestamp") {
+		return
-			return nil, errNotEnoughInfo
+	} else if err != nil {
-		}
+		logging.ExtractLogger(ctx).Error().Err(err).Msg("")
+		return
+	}
 
-		guildID := msg.GuildID
+	shouldDeleteSnippet := s.Owner != nil && s.Owner.DiscordDeleteSnippetOnMessageDelete
-		if guildID == nil {
+	if shouldDeleteSnippet {
-			/*
+		logging.ExtractLogger(ctx).Debug().
-				This is weird, but it can happen when we fetch messages from
+			Int("snippet_id", s.Snippet.ID).
-				history instead of receiving it from the gateway. In this case
+			Int("user_id", s.Owner.ID).
-				we just assume it's from the HMN server.
+			Msg("deleting snippet from Discord message")
-			*/
+		_, err = dbConn.Exec(ctx,
-			guildID = &config.Config.Discord.GuildID
-		}
-
-		_, err = tx.Exec(ctx,
 			`
-			INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created)
+			DELETE FROM handmade_snippet
-			VALUES ($1, $2, $3, $4, $5, $6, $7)
-			`,
-			msg.ID,
-			msg.ChannelID,
-			*guildID,
-			msg.JumpURL(),
-			msg.Author.ID,
-			msg.Time(),
-			false,
-		)
-		if err != nil {
-			return nil, oops.New(err, "failed to save new discord message")
-		}
-
-		/*
-			TODO(db): This is a spot where it would be really nice to be able
-			to use RETURNING, and avoid this second query.
-		*/
-		iDiscordMessage, err = db.QueryOne(ctx, tx, models.DiscordMessage{},
-			`
-			SELECT $columns
-			FROM handmade_discordmessage
 			WHERE id = $1
 			`,
-			msg.ID,
+			s.Snippet.ID,
 		)
 		if err != nil {
-			panic(err)
+			log.Error().Err(err).Msg("failed to delete snippet")
-		}
+			return
-	} else if err != nil {
-		return nil, oops.New(err, "failed to check for existing Discord message")
-	}
-
-	return iDiscordMessage.(*models.DiscordMessage), nil
-}
-
-/*
-Processes a single Discord message, saving as much of the message's content
-and attachments as allowed by our rules and user settings. Does NOT create
-snippets.
-
-Idempotent; can be called any time whether the message exists or not.
-*/
-func SaveMessageAndContents(
-	ctx context.Context,
-	tx db.ConnOrTx,
-	msg *Message,
-) (*models.DiscordMessage, error) {
-	newMsg, err := SaveMessage(ctx, tx, msg)
-	if err != nil {
-		return nil, err
-	}
-
-	// Check for linked Discord user
-	iDiscordUser, err := db.QueryOne(ctx, tx, models.DiscordUser{},
-		`
-		SELECT $columns
-		FROM handmade_discorduser
-		WHERE userid = $1
-		`,
-		newMsg.UserID,
-	)
-	if errors.Is(err, db.NotFound) {
-		return newMsg, nil
-	} else if err != nil {
-		return nil, oops.New(err, "failed to look up linked Discord user")
-	}
-	discordUser := iDiscordUser.(*models.DiscordUser)
-
-	// We have a linked Discord account, so save the message contents (regardless of
-	// whether we create a snippet or not).
-
-	if msg.OriginalHasFields("content") {
-		_, err = tx.Exec(ctx,
-			`
-			INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content)
-			VALUES ($1, $2, $3)
-			ON CONFLICT (message_id) DO UPDATE SET
-				discord_id = EXCLUDED.discord_id,
-				last_content = EXCLUDED.last_content
-			`,
-			newMsg.ID,
-			discordUser.ID,
-			CleanUpMarkdown(ctx, msg.Content),
-		)
-	}
-
-	// Save attachments
-	if msg.OriginalHasFields("attachments") {
-		for _, attachment := range msg.Attachments {
-			_, err := saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID)
-			if err != nil {
-				return nil, oops.New(err, "failed to save attachment")
-			}
 		}
 	}
-
-	// Save / delete embeds
-	if msg.OriginalHasFields("embeds") {
-		numSavedEmbeds, err := db.QueryInt(ctx, tx,
-			`
-			SELECT COUNT(*)
-			FROM handmade_discordmessageembed
-			WHERE message_id = $1
-			`,
-			msg.ID,
-		)
-		if err != nil {
-			return nil, oops.New(err, "failed to count existing embeds")
-		}
-		if numSavedEmbeds == 0 {
-			// No embeds yet, so save new ones
-			for _, embed := range msg.Embeds {
-				_, err := saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID)
-				if err != nil {
-					return nil, oops.New(err, "failed to save embed")
-				}
-			}
-		} else if len(msg.Embeds) > 0 {
-			// Embeds were removed from the message
-			_, err := tx.Exec(ctx,
-				`
-				DELETE FROM handmade_discordmessageembed
-				WHERE message_id = $1
-				`,
-				msg.ID,
-			)
-			if err != nil {
-				return nil, oops.New(err, "failed to delete embeds")
-			}
-		}
-	}
-
-	return newMsg, nil
-}
-
-var discordDownloadClient = &http.Client{
-	Timeout: 10 * time.Second,
-}
-
-type DiscordResourceBadStatusCode error
-
-func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
-	if err != nil {
-		return nil, "", oops.New(err, "failed to make Discord download request")
-	}
-	res, err := discordDownloadClient.Do(req)
-	if err != nil {
-		return nil, "", oops.New(err, "failed to fetch Discord resource data")
-	}
-	defer res.Body.Close()
-
-	if res.StatusCode < 200 || 299 < res.StatusCode {
-		return nil, "", DiscordResourceBadStatusCode(fmt.Errorf("status code %d from Discord resource: %s", res.StatusCode, url))
-	}
-
-	content, err := io.ReadAll(res.Body)
-	if err != nil {
-		panic(err)
-	}
-
-	return content, res.Header.Get("Content-Type"), nil
-}
-
-/*
-Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment
-that already exists
-*/
-func saveAttachment(
-	ctx context.Context,
-	tx db.ConnOrTx,
-	attachment *Attachment,
-	hmnUserID int,
-	discordMessageID string,
-) (*models.DiscordMessageAttachment, error) {
-	iexisting, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessageattachment
-		WHERE id = $1
-		`,
-		attachment.ID,
-	)
-	if err == nil {
-		return iexisting.(*models.DiscordMessageAttachment), nil
-	} else if errors.Is(err, db.NotFound) {
-		// this is fine, just create it
-	} else {
-		return nil, oops.New(err, "failed to check for existing attachment")
-	}
-
-	width := 0
-	height := 0
-	if attachment.Width != nil {
-		width = *attachment.Width
-	}
-	if attachment.Height != nil {
-		height = *attachment.Height
-	}
-
-	content, _, err := downloadDiscordResource(ctx, attachment.Url)
-	if err != nil {
-		return nil, oops.New(err, "failed to download Discord attachment")
-	}
-
-	contentType := "application/octet-stream"
-	if attachment.ContentType != nil {
-		contentType = *attachment.ContentType
-	}
-
-	asset, err := assets.Create(ctx, tx, assets.CreateInput{
-		Content:     content,
-		Filename:    attachment.Filename,
-		ContentType: contentType,
-
-		UploaderID: &hmnUserID,
-		Width:      width,
-		Height:     height,
-	})
-	if err != nil {
-		return nil, oops.New(err, "failed to save asset for Discord attachment")
-	}
-
-	// TODO(db): RETURNING plz thanks
-	_, err = tx.Exec(ctx,
-		`
-		INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id)
-		VALUES ($1, $2, $3)
-		`,
-		attachment.ID,
-		asset.ID,
-		discordMessageID,
-	)
-	if err != nil {
-		return nil, oops.New(err, "failed to save Discord attachment data")
-	}
-
-	iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessageattachment
-		WHERE id = $1
-		`,
-		attachment.ID,
-	)
-	if err != nil {
-		return nil, oops.New(err, "failed to fetch new Discord attachment data")
-	}
-
-	return iDiscordAttachment.(*models.DiscordMessageAttachment), nil
-}
-
-// Saves an embed from Discord. NOTE: This is _not_ idempotent, so only call it
-// if you do not have any embeds saved for this message yet.
-func saveEmbed(
-	ctx context.Context,
-	tx db.ConnOrTx,
-	embed *Embed,
-	hmnUserID int,
-	discordMessageID string,
-) (*models.DiscordMessageEmbed, error) {
-	isOkImageType := func(contentType string) bool {
-		return strings.HasPrefix(contentType, "image/")
-	}
-
-	isOkVideoType := func(contentType string) bool {
-		return strings.HasPrefix(contentType, "video/")
-	}
-
-	maybeSaveImageish := func(i EmbedImageish, contentTypeCheck func(string) bool) (*uuid.UUID, error) {
-		content, contentType, err := downloadDiscordResource(ctx, *i.Url)
-		if err != nil {
-			var statusError DiscordResourceBadStatusCode
-			if errors.As(err, &statusError) {
-				return nil, nil
-			} else {
-				return nil, oops.New(err, "failed to save Discord embed")
-			}
-		}
-		if contentTypeCheck(contentType) {
-			in := assets.CreateInput{
-				Content:     content,
-				Filename:    "embed",
-				ContentType: contentType,
-				UploaderID:  &hmnUserID,
-			}
-
-			if i.Width != nil {
-				in.Width = *i.Width
-			}
-			if i.Height != nil {
-				in.Height = *i.Height
-			}
-
-			asset, err := assets.Create(ctx, tx, in)
-			if err != nil {
-				return nil, oops.New(err, "failed to create asset from embed")
-			}
-			return &asset.ID, nil
-		}
-
-		return nil, nil
-	}
-
-	var imageAssetId *uuid.UUID
-	var videoAssetId *uuid.UUID
-	var err error
-
-	if embed.Video != nil && embed.Video.Url != nil {
-		videoAssetId, err = maybeSaveImageish(embed.Video.EmbedImageish, isOkVideoType)
-	} else if embed.Image != nil && embed.Image.Url != nil {
-		imageAssetId, err = maybeSaveImageish(embed.Image.EmbedImageish, isOkImageType)
-	} else if embed.Thumbnail != nil && embed.Thumbnail.Url != nil {
-		imageAssetId, err = maybeSaveImageish(embed.Thumbnail.EmbedImageish, isOkImageType)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	// Save the embed into the db
-	// TODO(db): Insert, RETURNING
-	var savedEmbedId int
-	err = tx.QueryRow(ctx,
-		`
-		INSERT INTO handmade_discordmessageembed (title, description, url, message_id, image_id, video_id)
-		VALUES ($1, $2, $3, $4, $5, $6)
-		RETURNING id
-		`,
-		embed.Title,
-		embed.Description,
-		embed.Url,
-		discordMessageID,
-		imageAssetId,
-		videoAssetId,
-	).Scan(&savedEmbedId)
-	if err != nil {
-		return nil, oops.New(err, "failed to insert new embed")
-	}
-
-	iDiscordEmbed, err := db.QueryOne(ctx, tx, models.DiscordMessageEmbed{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessageembed
-		WHERE id = $1
-		`,
-		savedEmbedId,
-	)
-	if err != nil {
-		return nil, oops.New(err, "failed to fetch new Discord embed data")
-	}
-
-	return iDiscordEmbed.(*models.DiscordMessageEmbed), nil
-}
-
-type DiscordUserAndStuff struct {
-	DiscordUser models.DiscordUser `db:"duser"`
-	HMNUser     models.User        `db:"u"`
-}
-
-func FetchDiscordUser(ctx context.Context, dbConn db.ConnOrTx, discordUserID string) (*DiscordUserAndStuff, error) {
-	iuser, err := db.QueryOne(ctx, dbConn, DiscordUserAndStuff{},
-		`
-		SELECT $columns
-		FROM
-			handmade_discorduser AS duser
-			JOIN auth_user AS u ON duser.hmn_user_id = u.id
-		WHERE
-			duser.userid = $1
-		`,
-		discordUserID,
-	)
-	if err != nil {
-		return nil, err
-	}
-	return iuser.(*DiscordUserAndStuff), nil
 }
 
 /*
@@ -542,7 +153,7 @@ Checks settings and permissions to decide whether we are allowed to create
 snippets for a user.
 */
 func AllowedToCreateMessageSnippets(ctx context.Context, tx db.ConnOrTx, discordUserId string) (bool, error) {
-	u, err := FetchDiscordUser(ctx, tx, discordUserId)
+	u, err := hmndata.FetchDiscordUser(ctx, tx, discordUserId)
 	if errors.Is(err, db.NotFound) {
 		return false, nil
 	} else if err != nil {
@@ -697,7 +308,7 @@ func UpdateSnippetTagsIfAny(ctx context.Context, dbConn db.ConnOrTx, msg *Messag
 
 	// Fetch the Discord user; we only process messages for users with linked
 	// Discord accounts
-	u, err := FetchDiscordUser(ctx, tx, msg.Author.ID)
+	u, err := hmndata.FetchDiscordUser(ctx, tx, msg.Author.ID)
 	if err == db.NotFound {
 		return nil
 	} else if err != nil {
@@ -798,68 +409,6 @@ func UpdateSnippetTagsIfAny(ctx context.Context, dbConn db.ConnOrTx, msg *Messag
 	return nil
 }
 
-// NOTE(ben): This is maybe redundant with the regexes we use for markdown. But
-// do we actually want to reuse those, or should we keep them separate?
-var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`)
-
-func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msg *models.DiscordMessage) (*uuid.UUID, *string, error) {
-	// Check attachments
-	attachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessageattachment
-		WHERE message_id = $1
-		`,
-		msg.ID,
-	)
-	if err != nil {
-		return nil, nil, oops.New(err, "failed to fetch message attachments")
-	}
-	for _, iattachment := range attachments {
-		attachment := iattachment.(*models.DiscordMessageAttachment)
-		return &attachment.AssetID, nil, nil
-	}
-
-	// Check embeds
-	embeds, err := db.Query(ctx, tx, models.DiscordMessageEmbed{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessageembed
-		WHERE message_id = $1
-		`,
-		msg.ID,
-	)
-	if err != nil {
-		return nil, nil, oops.New(err, "failed to fetch discord embeds")
-	}
-	for _, iembed := range embeds {
-		embed := iembed.(*models.DiscordMessageEmbed)
-		if embed.VideoID != nil {
-			return embed.VideoID, nil, nil
-		} else if embed.ImageID != nil {
-			return embed.ImageID, nil, nil
-		} else if embed.URL != nil {
-			if RESnippetableUrl.MatchString(*embed.URL) {
-				return nil, embed.URL, nil
-			}
-		}
-	}
-
-	return nil, nil, nil
-}
-
-func messageHasLinks(content string) bool {
-	links := reDiscordMessageLink.FindAllString(content, -1)
-	for _, link := range links {
-		_, err := url.Parse(strings.TrimSpace(link))
-		if err == nil {
-			return true
-		}
-	}
-
-	return false
-}
-
 var REDiscordTag = regexp.MustCompile(`&([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*)`)
 
 func getDiscordTags(content string) []string {

src/discord/snippets.go

@@ -0,0 +1,65 @@
+package discord
+
+import (
+	"context"
+	"regexp"
+
+	"git.handmade.network/hmn/hmn/src/db"
+	"git.handmade.network/hmn/hmn/src/models"
+	"git.handmade.network/hmn/hmn/src/oops"
+	"github.com/google/uuid"
+)
+
+// NOTE(ben): This is maybe redundant with the regexes we use for markdown. But
+// do we actually want to reuse those, or should we keep them separate?
+var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`)
+
+// Given a Discord message for which we have saved content, this function will
+// return either an asset ID or a URL with which to make a snippet.
+//
+// Be aware that this function may return absolutely nothing.
+func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msgID string) (*uuid.UUID, string, error) {
+	// Check attachments
+	attachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessageattachment
+		WHERE message_id = $1
+		`,
+		msgID,
+	)
+	if err != nil {
+		return nil, "", oops.New(err, "failed to fetch message attachments")
+	}
+	for _, iattachment := range attachments {
+		attachment := iattachment.(*models.DiscordMessageAttachment)
+		return &attachment.AssetID, "", nil
+	}
+
+	// Check embeds
+	embeds, err := db.Query(ctx, tx, models.DiscordMessageEmbed{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessageembed
+		WHERE message_id = $1
+		`,
+		msgID,
+	)
+	if err != nil {
+		return nil, "", oops.New(err, "failed to fetch discord embeds")
+	}
+	for _, iembed := range embeds {
+		embed := iembed.(*models.DiscordMessageEmbed)
+		if embed.VideoID != nil {
+			return embed.VideoID, "", nil
+		} else if embed.ImageID != nil {
+			return embed.ImageID, "", nil
+		} else if embed.URL != nil {
+			if RESnippetableUrl.MatchString(*embed.URL) {
+				return nil, *embed.URL, nil
+			}
+		}
+	}
+
+	return nil, "", nil
+}

src/hmndata/discord_helper.go

@@ -0,0 +1,65 @@
+package hmndata
+
+import (
+	"context"
+	"errors"
+
+	"git.handmade.network/hmn/hmn/src/db"
+	"git.handmade.network/hmn/hmn/src/models"
+	"git.handmade.network/hmn/hmn/src/oops"
+)
+
+type DiscordUserAndStuff struct {
+	DiscordUser models.DiscordUser `db:"duser"`
+	HMNUser     models.User        `db:"u"`
+}
+
+func FetchDiscordUser(ctx context.Context, dbConn db.ConnOrTx, discordUserID string) (*DiscordUserAndStuff, error) {
+	iuser, err := db.QueryOne(ctx, dbConn, DiscordUserAndStuff{},
+		`
+		SELECT $columns
+		FROM
+			handmade_discorduser AS duser
+			JOIN auth_user AS u ON duser.hmn_user_id = u.id
+		WHERE
+			duser.userid = $1
+		`,
+		discordUserID,
+	)
+	if err != nil {
+		return nil, err
+	}
+	return iuser.(*DiscordUserAndStuff), nil
+}
+
+type DiscordMessageAndStuff struct {
+	Message models.DiscordMessage         `db:"msg"`
+	Content *models.DiscordMessageContent `db:"content"`
+
+	DiscordUser *models.DiscordUser `db:"duser"`
+	HMNUser     *models.User        `db:"u"`
+}
+
+func FetchDiscordMessage(ctx context.Context, dbConn db.ConnOrTx, msgID string) (*DiscordMessageAndStuff, error) {
+	iresult, err := db.QueryOne(ctx, dbConn, DiscordMessageAndStuff{},
+		`
+		SELECT $columns
+		FROM
+			handmade_discordmessage AS msg
+			LEFT JOIN handmade_discordmessagecontent AS content ON content.message_id = msg.id
+			LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.id
+			LEFT JOIN auth_user AS u ON duser.hmn_user_id = u.id
+		WHERE
+			msg.id = $1
+		`,
+		msgID,
+	)
+	if errors.Is(err, db.NotFound) {
+		return nil, db.NotFound
+	} else if err != nil {
+		return nil, oops.New(err, "failed to fetch Discord message")
+	}
+	result := iresult.(*DiscordMessageAndStuff)
+
+	return result, nil
+}

src/models/discord.go

@@ -31,6 +31,7 @@ type DiscordMessage struct {
 	Url            string    `db:"url"`
 	UserID         string    `db:"user_id"`
 	SentAt         time.Time `db:"sent_at"`
+	EditedAt       time.Time `db:"edited_at"`
 	SnippetCreated bool      `db:"snippet_created"`
 }
 

src/models/snippet.go

@@ -18,6 +18,6 @@ type Snippet struct {
 	Url     *string    `db:"url"`
 	AssetID *uuid.UUID `db:"asset_id"`
 
-	EditedOnWebsite  bool    `db:"edited_on_website"`
+	EditedOnWebsite  bool    `db:"edited_on_website"` // TODO: Make this "ConnectedToDiscord"
 	DiscordMessageID *string `db:"discord_message_id"`
 }

src/utils/utils.go

@@ -73,3 +73,29 @@ func SleepContext(ctx context.Context, d time.Duration) error {
 		return nil
 	}
 }
+
+func IntSliceIndex(s []int, v int) int {
+	for i, vv := range s {
+		if vv == v {
+			return i
+		}
+	}
+	return -1
+}
+
+func StringSliceIndex(s []string, v string) int {
+	for i, vv := range s {
+		if vv == v {
+			return i
+		}
+	}
+	return -1
+}
+
+func IntSliceContains(s []int, v int) bool {
+	return IntSliceIndex(s, v) > -1
+}
+
+func StringSliceContains(s []string, v string) bool {
+	return StringSliceIndex(s, v) > -1
+}