Discord message handling reorganization.

src/discord/cmd/cmd.go

@@ -2,6 +2,9 @@ package cmd
 
 import (
 	"context"
+	"errors"
+	"fmt"
+	"os"
 	"time"
 
 	"git.handmade.network/hmn/hmn/src/db"
@@ -35,18 +38,45 @@ func init() {
 	rootCommand.AddCommand(scrapeCommand)
 
 	makeSnippetCommand := &cobra.Command{
-		Use:   "makesnippet [<message id>...]",
+		Use:   "makesnippet <channel id> [<message id>...]",
 		Short: "Make snippets from saved Discord messages",
 		Long:  "Make snippets from Discord messages whose content we have already saved. Useful for creating snippets from messages in non-showcase channels.",
 		Run: func(cmd *cobra.Command, args []string) {
+			if len(args) < 2 {
+				cmd.Usage()
+				os.Exit(1)
+			}
 			ctx := context.Background()
 			conn := db.NewConnPool(1, 1)
 			defer conn.Close()
 
-			err := discord.CreateMessageSnippets(ctx, conn, args...)
+			chanID := args[0]
-			if err != nil {
+
-				logging.Error().Err(err).Msg("failed to create snippets")
+			count := 0
+
+			for _, msgID := range args[1:] {
+				message, err := discord.GetChannelMessage(ctx, chanID, msgID)
+				if errors.Is(err, discord.NotFound) {
+					logging.Warn().Msg(fmt.Sprintf("no message found on discord for id %s", msgID))
+					continue
+				} else if err != nil {
+					logging.Error().Msg(fmt.Sprintf("failed to fetch discord message id %s", msgID))
+					continue
+				}
+				err = discord.InternMessage(ctx, conn, message)
+				if err != nil {
+					logging.Error().Msg(fmt.Sprintf("failed to intern discord message id %s", msgID))
+					continue
+				}
+				err = discord.HandleInternedMessage(ctx, conn, message, false, true)
+				if err != nil {
+					logging.Error().Msg(fmt.Sprintf("failed to handle interned message id %s", msgID))
+					continue
+				}
+				count += 1
 			}
+
+			logging.Info().Msg(fmt.Sprintf("Handled %d messages", count))
 		},
 	}
 	rootCommand.AddCommand(makeSnippetCommand)

src/discord/gateway.go

@@ -591,103 +591,31 @@ func (bot *botInstance) messageCreateOrUpdate(ctx context.Context, msg *Message)
 		return nil
 	}
 
-	if msg.ChannelID == config.Config.Discord.ShowcaseChannelID {
+	err := HandleIncomingMessage(ctx, bot.dbConn, msg, true)
-		err := bot.processShowcaseMsg(ctx, msg)
-		if err != nil {
-			logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to process showcase message")
-			return nil
-		}
-		return nil
-	}
-
-	if msg.ChannelID == config.Config.Discord.LibraryChannelID {
-		err := bot.processLibraryMsg(ctx, msg)
-		if err != nil {
-			logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to process library message")
-			return nil
-		}
-		return nil
-	}
-
-	err := UpdateSnippetTagsIfAny(ctx, bot.dbConn, msg)
 	if err != nil {
-		logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to update tags for Discord snippet")
+		logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to handle incoming message")
 	}
 
+	// NOTE(asaf): Since any error from HandleIncomingMessage is an internal error and not a discord
+	//			   error, we only want to log it and not restart the bot. So we're not returning the error.
 	return nil
 }
 
 func (bot *botInstance) messageDelete(ctx context.Context, msgDelete MessageDelete) {
 	log := logging.ExtractLogger(ctx)
 
-	tx, err := bot.dbConn.Begin(ctx)
+	interned, err := FetchInternedMessage(ctx, bot.dbConn, msgDelete.ID)
 	if err != nil {
-		log.Error().Err(err).Msg("failed to start transaction")
+		log.Error().Err(err).Msg("failed to fetch interned message")
 		return
 	}
-	defer tx.Rollback(ctx)
+	if interned != nil {
-
+		err = DeleteInternedMessage(ctx, bot.dbConn, interned)
-	type deleteMessageQuery struct {
-		Message     models.DiscordMessage `db:"msg"`
-		DiscordUser *models.DiscordUser   `db:"duser"`
-		HMNUser     *models.User          `db:"hmnuser"`
-		SnippetID   *int                  `db:"snippet.id"`
-	}
-	iresult, err := db.QueryOne(ctx, tx, deleteMessageQuery{},
-		`
-		SELECT $columns
-		FROM
-			handmade_discordmessage AS msg
-			LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid
-			LEFT JOIN auth_user AS hmnuser ON duser.hmn_user_id = hmnuser.id
-			LEFT JOIN handmade_asset AS hmnuser_avatar ON hmnuser_avatar.id = hmnuser.avatar_asset_id
-			LEFT JOIN handmade_snippet AS snippet ON snippet.discord_message_id = msg.id
-		WHERE msg.id = $1 AND msg.channel_id = $2
-		`,
-		msgDelete.ID, msgDelete.ChannelID,
-	)
-	if errors.Is(err, db.NotFound) {
-		return
-	} else if err != nil {
-		log.Error().Err(err).Msg("failed to check for message to delete")
-		return
-	}
-	result := iresult.(*deleteMessageQuery)
-
-	log.Debug().Msg("deleting Discord message")
-	_, err = tx.Exec(ctx,
-		`
-		DELETE FROM handmade_discordmessage
-		WHERE id = $1 AND channel_id = $2
-		`,
-		msgDelete.ID,
-		msgDelete.ChannelID,
-	)
-
-	shouldDeleteSnippet := result.HMNUser != nil && result.HMNUser.DiscordDeleteSnippetOnMessageDelete
-	if result.SnippetID != nil && shouldDeleteSnippet {
-		log.Debug().
-			Int("snippet_id", *result.SnippetID).
-			Int("user_id", result.HMNUser.ID).
-			Msg("deleting snippet from Discord message")
-		_, err = tx.Exec(ctx,
-			`
-			DELETE FROM handmade_snippet
-			WHERE id = $1
-			`,
-			result.SnippetID,
-		)
 		if err != nil {
-			log.Error().Err(err).Msg("failed to delete snippet")
+			log.Error().Err(err).Msg("failed to delete interned message")
 			return
 		}
 	}
-
-	err = tx.Commit(ctx)
-	if err != nil {
-		log.Error().Err(err).Msg("failed to delete Discord message")
-		return
-	}
 }
 
 type MessageToSend struct {
@@ -698,7 +626,7 @@ type MessageToSend struct {
 
 func SendMessages(
 	ctx context.Context,
-	conn *pgxpool.Pool,
+	conn db.ConnOrTx,
 	msgs ...MessageToSend,
 ) error {
 	tx, err := conn.Begin(ctx)

src/discord/history.go

@@ -9,7 +9,6 @@ import (
 	"git.handmade.network/hmn/hmn/src/db"
 	"git.handmade.network/hmn/hmn/src/logging"
 	"git.handmade.network/hmn/hmn/src/models"
-	"github.com/jackc/pgx/v4"
 	"github.com/jackc/pgx/v4/pgxpool"
 )
 
@@ -34,7 +33,10 @@ func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{
 		backfillInterval := 1 * time.Hour
 
 		newUserTicker := time.NewTicker(5 * time.Second)
-		backfillTicker := time.NewTicker(backfillInterval)
+
+		// NOTE(asaf): 5 seconds to ensure this runs on start, we then reset it to the correct
+		//		       interval after the first run.
+		backfillTicker := time.NewTicker(5 * time.Second)
 
 		lastBackfillTime := time.Now().Add(-backfillInterval)
 		for {
@@ -45,6 +47,10 @@ func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{
 				// Get content for messages when a user links their account (but do not create snippets)
 				fetchMissingContent(ctx, dbConn)
 			case <-backfillTicker.C:
+				backfillTicker.Reset(backfillInterval)
+				// TODO(asaf): Do we need to update lastBackfillTime here? Otherwise we'll be rescraping
+				//			   from (start up time - 1 hour) every time.
+
 				// Run a backfill to patch up places where the Discord bot missed (does create snippets)
 				Scrape(ctx, dbConn,
 					config.Config.Discord.ShowcaseChannelID,
@@ -99,13 +105,16 @@ func fetchMissingContent(ctx context.Context, dbConn *pgxpool.Pool) {
 			discordMsg, err := GetChannelMessage(ctx, msg.ChannelID, msg.ID)
 			if errors.Is(err, NotFound) {
 				// This message has apparently been deleted; delete it from our database
-				_, err = dbConn.Exec(ctx,
+				interned, err := FetchInternedMessage(ctx, dbConn, msg.ID)
-					`
+				if err != nil {
-					DELETE FROM handmade_discordmessage
+					log.Error().Err(err).Msg("failed to fetch interned message")
-					WHERE id = $1
+					continue
-					`,
+				}
-					msg.ID,
+				if interned == nil {
-				)
+					log.Error().Msg("couldn't find interned message")
+					continue
+				}
+				err = DeleteInternedMessage(ctx, dbConn, interned)
 				if err != nil {
 					log.Error().Err(err).Msg("failed to delete missing message")
 					continue
@@ -119,7 +128,7 @@ func fetchMissingContent(ctx context.Context, dbConn *pgxpool.Pool) {
 
 			log.Info().Str("msg", discordMsg.ShortString()).Msg("fetched message for content")
 
-			err = handleHistoryMessage(ctx, dbConn, discordMsg, false)
+			err = HandleInternedMessage(ctx, dbConn, discordMsg, false, false)
 			if err != nil {
 				log.Error().Err(err).Msg("failed to save content for message")
 				continue
@@ -166,12 +175,10 @@ func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earlies
 				return
 			}
 
-			err := handleHistoryMessage(ctx, dbConn, &msg, createSnippets)
+			err := HandleIncomingMessage(ctx, dbConn, &msg, createSnippets)
+
 			if err != nil {
 				errLog := logging.ExtractLogger(ctx).Error()
-				if errors.Is(err, errNotEnoughInfo) {
-					errLog = logging.ExtractLogger(ctx).Warn()
-				}
 				errLog.Err(err).Msg("failed to process Discord message")
 			}
 
@@ -179,37 +186,3 @@ func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earlies
 		}
 	}
 }
-
-func handleHistoryMessage(ctx context.Context, dbConn *pgxpool.Pool, msg *Message, createSnippets bool) error {
-	var tx pgx.Tx
-	for {
-		var err error
-		tx, err = dbConn.Begin(ctx)
-		if err != nil {
-			logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to start transaction for message")
-			time.Sleep(1 * time.Second)
-			continue
-		}
-		break
-	}
-
-	newMsg, err := SaveMessageAndContents(ctx, tx, msg)
-	if err != nil {
-		return err
-	}
-	if createSnippets {
-		if doSnippet, err := AllowedToCreateMessageSnippets(ctx, tx, newMsg.UserID); doSnippet && err == nil {
-			err := CreateMessageSnippets(ctx, tx, msg.ID)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	err = tx.Commit(ctx)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

src/discord/library.go

@@ -1,45 +0,0 @@
-package discord
-
-import (
-	"context"
-
-	"git.handmade.network/hmn/hmn/src/oops"
-)
-
-func (bot *botInstance) processLibraryMsg(ctx context.Context, msg *Message) error {
-	switch msg.Type {
-	case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
-	default:
-		return nil
-	}
-
-	if !msg.OriginalHasFields("content") {
-		return nil
-	}
-
-	if !messageHasLinks(msg.Content) {
-		err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
-		if err != nil {
-			return oops.New(err, "failed to delete message")
-		}
-
-		if !msg.Author.IsBot {
-			channel, err := CreateDM(ctx, msg.Author.ID)
-			if err != nil {
-				return oops.New(err, "failed to create DM channel")
-			}
-
-			err = SendMessages(ctx, bot.dbConn, MessageToSend{
-				ChannelID: channel.ID,
-				Req: CreateMessageRequest{
-					Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.",
-				},
-			})
-			if err != nil {
-				return oops.New(err, "failed to send showcase warning message")
-			}
-		}
-	}
-
-	return nil
-}

src/discord/message_handling.go

@@ -0,0 +1,957 @@
+package discord
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"git.handmade.network/hmn/hmn/src/assets"
+	"git.handmade.network/hmn/hmn/src/config"
+	"git.handmade.network/hmn/hmn/src/db"
+	"git.handmade.network/hmn/hmn/src/hmndata"
+	"git.handmade.network/hmn/hmn/src/logging"
+	"git.handmade.network/hmn/hmn/src/models"
+	"git.handmade.network/hmn/hmn/src/oops"
+	"git.handmade.network/hmn/hmn/src/parsing"
+	"github.com/google/uuid"
+)
+
+func HandleIncomingMessage(ctx context.Context, dbConn db.ConnOrTx, msg *Message, createSnippets bool) error {
+	deleted := false
+	var err error
+
+	// NOTE(asaf): All functions called here should verify that the message applies to them.
+
+	if !deleted && err == nil {
+		deleted, err = CleanUpLibrary(ctx, dbConn, msg)
+	}
+
+	if !deleted && err == nil {
+		deleted, err = CleanUpShowcase(ctx, dbConn, msg)
+	}
+
+	if !deleted && err == nil {
+		err = MaybeInternMessage(ctx, dbConn, msg)
+	}
+
+	if err == nil {
+		err = HandleInternedMessage(ctx, dbConn, msg, deleted, createSnippets)
+	}
+
+	return err
+}
+
+func CleanUpShowcase(ctx context.Context, dbConn db.ConnOrTx, msg *Message) (bool, error) {
+	deleted := false
+	if msg.ChannelID == config.Config.Discord.ShowcaseChannelID {
+		switch msg.Type {
+		case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
+		default:
+			return deleted, nil
+		}
+
+		hasGoodContent := true
+		if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) {
+			hasGoodContent = false
+		}
+
+		hasGoodAttachments := true
+		if msg.OriginalHasFields("attachments") && len(msg.Attachments) == 0 {
+			hasGoodAttachments = false
+		}
+
+		if !hasGoodContent && !hasGoodAttachments {
+			err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
+			if err != nil {
+				return deleted, oops.New(err, "failed to delete message")
+			}
+			deleted = true
+
+			if !msg.Author.IsBot {
+				channel, err := CreateDM(ctx, msg.Author.ID)
+				if err != nil {
+					return deleted, oops.New(err, "failed to create DM channel")
+				}
+
+				err = SendMessages(ctx, dbConn, MessageToSend{
+					ChannelID: channel.ID,
+					Req: CreateMessageRequest{
+						Content: "Posts in #project-showcase are required to have either an image/video or a link. Discuss showcase content in #projects.",
+					},
+				})
+				if err != nil {
+					return deleted, oops.New(err, "failed to send showcase warning message")
+				}
+			}
+		}
+	}
+
+	return deleted, nil
+}
+
+func CleanUpLibrary(ctx context.Context, dbConn db.ConnOrTx, msg *Message) (bool, error) {
+	deleted := false
+	if msg.ChannelID == config.Config.Discord.LibraryChannelID {
+		switch msg.Type {
+		case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
+		default:
+			return deleted, nil
+		}
+
+		if !msg.OriginalHasFields("content") {
+			return deleted, nil
+		}
+
+		if !messageHasLinks(msg.Content) {
+			err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
+			if err != nil {
+				return deleted, oops.New(err, "failed to delete message")
+			}
+			deleted = true
+
+			if !msg.Author.IsBot {
+				channel, err := CreateDM(ctx, msg.Author.ID)
+				if err != nil {
+					return deleted, oops.New(err, "failed to create DM channel")
+				}
+
+				err = SendMessages(ctx, dbConn, MessageToSend{
+					ChannelID: channel.ID,
+					Req: CreateMessageRequest{
+						Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.",
+					},
+				})
+				if err != nil {
+					return deleted, oops.New(err, "failed to send showcase warning message")
+				}
+			}
+		}
+	}
+
+	return deleted, nil
+}
+
+func MaybeInternMessage(ctx context.Context, dbConn db.ConnOrTx, msg *Message) error {
+	if msg.ChannelID == config.Config.Discord.ShowcaseChannelID {
+		err := InternMessage(ctx, dbConn, msg)
+		if errors.Is(err, errNotEnoughInfo) {
+			logging.ExtractLogger(ctx).Warn().
+				Interface("msg", msg).
+				Msg("didn't have enough info to intern Discord message")
+		} else if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+/*
+Ensures that a Discord message is stored in the database. This function is
+idempotent and can be called regardless of whether the item already exists in
+the database.
+
+This does not create snippets or save content or do anything besides save the message itself.
+*/
+var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this")
+
+func InternMessage(
+	ctx context.Context,
+	dbConn db.ConnOrTx,
+	msg *Message,
+) error {
+	_, err := db.QueryOne(ctx, dbConn, models.DiscordMessage{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessage
+		WHERE id = $1
+		`,
+		msg.ID,
+	)
+	if errors.Is(err, db.NotFound) {
+		if !msg.OriginalHasFields("author", "timestamp") {
+			return errNotEnoughInfo
+		}
+
+		guildID := msg.GuildID
+		if guildID == nil {
+			/*
+				This is weird, but it can happen when we fetch messages from
+				history instead of receiving it from the gateway. In this case
+				we just assume it's from the HMN server.
+			*/
+			guildID = &config.Config.Discord.GuildID
+		}
+
+		_, err = dbConn.Exec(ctx,
+			`
+			INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created)
+			VALUES ($1, $2, $3, $4, $5, $6, $7)
+			`,
+			msg.ID,
+			msg.ChannelID,
+			*guildID,
+			msg.JumpURL(),
+			msg.Author.ID,
+			msg.Time(),
+			false,
+		)
+		if err != nil {
+			return oops.New(err, "failed to save new discord message")
+		}
+	} else if err != nil {
+		return oops.New(err, "failed to check for existing Discord message")
+	}
+
+	return nil
+}
+
+type InternedMessage struct {
+	Message        models.DiscordMessage         `db:"message"`
+	MessageContent *models.DiscordMessageContent `db:"content"`
+	HMNUser        *models.User                  `db:"hmnuser"`
+	DiscordUser    *models.DiscordUser           `db:"duser"`
+}
+
+func FetchInternedMessage(ctx context.Context, dbConn db.ConnOrTx, msgId string) (*InternedMessage, error) {
+	result, err := db.QueryOne(ctx, dbConn, InternedMessage{},
+		`
+		SELECT $columns
+		FROM
+			handmade_discordmessage AS message
+			LEFT JOIN handmade_discordmessagecontent AS content ON content.message_id = message.id
+			LEFT JOIN handmade_discorduser AS duser ON duser.userid = message.user_id
+			LEFT JOIN auth_user AS hmnuser ON hmnuser.id = duser.hmn_user_id
+			LEFT JOIN handmade_asset AS hmnuser_avatar ON hmnuser_avatar.id = hmnuser.avatar_asset_id
+		WHERE message.id = $1
+		`,
+		msgId,
+	)
+	if err != nil {
+		if errors.Is(err, db.NotFound) {
+			return nil, nil
+		} else {
+			return nil, err
+		}
+	}
+
+	interned := result.(*InternedMessage)
+	return interned, nil
+}
+
+// Checks if a message is interned and handles it to the extent possible:
+// 1. Saves/updates content
+// 2. Saves/updates snippet
+// 3. Deletes content/snippet
+func HandleInternedMessage(ctx context.Context, dbConn db.ConnOrTx, msg *Message, deleted bool, createSnippet bool) error {
+	tx, err := dbConn.Begin(ctx)
+	if err != nil {
+		return oops.New(err, "failed to start transaction")
+	}
+	defer tx.Rollback(ctx)
+
+	interned, err := FetchInternedMessage(ctx, tx, msg.ID)
+	if err != nil {
+		return err
+	}
+
+	if interned != nil {
+		if !deleted {
+			err = SaveMessageContents(ctx, tx, interned, msg)
+			if err != nil {
+				return err
+
+			}
+			if createSnippet {
+				err = HandleSnippetForInternedMessage(ctx, tx, interned, false)
+				if err != nil {
+					return err
+				}
+			}
+		} else {
+			err = DeleteInternedMessage(ctx, tx, interned)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	err = tx.Commit(ctx)
+	if err != nil {
+		return oops.New(err, "failed to commit Discord message updates")
+	}
+
+	return nil
+}
+
+func DeleteInternedMessage(ctx context.Context, dbConn db.ConnOrTx, interned *InternedMessage) error {
+	isnippet, err := db.QueryOne(ctx, dbConn, models.Snippet{},
+		`
+		SELECT $columns
+		FROM handmade_snippet
+		WHERE discord_message_id = $1
+		`,
+		interned.Message.ID,
+	)
+	if err != nil && !errors.Is(err, db.NotFound) {
+		return oops.New(err, "failed to fetch snippet for discord message")
+	}
+	var snippet *models.Snippet
+	if !errors.Is(err, db.NotFound) {
+		snippet = isnippet.(*models.Snippet)
+	}
+
+	// NOTE(asaf): Also deletes the following through a db cascade:
+	//			   * handmade_discordmessageattachment
+	//			   * handmade_discordmessagecontent
+	//			   * handmade_discordmessageembed
+	//             DOES NOT DELETE ASSETS FOR CONTENT/EMBEDS
+	_, err = dbConn.Exec(ctx,
+		`
+		DELETE FROM handmade_discordmessage
+		WHERE id = $1
+		`,
+		interned.Message.ID,
+	)
+
+	if snippet != nil {
+		userApprovesDeletion := interned.HMNUser != nil && snippet.OwnerID == interned.HMNUser.ID && interned.HMNUser.DiscordDeleteSnippetOnMessageDelete
+		if !snippet.EditedOnWebsite && userApprovesDeletion {
+			// NOTE(asaf): Does not delete asset!
+			_, err = dbConn.Exec(ctx,
+				`
+				DELETE FROM handmade_snippet
+				WHERE id = $1
+				`,
+				snippet.ID,
+			)
+			if err != nil {
+				return oops.New(err, "failed to delete snippet")
+			}
+		}
+	}
+
+	return nil
+}
+
+/*
+Processes a single Discord message, saving as much of the message's content
+and attachments as allowed by our rules and user settings. Does NOT create
+snippets.
+
+Idempotent; can be called any time whether the contents exist or not.
+
+NOTE!!: Replaces interned.MessageContent if it was created or updated!!
+*/
+func SaveMessageContents(
+	ctx context.Context,
+	dbConn db.ConnOrTx,
+	interned *InternedMessage,
+	msg *Message,
+) error {
+	if interned.DiscordUser != nil {
+		// We have a linked Discord account, so save the message contents (regardless of
+		// whether we create a snippet or not).
+		if msg.OriginalHasFields("content") {
+			_, err := dbConn.Exec(ctx,
+				`
+			INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content)
+			VALUES ($1, $2, $3)
+			ON CONFLICT (message_id) DO UPDATE SET
+				discord_id = EXCLUDED.discord_id,
+				last_content = EXCLUDED.last_content
+			`,
+				interned.Message.ID,
+				interned.DiscordUser.ID,
+				CleanUpMarkdown(ctx, msg.Content),
+			)
+			if err != nil {
+				return oops.New(err, "failed to create or update message contents")
+			}
+
+			icontent, err := db.QueryOne(ctx, dbConn, models.DiscordMessageContent{},
+				`
+				SELECT $columns
+				FROM
+					handmade_discordmessagecontent
+				WHERE
+					handmade_discordmessagecontent.message_id = $1
+				`,
+				interned.Message.ID,
+			)
+			if err != nil {
+				return oops.New(err, "failed to fetch message contents")
+			}
+			interned.MessageContent = icontent.(*models.DiscordMessageContent)
+		} // TODO(asaf): What happens if we edit the message and delete the content but keep the attachment??
+
+		// Save attachments
+		if msg.OriginalHasFields("attachments") {
+			for _, attachment := range msg.Attachments {
+				_, err := saveAttachment(ctx, dbConn, &attachment, interned.DiscordUser.HMNUserId, msg.ID)
+				if err != nil {
+					return oops.New(err, "failed to save attachment")
+				}
+			}
+		}
+
+		// Save / delete embeds
+		if msg.OriginalHasFields("embeds") {
+			numSavedEmbeds, err := db.QueryInt(ctx, dbConn,
+				`
+			SELECT COUNT(*)
+			FROM handmade_discordmessageembed
+			WHERE message_id = $1
+			`,
+				msg.ID,
+			)
+			if err != nil {
+				return oops.New(err, "failed to count existing embeds")
+			}
+			if numSavedEmbeds == 0 {
+				// No embeds yet, so save new ones
+				for _, embed := range msg.Embeds {
+					_, err := saveEmbed(ctx, dbConn, &embed, interned.DiscordUser.HMNUserId, msg.ID)
+					if err != nil {
+						return oops.New(err, "failed to save embed")
+					}
+				}
+			} else if len(msg.Embeds) > 0 {
+				// Embeds were removed from the message
+				_, err := dbConn.Exec(ctx,
+					`
+				DELETE FROM handmade_discordmessageembed
+				WHERE message_id = $1
+				`,
+					msg.ID,
+				)
+				if err != nil {
+					return oops.New(err, "failed to delete embeds")
+				}
+			}
+		}
+	}
+	return nil
+}
+
+var discordDownloadClient = &http.Client{
+	Timeout: 10 * time.Second,
+}
+
+type DiscordResourceBadStatusCode error
+
+func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, "", oops.New(err, "failed to make Discord download request")
+	}
+	res, err := discordDownloadClient.Do(req)
+	if err != nil {
+		return nil, "", oops.New(err, "failed to fetch Discord resource data")
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode < 200 || 299 < res.StatusCode {
+		return nil, "", DiscordResourceBadStatusCode(fmt.Errorf("status code %d from Discord resource: %s", res.StatusCode, url))
+	}
+
+	content, err := io.ReadAll(res.Body)
+	if err != nil {
+		panic(err)
+	}
+
+	return content, res.Header.Get("Content-Type"), nil
+}
+
+/*
+Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment
+that already exists
+*/
+func saveAttachment(
+	ctx context.Context,
+	tx db.ConnOrTx,
+	attachment *Attachment,
+	hmnUserID int,
+	discordMessageID string,
+) (*models.DiscordMessageAttachment, error) {
+	iexisting, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessageattachment
+		WHERE id = $1
+		`,
+		attachment.ID,
+	)
+	if err == nil {
+		return iexisting.(*models.DiscordMessageAttachment), nil
+	} else if errors.Is(err, db.NotFound) {
+		// this is fine, just create it
+	} else {
+		return nil, oops.New(err, "failed to check for existing attachment")
+	}
+
+	width := 0
+	height := 0
+	if attachment.Width != nil {
+		width = *attachment.Width
+	}
+	if attachment.Height != nil {
+		height = *attachment.Height
+	}
+
+	content, _, err := downloadDiscordResource(ctx, attachment.Url)
+	if err != nil {
+		return nil, oops.New(err, "failed to download Discord attachment")
+	}
+
+	contentType := "application/octet-stream"
+	if attachment.ContentType != nil {
+		contentType = *attachment.ContentType
+	}
+
+	asset, err := assets.Create(ctx, tx, assets.CreateInput{
+		Content:     content,
+		Filename:    attachment.Filename,
+		ContentType: contentType,
+
+		UploaderID: &hmnUserID,
+		Width:      width,
+		Height:     height,
+	})
+	if err != nil {
+		return nil, oops.New(err, "failed to save asset for Discord attachment")
+	}
+
+	// TODO(db): RETURNING plz thanks
+	_, err = tx.Exec(ctx,
+		`
+		INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id)
+		VALUES ($1, $2, $3)
+		`,
+		attachment.ID,
+		asset.ID,
+		discordMessageID,
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to save Discord attachment data")
+	}
+
+	iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessageattachment
+		WHERE id = $1
+		`,
+		attachment.ID,
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to fetch new Discord attachment data")
+	}
+
+	return iDiscordAttachment.(*models.DiscordMessageAttachment), nil
+}
+
+// Saves an embed from Discord. NOTE: This is _not_ idempotent, so only call it
+// if you do not have any embeds saved for this message yet.
+func saveEmbed(
+	ctx context.Context,
+	tx db.ConnOrTx,
+	embed *Embed,
+	hmnUserID int,
+	discordMessageID string,
+) (*models.DiscordMessageEmbed, error) {
+	isOkImageType := func(contentType string) bool {
+		return strings.HasPrefix(contentType, "image/")
+	}
+
+	isOkVideoType := func(contentType string) bool {
+		return strings.HasPrefix(contentType, "video/")
+	}
+
+	maybeSaveImageish := func(i EmbedImageish, contentTypeCheck func(string) bool) (*uuid.UUID, error) {
+		content, contentType, err := downloadDiscordResource(ctx, *i.Url)
+		if err != nil {
+			var statusError DiscordResourceBadStatusCode
+			if errors.As(err, &statusError) {
+				return nil, nil
+			} else {
+				return nil, oops.New(err, "failed to save Discord embed")
+			}
+		}
+		if contentTypeCheck(contentType) {
+			in := assets.CreateInput{
+				Content:     content,
+				Filename:    "embed",
+				ContentType: contentType,
+				UploaderID:  &hmnUserID,
+			}
+
+			if i.Width != nil {
+				in.Width = *i.Width
+			}
+			if i.Height != nil {
+				in.Height = *i.Height
+			}
+
+			asset, err := assets.Create(ctx, tx, in)
+			if err != nil {
+				return nil, oops.New(err, "failed to create asset from embed")
+			}
+			return &asset.ID, nil
+		}
+
+		return nil, nil
+	}
+
+	var imageAssetId *uuid.UUID
+	var videoAssetId *uuid.UUID
+	var err error
+
+	if embed.Video != nil && embed.Video.Url != nil {
+		videoAssetId, err = maybeSaveImageish(embed.Video.EmbedImageish, isOkVideoType)
+	} else if embed.Image != nil && embed.Image.Url != nil {
+		imageAssetId, err = maybeSaveImageish(embed.Image.EmbedImageish, isOkImageType)
+	} else if embed.Thumbnail != nil && embed.Thumbnail.Url != nil {
+		imageAssetId, err = maybeSaveImageish(embed.Thumbnail.EmbedImageish, isOkImageType)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	// Save the embed into the db
+	// TODO(db): Insert, RETURNING
+	var savedEmbedId int
+	err = tx.QueryRow(ctx,
+		`
+		INSERT INTO handmade_discordmessageembed (title, description, url, message_id, image_id, video_id)
+		VALUES ($1, $2, $3, $4, $5, $6)
+		RETURNING id
+		`,
+		embed.Title,
+		embed.Description,
+		embed.Url,
+		discordMessageID,
+		imageAssetId,
+		videoAssetId,
+	).Scan(&savedEmbedId)
+	if err != nil {
+		return nil, oops.New(err, "failed to insert new embed")
+	}
+
+	iDiscordEmbed, err := db.QueryOne(ctx, tx, models.DiscordMessageEmbed{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessageembed
+		WHERE id = $1
+		`,
+		savedEmbedId,
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to fetch new Discord embed data")
+	}
+
+	return iDiscordEmbed.(*models.DiscordMessageEmbed), nil
+}
+
+func FetchSnippetForMessage(ctx context.Context, dbConn db.ConnOrTx, msgID string) (*models.Snippet, error) {
+	iresult, err := db.QueryOne(ctx, dbConn, models.Snippet{},
+		`
+		SELECT $columns
+		FROM handmade_snippet
+		WHERE discord_message_id = $1
+		`,
+		msgID,
+	)
+
+	if err != nil {
+		if errors.Is(err, db.NotFound) {
+			return nil, nil
+		} else {
+			return nil, oops.New(err, "failed to fetch existing snippet for message %s", msgID)
+		}
+	}
+
+	return iresult.(*models.Snippet), nil
+}
+
+/*
+Potentially creates or updates a snippet for the given interned message.
+It uses the content saved in the database to do this. If we do not have any
+content saved, nothing will happen.
+
+If a user does not have their Discord account linked, this function will
+naturally do nothing because we have no message content saved.
+If forceCreate is true, it does not check any user settings such as automatically creating snippets from
+#project-showcase. If we have the content, it will make a snippet for it, no
+questions asked. Bear that in mind.
+*/
+func HandleSnippetForInternedMessage(ctx context.Context, dbConn db.ConnOrTx, interned *InternedMessage, forceCreate bool) error {
+	if interned.HMNUser == nil {
+		// NOTE(asaf): Can't handle snippets when there's no linked user
+		return nil
+	}
+
+	if interned.MessageContent == nil {
+		// NOTE(asaf): Can't have a snippet without content
+		// NOTE(asaf): Messages that only have an attachment also have blank content
+		// TODO(asaf): Do we need to delete existing snippets in this case??? Not entirely sure how to trigger this through discord
+		return nil
+	}
+
+	tx, err := dbConn.Begin(ctx)
+	if err != nil {
+		oops.New(err, "failed to start transaction")
+	}
+	defer tx.Rollback(ctx)
+
+	existingSnippet, err := FetchSnippetForMessage(ctx, tx, interned.Message.ID)
+	if err != nil {
+		return oops.New(err, "failed to check for existing snippet for message %s", interned.Message.ID)
+	}
+
+	if existingSnippet != nil {
+		// TODO(asaf): We're not handling the case where embeds were removed or modified.
+		//             Also not handling the case where a message had both an attachment and an embed
+		//             and the attachment was removed (leaving only the embed).
+		LinkedUserIsSnippetOwner := existingSnippet.OwnerID == interned.DiscordUser.HMNUserId
+		if LinkedUserIsSnippetOwner && !existingSnippet.EditedOnWebsite {
+			contentMarkdown := interned.MessageContent.LastContent
+			contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
+
+			_, err := tx.Exec(ctx,
+				`
+				UPDATE handmade_snippet
+				SET
+					description = $1,
+					_description_html = $2
+				WHERE id = $3
+				`,
+				contentMarkdown,
+				contentHTML,
+				existingSnippet.ID,
+			)
+			if err != nil {
+				return oops.New(err, "failed to update content of snippet on message edit")
+			}
+			existingSnippet.Description = contentMarkdown
+			existingSnippet.DescriptionHtml = contentHTML
+		}
+	} else {
+		userAllowsSnippet := interned.HMNUser.DiscordSaveShowcase || forceCreate
+		shouldCreate := !interned.Message.SnippetCreated && userAllowsSnippet
+
+		if shouldCreate {
+			// Get an asset ID or URL to make a snippet from
+			assetId, url, err := getSnippetAssetOrUrl(ctx, tx, &interned.Message)
+			if assetId != nil || url != nil {
+				contentMarkdown := interned.MessageContent.LastContent
+				contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
+
+				_, err = tx.Exec(ctx,
+					`
+					INSERT INTO handmade_snippet (url, "when", description, _description_html, asset_id, discord_message_id, owner_id)
+					VALUES ($1, $2, $3, $4, $5, $6, $7)
+					`,
+					url,
+					interned.Message.SentAt,
+					contentMarkdown,
+					contentHTML,
+					assetId,
+					interned.Message.ID,
+					interned.HMNUser.ID,
+				)
+				if err != nil {
+					return oops.New(err, "failed to create snippet from attachment")
+				}
+
+				existingSnippet, err = FetchSnippetForMessage(ctx, tx, interned.Message.ID)
+				if err != nil {
+					return oops.New(err, "failed to fetch newly-created snippet")
+				}
+
+				_, err = tx.Exec(ctx,
+					`
+					UPDATE handmade_discordmessage
+					SET snippet_created = TRUE
+					WHERE id = $1
+					`,
+					interned.Message.ID,
+				)
+				if err != nil {
+					return oops.New(err, "failed to mark message as having snippet")
+				}
+			}
+		}
+	}
+
+	if existingSnippet != nil {
+		// Update tags
+
+		// Try to associate tags in the message with project tags in HMN.
+		// Match only tags for projects in which the current user is a collaborator.
+		messageTags := getDiscordTags(existingSnippet.Description)
+
+		var desiredTags []int
+		var allTags []int
+
+		if len(messageTags) > 0 {
+			// Fetch projects so we know what tags the user can apply to their snippet.
+			projects, err := hmndata.FetchProjects(ctx, tx, interned.HMNUser, hmndata.ProjectsQuery{
+				OwnerIDs: []int{interned.HMNUser.ID},
+			})
+			if err != nil {
+				return oops.New(err, "failed to look up user projects")
+			}
+
+			projectIDs := make([]int, len(projects))
+			for i, p := range projects {
+				projectIDs[i] = p.Project.ID
+			}
+
+			type tagsRow struct {
+				Tag models.Tag `db:"tags"`
+			}
+			iUserTags, err := db.Query(ctx, tx, tagsRow{},
+				`
+				SELECT $columns
+				FROM
+					tags
+					JOIN handmade_project AS project ON project.tag = tags.id
+				WHERE
+					project.id = ANY ($1)
+				`,
+				projectIDs,
+			)
+			if err != nil {
+				return oops.New(err, "failed to fetch tags for user projects")
+			}
+
+			for _, itag := range iUserTags {
+				tag := itag.(*tagsRow).Tag
+				for _, messageTag := range messageTags {
+					allTags = append(allTags, tag.ID)
+					if strings.EqualFold(tag.Text, messageTag) {
+						desiredTags = append(desiredTags, tag.ID)
+					}
+				}
+			}
+		}
+
+		_, err = tx.Exec(ctx,
+			`
+			DELETE FROM snippet_tags
+			WHERE
+				snippet_id = $1
+				AND tag_id = ANY ($2)
+			`,
+			existingSnippet.ID,
+			allTags,
+		)
+		if err != nil {
+			return oops.New(err, "failed to clear tags from snippet")
+		}
+
+		for _, tagID := range desiredTags {
+			_, err = tx.Exec(ctx,
+				`
+				INSERT INTO snippet_tags (snippet_id, tag_id)
+				VALUES ($1, $2)
+				ON CONFLICT DO NOTHING
+				`,
+				existingSnippet.ID,
+				tagID,
+			)
+			if err != nil {
+				return oops.New(err, "failed to associate snippet with tag")
+			}
+		}
+	}
+
+	err = tx.Commit(ctx)
+	if err != nil {
+		return oops.New(err, "failed to commit transaction")
+	}
+
+	return nil
+}
+
+// TODO(asaf): I believe this will also match https://example.com?hello=1&whatever=5
+//             Probably need to add word boundaries.
+var REDiscordTag = regexp.MustCompile(`&([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*)`)
+
+func getDiscordTags(content string) []string {
+	matches := REDiscordTag.FindAllStringSubmatch(content, -1)
+	result := make([]string, len(matches))
+	for i, m := range matches {
+		result[i] = m[1]
+	}
+	return result
+}
+
+// NOTE(ben): This is maybe redundant with the regexes we use for markdown. But
+// do we actually want to reuse those, or should we keep them separate?
+var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`)
+
+func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msg *models.DiscordMessage) (*uuid.UUID, *string, error) {
+	// Check attachments
+	attachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessageattachment
+		WHERE message_id = $1
+		`,
+		msg.ID,
+	)
+	if err != nil {
+		return nil, nil, oops.New(err, "failed to fetch message attachments")
+	}
+	for _, iattachment := range attachments {
+		attachment := iattachment.(*models.DiscordMessageAttachment)
+		return &attachment.AssetID, nil, nil
+	}
+
+	// Check embeds
+	embeds, err := db.Query(ctx, tx, models.DiscordMessageEmbed{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessageembed
+		WHERE message_id = $1
+		`,
+		msg.ID,
+	)
+	if err != nil {
+		return nil, nil, oops.New(err, "failed to fetch discord embeds")
+	}
+	for _, iembed := range embeds {
+		embed := iembed.(*models.DiscordMessageEmbed)
+		if embed.VideoID != nil {
+			return embed.VideoID, nil, nil
+		} else if embed.ImageID != nil {
+			return embed.ImageID, nil, nil
+		} else if embed.URL != nil {
+			if RESnippetableUrl.MatchString(*embed.URL) {
+				return nil, embed.URL, nil
+			}
+		}
+	}
+
+	return nil, nil, nil
+}
+
+var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`)
+
+func messageHasLinks(content string) bool {
+	links := reDiscordMessageLink.FindAllString(content, -1)
+	for _, link := range links {
+		_, err := url.Parse(strings.TrimSpace(link))
+		if err == nil {
+			return true
+		}
+	}
+
+	return false
+}

src/discord/showcase.go

@@ -1,873 +0,0 @@
-package discord
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"regexp"
-	"strings"
-	"time"
-
-	"git.handmade.network/hmn/hmn/src/assets"
-	"git.handmade.network/hmn/hmn/src/config"
-	"git.handmade.network/hmn/hmn/src/db"
-	"git.handmade.network/hmn/hmn/src/hmndata"
-	"git.handmade.network/hmn/hmn/src/logging"
-	"git.handmade.network/hmn/hmn/src/models"
-	"git.handmade.network/hmn/hmn/src/oops"
-	"git.handmade.network/hmn/hmn/src/parsing"
-	"github.com/google/uuid"
-)
-
-var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`)
-
-var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this")
-
-func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) error {
-	switch msg.Type {
-	case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
-	default:
-		return nil
-	}
-
-	didDelete, err := bot.maybeDeleteShowcaseMsg(ctx, msg)
-	if err != nil {
-		return err
-	}
-	if didDelete {
-		return nil
-	}
-
-	tx, err := bot.dbConn.Begin(ctx)
-	if err != nil {
-		panic(err)
-	}
-	defer tx.Rollback(ctx)
-
-	// save the message, maybe save its contents
-	newMsg, err := SaveMessageAndContents(ctx, tx, msg)
-	if errors.Is(err, errNotEnoughInfo) {
-		logging.ExtractLogger(ctx).Warn().
-			Interface("msg", msg).
-			Msg("didn't have enough info to process Discord message")
-		return nil
-	} else if err != nil {
-		return err
-	}
-
-	// ...and maybe make a snippet too, if the user wants us to
-	duser, err := FetchDiscordUser(ctx, tx, newMsg.UserID)
-	if err == nil && duser.HMNUser.DiscordSaveShowcase {
-		err = CreateMessageSnippets(ctx, tx, newMsg.UserID, msg.ID)
-		if err != nil {
-			return oops.New(err, "failed to create snippet in gateway")
-		}
-	} else {
-		if err == db.NotFound {
-			// this is fine, just don't create a snippet
-		} else {
-			return err
-		}
-	}
-
-	err = tx.Commit(ctx)
-	if err != nil {
-		return oops.New(err, "failed to commit Discord message updates")
-	}
-
-	return nil
-}
-
-func (bot *botInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message) (didDelete bool, err error) {
-	hasGoodContent := true
-	if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) {
-		hasGoodContent = false
-	}
-
-	hasGoodAttachments := true
-	if msg.OriginalHasFields("attachments") && len(msg.Attachments) == 0 {
-		hasGoodAttachments = false
-	}
-
-	didDelete = false
-	if !hasGoodContent && !hasGoodAttachments {
-		didDelete = true
-		err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
-		if err != nil {
-			return false, oops.New(err, "failed to delete message")
-		}
-
-		if !msg.Author.IsBot {
-			channel, err := CreateDM(ctx, msg.Author.ID)
-			if err != nil {
-				return false, oops.New(err, "failed to create DM channel")
-			}
-
-			err = SendMessages(ctx, bot.dbConn, MessageToSend{
-				ChannelID: channel.ID,
-				Req: CreateMessageRequest{
-					Content: "Posts in #project-showcase are required to have either an image/video or a link. Discuss showcase content in #projects.",
-				},
-			})
-			if err != nil {
-				return false, oops.New(err, "failed to send showcase warning message")
-			}
-		}
-	}
-
-	return didDelete, nil
-}
-
-/*
-Ensures that a Discord message is stored in the database. This function is
-idempotent and can be called regardless of whether the item already exists in
-the database.
-
-This does not create snippets or do anything besides save the message itself.
-*/
-func SaveMessage(
-	ctx context.Context,
-	tx db.ConnOrTx,
-	msg *Message,
-) (*models.DiscordMessage, error) {
-	iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessage
-		WHERE id = $1
-		`,
-		msg.ID,
-	)
-	if errors.Is(err, db.NotFound) {
-		if !msg.OriginalHasFields("author", "timestamp") {
-			return nil, errNotEnoughInfo
-		}
-
-		guildID := msg.GuildID
-		if guildID == nil {
-			/*
-				This is weird, but it can happen when we fetch messages from
-				history instead of receiving it from the gateway. In this case
-				we just assume it's from the HMN server.
-			*/
-			guildID = &config.Config.Discord.GuildID
-		}
-
-		_, err = tx.Exec(ctx,
-			`
-			INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created)
-			VALUES ($1, $2, $3, $4, $5, $6, $7)
-			`,
-			msg.ID,
-			msg.ChannelID,
-			*guildID,
-			msg.JumpURL(),
-			msg.Author.ID,
-			msg.Time(),
-			false,
-		)
-		if err != nil {
-			return nil, oops.New(err, "failed to save new discord message")
-		}
-
-		/*
-			TODO(db): This is a spot where it would be really nice to be able
-			to use RETURNING, and avoid this second query.
-		*/
-		iDiscordMessage, err = db.QueryOne(ctx, tx, models.DiscordMessage{},
-			`
-			SELECT $columns
-			FROM handmade_discordmessage
-			WHERE id = $1
-			`,
-			msg.ID,
-		)
-		if err != nil {
-			panic(err)
-		}
-	} else if err != nil {
-		return nil, oops.New(err, "failed to check for existing Discord message")
-	}
-
-	return iDiscordMessage.(*models.DiscordMessage), nil
-}
-
-/*
-Processes a single Discord message, saving as much of the message's content
-and attachments as allowed by our rules and user settings. Does NOT create
-snippets.
-
-Idempotent; can be called any time whether the message exists or not.
-*/
-func SaveMessageAndContents(
-	ctx context.Context,
-	tx db.ConnOrTx,
-	msg *Message,
-) (*models.DiscordMessage, error) {
-	newMsg, err := SaveMessage(ctx, tx, msg)
-	if err != nil {
-		return nil, err
-	}
-
-	// Check for linked Discord user
-	iDiscordUser, err := db.QueryOne(ctx, tx, models.DiscordUser{},
-		`
-		SELECT $columns
-		FROM handmade_discorduser
-		WHERE userid = $1
-		`,
-		newMsg.UserID,
-	)
-	if errors.Is(err, db.NotFound) {
-		return newMsg, nil
-	} else if err != nil {
-		return nil, oops.New(err, "failed to look up linked Discord user")
-	}
-	discordUser := iDiscordUser.(*models.DiscordUser)
-
-	// We have a linked Discord account, so save the message contents (regardless of
-	// whether we create a snippet or not).
-
-	if msg.OriginalHasFields("content") {
-		_, err = tx.Exec(ctx,
-			`
-			INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content)
-			VALUES ($1, $2, $3)
-			ON CONFLICT (message_id) DO UPDATE SET
-				discord_id = EXCLUDED.discord_id,
-				last_content = EXCLUDED.last_content
-			`,
-			newMsg.ID,
-			discordUser.ID,
-			CleanUpMarkdown(ctx, msg.Content),
-		)
-	}
-
-	// Save attachments
-	if msg.OriginalHasFields("attachments") {
-		for _, attachment := range msg.Attachments {
-			_, err := saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID)
-			if err != nil {
-				return nil, oops.New(err, "failed to save attachment")
-			}
-		}
-	}
-
-	// Save / delete embeds
-	if msg.OriginalHasFields("embeds") {
-		numSavedEmbeds, err := db.QueryInt(ctx, tx,
-			`
-			SELECT COUNT(*)
-			FROM handmade_discordmessageembed
-			WHERE message_id = $1
-			`,
-			msg.ID,
-		)
-		if err != nil {
-			return nil, oops.New(err, "failed to count existing embeds")
-		}
-		if numSavedEmbeds == 0 {
-			// No embeds yet, so save new ones
-			for _, embed := range msg.Embeds {
-				_, err := saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID)
-				if err != nil {
-					return nil, oops.New(err, "failed to save embed")
-				}
-			}
-		} else if len(msg.Embeds) > 0 {
-			// Embeds were removed from the message
-			_, err := tx.Exec(ctx,
-				`
-				DELETE FROM handmade_discordmessageembed
-				WHERE message_id = $1
-				`,
-				msg.ID,
-			)
-			if err != nil {
-				return nil, oops.New(err, "failed to delete embeds")
-			}
-		}
-	}
-
-	return newMsg, nil
-}
-
-var discordDownloadClient = &http.Client{
-	Timeout: 10 * time.Second,
-}
-
-type DiscordResourceBadStatusCode error
-
-func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
-	if err != nil {
-		return nil, "", oops.New(err, "failed to make Discord download request")
-	}
-	res, err := discordDownloadClient.Do(req)
-	if err != nil {
-		return nil, "", oops.New(err, "failed to fetch Discord resource data")
-	}
-	defer res.Body.Close()
-
-	if res.StatusCode < 200 || 299 < res.StatusCode {
-		return nil, "", DiscordResourceBadStatusCode(fmt.Errorf("status code %d from Discord resource: %s", res.StatusCode, url))
-	}
-
-	content, err := io.ReadAll(res.Body)
-	if err != nil {
-		panic(err)
-	}
-
-	return content, res.Header.Get("Content-Type"), nil
-}
-
-/*
-Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment
-that already exists
-*/
-func saveAttachment(
-	ctx context.Context,
-	tx db.ConnOrTx,
-	attachment *Attachment,
-	hmnUserID int,
-	discordMessageID string,
-) (*models.DiscordMessageAttachment, error) {
-	iexisting, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessageattachment
-		WHERE id = $1
-		`,
-		attachment.ID,
-	)
-	if err == nil {
-		return iexisting.(*models.DiscordMessageAttachment), nil
-	} else if errors.Is(err, db.NotFound) {
-		// this is fine, just create it
-	} else {
-		return nil, oops.New(err, "failed to check for existing attachment")
-	}
-
-	width := 0
-	height := 0
-	if attachment.Width != nil {
-		width = *attachment.Width
-	}
-	if attachment.Height != nil {
-		height = *attachment.Height
-	}
-
-	content, _, err := downloadDiscordResource(ctx, attachment.Url)
-	if err != nil {
-		return nil, oops.New(err, "failed to download Discord attachment")
-	}
-
-	contentType := "application/octet-stream"
-	if attachment.ContentType != nil {
-		contentType = *attachment.ContentType
-	}
-
-	asset, err := assets.Create(ctx, tx, assets.CreateInput{
-		Content:     content,
-		Filename:    attachment.Filename,
-		ContentType: contentType,
-
-		UploaderID: &hmnUserID,
-		Width:      width,
-		Height:     height,
-	})
-	if err != nil {
-		return nil, oops.New(err, "failed to save asset for Discord attachment")
-	}
-
-	// TODO(db): RETURNING plz thanks
-	_, err = tx.Exec(ctx,
-		`
-		INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id)
-		VALUES ($1, $2, $3)
-		`,
-		attachment.ID,
-		asset.ID,
-		discordMessageID,
-	)
-	if err != nil {
-		return nil, oops.New(err, "failed to save Discord attachment data")
-	}
-
-	iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessageattachment
-		WHERE id = $1
-		`,
-		attachment.ID,
-	)
-	if err != nil {
-		return nil, oops.New(err, "failed to fetch new Discord attachment data")
-	}
-
-	return iDiscordAttachment.(*models.DiscordMessageAttachment), nil
-}
-
-// Saves an embed from Discord. NOTE: This is _not_ idempotent, so only call it
-// if you do not have any embeds saved for this message yet.
-func saveEmbed(
-	ctx context.Context,
-	tx db.ConnOrTx,
-	embed *Embed,
-	hmnUserID int,
-	discordMessageID string,
-) (*models.DiscordMessageEmbed, error) {
-	isOkImageType := func(contentType string) bool {
-		return strings.HasPrefix(contentType, "image/")
-	}
-
-	isOkVideoType := func(contentType string) bool {
-		return strings.HasPrefix(contentType, "video/")
-	}
-
-	maybeSaveImageish := func(i EmbedImageish, contentTypeCheck func(string) bool) (*uuid.UUID, error) {
-		content, contentType, err := downloadDiscordResource(ctx, *i.Url)
-		if err != nil {
-			var statusError DiscordResourceBadStatusCode
-			if errors.As(err, &statusError) {
-				return nil, nil
-			} else {
-				return nil, oops.New(err, "failed to save Discord embed")
-			}
-		}
-		if contentTypeCheck(contentType) {
-			in := assets.CreateInput{
-				Content:     content,
-				Filename:    "embed",
-				ContentType: contentType,
-				UploaderID:  &hmnUserID,
-			}
-
-			if i.Width != nil {
-				in.Width = *i.Width
-			}
-			if i.Height != nil {
-				in.Height = *i.Height
-			}
-
-			asset, err := assets.Create(ctx, tx, in)
-			if err != nil {
-				return nil, oops.New(err, "failed to create asset from embed")
-			}
-			return &asset.ID, nil
-		}
-
-		return nil, nil
-	}
-
-	var imageAssetId *uuid.UUID
-	var videoAssetId *uuid.UUID
-	var err error
-
-	if embed.Video != nil && embed.Video.Url != nil {
-		videoAssetId, err = maybeSaveImageish(embed.Video.EmbedImageish, isOkVideoType)
-	} else if embed.Image != nil && embed.Image.Url != nil {
-		imageAssetId, err = maybeSaveImageish(embed.Image.EmbedImageish, isOkImageType)
-	} else if embed.Thumbnail != nil && embed.Thumbnail.Url != nil {
-		imageAssetId, err = maybeSaveImageish(embed.Thumbnail.EmbedImageish, isOkImageType)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	// Save the embed into the db
-	// TODO(db): Insert, RETURNING
-	var savedEmbedId int
-	err = tx.QueryRow(ctx,
-		`
-		INSERT INTO handmade_discordmessageembed (title, description, url, message_id, image_id, video_id)
-		VALUES ($1, $2, $3, $4, $5, $6)
-		RETURNING id
-		`,
-		embed.Title,
-		embed.Description,
-		embed.Url,
-		discordMessageID,
-		imageAssetId,
-		videoAssetId,
-	).Scan(&savedEmbedId)
-	if err != nil {
-		return nil, oops.New(err, "failed to insert new embed")
-	}
-
-	iDiscordEmbed, err := db.QueryOne(ctx, tx, models.DiscordMessageEmbed{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessageembed
-		WHERE id = $1
-		`,
-		savedEmbedId,
-	)
-	if err != nil {
-		return nil, oops.New(err, "failed to fetch new Discord embed data")
-	}
-
-	return iDiscordEmbed.(*models.DiscordMessageEmbed), nil
-}
-
-type DiscordUserAndStuff struct {
-	DiscordUser models.DiscordUser `db:"duser"`
-	HMNUser     models.User        `db:"u"`
-}
-
-func FetchDiscordUser(ctx context.Context, dbConn db.ConnOrTx, discordUserID string) (*DiscordUserAndStuff, error) {
-	iuser, err := db.QueryOne(ctx, dbConn, DiscordUserAndStuff{},
-		`
-		SELECT $columns
-		FROM
-			handmade_discorduser AS duser
-			JOIN auth_user AS u ON duser.hmn_user_id = u.id
-			LEFT JOIN handmade_asset AS u_avatar ON u_avatar.id = u.avatar_asset_id
-		WHERE
-			duser.userid = $1
-		`,
-		discordUserID,
-	)
-	if err != nil {
-		return nil, err
-	}
-	return iuser.(*DiscordUserAndStuff), nil
-}
-
-/*
-Checks settings and permissions to decide whether we are allowed to create
-snippets for a user.
-*/
-func AllowedToCreateMessageSnippets(ctx context.Context, tx db.ConnOrTx, discordUserId string) (bool, error) {
-	u, err := FetchDiscordUser(ctx, tx, discordUserId)
-	if errors.Is(err, db.NotFound) {
-		return false, nil
-	} else if err != nil {
-		return false, oops.New(err, "failed to check if we can save Discord message")
-	}
-
-	return u.HMNUser.DiscordSaveShowcase, nil
-}
-
-/*
-Attempts to create snippets from Discord messages. If a snippet already exists
-for any message, no new snippet will be created.
-
-It uses the content saved in the database to do this. If we do not have any
-content saved, nothing will happen.
-
-If a user does not have their Discord account linked, this function will
-naturally do nothing because we have no message content saved. However, it does
-not check any user settings such as automatically creating snippets from
-#project-showcase. If we have the content, it will make a snippet for it, no
-questions asked. Bear that in mind.
-*/
-func CreateMessageSnippets(ctx context.Context, dbConn db.ConnOrTx, msgIDs ...string) error {
-	tx, err := dbConn.Begin(ctx)
-	if err != nil {
-		return oops.New(err, "failed to begin transaction")
-	}
-	defer tx.Rollback(ctx)
-
-	for _, msgID := range msgIDs {
-		// Check for existing snippet
-		type existingSnippetResult struct {
-			Message        models.DiscordMessage         `db:"msg"`
-			MessageContent *models.DiscordMessageContent `db:"c"`
-			Snippet        *models.Snippet               `db:"snippet"`
-			DiscordUser    *models.DiscordUser           `db:"duser"`
-		}
-		iexisting, err := db.QueryOne(ctx, tx, existingSnippetResult{},
-			`
-			SELECT $columns
-			FROM
-				handmade_discordmessage AS msg
-				LEFT JOIN handmade_discordmessagecontent AS c ON c.message_id = msg.id
-				LEFT JOIN handmade_snippet AS snippet ON snippet.discord_message_id = msg.id
-				LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid
-			WHERE
-				msg.id = $1
-			`,
-			msgID,
-		)
-		if err != nil {
-			return oops.New(err, "failed to check for existing snippet for message %s", msgID)
-		}
-		existing := iexisting.(*existingSnippetResult)
-
-		if existing.Snippet != nil {
-			// A snippet already exists - maybe update its content.
-			if existing.MessageContent != nil && !existing.Snippet.EditedOnWebsite {
-				contentMarkdown := existing.MessageContent.LastContent
-				contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
-
-				_, err := tx.Exec(ctx,
-					`
-					UPDATE handmade_snippet
-					SET
-						description = $1,
-						_description_html = $2
-					WHERE id = $3
-					`,
-					contentMarkdown,
-					contentHTML,
-					existing.Snippet.ID,
-				)
-				if err != nil {
-					logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to update content of snippet on message edit")
-				}
-				continue
-			}
-		}
-
-		if existing.Message.SnippetCreated {
-			// A snippet once existed but no longer does
-			// (we do not create another one in this case)
-			return nil
-		}
-
-		if existing.MessageContent == nil || existing.DiscordUser == nil {
-			return nil
-		}
-
-		// Get an asset ID or URL to make a snippet from
-		assetId, url, err := getSnippetAssetOrUrl(ctx, tx, &existing.Message)
-		if assetId == nil && url == nil {
-			// Nothing to make a snippet from!
-			return nil
-		}
-
-		contentMarkdown := existing.MessageContent.LastContent
-		contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
-
-		// TODO(db): Insert
-		_, err = tx.Exec(ctx,
-			`
-			INSERT INTO handmade_snippet (url, "when", description, _description_html, asset_id, discord_message_id, owner_id)
-			VALUES ($1, $2, $3, $4, $5, $6, $7)
-			`,
-			url,
-			existing.Message.SentAt,
-			contentMarkdown,
-			contentHTML,
-			assetId,
-			msgID,
-			existing.DiscordUser.HMNUserId,
-		)
-		if err != nil {
-			return oops.New(err, "failed to create snippet from attachment")
-		}
-		_, err = tx.Exec(ctx,
-			`
-			UPDATE handmade_discordmessage
-			SET snippet_created = TRUE
-			WHERE id = $1
-			`,
-			msgID,
-		)
-		if err != nil {
-			return oops.New(err, "failed to mark message as having snippet")
-		}
-	}
-
-	err = tx.Commit(ctx)
-	if err != nil {
-		return oops.New(err, "failed to commit transaction")
-	}
-
-	return nil
-}
-
-/*
-Associates any Discord tags with website tags for projects. Idempotent; will
-clear out any existing project tags and then add new ones.
-
-If no Discord user is linked, or no snippet exists, or whatever, this will do
-nothing and return no error.
-*/
-func UpdateSnippetTagsIfAny(ctx context.Context, dbConn db.ConnOrTx, msg *Message) error {
-	tx, err := dbConn.Begin(ctx)
-	if err != nil {
-		return oops.New(err, "failed to start transaction")
-	}
-	defer tx.Rollback(ctx)
-
-	// Fetch the Discord user; we only process messages for users with linked
-	// Discord accounts
-	u, err := FetchDiscordUser(ctx, tx, msg.Author.ID)
-	if err == db.NotFound {
-		return nil
-	} else if err != nil {
-		return oops.New(err, "failed to look up HMN user information from Discord user")
-	}
-
-	// Fetch the s associated with this Discord message (if any). If the
-	// s has already been edited on the website we'll skip it.
-	s, err := hmndata.FetchSnippetForDiscordMessage(ctx, tx, &u.HMNUser, msg.ID, hmndata.SnippetQuery{})
-	if err == db.NotFound {
-		return nil
-	} else if err != nil {
-		return err
-	}
-
-	// Fetch projects so we know what tags the user can apply to their snippet.
-	projects, err := hmndata.FetchProjects(ctx, tx, &u.HMNUser, hmndata.ProjectsQuery{
-		OwnerIDs: []int{u.HMNUser.ID},
-	})
-	if err != nil {
-		return oops.New(err, "failed to look up user projects")
-	}
-	projectIDs := make([]int, len(projects))
-	for i, p := range projects {
-		projectIDs[i] = p.Project.ID
-	}
-
-	// Delete any existing project tags for this snippet. We don't want to
-	// delete other tags in case in the future we have manual tagging on the
-	// website or whatever, and this would clear those out.
-	_, err = tx.Exec(ctx,
-		`
-		DELETE FROM snippet_tags
-		WHERE
-			snippet_id = $1
-			AND tag_id IN (
-				SELECT tag FROM handmade_project
-			)
-		`,
-		s.Snippet.ID,
-	)
-	if err != nil {
-		return oops.New(err, "failed to delete existing snippet tags")
-	}
-
-	// Try to associate tags in the message with project tags in HMN.
-	// Match only tags for projects in which the current user is a collaborator.
-	messageTags := getDiscordTags(s.Snippet.Description)
-	type tagsRow struct {
-		Tag models.Tag `db:"tags"`
-	}
-	iUserTags, err := db.Query(ctx, tx, tagsRow{},
-		`
-		SELECT $columns
-		FROM
-			tags
-			JOIN handmade_project AS project ON project.tag = tags.id
-			JOIN handmade_user_projects AS user_project ON user_project.project_id = project.id
-		WHERE
-			project.id = ANY ($1)
-		`,
-		projectIDs,
-	)
-	if err != nil {
-		return oops.New(err, "failed to fetch tags for user projects")
-	}
-
-	var tagIDs []int
-	for _, itag := range iUserTags {
-		tag := itag.(*tagsRow).Tag
-		for _, messageTag := range messageTags {
-			if tag.Text == messageTag {
-				tagIDs = append(tagIDs, tag.ID)
-			}
-		}
-	}
-
-	for _, tagID := range tagIDs {
-		_, err = tx.Exec(ctx,
-			`
-			INSERT INTO snippet_tags (snippet_id, tag_id)
-			VALUES ($1, $2)
-			ON CONFLICT DO NOTHING
-			`,
-			s.Snippet.ID,
-			tagID,
-		)
-		if err != nil {
-			return oops.New(err, "failed to add tag to snippet")
-		}
-	}
-
-	err = tx.Commit(ctx)
-	if err != nil {
-		return oops.New(err, "failed to commit transaction")
-	}
-
-	return nil
-}
-
-// NOTE(ben): This is maybe redundant with the regexes we use for markdown. But
-// do we actually want to reuse those, or should we keep them separate?
-var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`)
-
-func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msg *models.DiscordMessage) (*uuid.UUID, *string, error) {
-	// Check attachments
-	attachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessageattachment
-		WHERE message_id = $1
-		`,
-		msg.ID,
-	)
-	if err != nil {
-		return nil, nil, oops.New(err, "failed to fetch message attachments")
-	}
-	for _, iattachment := range attachments {
-		attachment := iattachment.(*models.DiscordMessageAttachment)
-		return &attachment.AssetID, nil, nil
-	}
-
-	// Check embeds
-	embeds, err := db.Query(ctx, tx, models.DiscordMessageEmbed{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessageembed
-		WHERE message_id = $1
-		`,
-		msg.ID,
-	)
-	if err != nil {
-		return nil, nil, oops.New(err, "failed to fetch discord embeds")
-	}
-	for _, iembed := range embeds {
-		embed := iembed.(*models.DiscordMessageEmbed)
-		if embed.VideoID != nil {
-			return embed.VideoID, nil, nil
-		} else if embed.ImageID != nil {
-			return embed.ImageID, nil, nil
-		} else if embed.URL != nil {
-			if RESnippetableUrl.MatchString(*embed.URL) {
-				return nil, embed.URL, nil
-			}
-		}
-	}
-
-	return nil, nil, nil
-}
-
-func messageHasLinks(content string) bool {
-	links := reDiscordMessageLink.FindAllString(content, -1)
-	for _, link := range links {
-		_, err := url.Parse(strings.TrimSpace(link))
-		if err == nil {
-			return true
-		}
-	}
-
-	return false
-}
-
-var REDiscordTag = regexp.MustCompile(`&([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*)`)
-
-func getDiscordTags(content string) []string {
-	matches := REDiscordTag.FindAllStringSubmatch(content, -1)
-	result := make([]string, len(matches))
-	for i, m := range matches {
-		result[i] = strings.ToLower(m[1])
-	}
-	return result
-}

src/website/discord.go

@@ -143,17 +143,6 @@ func DiscordShowcaseBacklog(c *RequestContext) ResponseData {
 	}
 	duser := iduser.(*models.DiscordUser)
 
-	ok, err := discord.AllowedToCreateMessageSnippets(c.Context(), c.Conn, duser.UserID)
-	if err != nil {
-		return c.ErrorResponse(http.StatusInternalServerError, err)
-	}
-
-	if !ok {
-		// Not allowed to do this, bail out
-		c.Logger.Warn().Msg("was not allowed to save user snippets")
-		return c.Redirect(hmnurl.BuildUserProfile(c.CurrentUser.Username), http.StatusSeeOther)
-	}
-
 	type messageIdQuery struct {
 		MessageID string `db:"msg.id"`
 	}
@@ -177,9 +166,18 @@ func DiscordShowcaseBacklog(c *RequestContext) ResponseData {
 	for _, imsgId := range iMsgIDs {
 		msgIDs = append(msgIDs, imsgId.(*messageIdQuery).MessageID)
 	}
-	err = discord.CreateMessageSnippets(c.Context(), c.Conn, msgIDs...)
+	for _, msgID := range msgIDs {
-	if err != nil {
+		interned, err := discord.FetchInternedMessage(c.Context(), c.Conn, msgID)
-		return c.ErrorResponse(http.StatusInternalServerError, err)
+		if err != nil {
+			return c.ErrorResponse(http.StatusInternalServerError, err)
+		}
+		if interned != nil {
+			// NOTE(asaf): Creating snippet even if the checkbox is off because the user asked us to.
+			err = discord.HandleSnippetForInternedMessage(c.Context(), c.Conn, interned, true)
+			if err != nil {
+				return c.ErrorResponse(http.StatusInternalServerError, err)
+			}
+		}
 	}
 
 	return c.Redirect(hmnurl.BuildUserProfile(c.CurrentUser.Username), http.StatusSeeOther)