diff --git a/src/discord/cmd/cmd.go b/src/discord/cmd/cmd.go index 557dbd7..7eb85f0 100644 --- a/src/discord/cmd/cmd.go +++ b/src/discord/cmd/cmd.go @@ -2,6 +2,9 @@ package cmd import ( "context" + "errors" + "fmt" + "os" "time" "git.handmade.network/hmn/hmn/src/db" @@ -35,18 +38,45 @@ func init() { rootCommand.AddCommand(scrapeCommand) makeSnippetCommand := &cobra.Command{ - Use: "makesnippet [...]", + Use: "makesnippet [...]", Short: "Make snippets from saved Discord messages", Long: "Make snippets from Discord messages whose content we have already saved. Useful for creating snippets from messages in non-showcase channels.", Run: func(cmd *cobra.Command, args []string) { + if len(args) < 2 { + cmd.Usage() + os.Exit(1) + } ctx := context.Background() conn := db.NewConnPool(1, 1) defer conn.Close() - err := discord.CreateMessageSnippets(ctx, conn, args...) - if err != nil { - logging.Error().Err(err).Msg("failed to create snippets") + chanID := args[0] + + count := 0 + + for _, msgID := range args[1:] { + message, err := discord.GetChannelMessage(ctx, chanID, msgID) + if errors.Is(err, discord.NotFound) { + logging.Warn().Msg(fmt.Sprintf("no message found on discord for id %s", msgID)) + continue + } else if err != nil { + logging.Error().Msg(fmt.Sprintf("failed to fetch discord message id %s", msgID)) + continue + } + err = discord.InternMessage(ctx, conn, message) + if err != nil { + logging.Error().Msg(fmt.Sprintf("failed to intern discord message id %s", msgID)) + continue + } + err = discord.HandleInternedMessage(ctx, conn, message, false, true) + if err != nil { + logging.Error().Msg(fmt.Sprintf("failed to handle interned message id %s", msgID)) + continue + } + count += 1 } + + logging.Info().Msg(fmt.Sprintf("Handled %d messages", count)) }, } rootCommand.AddCommand(makeSnippetCommand) diff --git a/src/discord/gateway.go b/src/discord/gateway.go index 0fca35f..0d6cf44 100644 --- a/src/discord/gateway.go +++ b/src/discord/gateway.go @@ -591,103 +591,31 @@ func (bot *botInstance) messageCreateOrUpdate(ctx context.Context, msg *Message) return nil } - if msg.ChannelID == config.Config.Discord.ShowcaseChannelID { - err := bot.processShowcaseMsg(ctx, msg) - if err != nil { - logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to process showcase message") - return nil - } - return nil - } - - if msg.ChannelID == config.Config.Discord.LibraryChannelID { - err := bot.processLibraryMsg(ctx, msg) - if err != nil { - logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to process library message") - return nil - } - return nil - } - - err := UpdateSnippetTagsIfAny(ctx, bot.dbConn, msg) + err := HandleIncomingMessage(ctx, bot.dbConn, msg, true) if err != nil { - logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to update tags for Discord snippet") + logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to handle incoming message") } + // NOTE(asaf): Since any error from HandleIncomingMessage is an internal error and not a discord + // error, we only want to log it and not restart the bot. So we're not returning the error. return nil } func (bot *botInstance) messageDelete(ctx context.Context, msgDelete MessageDelete) { log := logging.ExtractLogger(ctx) - tx, err := bot.dbConn.Begin(ctx) + interned, err := FetchInternedMessage(ctx, bot.dbConn, msgDelete.ID) if err != nil { - log.Error().Err(err).Msg("failed to start transaction") + log.Error().Err(err).Msg("failed to fetch interned message") return } - defer tx.Rollback(ctx) - - type deleteMessageQuery struct { - Message models.DiscordMessage `db:"msg"` - DiscordUser *models.DiscordUser `db:"duser"` - HMNUser *models.User `db:"hmnuser"` - SnippetID *int `db:"snippet.id"` - } - iresult, err := db.QueryOne(ctx, tx, deleteMessageQuery{}, - ` - SELECT $columns - FROM - handmade_discordmessage AS msg - LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid - LEFT JOIN auth_user AS hmnuser ON duser.hmn_user_id = hmnuser.id - LEFT JOIN handmade_asset AS hmnuser_avatar ON hmnuser_avatar.id = hmnuser.avatar_asset_id - LEFT JOIN handmade_snippet AS snippet ON snippet.discord_message_id = msg.id - WHERE msg.id = $1 AND msg.channel_id = $2 - `, - msgDelete.ID, msgDelete.ChannelID, - ) - if errors.Is(err, db.NotFound) { - return - } else if err != nil { - log.Error().Err(err).Msg("failed to check for message to delete") - return - } - result := iresult.(*deleteMessageQuery) - - log.Debug().Msg("deleting Discord message") - _, err = tx.Exec(ctx, - ` - DELETE FROM handmade_discordmessage - WHERE id = $1 AND channel_id = $2 - `, - msgDelete.ID, - msgDelete.ChannelID, - ) - - shouldDeleteSnippet := result.HMNUser != nil && result.HMNUser.DiscordDeleteSnippetOnMessageDelete - if result.SnippetID != nil && shouldDeleteSnippet { - log.Debug(). - Int("snippet_id", *result.SnippetID). - Int("user_id", result.HMNUser.ID). - Msg("deleting snippet from Discord message") - _, err = tx.Exec(ctx, - ` - DELETE FROM handmade_snippet - WHERE id = $1 - `, - result.SnippetID, - ) + if interned != nil { + err = DeleteInternedMessage(ctx, bot.dbConn, interned) if err != nil { - log.Error().Err(err).Msg("failed to delete snippet") + log.Error().Err(err).Msg("failed to delete interned message") return } } - - err = tx.Commit(ctx) - if err != nil { - log.Error().Err(err).Msg("failed to delete Discord message") - return - } } type MessageToSend struct { @@ -698,7 +626,7 @@ type MessageToSend struct { func SendMessages( ctx context.Context, - conn *pgxpool.Pool, + conn db.ConnOrTx, msgs ...MessageToSend, ) error { tx, err := conn.Begin(ctx) diff --git a/src/discord/history.go b/src/discord/history.go index 4573dff..b5b94ca 100644 --- a/src/discord/history.go +++ b/src/discord/history.go @@ -9,7 +9,6 @@ import ( "git.handmade.network/hmn/hmn/src/db" "git.handmade.network/hmn/hmn/src/logging" "git.handmade.network/hmn/hmn/src/models" - "github.com/jackc/pgx/v4" "github.com/jackc/pgx/v4/pgxpool" ) @@ -34,7 +33,10 @@ func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{ backfillInterval := 1 * time.Hour newUserTicker := time.NewTicker(5 * time.Second) - backfillTicker := time.NewTicker(backfillInterval) + + // NOTE(asaf): 5 seconds to ensure this runs on start, we then reset it to the correct + // interval after the first run. + backfillTicker := time.NewTicker(5 * time.Second) lastBackfillTime := time.Now().Add(-backfillInterval) for { @@ -45,6 +47,10 @@ func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{ // Get content for messages when a user links their account (but do not create snippets) fetchMissingContent(ctx, dbConn) case <-backfillTicker.C: + backfillTicker.Reset(backfillInterval) + // TODO(asaf): Do we need to update lastBackfillTime here? Otherwise we'll be rescraping + // from (start up time - 1 hour) every time. + // Run a backfill to patch up places where the Discord bot missed (does create snippets) Scrape(ctx, dbConn, config.Config.Discord.ShowcaseChannelID, @@ -99,13 +105,16 @@ func fetchMissingContent(ctx context.Context, dbConn *pgxpool.Pool) { discordMsg, err := GetChannelMessage(ctx, msg.ChannelID, msg.ID) if errors.Is(err, NotFound) { // This message has apparently been deleted; delete it from our database - _, err = dbConn.Exec(ctx, - ` - DELETE FROM handmade_discordmessage - WHERE id = $1 - `, - msg.ID, - ) + interned, err := FetchInternedMessage(ctx, dbConn, msg.ID) + if err != nil { + log.Error().Err(err).Msg("failed to fetch interned message") + continue + } + if interned == nil { + log.Error().Msg("couldn't find interned message") + continue + } + err = DeleteInternedMessage(ctx, dbConn, interned) if err != nil { log.Error().Err(err).Msg("failed to delete missing message") continue @@ -119,7 +128,7 @@ func fetchMissingContent(ctx context.Context, dbConn *pgxpool.Pool) { log.Info().Str("msg", discordMsg.ShortString()).Msg("fetched message for content") - err = handleHistoryMessage(ctx, dbConn, discordMsg, false) + err = HandleInternedMessage(ctx, dbConn, discordMsg, false, false) if err != nil { log.Error().Err(err).Msg("failed to save content for message") continue @@ -166,12 +175,10 @@ func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earlies return } - err := handleHistoryMessage(ctx, dbConn, &msg, createSnippets) + err := HandleIncomingMessage(ctx, dbConn, &msg, createSnippets) + if err != nil { errLog := logging.ExtractLogger(ctx).Error() - if errors.Is(err, errNotEnoughInfo) { - errLog = logging.ExtractLogger(ctx).Warn() - } errLog.Err(err).Msg("failed to process Discord message") } @@ -179,37 +186,3 @@ func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earlies } } } - -func handleHistoryMessage(ctx context.Context, dbConn *pgxpool.Pool, msg *Message, createSnippets bool) error { - var tx pgx.Tx - for { - var err error - tx, err = dbConn.Begin(ctx) - if err != nil { - logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to start transaction for message") - time.Sleep(1 * time.Second) - continue - } - break - } - - newMsg, err := SaveMessageAndContents(ctx, tx, msg) - if err != nil { - return err - } - if createSnippets { - if doSnippet, err := AllowedToCreateMessageSnippets(ctx, tx, newMsg.UserID); doSnippet && err == nil { - err := CreateMessageSnippets(ctx, tx, msg.ID) - if err != nil { - return err - } - } - } - - err = tx.Commit(ctx) - if err != nil { - return err - } - - return nil -} diff --git a/src/discord/library.go b/src/discord/library.go deleted file mode 100644 index bff896c..0000000 --- a/src/discord/library.go +++ /dev/null @@ -1,45 +0,0 @@ -package discord - -import ( - "context" - - "git.handmade.network/hmn/hmn/src/oops" -) - -func (bot *botInstance) processLibraryMsg(ctx context.Context, msg *Message) error { - switch msg.Type { - case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: - default: - return nil - } - - if !msg.OriginalHasFields("content") { - return nil - } - - if !messageHasLinks(msg.Content) { - err := DeleteMessage(ctx, msg.ChannelID, msg.ID) - if err != nil { - return oops.New(err, "failed to delete message") - } - - if !msg.Author.IsBot { - channel, err := CreateDM(ctx, msg.Author.ID) - if err != nil { - return oops.New(err, "failed to create DM channel") - } - - err = SendMessages(ctx, bot.dbConn, MessageToSend{ - ChannelID: channel.ID, - Req: CreateMessageRequest{ - Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.", - }, - }) - if err != nil { - return oops.New(err, "failed to send showcase warning message") - } - } - } - - return nil -} diff --git a/src/discord/message_handling.go b/src/discord/message_handling.go new file mode 100644 index 0000000..2fbd9af --- /dev/null +++ b/src/discord/message_handling.go @@ -0,0 +1,957 @@ +package discord + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "git.handmade.network/hmn/hmn/src/assets" + "git.handmade.network/hmn/hmn/src/config" + "git.handmade.network/hmn/hmn/src/db" + "git.handmade.network/hmn/hmn/src/hmndata" + "git.handmade.network/hmn/hmn/src/logging" + "git.handmade.network/hmn/hmn/src/models" + "git.handmade.network/hmn/hmn/src/oops" + "git.handmade.network/hmn/hmn/src/parsing" + "github.com/google/uuid" +) + +func HandleIncomingMessage(ctx context.Context, dbConn db.ConnOrTx, msg *Message, createSnippets bool) error { + deleted := false + var err error + + // NOTE(asaf): All functions called here should verify that the message applies to them. + + if !deleted && err == nil { + deleted, err = CleanUpLibrary(ctx, dbConn, msg) + } + + if !deleted && err == nil { + deleted, err = CleanUpShowcase(ctx, dbConn, msg) + } + + if !deleted && err == nil { + err = MaybeInternMessage(ctx, dbConn, msg) + } + + if err == nil { + err = HandleInternedMessage(ctx, dbConn, msg, deleted, createSnippets) + } + + return err +} + +func CleanUpShowcase(ctx context.Context, dbConn db.ConnOrTx, msg *Message) (bool, error) { + deleted := false + if msg.ChannelID == config.Config.Discord.ShowcaseChannelID { + switch msg.Type { + case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: + default: + return deleted, nil + } + + hasGoodContent := true + if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) { + hasGoodContent = false + } + + hasGoodAttachments := true + if msg.OriginalHasFields("attachments") && len(msg.Attachments) == 0 { + hasGoodAttachments = false + } + + if !hasGoodContent && !hasGoodAttachments { + err := DeleteMessage(ctx, msg.ChannelID, msg.ID) + if err != nil { + return deleted, oops.New(err, "failed to delete message") + } + deleted = true + + if !msg.Author.IsBot { + channel, err := CreateDM(ctx, msg.Author.ID) + if err != nil { + return deleted, oops.New(err, "failed to create DM channel") + } + + err = SendMessages(ctx, dbConn, MessageToSend{ + ChannelID: channel.ID, + Req: CreateMessageRequest{ + Content: "Posts in #project-showcase are required to have either an image/video or a link. Discuss showcase content in #projects.", + }, + }) + if err != nil { + return deleted, oops.New(err, "failed to send showcase warning message") + } + } + } + } + + return deleted, nil +} + +func CleanUpLibrary(ctx context.Context, dbConn db.ConnOrTx, msg *Message) (bool, error) { + deleted := false + if msg.ChannelID == config.Config.Discord.LibraryChannelID { + switch msg.Type { + case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: + default: + return deleted, nil + } + + if !msg.OriginalHasFields("content") { + return deleted, nil + } + + if !messageHasLinks(msg.Content) { + err := DeleteMessage(ctx, msg.ChannelID, msg.ID) + if err != nil { + return deleted, oops.New(err, "failed to delete message") + } + deleted = true + + if !msg.Author.IsBot { + channel, err := CreateDM(ctx, msg.Author.ID) + if err != nil { + return deleted, oops.New(err, "failed to create DM channel") + } + + err = SendMessages(ctx, dbConn, MessageToSend{ + ChannelID: channel.ID, + Req: CreateMessageRequest{ + Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.", + }, + }) + if err != nil { + return deleted, oops.New(err, "failed to send showcase warning message") + } + } + } + } + + return deleted, nil +} + +func MaybeInternMessage(ctx context.Context, dbConn db.ConnOrTx, msg *Message) error { + if msg.ChannelID == config.Config.Discord.ShowcaseChannelID { + err := InternMessage(ctx, dbConn, msg) + if errors.Is(err, errNotEnoughInfo) { + logging.ExtractLogger(ctx).Warn(). + Interface("msg", msg). + Msg("didn't have enough info to intern Discord message") + } else if err != nil { + return err + } + } + return nil +} + +/* +Ensures that a Discord message is stored in the database. This function is +idempotent and can be called regardless of whether the item already exists in +the database. + +This does not create snippets or save content or do anything besides save the message itself. +*/ +var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this") + +func InternMessage( + ctx context.Context, + dbConn db.ConnOrTx, + msg *Message, +) error { + _, err := db.QueryOne(ctx, dbConn, models.DiscordMessage{}, + ` + SELECT $columns + FROM handmade_discordmessage + WHERE id = $1 + `, + msg.ID, + ) + if errors.Is(err, db.NotFound) { + if !msg.OriginalHasFields("author", "timestamp") { + return errNotEnoughInfo + } + + guildID := msg.GuildID + if guildID == nil { + /* + This is weird, but it can happen when we fetch messages from + history instead of receiving it from the gateway. In this case + we just assume it's from the HMN server. + */ + guildID = &config.Config.Discord.GuildID + } + + _, err = dbConn.Exec(ctx, + ` + INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created) + VALUES ($1, $2, $3, $4, $5, $6, $7) + `, + msg.ID, + msg.ChannelID, + *guildID, + msg.JumpURL(), + msg.Author.ID, + msg.Time(), + false, + ) + if err != nil { + return oops.New(err, "failed to save new discord message") + } + } else if err != nil { + return oops.New(err, "failed to check for existing Discord message") + } + + return nil +} + +type InternedMessage struct { + Message models.DiscordMessage `db:"message"` + MessageContent *models.DiscordMessageContent `db:"content"` + HMNUser *models.User `db:"hmnuser"` + DiscordUser *models.DiscordUser `db:"duser"` +} + +func FetchInternedMessage(ctx context.Context, dbConn db.ConnOrTx, msgId string) (*InternedMessage, error) { + result, err := db.QueryOne(ctx, dbConn, InternedMessage{}, + ` + SELECT $columns + FROM + handmade_discordmessage AS message + LEFT JOIN handmade_discordmessagecontent AS content ON content.message_id = message.id + LEFT JOIN handmade_discorduser AS duser ON duser.userid = message.user_id + LEFT JOIN auth_user AS hmnuser ON hmnuser.id = duser.hmn_user_id + LEFT JOIN handmade_asset AS hmnuser_avatar ON hmnuser_avatar.id = hmnuser.avatar_asset_id + WHERE message.id = $1 + `, + msgId, + ) + if err != nil { + if errors.Is(err, db.NotFound) { + return nil, nil + } else { + return nil, err + } + } + + interned := result.(*InternedMessage) + return interned, nil +} + +// Checks if a message is interned and handles it to the extent possible: +// 1. Saves/updates content +// 2. Saves/updates snippet +// 3. Deletes content/snippet +func HandleInternedMessage(ctx context.Context, dbConn db.ConnOrTx, msg *Message, deleted bool, createSnippet bool) error { + tx, err := dbConn.Begin(ctx) + if err != nil { + return oops.New(err, "failed to start transaction") + } + defer tx.Rollback(ctx) + + interned, err := FetchInternedMessage(ctx, tx, msg.ID) + if err != nil { + return err + } + + if interned != nil { + if !deleted { + err = SaveMessageContents(ctx, tx, interned, msg) + if err != nil { + return err + + } + if createSnippet { + err = HandleSnippetForInternedMessage(ctx, tx, interned, false) + if err != nil { + return err + } + } + } else { + err = DeleteInternedMessage(ctx, tx, interned) + if err != nil { + return err + } + } + } + err = tx.Commit(ctx) + if err != nil { + return oops.New(err, "failed to commit Discord message updates") + } + + return nil +} + +func DeleteInternedMessage(ctx context.Context, dbConn db.ConnOrTx, interned *InternedMessage) error { + isnippet, err := db.QueryOne(ctx, dbConn, models.Snippet{}, + ` + SELECT $columns + FROM handmade_snippet + WHERE discord_message_id = $1 + `, + interned.Message.ID, + ) + if err != nil && !errors.Is(err, db.NotFound) { + return oops.New(err, "failed to fetch snippet for discord message") + } + var snippet *models.Snippet + if !errors.Is(err, db.NotFound) { + snippet = isnippet.(*models.Snippet) + } + + // NOTE(asaf): Also deletes the following through a db cascade: + // * handmade_discordmessageattachment + // * handmade_discordmessagecontent + // * handmade_discordmessageembed + // DOES NOT DELETE ASSETS FOR CONTENT/EMBEDS + _, err = dbConn.Exec(ctx, + ` + DELETE FROM handmade_discordmessage + WHERE id = $1 + `, + interned.Message.ID, + ) + + if snippet != nil { + userApprovesDeletion := interned.HMNUser != nil && snippet.OwnerID == interned.HMNUser.ID && interned.HMNUser.DiscordDeleteSnippetOnMessageDelete + if !snippet.EditedOnWebsite && userApprovesDeletion { + // NOTE(asaf): Does not delete asset! + _, err = dbConn.Exec(ctx, + ` + DELETE FROM handmade_snippet + WHERE id = $1 + `, + snippet.ID, + ) + if err != nil { + return oops.New(err, "failed to delete snippet") + } + } + } + + return nil +} + +/* +Processes a single Discord message, saving as much of the message's content +and attachments as allowed by our rules and user settings. Does NOT create +snippets. + +Idempotent; can be called any time whether the contents exist or not. + +NOTE!!: Replaces interned.MessageContent if it was created or updated!! +*/ +func SaveMessageContents( + ctx context.Context, + dbConn db.ConnOrTx, + interned *InternedMessage, + msg *Message, +) error { + if interned.DiscordUser != nil { + // We have a linked Discord account, so save the message contents (regardless of + // whether we create a snippet or not). + if msg.OriginalHasFields("content") { + _, err := dbConn.Exec(ctx, + ` + INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content) + VALUES ($1, $2, $3) + ON CONFLICT (message_id) DO UPDATE SET + discord_id = EXCLUDED.discord_id, + last_content = EXCLUDED.last_content + `, + interned.Message.ID, + interned.DiscordUser.ID, + CleanUpMarkdown(ctx, msg.Content), + ) + if err != nil { + return oops.New(err, "failed to create or update message contents") + } + + icontent, err := db.QueryOne(ctx, dbConn, models.DiscordMessageContent{}, + ` + SELECT $columns + FROM + handmade_discordmessagecontent + WHERE + handmade_discordmessagecontent.message_id = $1 + `, + interned.Message.ID, + ) + if err != nil { + return oops.New(err, "failed to fetch message contents") + } + interned.MessageContent = icontent.(*models.DiscordMessageContent) + } // TODO(asaf): What happens if we edit the message and delete the content but keep the attachment?? + + // Save attachments + if msg.OriginalHasFields("attachments") { + for _, attachment := range msg.Attachments { + _, err := saveAttachment(ctx, dbConn, &attachment, interned.DiscordUser.HMNUserId, msg.ID) + if err != nil { + return oops.New(err, "failed to save attachment") + } + } + } + + // Save / delete embeds + if msg.OriginalHasFields("embeds") { + numSavedEmbeds, err := db.QueryInt(ctx, dbConn, + ` + SELECT COUNT(*) + FROM handmade_discordmessageembed + WHERE message_id = $1 + `, + msg.ID, + ) + if err != nil { + return oops.New(err, "failed to count existing embeds") + } + if numSavedEmbeds == 0 { + // No embeds yet, so save new ones + for _, embed := range msg.Embeds { + _, err := saveEmbed(ctx, dbConn, &embed, interned.DiscordUser.HMNUserId, msg.ID) + if err != nil { + return oops.New(err, "failed to save embed") + } + } + } else if len(msg.Embeds) > 0 { + // Embeds were removed from the message + _, err := dbConn.Exec(ctx, + ` + DELETE FROM handmade_discordmessageembed + WHERE message_id = $1 + `, + msg.ID, + ) + if err != nil { + return oops.New(err, "failed to delete embeds") + } + } + } + } + return nil +} + +var discordDownloadClient = &http.Client{ + Timeout: 10 * time.Second, +} + +type DiscordResourceBadStatusCode error + +func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, "", oops.New(err, "failed to make Discord download request") + } + res, err := discordDownloadClient.Do(req) + if err != nil { + return nil, "", oops.New(err, "failed to fetch Discord resource data") + } + defer res.Body.Close() + + if res.StatusCode < 200 || 299 < res.StatusCode { + return nil, "", DiscordResourceBadStatusCode(fmt.Errorf("status code %d from Discord resource: %s", res.StatusCode, url)) + } + + content, err := io.ReadAll(res.Body) + if err != nil { + panic(err) + } + + return content, res.Header.Get("Content-Type"), nil +} + +/* +Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment +that already exists +*/ +func saveAttachment( + ctx context.Context, + tx db.ConnOrTx, + attachment *Attachment, + hmnUserID int, + discordMessageID string, +) (*models.DiscordMessageAttachment, error) { + iexisting, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{}, + ` + SELECT $columns + FROM handmade_discordmessageattachment + WHERE id = $1 + `, + attachment.ID, + ) + if err == nil { + return iexisting.(*models.DiscordMessageAttachment), nil + } else if errors.Is(err, db.NotFound) { + // this is fine, just create it + } else { + return nil, oops.New(err, "failed to check for existing attachment") + } + + width := 0 + height := 0 + if attachment.Width != nil { + width = *attachment.Width + } + if attachment.Height != nil { + height = *attachment.Height + } + + content, _, err := downloadDiscordResource(ctx, attachment.Url) + if err != nil { + return nil, oops.New(err, "failed to download Discord attachment") + } + + contentType := "application/octet-stream" + if attachment.ContentType != nil { + contentType = *attachment.ContentType + } + + asset, err := assets.Create(ctx, tx, assets.CreateInput{ + Content: content, + Filename: attachment.Filename, + ContentType: contentType, + + UploaderID: &hmnUserID, + Width: width, + Height: height, + }) + if err != nil { + return nil, oops.New(err, "failed to save asset for Discord attachment") + } + + // TODO(db): RETURNING plz thanks + _, err = tx.Exec(ctx, + ` + INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id) + VALUES ($1, $2, $3) + `, + attachment.ID, + asset.ID, + discordMessageID, + ) + if err != nil { + return nil, oops.New(err, "failed to save Discord attachment data") + } + + iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{}, + ` + SELECT $columns + FROM handmade_discordmessageattachment + WHERE id = $1 + `, + attachment.ID, + ) + if err != nil { + return nil, oops.New(err, "failed to fetch new Discord attachment data") + } + + return iDiscordAttachment.(*models.DiscordMessageAttachment), nil +} + +// Saves an embed from Discord. NOTE: This is _not_ idempotent, so only call it +// if you do not have any embeds saved for this message yet. +func saveEmbed( + ctx context.Context, + tx db.ConnOrTx, + embed *Embed, + hmnUserID int, + discordMessageID string, +) (*models.DiscordMessageEmbed, error) { + isOkImageType := func(contentType string) bool { + return strings.HasPrefix(contentType, "image/") + } + + isOkVideoType := func(contentType string) bool { + return strings.HasPrefix(contentType, "video/") + } + + maybeSaveImageish := func(i EmbedImageish, contentTypeCheck func(string) bool) (*uuid.UUID, error) { + content, contentType, err := downloadDiscordResource(ctx, *i.Url) + if err != nil { + var statusError DiscordResourceBadStatusCode + if errors.As(err, &statusError) { + return nil, nil + } else { + return nil, oops.New(err, "failed to save Discord embed") + } + } + if contentTypeCheck(contentType) { + in := assets.CreateInput{ + Content: content, + Filename: "embed", + ContentType: contentType, + UploaderID: &hmnUserID, + } + + if i.Width != nil { + in.Width = *i.Width + } + if i.Height != nil { + in.Height = *i.Height + } + + asset, err := assets.Create(ctx, tx, in) + if err != nil { + return nil, oops.New(err, "failed to create asset from embed") + } + return &asset.ID, nil + } + + return nil, nil + } + + var imageAssetId *uuid.UUID + var videoAssetId *uuid.UUID + var err error + + if embed.Video != nil && embed.Video.Url != nil { + videoAssetId, err = maybeSaveImageish(embed.Video.EmbedImageish, isOkVideoType) + } else if embed.Image != nil && embed.Image.Url != nil { + imageAssetId, err = maybeSaveImageish(embed.Image.EmbedImageish, isOkImageType) + } else if embed.Thumbnail != nil && embed.Thumbnail.Url != nil { + imageAssetId, err = maybeSaveImageish(embed.Thumbnail.EmbedImageish, isOkImageType) + } + if err != nil { + return nil, err + } + + // Save the embed into the db + // TODO(db): Insert, RETURNING + var savedEmbedId int + err = tx.QueryRow(ctx, + ` + INSERT INTO handmade_discordmessageembed (title, description, url, message_id, image_id, video_id) + VALUES ($1, $2, $3, $4, $5, $6) + RETURNING id + `, + embed.Title, + embed.Description, + embed.Url, + discordMessageID, + imageAssetId, + videoAssetId, + ).Scan(&savedEmbedId) + if err != nil { + return nil, oops.New(err, "failed to insert new embed") + } + + iDiscordEmbed, err := db.QueryOne(ctx, tx, models.DiscordMessageEmbed{}, + ` + SELECT $columns + FROM handmade_discordmessageembed + WHERE id = $1 + `, + savedEmbedId, + ) + if err != nil { + return nil, oops.New(err, "failed to fetch new Discord embed data") + } + + return iDiscordEmbed.(*models.DiscordMessageEmbed), nil +} + +func FetchSnippetForMessage(ctx context.Context, dbConn db.ConnOrTx, msgID string) (*models.Snippet, error) { + iresult, err := db.QueryOne(ctx, dbConn, models.Snippet{}, + ` + SELECT $columns + FROM handmade_snippet + WHERE discord_message_id = $1 + `, + msgID, + ) + + if err != nil { + if errors.Is(err, db.NotFound) { + return nil, nil + } else { + return nil, oops.New(err, "failed to fetch existing snippet for message %s", msgID) + } + } + + return iresult.(*models.Snippet), nil +} + +/* +Potentially creates or updates a snippet for the given interned message. +It uses the content saved in the database to do this. If we do not have any +content saved, nothing will happen. + +If a user does not have their Discord account linked, this function will +naturally do nothing because we have no message content saved. +If forceCreate is true, it does not check any user settings such as automatically creating snippets from +#project-showcase. If we have the content, it will make a snippet for it, no +questions asked. Bear that in mind. +*/ +func HandleSnippetForInternedMessage(ctx context.Context, dbConn db.ConnOrTx, interned *InternedMessage, forceCreate bool) error { + if interned.HMNUser == nil { + // NOTE(asaf): Can't handle snippets when there's no linked user + return nil + } + + if interned.MessageContent == nil { + // NOTE(asaf): Can't have a snippet without content + // NOTE(asaf): Messages that only have an attachment also have blank content + // TODO(asaf): Do we need to delete existing snippets in this case??? Not entirely sure how to trigger this through discord + return nil + } + + tx, err := dbConn.Begin(ctx) + if err != nil { + oops.New(err, "failed to start transaction") + } + defer tx.Rollback(ctx) + + existingSnippet, err := FetchSnippetForMessage(ctx, tx, interned.Message.ID) + if err != nil { + return oops.New(err, "failed to check for existing snippet for message %s", interned.Message.ID) + } + + if existingSnippet != nil { + // TODO(asaf): We're not handling the case where embeds were removed or modified. + // Also not handling the case where a message had both an attachment and an embed + // and the attachment was removed (leaving only the embed). + LinkedUserIsSnippetOwner := existingSnippet.OwnerID == interned.DiscordUser.HMNUserId + if LinkedUserIsSnippetOwner && !existingSnippet.EditedOnWebsite { + contentMarkdown := interned.MessageContent.LastContent + contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown) + + _, err := tx.Exec(ctx, + ` + UPDATE handmade_snippet + SET + description = $1, + _description_html = $2 + WHERE id = $3 + `, + contentMarkdown, + contentHTML, + existingSnippet.ID, + ) + if err != nil { + return oops.New(err, "failed to update content of snippet on message edit") + } + existingSnippet.Description = contentMarkdown + existingSnippet.DescriptionHtml = contentHTML + } + } else { + userAllowsSnippet := interned.HMNUser.DiscordSaveShowcase || forceCreate + shouldCreate := !interned.Message.SnippetCreated && userAllowsSnippet + + if shouldCreate { + // Get an asset ID or URL to make a snippet from + assetId, url, err := getSnippetAssetOrUrl(ctx, tx, &interned.Message) + if assetId != nil || url != nil { + contentMarkdown := interned.MessageContent.LastContent + contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown) + + _, err = tx.Exec(ctx, + ` + INSERT INTO handmade_snippet (url, "when", description, _description_html, asset_id, discord_message_id, owner_id) + VALUES ($1, $2, $3, $4, $5, $6, $7) + `, + url, + interned.Message.SentAt, + contentMarkdown, + contentHTML, + assetId, + interned.Message.ID, + interned.HMNUser.ID, + ) + if err != nil { + return oops.New(err, "failed to create snippet from attachment") + } + + existingSnippet, err = FetchSnippetForMessage(ctx, tx, interned.Message.ID) + if err != nil { + return oops.New(err, "failed to fetch newly-created snippet") + } + + _, err = tx.Exec(ctx, + ` + UPDATE handmade_discordmessage + SET snippet_created = TRUE + WHERE id = $1 + `, + interned.Message.ID, + ) + if err != nil { + return oops.New(err, "failed to mark message as having snippet") + } + } + } + } + + if existingSnippet != nil { + // Update tags + + // Try to associate tags in the message with project tags in HMN. + // Match only tags for projects in which the current user is a collaborator. + messageTags := getDiscordTags(existingSnippet.Description) + + var desiredTags []int + var allTags []int + + if len(messageTags) > 0 { + // Fetch projects so we know what tags the user can apply to their snippet. + projects, err := hmndata.FetchProjects(ctx, tx, interned.HMNUser, hmndata.ProjectsQuery{ + OwnerIDs: []int{interned.HMNUser.ID}, + }) + if err != nil { + return oops.New(err, "failed to look up user projects") + } + + projectIDs := make([]int, len(projects)) + for i, p := range projects { + projectIDs[i] = p.Project.ID + } + + type tagsRow struct { + Tag models.Tag `db:"tags"` + } + iUserTags, err := db.Query(ctx, tx, tagsRow{}, + ` + SELECT $columns + FROM + tags + JOIN handmade_project AS project ON project.tag = tags.id + WHERE + project.id = ANY ($1) + `, + projectIDs, + ) + if err != nil { + return oops.New(err, "failed to fetch tags for user projects") + } + + for _, itag := range iUserTags { + tag := itag.(*tagsRow).Tag + for _, messageTag := range messageTags { + allTags = append(allTags, tag.ID) + if strings.EqualFold(tag.Text, messageTag) { + desiredTags = append(desiredTags, tag.ID) + } + } + } + } + + _, err = tx.Exec(ctx, + ` + DELETE FROM snippet_tags + WHERE + snippet_id = $1 + AND tag_id = ANY ($2) + `, + existingSnippet.ID, + allTags, + ) + if err != nil { + return oops.New(err, "failed to clear tags from snippet") + } + + for _, tagID := range desiredTags { + _, err = tx.Exec(ctx, + ` + INSERT INTO snippet_tags (snippet_id, tag_id) + VALUES ($1, $2) + ON CONFLICT DO NOTHING + `, + existingSnippet.ID, + tagID, + ) + if err != nil { + return oops.New(err, "failed to associate snippet with tag") + } + } + } + + err = tx.Commit(ctx) + if err != nil { + return oops.New(err, "failed to commit transaction") + } + + return nil +} + +// TODO(asaf): I believe this will also match https://example.com?hello=1&whatever=5 +// Probably need to add word boundaries. +var REDiscordTag = regexp.MustCompile(`&([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*)`) + +func getDiscordTags(content string) []string { + matches := REDiscordTag.FindAllStringSubmatch(content, -1) + result := make([]string, len(matches)) + for i, m := range matches { + result[i] = m[1] + } + return result +} + +// NOTE(ben): This is maybe redundant with the regexes we use for markdown. But +// do we actually want to reuse those, or should we keep them separate? +var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`) + +func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msg *models.DiscordMessage) (*uuid.UUID, *string, error) { + // Check attachments + attachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{}, + ` + SELECT $columns + FROM handmade_discordmessageattachment + WHERE message_id = $1 + `, + msg.ID, + ) + if err != nil { + return nil, nil, oops.New(err, "failed to fetch message attachments") + } + for _, iattachment := range attachments { + attachment := iattachment.(*models.DiscordMessageAttachment) + return &attachment.AssetID, nil, nil + } + + // Check embeds + embeds, err := db.Query(ctx, tx, models.DiscordMessageEmbed{}, + ` + SELECT $columns + FROM handmade_discordmessageembed + WHERE message_id = $1 + `, + msg.ID, + ) + if err != nil { + return nil, nil, oops.New(err, "failed to fetch discord embeds") + } + for _, iembed := range embeds { + embed := iembed.(*models.DiscordMessageEmbed) + if embed.VideoID != nil { + return embed.VideoID, nil, nil + } else if embed.ImageID != nil { + return embed.ImageID, nil, nil + } else if embed.URL != nil { + if RESnippetableUrl.MatchString(*embed.URL) { + return nil, embed.URL, nil + } + } + } + + return nil, nil, nil +} + +var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`) + +func messageHasLinks(content string) bool { + links := reDiscordMessageLink.FindAllString(content, -1) + for _, link := range links { + _, err := url.Parse(strings.TrimSpace(link)) + if err == nil { + return true + } + } + + return false +} diff --git a/src/discord/showcase.go b/src/discord/showcase.go deleted file mode 100644 index 3fa79a2..0000000 --- a/src/discord/showcase.go +++ /dev/null @@ -1,873 +0,0 @@ -package discord - -import ( - "context" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "regexp" - "strings" - "time" - - "git.handmade.network/hmn/hmn/src/assets" - "git.handmade.network/hmn/hmn/src/config" - "git.handmade.network/hmn/hmn/src/db" - "git.handmade.network/hmn/hmn/src/hmndata" - "git.handmade.network/hmn/hmn/src/logging" - "git.handmade.network/hmn/hmn/src/models" - "git.handmade.network/hmn/hmn/src/oops" - "git.handmade.network/hmn/hmn/src/parsing" - "github.com/google/uuid" -) - -var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`) - -var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this") - -func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) error { - switch msg.Type { - case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: - default: - return nil - } - - didDelete, err := bot.maybeDeleteShowcaseMsg(ctx, msg) - if err != nil { - return err - } - if didDelete { - return nil - } - - tx, err := bot.dbConn.Begin(ctx) - if err != nil { - panic(err) - } - defer tx.Rollback(ctx) - - // save the message, maybe save its contents - newMsg, err := SaveMessageAndContents(ctx, tx, msg) - if errors.Is(err, errNotEnoughInfo) { - logging.ExtractLogger(ctx).Warn(). - Interface("msg", msg). - Msg("didn't have enough info to process Discord message") - return nil - } else if err != nil { - return err - } - - // ...and maybe make a snippet too, if the user wants us to - duser, err := FetchDiscordUser(ctx, tx, newMsg.UserID) - if err == nil && duser.HMNUser.DiscordSaveShowcase { - err = CreateMessageSnippets(ctx, tx, newMsg.UserID, msg.ID) - if err != nil { - return oops.New(err, "failed to create snippet in gateway") - } - } else { - if err == db.NotFound { - // this is fine, just don't create a snippet - } else { - return err - } - } - - err = tx.Commit(ctx) - if err != nil { - return oops.New(err, "failed to commit Discord message updates") - } - - return nil -} - -func (bot *botInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message) (didDelete bool, err error) { - hasGoodContent := true - if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) { - hasGoodContent = false - } - - hasGoodAttachments := true - if msg.OriginalHasFields("attachments") && len(msg.Attachments) == 0 { - hasGoodAttachments = false - } - - didDelete = false - if !hasGoodContent && !hasGoodAttachments { - didDelete = true - err := DeleteMessage(ctx, msg.ChannelID, msg.ID) - if err != nil { - return false, oops.New(err, "failed to delete message") - } - - if !msg.Author.IsBot { - channel, err := CreateDM(ctx, msg.Author.ID) - if err != nil { - return false, oops.New(err, "failed to create DM channel") - } - - err = SendMessages(ctx, bot.dbConn, MessageToSend{ - ChannelID: channel.ID, - Req: CreateMessageRequest{ - Content: "Posts in #project-showcase are required to have either an image/video or a link. Discuss showcase content in #projects.", - }, - }) - if err != nil { - return false, oops.New(err, "failed to send showcase warning message") - } - } - } - - return didDelete, nil -} - -/* -Ensures that a Discord message is stored in the database. This function is -idempotent and can be called regardless of whether the item already exists in -the database. - -This does not create snippets or do anything besides save the message itself. -*/ -func SaveMessage( - ctx context.Context, - tx db.ConnOrTx, - msg *Message, -) (*models.DiscordMessage, error) { - iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{}, - ` - SELECT $columns - FROM handmade_discordmessage - WHERE id = $1 - `, - msg.ID, - ) - if errors.Is(err, db.NotFound) { - if !msg.OriginalHasFields("author", "timestamp") { - return nil, errNotEnoughInfo - } - - guildID := msg.GuildID - if guildID == nil { - /* - This is weird, but it can happen when we fetch messages from - history instead of receiving it from the gateway. In this case - we just assume it's from the HMN server. - */ - guildID = &config.Config.Discord.GuildID - } - - _, err = tx.Exec(ctx, - ` - INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created) - VALUES ($1, $2, $3, $4, $5, $6, $7) - `, - msg.ID, - msg.ChannelID, - *guildID, - msg.JumpURL(), - msg.Author.ID, - msg.Time(), - false, - ) - if err != nil { - return nil, oops.New(err, "failed to save new discord message") - } - - /* - TODO(db): This is a spot where it would be really nice to be able - to use RETURNING, and avoid this second query. - */ - iDiscordMessage, err = db.QueryOne(ctx, tx, models.DiscordMessage{}, - ` - SELECT $columns - FROM handmade_discordmessage - WHERE id = $1 - `, - msg.ID, - ) - if err != nil { - panic(err) - } - } else if err != nil { - return nil, oops.New(err, "failed to check for existing Discord message") - } - - return iDiscordMessage.(*models.DiscordMessage), nil -} - -/* -Processes a single Discord message, saving as much of the message's content -and attachments as allowed by our rules and user settings. Does NOT create -snippets. - -Idempotent; can be called any time whether the message exists or not. -*/ -func SaveMessageAndContents( - ctx context.Context, - tx db.ConnOrTx, - msg *Message, -) (*models.DiscordMessage, error) { - newMsg, err := SaveMessage(ctx, tx, msg) - if err != nil { - return nil, err - } - - // Check for linked Discord user - iDiscordUser, err := db.QueryOne(ctx, tx, models.DiscordUser{}, - ` - SELECT $columns - FROM handmade_discorduser - WHERE userid = $1 - `, - newMsg.UserID, - ) - if errors.Is(err, db.NotFound) { - return newMsg, nil - } else if err != nil { - return nil, oops.New(err, "failed to look up linked Discord user") - } - discordUser := iDiscordUser.(*models.DiscordUser) - - // We have a linked Discord account, so save the message contents (regardless of - // whether we create a snippet or not). - - if msg.OriginalHasFields("content") { - _, err = tx.Exec(ctx, - ` - INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content) - VALUES ($1, $2, $3) - ON CONFLICT (message_id) DO UPDATE SET - discord_id = EXCLUDED.discord_id, - last_content = EXCLUDED.last_content - `, - newMsg.ID, - discordUser.ID, - CleanUpMarkdown(ctx, msg.Content), - ) - } - - // Save attachments - if msg.OriginalHasFields("attachments") { - for _, attachment := range msg.Attachments { - _, err := saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID) - if err != nil { - return nil, oops.New(err, "failed to save attachment") - } - } - } - - // Save / delete embeds - if msg.OriginalHasFields("embeds") { - numSavedEmbeds, err := db.QueryInt(ctx, tx, - ` - SELECT COUNT(*) - FROM handmade_discordmessageembed - WHERE message_id = $1 - `, - msg.ID, - ) - if err != nil { - return nil, oops.New(err, "failed to count existing embeds") - } - if numSavedEmbeds == 0 { - // No embeds yet, so save new ones - for _, embed := range msg.Embeds { - _, err := saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID) - if err != nil { - return nil, oops.New(err, "failed to save embed") - } - } - } else if len(msg.Embeds) > 0 { - // Embeds were removed from the message - _, err := tx.Exec(ctx, - ` - DELETE FROM handmade_discordmessageembed - WHERE message_id = $1 - `, - msg.ID, - ) - if err != nil { - return nil, oops.New(err, "failed to delete embeds") - } - } - } - - return newMsg, nil -} - -var discordDownloadClient = &http.Client{ - Timeout: 10 * time.Second, -} - -type DiscordResourceBadStatusCode error - -func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, "", oops.New(err, "failed to make Discord download request") - } - res, err := discordDownloadClient.Do(req) - if err != nil { - return nil, "", oops.New(err, "failed to fetch Discord resource data") - } - defer res.Body.Close() - - if res.StatusCode < 200 || 299 < res.StatusCode { - return nil, "", DiscordResourceBadStatusCode(fmt.Errorf("status code %d from Discord resource: %s", res.StatusCode, url)) - } - - content, err := io.ReadAll(res.Body) - if err != nil { - panic(err) - } - - return content, res.Header.Get("Content-Type"), nil -} - -/* -Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment -that already exists -*/ -func saveAttachment( - ctx context.Context, - tx db.ConnOrTx, - attachment *Attachment, - hmnUserID int, - discordMessageID string, -) (*models.DiscordMessageAttachment, error) { - iexisting, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{}, - ` - SELECT $columns - FROM handmade_discordmessageattachment - WHERE id = $1 - `, - attachment.ID, - ) - if err == nil { - return iexisting.(*models.DiscordMessageAttachment), nil - } else if errors.Is(err, db.NotFound) { - // this is fine, just create it - } else { - return nil, oops.New(err, "failed to check for existing attachment") - } - - width := 0 - height := 0 - if attachment.Width != nil { - width = *attachment.Width - } - if attachment.Height != nil { - height = *attachment.Height - } - - content, _, err := downloadDiscordResource(ctx, attachment.Url) - if err != nil { - return nil, oops.New(err, "failed to download Discord attachment") - } - - contentType := "application/octet-stream" - if attachment.ContentType != nil { - contentType = *attachment.ContentType - } - - asset, err := assets.Create(ctx, tx, assets.CreateInput{ - Content: content, - Filename: attachment.Filename, - ContentType: contentType, - - UploaderID: &hmnUserID, - Width: width, - Height: height, - }) - if err != nil { - return nil, oops.New(err, "failed to save asset for Discord attachment") - } - - // TODO(db): RETURNING plz thanks - _, err = tx.Exec(ctx, - ` - INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id) - VALUES ($1, $2, $3) - `, - attachment.ID, - asset.ID, - discordMessageID, - ) - if err != nil { - return nil, oops.New(err, "failed to save Discord attachment data") - } - - iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{}, - ` - SELECT $columns - FROM handmade_discordmessageattachment - WHERE id = $1 - `, - attachment.ID, - ) - if err != nil { - return nil, oops.New(err, "failed to fetch new Discord attachment data") - } - - return iDiscordAttachment.(*models.DiscordMessageAttachment), nil -} - -// Saves an embed from Discord. NOTE: This is _not_ idempotent, so only call it -// if you do not have any embeds saved for this message yet. -func saveEmbed( - ctx context.Context, - tx db.ConnOrTx, - embed *Embed, - hmnUserID int, - discordMessageID string, -) (*models.DiscordMessageEmbed, error) { - isOkImageType := func(contentType string) bool { - return strings.HasPrefix(contentType, "image/") - } - - isOkVideoType := func(contentType string) bool { - return strings.HasPrefix(contentType, "video/") - } - - maybeSaveImageish := func(i EmbedImageish, contentTypeCheck func(string) bool) (*uuid.UUID, error) { - content, contentType, err := downloadDiscordResource(ctx, *i.Url) - if err != nil { - var statusError DiscordResourceBadStatusCode - if errors.As(err, &statusError) { - return nil, nil - } else { - return nil, oops.New(err, "failed to save Discord embed") - } - } - if contentTypeCheck(contentType) { - in := assets.CreateInput{ - Content: content, - Filename: "embed", - ContentType: contentType, - UploaderID: &hmnUserID, - } - - if i.Width != nil { - in.Width = *i.Width - } - if i.Height != nil { - in.Height = *i.Height - } - - asset, err := assets.Create(ctx, tx, in) - if err != nil { - return nil, oops.New(err, "failed to create asset from embed") - } - return &asset.ID, nil - } - - return nil, nil - } - - var imageAssetId *uuid.UUID - var videoAssetId *uuid.UUID - var err error - - if embed.Video != nil && embed.Video.Url != nil { - videoAssetId, err = maybeSaveImageish(embed.Video.EmbedImageish, isOkVideoType) - } else if embed.Image != nil && embed.Image.Url != nil { - imageAssetId, err = maybeSaveImageish(embed.Image.EmbedImageish, isOkImageType) - } else if embed.Thumbnail != nil && embed.Thumbnail.Url != nil { - imageAssetId, err = maybeSaveImageish(embed.Thumbnail.EmbedImageish, isOkImageType) - } - if err != nil { - return nil, err - } - - // Save the embed into the db - // TODO(db): Insert, RETURNING - var savedEmbedId int - err = tx.QueryRow(ctx, - ` - INSERT INTO handmade_discordmessageembed (title, description, url, message_id, image_id, video_id) - VALUES ($1, $2, $3, $4, $5, $6) - RETURNING id - `, - embed.Title, - embed.Description, - embed.Url, - discordMessageID, - imageAssetId, - videoAssetId, - ).Scan(&savedEmbedId) - if err != nil { - return nil, oops.New(err, "failed to insert new embed") - } - - iDiscordEmbed, err := db.QueryOne(ctx, tx, models.DiscordMessageEmbed{}, - ` - SELECT $columns - FROM handmade_discordmessageembed - WHERE id = $1 - `, - savedEmbedId, - ) - if err != nil { - return nil, oops.New(err, "failed to fetch new Discord embed data") - } - - return iDiscordEmbed.(*models.DiscordMessageEmbed), nil -} - -type DiscordUserAndStuff struct { - DiscordUser models.DiscordUser `db:"duser"` - HMNUser models.User `db:"u"` -} - -func FetchDiscordUser(ctx context.Context, dbConn db.ConnOrTx, discordUserID string) (*DiscordUserAndStuff, error) { - iuser, err := db.QueryOne(ctx, dbConn, DiscordUserAndStuff{}, - ` - SELECT $columns - FROM - handmade_discorduser AS duser - JOIN auth_user AS u ON duser.hmn_user_id = u.id - LEFT JOIN handmade_asset AS u_avatar ON u_avatar.id = u.avatar_asset_id - WHERE - duser.userid = $1 - `, - discordUserID, - ) - if err != nil { - return nil, err - } - return iuser.(*DiscordUserAndStuff), nil -} - -/* -Checks settings and permissions to decide whether we are allowed to create -snippets for a user. -*/ -func AllowedToCreateMessageSnippets(ctx context.Context, tx db.ConnOrTx, discordUserId string) (bool, error) { - u, err := FetchDiscordUser(ctx, tx, discordUserId) - if errors.Is(err, db.NotFound) { - return false, nil - } else if err != nil { - return false, oops.New(err, "failed to check if we can save Discord message") - } - - return u.HMNUser.DiscordSaveShowcase, nil -} - -/* -Attempts to create snippets from Discord messages. If a snippet already exists -for any message, no new snippet will be created. - -It uses the content saved in the database to do this. If we do not have any -content saved, nothing will happen. - -If a user does not have their Discord account linked, this function will -naturally do nothing because we have no message content saved. However, it does -not check any user settings such as automatically creating snippets from -#project-showcase. If we have the content, it will make a snippet for it, no -questions asked. Bear that in mind. -*/ -func CreateMessageSnippets(ctx context.Context, dbConn db.ConnOrTx, msgIDs ...string) error { - tx, err := dbConn.Begin(ctx) - if err != nil { - return oops.New(err, "failed to begin transaction") - } - defer tx.Rollback(ctx) - - for _, msgID := range msgIDs { - // Check for existing snippet - type existingSnippetResult struct { - Message models.DiscordMessage `db:"msg"` - MessageContent *models.DiscordMessageContent `db:"c"` - Snippet *models.Snippet `db:"snippet"` - DiscordUser *models.DiscordUser `db:"duser"` - } - iexisting, err := db.QueryOne(ctx, tx, existingSnippetResult{}, - ` - SELECT $columns - FROM - handmade_discordmessage AS msg - LEFT JOIN handmade_discordmessagecontent AS c ON c.message_id = msg.id - LEFT JOIN handmade_snippet AS snippet ON snippet.discord_message_id = msg.id - LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid - WHERE - msg.id = $1 - `, - msgID, - ) - if err != nil { - return oops.New(err, "failed to check for existing snippet for message %s", msgID) - } - existing := iexisting.(*existingSnippetResult) - - if existing.Snippet != nil { - // A snippet already exists - maybe update its content. - if existing.MessageContent != nil && !existing.Snippet.EditedOnWebsite { - contentMarkdown := existing.MessageContent.LastContent - contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown) - - _, err := tx.Exec(ctx, - ` - UPDATE handmade_snippet - SET - description = $1, - _description_html = $2 - WHERE id = $3 - `, - contentMarkdown, - contentHTML, - existing.Snippet.ID, - ) - if err != nil { - logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to update content of snippet on message edit") - } - continue - } - } - - if existing.Message.SnippetCreated { - // A snippet once existed but no longer does - // (we do not create another one in this case) - return nil - } - - if existing.MessageContent == nil || existing.DiscordUser == nil { - return nil - } - - // Get an asset ID or URL to make a snippet from - assetId, url, err := getSnippetAssetOrUrl(ctx, tx, &existing.Message) - if assetId == nil && url == nil { - // Nothing to make a snippet from! - return nil - } - - contentMarkdown := existing.MessageContent.LastContent - contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown) - - // TODO(db): Insert - _, err = tx.Exec(ctx, - ` - INSERT INTO handmade_snippet (url, "when", description, _description_html, asset_id, discord_message_id, owner_id) - VALUES ($1, $2, $3, $4, $5, $6, $7) - `, - url, - existing.Message.SentAt, - contentMarkdown, - contentHTML, - assetId, - msgID, - existing.DiscordUser.HMNUserId, - ) - if err != nil { - return oops.New(err, "failed to create snippet from attachment") - } - _, err = tx.Exec(ctx, - ` - UPDATE handmade_discordmessage - SET snippet_created = TRUE - WHERE id = $1 - `, - msgID, - ) - if err != nil { - return oops.New(err, "failed to mark message as having snippet") - } - } - - err = tx.Commit(ctx) - if err != nil { - return oops.New(err, "failed to commit transaction") - } - - return nil -} - -/* -Associates any Discord tags with website tags for projects. Idempotent; will -clear out any existing project tags and then add new ones. - -If no Discord user is linked, or no snippet exists, or whatever, this will do -nothing and return no error. -*/ -func UpdateSnippetTagsIfAny(ctx context.Context, dbConn db.ConnOrTx, msg *Message) error { - tx, err := dbConn.Begin(ctx) - if err != nil { - return oops.New(err, "failed to start transaction") - } - defer tx.Rollback(ctx) - - // Fetch the Discord user; we only process messages for users with linked - // Discord accounts - u, err := FetchDiscordUser(ctx, tx, msg.Author.ID) - if err == db.NotFound { - return nil - } else if err != nil { - return oops.New(err, "failed to look up HMN user information from Discord user") - } - - // Fetch the s associated with this Discord message (if any). If the - // s has already been edited on the website we'll skip it. - s, err := hmndata.FetchSnippetForDiscordMessage(ctx, tx, &u.HMNUser, msg.ID, hmndata.SnippetQuery{}) - if err == db.NotFound { - return nil - } else if err != nil { - return err - } - - // Fetch projects so we know what tags the user can apply to their snippet. - projects, err := hmndata.FetchProjects(ctx, tx, &u.HMNUser, hmndata.ProjectsQuery{ - OwnerIDs: []int{u.HMNUser.ID}, - }) - if err != nil { - return oops.New(err, "failed to look up user projects") - } - projectIDs := make([]int, len(projects)) - for i, p := range projects { - projectIDs[i] = p.Project.ID - } - - // Delete any existing project tags for this snippet. We don't want to - // delete other tags in case in the future we have manual tagging on the - // website or whatever, and this would clear those out. - _, err = tx.Exec(ctx, - ` - DELETE FROM snippet_tags - WHERE - snippet_id = $1 - AND tag_id IN ( - SELECT tag FROM handmade_project - ) - `, - s.Snippet.ID, - ) - if err != nil { - return oops.New(err, "failed to delete existing snippet tags") - } - - // Try to associate tags in the message with project tags in HMN. - // Match only tags for projects in which the current user is a collaborator. - messageTags := getDiscordTags(s.Snippet.Description) - type tagsRow struct { - Tag models.Tag `db:"tags"` - } - iUserTags, err := db.Query(ctx, tx, tagsRow{}, - ` - SELECT $columns - FROM - tags - JOIN handmade_project AS project ON project.tag = tags.id - JOIN handmade_user_projects AS user_project ON user_project.project_id = project.id - WHERE - project.id = ANY ($1) - `, - projectIDs, - ) - if err != nil { - return oops.New(err, "failed to fetch tags for user projects") - } - - var tagIDs []int - for _, itag := range iUserTags { - tag := itag.(*tagsRow).Tag - for _, messageTag := range messageTags { - if tag.Text == messageTag { - tagIDs = append(tagIDs, tag.ID) - } - } - } - - for _, tagID := range tagIDs { - _, err = tx.Exec(ctx, - ` - INSERT INTO snippet_tags (snippet_id, tag_id) - VALUES ($1, $2) - ON CONFLICT DO NOTHING - `, - s.Snippet.ID, - tagID, - ) - if err != nil { - return oops.New(err, "failed to add tag to snippet") - } - } - - err = tx.Commit(ctx) - if err != nil { - return oops.New(err, "failed to commit transaction") - } - - return nil -} - -// NOTE(ben): This is maybe redundant with the regexes we use for markdown. But -// do we actually want to reuse those, or should we keep them separate? -var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`) - -func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msg *models.DiscordMessage) (*uuid.UUID, *string, error) { - // Check attachments - attachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{}, - ` - SELECT $columns - FROM handmade_discordmessageattachment - WHERE message_id = $1 - `, - msg.ID, - ) - if err != nil { - return nil, nil, oops.New(err, "failed to fetch message attachments") - } - for _, iattachment := range attachments { - attachment := iattachment.(*models.DiscordMessageAttachment) - return &attachment.AssetID, nil, nil - } - - // Check embeds - embeds, err := db.Query(ctx, tx, models.DiscordMessageEmbed{}, - ` - SELECT $columns - FROM handmade_discordmessageembed - WHERE message_id = $1 - `, - msg.ID, - ) - if err != nil { - return nil, nil, oops.New(err, "failed to fetch discord embeds") - } - for _, iembed := range embeds { - embed := iembed.(*models.DiscordMessageEmbed) - if embed.VideoID != nil { - return embed.VideoID, nil, nil - } else if embed.ImageID != nil { - return embed.ImageID, nil, nil - } else if embed.URL != nil { - if RESnippetableUrl.MatchString(*embed.URL) { - return nil, embed.URL, nil - } - } - } - - return nil, nil, nil -} - -func messageHasLinks(content string) bool { - links := reDiscordMessageLink.FindAllString(content, -1) - for _, link := range links { - _, err := url.Parse(strings.TrimSpace(link)) - if err == nil { - return true - } - } - - return false -} - -var REDiscordTag = regexp.MustCompile(`&([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*)`) - -func getDiscordTags(content string) []string { - matches := REDiscordTag.FindAllStringSubmatch(content, -1) - result := make([]string, len(matches)) - for i, m := range matches { - result[i] = strings.ToLower(m[1]) - } - return result -} diff --git a/src/website/discord.go b/src/website/discord.go index 32bf0c8..a5f774e 100644 --- a/src/website/discord.go +++ b/src/website/discord.go @@ -143,17 +143,6 @@ func DiscordShowcaseBacklog(c *RequestContext) ResponseData { } duser := iduser.(*models.DiscordUser) - ok, err := discord.AllowedToCreateMessageSnippets(c.Context(), c.Conn, duser.UserID) - if err != nil { - return c.ErrorResponse(http.StatusInternalServerError, err) - } - - if !ok { - // Not allowed to do this, bail out - c.Logger.Warn().Msg("was not allowed to save user snippets") - return c.Redirect(hmnurl.BuildUserProfile(c.CurrentUser.Username), http.StatusSeeOther) - } - type messageIdQuery struct { MessageID string `db:"msg.id"` } @@ -177,9 +166,18 @@ func DiscordShowcaseBacklog(c *RequestContext) ResponseData { for _, imsgId := range iMsgIDs { msgIDs = append(msgIDs, imsgId.(*messageIdQuery).MessageID) } - err = discord.CreateMessageSnippets(c.Context(), c.Conn, msgIDs...) - if err != nil { - return c.ErrorResponse(http.StatusInternalServerError, err) + for _, msgID := range msgIDs { + interned, err := discord.FetchInternedMessage(c.Context(), c.Conn, msgID) + if err != nil { + return c.ErrorResponse(http.StatusInternalServerError, err) + } + if interned != nil { + // NOTE(asaf): Creating snippet even if the checkbox is off because the user asked us to. + err = discord.HandleSnippetForInternedMessage(c.Context(), c.Conn, interned, true) + if err != nil { + return c.ErrorResponse(http.StatusInternalServerError, err) + } + } } return c.Redirect(hmnurl.BuildUserProfile(c.CurrentUser.Username), http.StatusSeeOther)