package discord import ( "context" "errors" "fmt" "io" "net/http" "net/url" "regexp" "strings" "time" "git.handmade.network/hmn/hmn/src/assets" "git.handmade.network/hmn/hmn/src/config" "git.handmade.network/hmn/hmn/src/db" "git.handmade.network/hmn/hmn/src/hmndata" "git.handmade.network/hmn/hmn/src/logging" "git.handmade.network/hmn/hmn/src/models" "git.handmade.network/hmn/hmn/src/oops" "git.handmade.network/hmn/hmn/src/parsing" "github.com/google/uuid" ) var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`) var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this") func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) error { switch msg.Type { case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand: default: return nil } didDelete, err := bot.maybeDeleteShowcaseMsg(ctx, msg) if err != nil { return err } if didDelete { return nil } tx, err := bot.dbConn.Begin(ctx) if err != nil { panic(err) } defer tx.Rollback(ctx) // save the message, maybe save its contents, and maybe make a snippet too newMsg, err := SaveMessageAndContents(ctx, tx, msg) if errors.Is(err, errNotEnoughInfo) { logging.ExtractLogger(ctx).Warn(). Interface("msg", msg). Msg("didn't have enough info to process Discord message") return nil } else if err != nil { return err } if doSnippet, err := AllowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil { _, err := CreateMessageSnippet(ctx, tx, newMsg.UserID, msg.ID) if err != nil { return oops.New(err, "failed to create snippet in gateway") } } else if err != nil { return oops.New(err, "failed to check snippet permissions in gateway") } err = tx.Commit(ctx) if err != nil { return oops.New(err, "failed to commit Discord message updates") } return nil } func (bot *botInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message) (didDelete bool, err error) { hasGoodContent := true if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) { hasGoodContent = false } hasGoodAttachments := true if msg.OriginalHasFields("attachments") && len(msg.Attachments) == 0 { hasGoodAttachments = false } didDelete = false if !hasGoodContent && !hasGoodAttachments { didDelete = true err := DeleteMessage(ctx, msg.ChannelID, msg.ID) if err != nil { return false, oops.New(err, "failed to delete message") } if !msg.Author.IsBot { channel, err := CreateDM(ctx, msg.Author.ID) if err != nil { return false, oops.New(err, "failed to create DM channel") } err = SendMessages(ctx, bot.dbConn, MessageToSend{ ChannelID: channel.ID, Req: CreateMessageRequest{ Content: "Posts in #project-showcase are required to have either an image/video or a link. Discuss showcase content in #projects.", }, }) if err != nil { return false, oops.New(err, "failed to send showcase warning message") } } } return didDelete, nil } /* Ensures that a Discord message is stored in the database. This function is idempotent and can be called regardless of whether the item already exists in the database. This does not create snippets or do anything besides save the message itself. */ func SaveMessage( ctx context.Context, tx db.ConnOrTx, msg *Message, ) (*models.DiscordMessage, error) { iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{}, ` SELECT $columns FROM handmade_discordmessage WHERE id = $1 `, msg.ID, ) if errors.Is(err, db.NotFound) { if !msg.OriginalHasFields("author", "timestamp") { return nil, errNotEnoughInfo } guildID := msg.GuildID if guildID == nil { /* This is weird, but it can happen when we fetch messages from history instead of receiving it from the gateway. In this case we just assume it's from the HMN server. */ guildID = &config.Config.Discord.GuildID } _, err = tx.Exec(ctx, ` INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created) VALUES ($1, $2, $3, $4, $5, $6, $7) `, msg.ID, msg.ChannelID, *guildID, msg.JumpURL(), msg.Author.ID, msg.Time(), false, ) if err != nil { return nil, oops.New(err, "failed to save new discord message") } /* TODO(db): This is a spot where it would be really nice to be able to use RETURNING, and avoid this second query. */ iDiscordMessage, err = db.QueryOne(ctx, tx, models.DiscordMessage{}, ` SELECT $columns FROM handmade_discordmessage WHERE id = $1 `, msg.ID, ) if err != nil { panic(err) } } else if err != nil { return nil, oops.New(err, "failed to check for existing Discord message") } return iDiscordMessage.(*models.DiscordMessage), nil } /* Processes a single Discord message, saving as much of the message's content and attachments as allowed by our rules and user settings. Does NOT create snippets. Idempotent; can be called any time whether the message exists or not. */ func SaveMessageAndContents( ctx context.Context, tx db.ConnOrTx, msg *Message, ) (*models.DiscordMessage, error) { newMsg, err := SaveMessage(ctx, tx, msg) if err != nil { return nil, err } // Check for linked Discord user iDiscordUser, err := db.QueryOne(ctx, tx, models.DiscordUser{}, ` SELECT $columns FROM handmade_discorduser WHERE userid = $1 `, newMsg.UserID, ) if errors.Is(err, db.NotFound) { return newMsg, nil } else if err != nil { return nil, oops.New(err, "failed to look up linked Discord user") } discordUser := iDiscordUser.(*models.DiscordUser) // We have a linked Discord account, so save the message contents (regardless of // whether we create a snippet or not). if msg.OriginalHasFields("content") { _, err = tx.Exec(ctx, ` INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content) VALUES ($1, $2, $3) ON CONFLICT (message_id) DO UPDATE SET discord_id = EXCLUDED.discord_id, last_content = EXCLUDED.last_content `, newMsg.ID, discordUser.ID, CleanUpMarkdown(ctx, msg.Content), ) } // Save attachments if msg.OriginalHasFields("attachments") { for _, attachment := range msg.Attachments { _, err := saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID) if err != nil { return nil, oops.New(err, "failed to save attachment") } } } // Save / delete embeds if msg.OriginalHasFields("embeds") { numSavedEmbeds, err := db.QueryInt(ctx, tx, ` SELECT COUNT(*) FROM handmade_discordmessageembed WHERE message_id = $1 `, msg.ID, ) if err != nil { return nil, oops.New(err, "failed to count existing embeds") } if numSavedEmbeds == 0 { // No embeds yet, so save new ones for _, embed := range msg.Embeds { _, err := saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID) if err != nil { return nil, oops.New(err, "failed to save embed") } } } else if len(msg.Embeds) > 0 { // Embeds were removed from the message _, err := tx.Exec(ctx, ` DELETE FROM handmade_discordmessageembed WHERE message_id = $1 `, msg.ID, ) if err != nil { return nil, oops.New(err, "failed to delete embeds") } } } return newMsg, nil } var discordDownloadClient = &http.Client{ Timeout: 10 * time.Second, } type DiscordResourceBadStatusCode error func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, "", oops.New(err, "failed to make Discord download request") } res, err := discordDownloadClient.Do(req) if err != nil { return nil, "", oops.New(err, "failed to fetch Discord resource data") } defer res.Body.Close() if res.StatusCode < 200 || 299 < res.StatusCode { return nil, "", DiscordResourceBadStatusCode(fmt.Errorf("status code %d from Discord resource: %s", res.StatusCode, url)) } content, err := io.ReadAll(res.Body) if err != nil { panic(err) } return content, res.Header.Get("Content-Type"), nil } /* Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment that already exists */ func saveAttachment( ctx context.Context, tx db.ConnOrTx, attachment *Attachment, hmnUserID int, discordMessageID string, ) (*models.DiscordMessageAttachment, error) { iexisting, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{}, ` SELECT $columns FROM handmade_discordmessageattachment WHERE id = $1 `, attachment.ID, ) if err == nil { return iexisting.(*models.DiscordMessageAttachment), nil } else if errors.Is(err, db.NotFound) { // this is fine, just create it } else { return nil, oops.New(err, "failed to check for existing attachment") } width := 0 height := 0 if attachment.Width != nil { width = *attachment.Width } if attachment.Height != nil { height = *attachment.Height } content, _, err := downloadDiscordResource(ctx, attachment.Url) if err != nil { return nil, oops.New(err, "failed to download Discord attachment") } contentType := "application/octet-stream" if attachment.ContentType != nil { contentType = *attachment.ContentType } asset, err := assets.Create(ctx, tx, assets.CreateInput{ Content: content, Filename: attachment.Filename, ContentType: contentType, UploaderID: &hmnUserID, Width: width, Height: height, }) if err != nil { return nil, oops.New(err, "failed to save asset for Discord attachment") } // TODO(db): RETURNING plz thanks _, err = tx.Exec(ctx, ` INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id) VALUES ($1, $2, $3) `, attachment.ID, asset.ID, discordMessageID, ) if err != nil { return nil, oops.New(err, "failed to save Discord attachment data") } iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{}, ` SELECT $columns FROM handmade_discordmessageattachment WHERE id = $1 `, attachment.ID, ) if err != nil { return nil, oops.New(err, "failed to fetch new Discord attachment data") } return iDiscordAttachment.(*models.DiscordMessageAttachment), nil } // Saves an embed from Discord. NOTE: This is _not_ idempotent, so only call it // if you do not have any embeds saved for this message yet. func saveEmbed( ctx context.Context, tx db.ConnOrTx, embed *Embed, hmnUserID int, discordMessageID string, ) (*models.DiscordMessageEmbed, error) { isOkImageType := func(contentType string) bool { return strings.HasPrefix(contentType, "image/") } isOkVideoType := func(contentType string) bool { return strings.HasPrefix(contentType, "video/") } maybeSaveImageish := func(i EmbedImageish, contentTypeCheck func(string) bool) (*uuid.UUID, error) { content, contentType, err := downloadDiscordResource(ctx, *i.Url) if err != nil { var statusError DiscordResourceBadStatusCode if errors.As(err, &statusError) { return nil, nil } else { return nil, oops.New(err, "failed to save Discord embed") } } if contentTypeCheck(contentType) { in := assets.CreateInput{ Content: content, Filename: "embed", ContentType: contentType, UploaderID: &hmnUserID, } if i.Width != nil { in.Width = *i.Width } if i.Height != nil { in.Height = *i.Height } asset, err := assets.Create(ctx, tx, in) if err != nil { return nil, oops.New(err, "failed to create asset from embed") } return &asset.ID, nil } return nil, nil } var imageAssetId *uuid.UUID var videoAssetId *uuid.UUID var err error if embed.Video != nil && embed.Video.Url != nil { videoAssetId, err = maybeSaveImageish(embed.Video.EmbedImageish, isOkVideoType) } else if embed.Image != nil && embed.Image.Url != nil { imageAssetId, err = maybeSaveImageish(embed.Image.EmbedImageish, isOkImageType) } else if embed.Thumbnail != nil && embed.Thumbnail.Url != nil { imageAssetId, err = maybeSaveImageish(embed.Thumbnail.EmbedImageish, isOkImageType) } if err != nil { return nil, err } // Save the embed into the db // TODO(db): Insert, RETURNING var savedEmbedId int err = tx.QueryRow(ctx, ` INSERT INTO handmade_discordmessageembed (title, description, url, message_id, image_id, video_id) VALUES ($1, $2, $3, $4, $5, $6) RETURNING id `, embed.Title, embed.Description, embed.Url, discordMessageID, imageAssetId, videoAssetId, ).Scan(&savedEmbedId) if err != nil { return nil, oops.New(err, "failed to insert new embed") } iDiscordEmbed, err := db.QueryOne(ctx, tx, models.DiscordMessageEmbed{}, ` SELECT $columns FROM handmade_discordmessageembed WHERE id = $1 `, savedEmbedId, ) if err != nil { return nil, oops.New(err, "failed to fetch new Discord embed data") } return iDiscordEmbed.(*models.DiscordMessageEmbed), nil } type DiscordUserAndStuff struct { DiscordUser models.DiscordUser `db:"duser"` HMNUser models.User `db:"u"` } func FetchDiscordUser(ctx context.Context, dbConn db.ConnOrTx, discordUserID string) (*DiscordUserAndStuff, error) { iuser, err := db.QueryOne(ctx, dbConn, DiscordUserAndStuff{}, ` SELECT $columns FROM handmade_discorduser AS duser JOIN auth_user AS u ON duser.hmn_user_id = u.id WHERE duser.userid = $1 `, discordUserID, ) if err != nil { return nil, err } return iuser.(*DiscordUserAndStuff), nil } /* Checks settings and permissions to decide whether we are allowed to create snippets for a user. */ func AllowedToCreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, discordUserId string) (bool, error) { u, err := FetchDiscordUser(ctx, tx, discordUserId) if errors.Is(err, db.NotFound) { return false, nil } else if err != nil { return false, oops.New(err, "failed to check if we can save Discord message") } return u.HMNUser.DiscordSaveShowcase, nil } /* Attempts to create a snippet from a Discord message. If a snippet already exists, it will be returned and no new snippets will be created. It uses the content saved in the database to do this. If we do not have any content saved, nothing will happen. Does not check user preferences around snippets. */ func CreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, userID, msgID string) (snippet *models.Snippet, err error) { defer func() { err := updateSnippetTags(ctx, tx, userID, snippet) if err != nil { logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to update tags for Discord snippet") } }() // Check for existing snippet, maybe return it type existingSnippetResult struct { Message models.DiscordMessage `db:"msg"` MessageContent *models.DiscordMessageContent `db:"c"` Snippet *models.Snippet `db:"snippet"` DiscordUser *models.DiscordUser `db:"duser"` } iexisting, err := db.QueryOne(ctx, tx, existingSnippetResult{}, ` SELECT $columns FROM handmade_discordmessage AS msg LEFT JOIN handmade_discordmessagecontent AS c ON c.message_id = msg.id LEFT JOIN handmade_snippet AS snippet ON snippet.discord_message_id = msg.id LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid WHERE msg.id = $1 `, msgID, ) if err != nil { return nil, oops.New(err, "failed to check for existing snippet for message %s", msgID) } existing := iexisting.(*existingSnippetResult) if existing.Snippet != nil { // A snippet already exists - maybe update its content, then return it if existing.MessageContent != nil && !existing.Snippet.EditedOnWebsite { contentMarkdown := existing.MessageContent.LastContent contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown) iSnippet, err := db.QueryOne(ctx, tx, models.Snippet{}, ` UPDATE handmade_snippet SET description = $1, _description_html = $2 WHERE id = $3 RETURNING $columns `, contentMarkdown, contentHTML, existing.Snippet.ID, ) if err != nil { logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to update content of snippet on message edit") } return iSnippet.(*models.Snippet), nil } else { return existing.Snippet, nil } } if existing.Message.SnippetCreated { // A snippet once existed but no longer does // (we do not create another one in this case) return nil, nil } if existing.MessageContent == nil || existing.DiscordUser == nil { return nil, nil } // Get an asset ID or URL to make a snippet from assetId, url, err := getSnippetAssetOrUrl(ctx, tx, &existing.Message) if assetId == nil && url == nil { // Nothing to make a snippet from! return nil, nil } contentMarkdown := existing.MessageContent.LastContent contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown) // TODO(db): Insert isnippet, err := db.QueryOne(ctx, tx, models.Snippet{}, ` INSERT INTO handmade_snippet (url, "when", description, _description_html, asset_id, discord_message_id, owner_id) VALUES ($1, $2, $3, $4, $5, $6, $7) RETURNING $columns `, url, existing.Message.SentAt, contentMarkdown, contentHTML, assetId, msgID, existing.DiscordUser.HMNUserId, ) if err != nil { return nil, oops.New(err, "failed to create snippet from attachment") } _, err = tx.Exec(ctx, ` UPDATE handmade_discordmessage SET snippet_created = TRUE WHERE id = $1 `, msgID, ) if err != nil { return nil, oops.New(err, "failed to mark message as having snippet") } return isnippet.(*models.Snippet), nil } /* Associates any Discord tags with website tags. Idempotent; will clear out any existing tags and then add new ones. */ func updateSnippetTags(ctx context.Context, dbConn db.ConnOrTx, userID string, snippet *models.Snippet) error { tx, err := dbConn.Begin(ctx) if err != nil { return oops.New(err, "failed to start transaction") } defer tx.Rollback(ctx) u, err := FetchDiscordUser(ctx, tx, userID) if err != nil { return oops.New(err, "failed to look up HMN user information from Discord user") // we shouldn't see a "not found" here because of the AllowedToBlahBlahBlah check earlier in the process } projects, err := hmndata.FetchProjects(ctx, tx, &u.HMNUser, hmndata.ProjectsQuery{ OwnerIDs: []int{u.HMNUser.ID}, }) if err != nil { return oops.New(err, "failed to look up user projects") } projectIDs := make([]int, len(projects)) for i, p := range projects { projectIDs[i] = p.Project.ID } // Delete any existing tags for this snippet _, err = tx.Exec(ctx, ` DELETE FROM snippet_tags WHERE snippet_id = $1 `, snippet.ID, ) if err != nil { return oops.New(err, "failed to delete existing snippet tags") } // Try to associate tags in the message with project tags in HMN. // Match only tags for projects in which the current user is a collaborator. messageTags := getDiscordTags(snippet.Description) type tagsRow struct { Tag models.Tag `db:"tags"` } itUserTags, err := db.Query(ctx, tx, tagsRow{}, ` SELECT $columns FROM tags JOIN handmade_project AS project ON project.tag = tags.id JOIN handmade_user_projects AS user_project ON user_project.project_id = project.id WHERE project.id = ANY ($1) `, projectIDs, ) if err != nil { return oops.New(err, "failed to fetch tags for user projects") } iUserTags := itUserTags.ToSlice() var tagIDs []int for _, itag := range iUserTags { tag := itag.(*tagsRow).Tag for _, messageTag := range messageTags { if tag.Text == messageTag { tagIDs = append(tagIDs, tag.ID) } } } for _, tagID := range tagIDs { _, err = tx.Exec(ctx, `INSERT INTO snippet_tags (snippet_id, tag_id) VALUES ($1, $2)`, snippet.ID, tagID) if err != nil { return oops.New(err, "failed to add tag to snippet") } } err = tx.Commit(ctx) if err != nil { return oops.New(err, "failed to commit transaction") } return nil } // NOTE(ben): This is maybe redundant with the regexes we use for markdown. But // do we actually want to reuse those, or should we keep them separate? var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`) func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msg *models.DiscordMessage) (*uuid.UUID, *string, error) { // Check attachments itAttachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{}, ` SELECT $columns FROM handmade_discordmessageattachment WHERE message_id = $1 `, msg.ID, ) if err != nil { return nil, nil, oops.New(err, "failed to fetch message attachments") } attachments := itAttachments.ToSlice() for _, iattachment := range attachments { attachment := iattachment.(*models.DiscordMessageAttachment) return &attachment.AssetID, nil, nil } // Check embeds itEmbeds, err := db.Query(ctx, tx, models.DiscordMessageEmbed{}, ` SELECT $columns FROM handmade_discordmessageembed WHERE message_id = $1 `, msg.ID, ) if err != nil { return nil, nil, oops.New(err, "failed to fetch discord embeds") } embeds := itEmbeds.ToSlice() for _, iembed := range embeds { embed := iembed.(*models.DiscordMessageEmbed) if embed.VideoID != nil { return embed.VideoID, nil, nil } else if embed.ImageID != nil { return embed.ImageID, nil, nil } else if embed.URL != nil { if RESnippetableUrl.MatchString(*embed.URL) { return nil, embed.URL, nil } } } return nil, nil, nil } func messageHasLinks(content string) bool { links := reDiscordMessageLink.FindAllString(content, -1) for _, link := range links { _, err := url.Parse(strings.TrimSpace(link)) if err == nil { return true } } return false } var REDiscordTag = regexp.MustCompile(`&([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*)`) func getDiscordTags(content string) []string { matches := REDiscordTag.FindAllStringSubmatch(content, -1) result := make([]string, len(matches)) for i, m := range matches { result[i] = strings.ToLower(m[1]) } return result }