WIP. Rethinking things. See the README

This commit is contained in:
Ben Visness 2021-12-23 17:28:40 -06:00
parent f8e7779b7d
commit 8c8fdcce3f
14 changed files with 1045 additions and 606 deletions

View File

@ -438,3 +438,35 @@ func QueryBool(ctx context.Context, conn ConnOrTx, query string, args ...interfa
return false, oops.New(nil, "QueryBool got a non-bool result: %v", result)
}
}
// Begins a transaction, runs the provided function, and then commits the
// transaction. If the provided function returns an error, the transaction will
// be rolled back, and the original error will be returned.
func DoWithTransaction(ctx context.Context, conn ConnOrTx, f func(tx pgx.Tx) error) error {
tx, err := conn.Begin(ctx)
if err != nil {
return oops.New(err, "failed to begin transaction")
}
defer func() {
err := tx.Rollback(ctx)
if err != nil {
// TODO: Maybe in the future we could have a "multi-error" so we
// can actually return this error instead of having to log it.
logging.ExtractLogger(ctx).Error().
Err(err).
Msg("failed to roll back transaction")
}
}()
err = f(tx)
if err != nil {
return err
}
err = tx.Commit(ctx)
if err != nil {
return oops.New(err, "failed to commit transaction")
}
return nil
}

168
src/discord/README.md Normal file
View File

@ -0,0 +1,168 @@
# Discord is crazy
This document describes the entire process of persisting Discord messages and handling:
- showcase messages
- library messages
- snippets
- backfills
- admin commands
- who even knows what else
If you want to just hop in and make a quick change to the Discord logic, don't. Not without reading this first, and in particular not without going through the user stories.
## Constraints and design guidelines
We must never save message content for anyone without a linked Discord account.
We always have to respect rate limits, heartbeats, other gateway nuances.
Actions on Discord should never affect the website if the Discord account is not linked to an HMN profile.
Nothing should be added to a user's HMN profile without explicit user action. For example, linking your account should not result in snippets being automatically created from old #project-showcase posts.
We shouldn't assume that just because we have a message (or its contents) stored, we want to create a snippet from it. Sometimes messages could be saved for other reasons.
## All the logic
On new/updated message:
- if in a showcase channel and it doesn't fit the format, delete and return
- if in a library channel and it doesn't fit the format, delete and return
- if we care about the content of this message (see below):
- persist mirror of message data (incl. content if allowed)
- if persisting content, clean up message markdown
- if user has a linked Discord account:
- if this message is snippet-able (see below):
- if snippet exists (in any channel):
- update snippet
- if no snippet exists:
- check conditions for a new snippet:
- the flag for snippet creation is set
- any of the following is true:
- in "showcase" channel, and the user has the setting enabled
- an admin wants a snippet here
- we have message content
- SnippetCreated is false on the message
- if conditions are met, create snippet
- if this message is not snippet-able (see below):
- delete any existing snippets that are connected to Discord (huzzah for edge cases)
On deleted message:
- delete snippets, if the following conditions are true:
- snippet is connected to Discord
- the author has a Discord account linked
- the author chooses not to keep captured snippets on Discord deletes (profile setting)
- delete mirror of message data
On Discord account unlink:
- DO NOT disconnect snippets from Discord.
- There's no reason to.
We care about the content of the message if:
- we already have content for this message, or
- it is in a showcase channel or whatever, or
- an admin said so (CLI command)
A message is snippet-able if:
- It contains enough information to make a snippet. Namely:
- it has media
- Things that DO NOT affect snippetability:
- whether a Discord account is linked
- the user's settings for automatic snippet creation
- other permissions or whatever
### Things we do
These actions all fall into one of the two flows above - the create/update flow, or the delete flow. Some may require flags to modify the main flow behavior slightly.
Stuff for the create/update flow:
- receive message creates/updates in real time
- general backfill (1 hour) (exactly the same behavior as real-time)
- new user backfill (5 seconds) (message content only, no snippets)
- make snippets from #project-showcase posts (exactly the same behavior as real-time)
- admin scrapechannel (fetch messages and content in arbitrary channels, no snippets)
- admin makesnippet
Stuff for the delete flow:
- receive message deletes in real time
- IN THE FUTURE: delete messages in history backfill
## ✨ :D user story time :D ✨
Whenever touching the logic, consider all of these lovely edge cases!
- bad showcase post
- expected: post is immediately deleted
- bad library post
- expected: post is immediately deleted
- good showcase post from an unlinked user
- expected: lightweight message record is created, but no snippet is created
- good showcase post, then the user unlinks their discord account
- expected: the snippet remains on the user's profile
- good showcase post, then the user unlinks their discord account, then edits the message
- expected: the message edit has no effect on the snippet
- good showcase post, then the user unlinks their discord account, then deletes the message
- expected: the message delete has no effect on the snippet
- good showcase post, then the user unlinks and re-links their discord account, then edits the showcase post
- expected: the snippet remained connected to the discord message, and is updated
- good showcase post, then the user unlinks their discord account and links a different one
- expected: nothing happens to old snippets; new snippets get added as usual
- good showcase post, then the user unlinks their discord account, then their discord account is linked to a different website profile
- expected: snippets may be re-created on the new HMN profile
- good showcase post, then the user unlinks their discord account, then we run a backfill on that specific message
- expected: nothing happens because there is no discord account linked for that message
- good showcase post gets edited to be bad
- expected: the message and associated snippet are deleted
- good library post gets edited to be bad
- expected: the message is deleted
- good showcase post gets edited and is still good
- expected: the message record and associated snippet are updated
- good library post gets edited and is still good
- expected: nothing happens
- Ryan posts like 50 things in #projects, and we want them to be on his profile
- expected: an admin can run `admin makesnippet` on all the messages, and they will be added to Ryan's profile
- Ryan edits his old posts in #projects
- same as: someone edits their posts in #jam-showcase
- expected: any associated snippets are updated, even though the edit was not in a showcase channel
- Ryan edits an old post in #projects to no longer have a link
- expected: the associated snippet is deleted, but the message persists.
- Ryan edits an old post in #projects to no longer have a link, then edits it again to have a link again.
- expected: the associated snippet is deleted, and not reinstated on the second edit. (oh well!)
- an admin attempts to make a snippet for someone without a linked discord account
- expected: nothing happens
- an admin attempts to make a snippet for a message without media
- expected: nothing happens
- good showcase post is deleted
- expected: any snippets are deleted, if the user wishes.
- good showcase post, snippet is deleted from website, post is edited
- expected: no new snippet is created
- good showcase post, snippet is deleted from website, backfill runs
- expected: no new snippet is created
- good showcase post while the user has the showcase integration profile setting disabled, then they enable the showcase integration, then a backfill runs
- expected: no new snippets are created for old showcase posts
- good showcase post while the user has the integration disabled, then they enable the integration, then they explicitly create snippets from their showcase posts
- expected: all user messages in showcase without snippets get snippets (subject to SnippetCreated)

View File

@ -548,14 +548,14 @@ func (bot *botInstance) processEventMsg(ctx context.Context, msg *GatewayMessage
case "MESSAGE_CREATE":
newMessage := *MessageFromMap(msg.Data, "")
err := bot.messageCreateOrUpdate(ctx, &newMessage)
err := bot.gatewayMessageCreateOrUpdate(ctx, &newMessage)
if err != nil {
return oops.New(err, "error on new message")
}
case "MESSAGE_UPDATE":
newMessage := *MessageFromMap(msg.Data, "")
err := bot.messageCreateOrUpdate(ctx, &newMessage)
err := bot.gatewayMessageCreateOrUpdate(ctx, &newMessage)
if err != nil {
return oops.New(err, "error on updated message")
}
@ -585,76 +585,35 @@ func (bot *botInstance) processEventMsg(ctx context.Context, msg *GatewayMessage
}
// Only return an error if we want to restart the bot.
func (bot *botInstance) messageCreateOrUpdate(ctx context.Context, msg *Message) error {
if msg.OriginalHasFields("author") && msg.Author.ID == config.Config.Discord.BotUserID {
// Don't process your own messages
return nil
}
func (bot *botInstance) gatewayMessageCreateOrUpdate(ctx context.Context, msg *Message) error {
log := logging.ExtractLogger(ctx).With().
Str("msgID", msg.ID).
Str("channelID", msg.ChannelID).
Logger()
ctx = logging.AttachLoggerToContext(&log, ctx)
if msg.ChannelID == config.Config.Discord.ShowcaseChannelID {
err := bot.processShowcaseMsg(ctx, msg)
if err != nil {
logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to process showcase message")
return nil
}
err := MessageCreatedOrUpdated(ctx, bot.dbConn, msg, 0)
if errors.Is(err, context.Canceled) {
log.Warn().Err(err).Msg("context canceled when handling new Discord message")
return nil
}
if msg.ChannelID == config.Config.Discord.LibraryChannelID {
err := bot.processLibraryMsg(ctx, msg)
if err != nil {
logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to process library message")
return nil
}
} else if err != nil {
log.Error().Err(err).Msg("error while handling new Discord message")
return nil
}
err := UpdateSnippetTagsIfAny(ctx, bot.dbConn, msg)
if err != nil {
logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to update tags for Discord snippet")
}
return nil
}
// TODO: Fold this into MessageDeleted and get rid of it
func (bot *botInstance) messageDelete(ctx context.Context, msgDelete MessageDelete) {
log := logging.ExtractLogger(ctx)
tx, err := bot.dbConn.Begin(ctx)
if err != nil {
log.Error().Err(err).Msg("failed to start transaction")
return
}
defer tx.Rollback(ctx)
type deleteMessageQuery struct {
Message models.DiscordMessage `db:"msg"`
DiscordUser *models.DiscordUser `db:"duser"`
HMNUser *models.User `db:"hmnuser"`
SnippetID *int `db:"snippet.id"`
}
iresult, err := db.QueryOne(ctx, tx, deleteMessageQuery{},
`
SELECT $columns
FROM
handmade_discordmessage AS msg
LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid
LEFT JOIN auth_user AS hmnuser ON duser.hmn_user_id = hmnuser.id
LEFT JOIN handmade_snippet AS snippet ON snippet.discord_message_id = msg.id
WHERE msg.id = $1 AND msg.channel_id = $2
`,
msgDelete.ID, msgDelete.ChannelID,
)
if errors.Is(err, db.NotFound) {
return
} else if err != nil {
log.Error().Err(err).Msg("failed to check for message to delete")
return
}
result := iresult.(*deleteMessageQuery)
log := logging.ExtractLogger(ctx).With().
Str("msgID", msgDelete.ID).
Str("channelID", msgDelete.ChannelID).
Logger()
ctx = logging.AttachLoggerToContext(&log, ctx)
log.Debug().Msg("deleting Discord message")
_, err = tx.Exec(ctx,
_, err := bot.dbConn.Exec(ctx,
`
DELETE FROM handmade_discordmessage
WHERE id = $1 AND channel_id = $2
@ -662,30 +621,8 @@ func (bot *botInstance) messageDelete(ctx context.Context, msgDelete MessageDele
msgDelete.ID,
msgDelete.ChannelID,
)
shouldDeleteSnippet := result.HMNUser != nil && result.HMNUser.DiscordDeleteSnippetOnMessageDelete
if result.SnippetID != nil && shouldDeleteSnippet {
log.Debug().
Int("snippet_id", *result.SnippetID).
Int("user_id", result.HMNUser.ID).
Msg("deleting snippet from Discord message")
_, err = tx.Exec(ctx,
`
DELETE FROM handmade_snippet
WHERE id = $1
`,
result.SnippetID,
)
if err != nil {
log.Error().Err(err).Msg("failed to delete snippet")
return
}
}
err = tx.Commit(ctx)
if err != nil {
log.Error().Err(err).Msg("failed to delete Discord message")
return
logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to delete Discord message")
}
}
@ -697,10 +634,10 @@ type MessageToSend struct {
func SendMessages(
ctx context.Context,
conn *pgxpool.Pool,
dbConn db.ConnOrTx,
msgs ...MessageToSend,
) error {
tx, err := conn.Begin(ctx)
tx, err := dbConn.Begin(ctx)
if err != nil {
return oops.New(err, "failed to start transaction")
}

View File

@ -193,7 +193,7 @@ func handleHistoryMessage(ctx context.Context, dbConn *pgxpool.Pool, msg *Messag
break
}
newMsg, err := SaveMessageAndContents(ctx, tx, msg)
newMsg, err := PersistMessage(ctx, tx, msg, true)
if err != nil {
return err
}

210
src/discord/integration.go Normal file
View File

@ -0,0 +1,210 @@
package discord
import (
"context"
"errors"
"net/url"
"strings"
"git.handmade.network/hmn/hmn/src/config"
"git.handmade.network/hmn/hmn/src/db"
"git.handmade.network/hmn/hmn/src/models"
"git.handmade.network/hmn/hmn/src/oops"
"github.com/jackc/pgx/v4"
)
//
// READ THE README!
// (./README.md)
//
type MessageCreatedOrUpdatedFlags int
const (
// If set, snippet creation will generally be enabled. We will attempt to
// make snippets for new messages shared in #project-showcase or whatever.
//
// If not set, snippet updates and deletes will still occur where
// appropriate. But new snippets will never be created.
CreateSnippetsOnNewMessages MessageCreatedOrUpdatedFlags = 1 << iota
// Create a snippet even if a message is not in the usual channels. This
// will NOT, in fact, create a snippet if the message doesn't meet the
// conditions - e.g. the message has media, and the user has a linked
// Discord account.
//
// Still requires CreateSnippetsOnNewMessages to be set as well.
ForceSnippet
)
// Call whenever you have a new or updated Discord message, in any channel.
// For example, when:
// - You receive a message create/update event from the gateway
// - You see a new message in a history backfill
// - An admin command wants to make a snippet from some arbitrary message
//
// This implements the "on new/updated message" flow in the readme.
func MessageCreatedOrUpdated(
ctx context.Context,
dbConn db.ConnOrTx,
msg *Message,
flags MessageCreatedOrUpdatedFlags,
) error {
return db.DoWithTransaction(ctx, dbConn, func(tx pgx.Tx) error {
if msg.OriginalHasFields("author") && msg.Author.ID == config.Config.Discord.BotUserID {
// Don't process messages from the bot
return nil
}
didDelete, err := maybeDeleteShowcaseMsg(ctx, dbConn, msg)
if err != nil {
return err
} else if didDelete {
return nil
}
didDelete, err = maybeDeleteLibraryMsg(ctx, dbConn, msg)
if err != nil {
return err
} else if didDelete {
return nil
}
weCareAboutThisMessagesContents := false ||
flags&ForceSnippet != 0 ||
msg.ChannelID == config.Config.Discord.ShowcaseChannelID ||
weAlreadyHaveMessageContent(ctx, dbConn, msg)
if weCareAboutThisMessagesContents {
_, err := PersistMessage(ctx, dbConn, msg)
if err != nil {
return oops.New(err, "failed to persist new/updated Discord message")
}
}
//
// It's snippet time.
//
{
// Fetch the data we need to evaluate what to do from here.
type savedMessageAndSnippet struct {
Message models.DiscordMessage `db:"msg"`
MessageContent *models.DiscordMessageContent `db:"c"`
Snippet *models.Snippet `db:"snippet"`
DiscordUser *models.DiscordUser `db:"duser"`
HMNUser *models.User `db:"u"`
}
isaved, err := db.QueryOne(ctx, tx, savedMessageAndSnippet{},
`
SELECT $columns
FROM
handmade_discordmessage AS msg
LEFT JOIN handmade_discordmessagecontent AS c ON c.message_id = msg.id
LEFT JOIN handmade_snippet AS snippet ON snippet.discord_message_id = msg.id
LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid
LEFT JOIN auth_user AS u ON duser.hmn_user_id = u.id
WHERE
msg.id = $1
`,
msg.ID,
)
if errors.Is(err, db.NotFound) {
goto nosnippet
} else if err != nil {
return oops.New(err, "failed to fetch snippet data and stuff")
}
saved := isaved.(*savedMessageAndSnippet)
if saved.DiscordUser == nil {
goto nosnippet
}
snippetableAsset, snippetableUrl, err := getSnippetAssetOrUrl(ctx, tx, msg.ID)
if err != nil {
return err
}
messageIsSnippetable := snippetableAsset != nil || snippetableUrl != ""
if messageIsSnippetable {
createSnippet, updateSnippet := false, false
if saved.Snippet == nil {
// No snippet exists. Maybe create one?
wantSnippetBecauseShowcase := true &&
msg.ChannelID == config.Config.Discord.ShowcaseChannelID &&
saved.HMNUser.DiscordSaveShowcase
wantNewSnippet := false ||
wantSnippetBecauseShowcase ||
flags&ForceSnippet != 0
createSnippet = true &&
flags&CreateSnippetsOnNewMessages != 0 &&
wantNewSnippet &&
saved.MessageContent != nil &&
saved.Message.SnippetCreated == false
} else {
// Update the snippet
updateSnippet = true
}
if createSnippet {
// TODO
} else if updateSnippet {
// TODO
}
} else {
// Delete any existing snippets (don't restrict this by channel!)
// Message edits can happen anywhere, even in channels where we don't
// delete messages like we do in showcase.
// TODO
}
}
nosnippet:
return nil
})
}
// Call whenever a Discord message is deleted, in any channel.
func MessageDeleted(ctx context.Context, dbConn db.ConnOrTx, msg *Message) {
}
func messageHasLinks(content string) bool {
links := reDiscordMessageLink.FindAllString(content, -1)
for _, link := range links {
_, err := url.Parse(strings.TrimSpace(link))
if err == nil {
return true
}
}
return false
}
func weAlreadyHaveMessageContent(
ctx context.Context,
dbConn db.ConnOrTx,
msg *Message,
) bool {
hasContent, err := db.QueryBool(ctx, dbConn,
`
SELECT COUNT(*) > 0
FROM
handmade_discordmessagecontent
WHERE
message_id = $1
`,
msg.ID,
)
if err != nil {
panic(err)
}
return hasContent
}
// Yields HTML and tags for a snippet, but doesn't actually create the snippet.
func messageToSnippetData() {
}

View File

@ -3,43 +3,55 @@ package discord
import (
"context"
"git.handmade.network/hmn/hmn/src/config"
"git.handmade.network/hmn/hmn/src/db"
"git.handmade.network/hmn/hmn/src/oops"
)
func (bot *botInstance) processLibraryMsg(ctx context.Context, msg *Message) error {
func maybeDeleteLibraryMsg(
ctx context.Context,
dbConn db.ConnOrTx,
msg *Message,
) (didDelete bool, err error) {
// Ensure basic info about the message
if msg.ChannelID != config.Config.Discord.LibraryChannelID {
return false, nil
}
switch msg.Type {
case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
default:
return nil
return false, nil
}
if !msg.OriginalHasFields("content") {
return nil
return false, nil
}
// Maybe delete it if it's bad
didDelete = false
if !messageHasLinks(msg.Content) {
err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
if err != nil {
return oops.New(err, "failed to delete message")
return false, oops.New(err, "failed to delete message")
}
didDelete = true
if !msg.Author.IsBot {
channel, err := CreateDM(ctx, msg.Author.ID)
if err != nil {
return oops.New(err, "failed to create DM channel")
return false, oops.New(err, "failed to create DM channel")
}
err = SendMessages(ctx, bot.dbConn, MessageToSend{
err = SendMessages(ctx, dbConn, MessageToSend{
ChannelID: channel.ID,
Req: CreateMessageRequest{
Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.",
},
})
if err != nil {
return oops.New(err, "failed to send showcase warning message")
return false, oops.New(err, "failed to send showcase warning message")
}
}
}
return nil
return didDelete, nil
}

View File

@ -246,13 +246,14 @@ const (
// https://discord.com/developers/docs/resources/channel#message-object
type Message struct {
ID string `json:"id"`
ChannelID string `json:"channel_id"`
GuildID *string `json:"guild_id"`
Content string `json:"content"`
Author *User `json:"author"` // note that this may not be an actual valid user (see the docs)
Timestamp string `json:"timestamp"`
Type MessageType `json:"type"`
ID string `json:"id"`
ChannelID string `json:"channel_id"`
GuildID *string `json:"guild_id"`
Content string `json:"content"`
Author *User `json:"author"` // note that this may not be an actual valid user (see the docs)
Timestamp string `json:"timestamp"`
EditedTimestamp *string `json:"edited_timestamp"`
Type MessageType `json:"type"`
Attachments []Attachment `json:"attachments"`
Embeds []Embed `json:"embeds"`

373
src/discord/persistence.go Normal file
View File

@ -0,0 +1,373 @@
package discord
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"strings"
"time"
"git.handmade.network/hmn/hmn/src/assets"
"git.handmade.network/hmn/hmn/src/config"
"git.handmade.network/hmn/hmn/src/db"
"git.handmade.network/hmn/hmn/src/models"
"git.handmade.network/hmn/hmn/src/oops"
"github.com/google/uuid"
)
/*
Ensures that a Discord message is stored in the database. It will also store
the message's contents, if allowed. This function is idempotent and can be
called regardless of whether the item already exists in the database.
This does not create snippets or do anything besides save the message itself.
*/
func PersistMessage(
ctx context.Context,
dbConn db.ConnOrTx,
msg *Message,
) (*models.DiscordMessage, error) {
tx, err := dbConn.Begin(ctx)
if err != nil {
return nil, oops.New(err, "failed to start transaction")
}
defer tx.Rollback(ctx)
// For every single Discord message on the server, we store a lightweight
// record of its existence. This allows us to efficiently retrieve message
// contents when someone links their Discord account. Naturally, we do this
// regardless of whether a user has linked their Discord account.
//
// Unless a user has linked their Discord account, no message content is
// ever saved.
iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{},
`
SELECT $columns
FROM handmade_discordmessage
WHERE id = $1
`,
msg.ID,
)
if errors.Is(err, db.NotFound) {
if !msg.OriginalHasFields("author", "timestamp") {
return nil, errNotEnoughInfo
}
guildID := msg.GuildID
if guildID == nil {
/*
This is weird, but it can happen when we fetch messages from
history instead of receiving it from the gateway. In this case
we just assume it's from the HMN server.
*/
guildID = &config.Config.Discord.GuildID
}
iDiscordMessage, err = db.QueryOne(ctx, tx, models.DiscordMessage{},
`
INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created)
VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING $columns
`,
msg.ID,
msg.ChannelID,
*guildID,
msg.JumpURL(),
msg.Author.ID,
msg.Time(),
false,
)
if err != nil {
return nil, oops.New(err, "failed to save new discord message")
}
} else if err != nil {
return nil, oops.New(err, "failed to check for existing Discord message")
}
discordMessage := iDiscordMessage.(*models.DiscordMessage)
//
// Time to save content!
//
{
// Check for linked Discord user
iDiscordUser, err := db.QueryOne(ctx, tx, models.DiscordUser{},
`
SELECT $columns
FROM handmade_discorduser
WHERE userid = $1
`,
discordMessage.UserID,
)
if errors.Is(err, db.NotFound) {
goto cancelSavingContent // considered harmful??
} else if err != nil {
return nil, oops.New(err, "failed to look up linked Discord user")
}
discordUser := iDiscordUser.(*models.DiscordUser)
// Save message text
if msg.OriginalHasFields("content") {
_, err = tx.Exec(ctx,
`
INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content)
VALUES ($1, $2, $3)
ON CONFLICT (message_id) DO UPDATE SET
discord_id = EXCLUDED.discord_id,
last_content = EXCLUDED.last_content
`,
discordMessage.ID,
discordUser.ID,
CleanUpMarkdown(ctx, msg.Content),
)
}
// Save attachments
if msg.OriginalHasFields("attachments") {
for _, attachment := range msg.Attachments {
_, err := saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID)
if err != nil {
return nil, oops.New(err, "failed to save attachment")
}
}
}
// Save / delete embeds
if msg.OriginalHasFields("embeds") {
numSavedEmbeds, err := db.QueryInt(ctx, tx,
`
SELECT COUNT(*)
FROM handmade_discordmessageembed
WHERE message_id = $1
`,
msg.ID,
)
if err != nil {
return nil, oops.New(err, "failed to count existing embeds")
}
if numSavedEmbeds == 0 {
// No embeds yet, so save new ones
for _, embed := range msg.Embeds {
_, err := saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID)
if err != nil {
return nil, oops.New(err, "failed to save embed")
}
}
} else if len(msg.Embeds) > 0 {
// Embeds were removed from the message
_, err := tx.Exec(ctx,
`
DELETE FROM handmade_discordmessageembed
WHERE message_id = $1
`,
msg.ID,
)
if err != nil {
return nil, oops.New(err, "failed to delete embeds")
}
}
}
}
cancelSavingContent:
err = tx.Commit(ctx)
if err != nil {
return nil, oops.New(err, "failed to commit transaction")
}
return discordMessage, nil
}
/*
Saves a Discord attachment as an HMN asset. Idempotent; will not create an
attachment that already exists.
*/
func saveAttachment(
ctx context.Context,
tx db.ConnOrTx,
attachment *Attachment,
hmnUserID int,
discordMessageID string,
) (*models.DiscordMessageAttachment, error) {
iexisting, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
`
SELECT $columns
FROM handmade_discordmessageattachment
WHERE id = $1
`,
attachment.ID,
)
if err == nil {
return iexisting.(*models.DiscordMessageAttachment), nil
} else if errors.Is(err, db.NotFound) {
// this is fine, just create it
} else {
return nil, oops.New(err, "failed to check for existing attachment")
}
width := 0
height := 0
if attachment.Width != nil {
width = *attachment.Width
}
if attachment.Height != nil {
height = *attachment.Height
}
content, _, err := downloadDiscordResource(ctx, attachment.Url)
if err != nil {
return nil, oops.New(err, "failed to download Discord attachment")
}
contentType := "application/octet-stream"
if attachment.ContentType != nil {
contentType = *attachment.ContentType
}
asset, err := assets.Create(ctx, tx, assets.CreateInput{
Content: content,
Filename: attachment.Filename,
ContentType: contentType,
UploaderID: &hmnUserID,
Width: width,
Height: height,
})
if err != nil {
return nil, oops.New(err, "failed to save asset for Discord attachment")
}
iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
`
INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id)
VALUES ($1, $2, $3)
RETURNING $columns
`,
attachment.ID,
asset.ID,
discordMessageID,
)
if err != nil {
return nil, oops.New(err, "failed to save Discord attachment data")
}
return iDiscordAttachment.(*models.DiscordMessageAttachment), nil
}
// Saves an embed from Discord. NOTE: This is _not_ idempotent, so only call it
// if you do not have any embeds saved for this message yet.
func saveEmbed(
ctx context.Context,
tx db.ConnOrTx,
embed *Embed,
hmnUserID int,
discordMessageID string,
) (*models.DiscordMessageEmbed, error) {
isOkImageType := func(contentType string) bool {
return strings.HasPrefix(contentType, "image/")
}
isOkVideoType := func(contentType string) bool {
return strings.HasPrefix(contentType, "video/")
}
maybeSaveImageish := func(i EmbedImageish, contentTypeCheck func(string) bool) (*uuid.UUID, error) {
content, contentType, err := downloadDiscordResource(ctx, *i.Url)
if err != nil {
var statusError DiscordResourceBadStatusCode
if errors.As(err, &statusError) {
return nil, nil
} else {
return nil, oops.New(err, "failed to save Discord embed")
}
}
if contentTypeCheck(contentType) {
in := assets.CreateInput{
Content: content,
Filename: "embed",
ContentType: contentType,
UploaderID: &hmnUserID,
}
if i.Width != nil {
in.Width = *i.Width
}
if i.Height != nil {
in.Height = *i.Height
}
asset, err := assets.Create(ctx, tx, in)
if err != nil {
return nil, oops.New(err, "failed to create asset from embed")
}
return &asset.ID, nil
}
return nil, nil
}
var imageAssetId *uuid.UUID
var videoAssetId *uuid.UUID
var err error
if embed.Video != nil && embed.Video.Url != nil {
videoAssetId, err = maybeSaveImageish(embed.Video.EmbedImageish, isOkVideoType)
} else if embed.Image != nil && embed.Image.Url != nil {
imageAssetId, err = maybeSaveImageish(embed.Image.EmbedImageish, isOkImageType)
} else if embed.Thumbnail != nil && embed.Thumbnail.Url != nil {
imageAssetId, err = maybeSaveImageish(embed.Thumbnail.EmbedImageish, isOkImageType)
}
if err != nil {
return nil, err
}
// Save the embed into the db
iDiscordEmbed, err := db.QueryOne(ctx, tx, models.DiscordMessageEmbed{},
`
INSERT INTO handmade_discordmessageembed (title, description, url, message_id, image_id, video_id)
VALUES ($1, $2, $3, $4, $5, $6)
RETURNING id
`,
embed.Title,
embed.Description,
embed.Url,
discordMessageID,
imageAssetId,
videoAssetId,
)
if err != nil {
return nil, oops.New(err, "failed to insert new embed")
}
return iDiscordEmbed.(*models.DiscordMessageEmbed), nil
}
var discordDownloadClient = &http.Client{
Timeout: 10 * time.Second,
}
type DiscordResourceBadStatusCode error
func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, "", oops.New(err, "failed to make Discord download request")
}
res, err := discordDownloadClient.Do(req)
if err != nil {
return nil, "", oops.New(err, "failed to fetch Discord resource data")
}
defer res.Body.Close()
if res.StatusCode < 200 || 299 < res.StatusCode {
return nil, "", DiscordResourceBadStatusCode(fmt.Errorf("status code %d from Discord resource: %s", res.StatusCode, url))
}
content, err := io.ReadAll(res.Body)
if err != nil {
panic(err)
}
return content, res.Header.Get("Content-Type"), nil
}

View File

@ -3,15 +3,9 @@ package discord
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"git.handmade.network/hmn/hmn/src/assets"
"git.handmade.network/hmn/hmn/src/config"
"git.handmade.network/hmn/hmn/src/db"
"git.handmade.network/hmn/hmn/src/hmndata"
@ -19,49 +13,40 @@ import (
"git.handmade.network/hmn/hmn/src/models"
"git.handmade.network/hmn/hmn/src/oops"
"git.handmade.network/hmn/hmn/src/parsing"
"github.com/google/uuid"
"github.com/rs/zerolog/log"
)
var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`)
var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this")
func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) error {
switch msg.Type {
// For any persisted Discord message, attempt to create or update a snippet.
func makeSnippet(
ctx context.Context,
dbConn db.ConnOrTx,
discordMsg *Message,
hmnMsg *models.DiscordMessage,
) error {
if discordMsg.ChannelID != config.Config.Discord.ShowcaseChannelID {
return nil
}
switch discordMsg.Type {
case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
default:
return nil
}
didDelete, err := bot.maybeDeleteShowcaseMsg(ctx, msg)
if err != nil {
return err
}
if didDelete {
return nil
}
tx, err := bot.dbConn.Begin(ctx)
tx, err := dbConn.Begin(ctx)
if err != nil {
panic(err)
}
defer tx.Rollback(ctx)
// save the message, maybe save its contents
newMsg, err := SaveMessageAndContents(ctx, tx, msg)
if errors.Is(err, errNotEnoughInfo) {
logging.ExtractLogger(ctx).Warn().
Interface("msg", msg).
Msg("didn't have enough info to process Discord message")
return nil
} else if err != nil {
return err
}
// ...and maybe make a snippet too, if the user wants us to
duser, err := FetchDiscordUser(ctx, tx, newMsg.UserID)
duser, err := hmndata.FetchDiscordUser(ctx, tx, hmnMsg.UserID)
if err == nil && duser.HMNUser.DiscordSaveShowcase {
err = CreateMessageSnippets(ctx, tx, newMsg.UserID, msg.ID)
err = CreateMessageSnippets(ctx, tx, hmnMsg.UserID, discordMsg.ID)
if err != nil {
return oops.New(err, "failed to create snippet in gateway")
}
@ -81,7 +66,19 @@ func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) er
return nil
}
func (bot *botInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message) (didDelete bool, err error) {
func maybeDeleteShowcaseMsg(ctx context.Context, dbConn db.ConnOrTx, msg *Message) (didDelete bool, err error) {
// Ensure basics about the message
if msg.ChannelID != config.Config.Discord.ShowcaseChannelID {
return false, nil
}
switch msg.Type {
case MessageTypeDefault, MessageTypeReply:
// proceed
default:
return false, nil
}
// Check message content and stuff
hasGoodContent := true
if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) {
hasGoodContent = false
@ -106,7 +103,7 @@ func (bot *botInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message
return false, oops.New(err, "failed to create DM channel")
}
err = SendMessages(ctx, bot.dbConn, MessageToSend{
err = SendMessages(ctx, dbConn, MessageToSend{
ChannelID: channel.ID,
Req: CreateMessageRequest{
Content: "Posts in #project-showcase are required to have either an image/video or a link. Discuss showcase content in #projects.",
@ -121,420 +118,34 @@ func (bot *botInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message
return didDelete, nil
}
/*
Ensures that a Discord message is stored in the database. This function is
idempotent and can be called regardless of whether the item already exists in
the database.
This does not create snippets or do anything besides save the message itself.
*/
func SaveMessage(
ctx context.Context,
tx db.ConnOrTx,
msg *Message,
) (*models.DiscordMessage, error) {
iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{},
`
SELECT $columns
FROM handmade_discordmessage
WHERE id = $1
`,
msg.ID,
)
// For any Discord message, attempt to delete any snippets.
func deleteSnippet(ctx context.Context, dbConn db.ConnOrTx, msgDelete MessageDelete) {
s, err := hmndata.FetchSnippetForDiscordMessage(ctx, dbConn, nil, msgDelete.ID, hmndata.SnippetQuery{})
if errors.Is(err, db.NotFound) {
if !msg.OriginalHasFields("author", "timestamp") {
return nil, errNotEnoughInfo
}
return
} else if err != nil {
logging.ExtractLogger(ctx).Error().Err(err).Msg("")
return
}
guildID := msg.GuildID
if guildID == nil {
/*
This is weird, but it can happen when we fetch messages from
history instead of receiving it from the gateway. In this case
we just assume it's from the HMN server.
*/
guildID = &config.Config.Discord.GuildID
}
_, err = tx.Exec(ctx,
shouldDeleteSnippet := s.Owner != nil && s.Owner.DiscordDeleteSnippetOnMessageDelete
if shouldDeleteSnippet {
logging.ExtractLogger(ctx).Debug().
Int("snippet_id", s.Snippet.ID).
Int("user_id", s.Owner.ID).
Msg("deleting snippet from Discord message")
_, err = dbConn.Exec(ctx,
`
INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created)
VALUES ($1, $2, $3, $4, $5, $6, $7)
`,
msg.ID,
msg.ChannelID,
*guildID,
msg.JumpURL(),
msg.Author.ID,
msg.Time(),
false,
)
if err != nil {
return nil, oops.New(err, "failed to save new discord message")
}
/*
TODO(db): This is a spot where it would be really nice to be able
to use RETURNING, and avoid this second query.
*/
iDiscordMessage, err = db.QueryOne(ctx, tx, models.DiscordMessage{},
`
SELECT $columns
FROM handmade_discordmessage
DELETE FROM handmade_snippet
WHERE id = $1
`,
msg.ID,
s.Snippet.ID,
)
if err != nil {
panic(err)
}
} else if err != nil {
return nil, oops.New(err, "failed to check for existing Discord message")
}
return iDiscordMessage.(*models.DiscordMessage), nil
}
/*
Processes a single Discord message, saving as much of the message's content
and attachments as allowed by our rules and user settings. Does NOT create
snippets.
Idempotent; can be called any time whether the message exists or not.
*/
func SaveMessageAndContents(
ctx context.Context,
tx db.ConnOrTx,
msg *Message,
) (*models.DiscordMessage, error) {
newMsg, err := SaveMessage(ctx, tx, msg)
if err != nil {
return nil, err
}
// Check for linked Discord user
iDiscordUser, err := db.QueryOne(ctx, tx, models.DiscordUser{},
`
SELECT $columns
FROM handmade_discorduser
WHERE userid = $1
`,
newMsg.UserID,
)
if errors.Is(err, db.NotFound) {
return newMsg, nil
} else if err != nil {
return nil, oops.New(err, "failed to look up linked Discord user")
}
discordUser := iDiscordUser.(*models.DiscordUser)
// We have a linked Discord account, so save the message contents (regardless of
// whether we create a snippet or not).
if msg.OriginalHasFields("content") {
_, err = tx.Exec(ctx,
`
INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content)
VALUES ($1, $2, $3)
ON CONFLICT (message_id) DO UPDATE SET
discord_id = EXCLUDED.discord_id,
last_content = EXCLUDED.last_content
`,
newMsg.ID,
discordUser.ID,
CleanUpMarkdown(ctx, msg.Content),
)
}
// Save attachments
if msg.OriginalHasFields("attachments") {
for _, attachment := range msg.Attachments {
_, err := saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID)
if err != nil {
return nil, oops.New(err, "failed to save attachment")
}
log.Error().Err(err).Msg("failed to delete snippet")
return
}
}
// Save / delete embeds
if msg.OriginalHasFields("embeds") {
numSavedEmbeds, err := db.QueryInt(ctx, tx,
`
SELECT COUNT(*)
FROM handmade_discordmessageembed
WHERE message_id = $1
`,
msg.ID,
)
if err != nil {
return nil, oops.New(err, "failed to count existing embeds")
}
if numSavedEmbeds == 0 {
// No embeds yet, so save new ones
for _, embed := range msg.Embeds {
_, err := saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID)
if err != nil {
return nil, oops.New(err, "failed to save embed")
}
}
} else if len(msg.Embeds) > 0 {
// Embeds were removed from the message
_, err := tx.Exec(ctx,
`
DELETE FROM handmade_discordmessageembed
WHERE message_id = $1
`,
msg.ID,
)
if err != nil {
return nil, oops.New(err, "failed to delete embeds")
}
}
}
return newMsg, nil
}
var discordDownloadClient = &http.Client{
Timeout: 10 * time.Second,
}
type DiscordResourceBadStatusCode error
func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, "", oops.New(err, "failed to make Discord download request")
}
res, err := discordDownloadClient.Do(req)
if err != nil {
return nil, "", oops.New(err, "failed to fetch Discord resource data")
}
defer res.Body.Close()
if res.StatusCode < 200 || 299 < res.StatusCode {
return nil, "", DiscordResourceBadStatusCode(fmt.Errorf("status code %d from Discord resource: %s", res.StatusCode, url))
}
content, err := io.ReadAll(res.Body)
if err != nil {
panic(err)
}
return content, res.Header.Get("Content-Type"), nil
}
/*
Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment
that already exists
*/
func saveAttachment(
ctx context.Context,
tx db.ConnOrTx,
attachment *Attachment,
hmnUserID int,
discordMessageID string,
) (*models.DiscordMessageAttachment, error) {
iexisting, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
`
SELECT $columns
FROM handmade_discordmessageattachment
WHERE id = $1
`,
attachment.ID,
)
if err == nil {
return iexisting.(*models.DiscordMessageAttachment), nil
} else if errors.Is(err, db.NotFound) {
// this is fine, just create it
} else {
return nil, oops.New(err, "failed to check for existing attachment")
}
width := 0
height := 0
if attachment.Width != nil {
width = *attachment.Width
}
if attachment.Height != nil {
height = *attachment.Height
}
content, _, err := downloadDiscordResource(ctx, attachment.Url)
if err != nil {
return nil, oops.New(err, "failed to download Discord attachment")
}
contentType := "application/octet-stream"
if attachment.ContentType != nil {
contentType = *attachment.ContentType
}
asset, err := assets.Create(ctx, tx, assets.CreateInput{
Content: content,
Filename: attachment.Filename,
ContentType: contentType,
UploaderID: &hmnUserID,
Width: width,
Height: height,
})
if err != nil {
return nil, oops.New(err, "failed to save asset for Discord attachment")
}
// TODO(db): RETURNING plz thanks
_, err = tx.Exec(ctx,
`
INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id)
VALUES ($1, $2, $3)
`,
attachment.ID,
asset.ID,
discordMessageID,
)
if err != nil {
return nil, oops.New(err, "failed to save Discord attachment data")
}
iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
`
SELECT $columns
FROM handmade_discordmessageattachment
WHERE id = $1
`,
attachment.ID,
)
if err != nil {
return nil, oops.New(err, "failed to fetch new Discord attachment data")
}
return iDiscordAttachment.(*models.DiscordMessageAttachment), nil
}
// Saves an embed from Discord. NOTE: This is _not_ idempotent, so only call it
// if you do not have any embeds saved for this message yet.
func saveEmbed(
ctx context.Context,
tx db.ConnOrTx,
embed *Embed,
hmnUserID int,
discordMessageID string,
) (*models.DiscordMessageEmbed, error) {
isOkImageType := func(contentType string) bool {
return strings.HasPrefix(contentType, "image/")
}
isOkVideoType := func(contentType string) bool {
return strings.HasPrefix(contentType, "video/")
}
maybeSaveImageish := func(i EmbedImageish, contentTypeCheck func(string) bool) (*uuid.UUID, error) {
content, contentType, err := downloadDiscordResource(ctx, *i.Url)
if err != nil {
var statusError DiscordResourceBadStatusCode
if errors.As(err, &statusError) {
return nil, nil
} else {
return nil, oops.New(err, "failed to save Discord embed")
}
}
if contentTypeCheck(contentType) {
in := assets.CreateInput{
Content: content,
Filename: "embed",
ContentType: contentType,
UploaderID: &hmnUserID,
}
if i.Width != nil {
in.Width = *i.Width
}
if i.Height != nil {
in.Height = *i.Height
}
asset, err := assets.Create(ctx, tx, in)
if err != nil {
return nil, oops.New(err, "failed to create asset from embed")
}
return &asset.ID, nil
}
return nil, nil
}