nick12
/
hmn
forked from hmn/hmn
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Discord message handling reorganization.

This commit is contained in:
Asaf Gartner 2022-01-31 08:46:43 +02:00
parent f4f439489d
commit ad888346ef
7 changed files with 1034 additions and 1066 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
src/discord/cmd/cmd.go
View File

@ -2,6 +2,9 @@ package cmd
import (
"context"
"errors"
"fmt"
"os"
"time"
"git.handmade.network/hmn/hmn/src/db"
@ -35,18 +38,45 @@ func init() {
rootCommand.AddCommand(scrapeCommand)
makeSnippetCommand := &cobra.Command{
Use: "makesnippet [<message id>...]",
Use: "makesnippet <channel id> [<message id>...]",
Short: "Make snippets from saved Discord messages",
Long: "Make snippets from Discord messages whose content we have already saved. Useful for creating snippets from messages in non-showcase channels.",
Run: func(cmd *cobra.Command, args []string) {
if len(args) < 2 {
cmd.Usage()
os.Exit(1)
}
ctx := context.Background()
conn := db.NewConnPool(1, 1)
defer conn.Close()
err := discord.CreateMessageSnippets(ctx, conn, args...)
if err != nil {
logging.Error().Err(err).Msg("failed to create snippets")
chanID := args[0]
count := 0
for _, msgID := range args[1:] {
message, err := discord.GetChannelMessage(ctx, chanID, msgID)
if errors.Is(err, discord.NotFound) {
logging.Warn().Msg(fmt.Sprintf("no message found on discord for id %s", msgID))
continue
} else if err != nil {
logging.Error().Msg(fmt.Sprintf("failed to fetch discord message id %s", msgID))
continue
}
err = discord.InternMessage(ctx, conn, message)
if err != nil {
logging.Error().Msg(fmt.Sprintf("failed to intern discord message id %s", msgID))
continue
}
err = discord.HandleInternedMessage(ctx, conn, message, false, true)
if err != nil {
logging.Error().Msg(fmt.Sprintf("failed to handle interned message id %s", msgID))
continue
}
count += 1
}
logging.Info().Msg(fmt.Sprintf("Handled %d messages", count))
},
}
rootCommand.AddCommand(makeSnippetCommand)

92
src/discord/gateway.go
View File

@ -591,103 +591,31 @@ func (bot *botInstance) messageCreateOrUpdate(ctx context.Context, msg *Message)
return nil
}
if msg.ChannelID == config.Config.Discord.ShowcaseChannelID {
err := bot.processShowcaseMsg(ctx, msg)
if err != nil {
logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to process showcase message")
return nil
}
return nil
}
if msg.ChannelID == config.Config.Discord.LibraryChannelID {
err := bot.processLibraryMsg(ctx, msg)
if err != nil {
logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to process library message")
return nil
}
return nil
}
err := UpdateSnippetTagsIfAny(ctx, bot.dbConn, msg)
err := HandleIncomingMessage(ctx, bot.dbConn, msg, true)
if err != nil {
logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to update tags for Discord snippet")
logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to handle incoming message")
}
// NOTE(asaf): Since any error from HandleIncomingMessage is an internal error and not a discord
// error, we only want to log it and not restart the bot. So we're not returning the error.
return nil
}
func (bot *botInstance) messageDelete(ctx context.Context, msgDelete MessageDelete) {
log := logging.ExtractLogger(ctx)
tx, err := bot.dbConn.Begin(ctx)
interned, err := FetchInternedMessage(ctx, bot.dbConn, msgDelete.ID)
if err != nil {
log.Error().Err(err).Msg("failed to start transaction")
log.Error().Err(err).Msg("failed to fetch interned message")
return
}
defer tx.Rollback(ctx)
type deleteMessageQuery struct {
Message models.DiscordMessage `db:"msg"`
DiscordUser *models.DiscordUser `db:"duser"`
HMNUser *models.User `db:"hmnuser"`
SnippetID *int `db:"snippet.id"`
}
iresult, err := db.QueryOne(ctx, tx, deleteMessageQuery{},
`
SELECT $columns
FROM
handmade_discordmessage AS msg
LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid
LEFT JOIN auth_user AS hmnuser ON duser.hmn_user_id = hmnuser.id
LEFT JOIN handmade_asset AS hmnuser_avatar ON hmnuser_avatar.id = hmnuser.avatar_asset_id
LEFT JOIN handmade_snippet AS snippet ON snippet.discord_message_id = msg.id
WHERE msg.id = $1 AND msg.channel_id = $2
`,
msgDelete.ID, msgDelete.ChannelID,
)
if errors.Is(err, db.NotFound) {
return
} else if err != nil {
log.Error().Err(err).Msg("failed to check for message to delete")
return
}
result := iresult.(*deleteMessageQuery)
log.Debug().Msg("deleting Discord message")
_, err = tx.Exec(ctx,
`
DELETE FROM handmade_discordmessage
WHERE id = $1 AND channel_id = $2
`,
msgDelete.ID,
msgDelete.ChannelID,
)
shouldDeleteSnippet := result.HMNUser != nil && result.HMNUser.DiscordDeleteSnippetOnMessageDelete
if result.SnippetID != nil && shouldDeleteSnippet {
log.Debug().
Int("snippet_id", *result.SnippetID).
Int("user_id", result.HMNUser.ID).
Msg("deleting snippet from Discord message")
_, err = tx.Exec(ctx,
`
DELETE FROM handmade_snippet
WHERE id = $1
`,
result.SnippetID,
)
if interned != nil {
err = DeleteInternedMessage(ctx, bot.dbConn, interned)
if err != nil {
log.Error().Err(err).Msg("failed to delete snippet")
log.Error().Err(err).Msg("failed to delete interned message")
return
}
}
err = tx.Commit(ctx)
if err != nil {
log.Error().Err(err).Msg("failed to delete Discord message")
return
}
}
type MessageToSend struct {
@ -698,7 +626,7 @@ type MessageToSend struct {
func SendMessages(
ctx context.Context,
conn *pgxpool.Pool,
conn db.ConnOrTx,
msgs ...MessageToSend,
) error {
tx, err := conn.Begin(ctx)

69
src/discord/history.go
View File

@ -9,7 +9,6 @@ import (
"git.handmade.network/hmn/hmn/src/db"
"git.handmade.network/hmn/hmn/src/logging"
"git.handmade.network/hmn/hmn/src/models"
"github.com/jackc/pgx/v4"
"github.com/jackc/pgx/v4/pgxpool"
)
@ -34,7 +33,10 @@ func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{
backfillInterval := 1 * time.Hour
newUserTicker := time.NewTicker(5 * time.Second)
backfillTicker := time.NewTicker(backfillInterval)
// NOTE(asaf): 5 seconds to ensure this runs on start, we then reset it to the correct
// interval after the first run.
backfillTicker := time.NewTicker(5 * time.Second)
lastBackfillTime := time.Now().Add(-backfillInterval)
for {
@ -45,6 +47,10 @@ func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{
// Get content for messages when a user links their account (but do not create snippets)
fetchMissingContent(ctx, dbConn)
case <-backfillTicker.C:
backfillTicker.Reset(backfillInterval)
// TODO(asaf): Do we need to update lastBackfillTime here? Otherwise we'll be rescraping
// from (start up time - 1 hour) every time.
// Run a backfill to patch up places where the Discord bot missed (does create snippets)
Scrape(ctx, dbConn,
config.Config.Discord.ShowcaseChannelID,
@ -99,13 +105,16 @@ func fetchMissingContent(ctx context.Context, dbConn *pgxpool.Pool) {
discordMsg, err := GetChannelMessage(ctx, msg.ChannelID, msg.ID)
if errors.Is(err, NotFound) {
// This message has apparently been deleted; delete it from our database
_, err = dbConn.Exec(ctx,
`
DELETE FROM handmade_discordmessage
WHERE id = $1
`,
msg.ID,
)
interned, err := FetchInternedMessage(ctx, dbConn, msg.ID)
if err != nil {
log.Error().Err(err).Msg("failed to fetch interned message")
continue
}
if interned == nil {
log.Error().Msg("couldn't find interned message")
continue
}
err = DeleteInternedMessage(ctx, dbConn, interned)
if err != nil {
log.Error().Err(err).Msg("failed to delete missing message")
continue
@ -119,7 +128,7 @@ func fetchMissingContent(ctx context.Context, dbConn *pgxpool.Pool) {
log.Info().Str("msg", discordMsg.ShortString()).Msg("fetched message for content")
err = handleHistoryMessage(ctx, dbConn, discordMsg, false)
err = HandleInternedMessage(ctx, dbConn, discordMsg, false, false)
if err != nil {
log.Error().Err(err).Msg("failed to save content for message")
continue
@ -166,12 +175,10 @@ func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earlies
return
}
err := handleHistoryMessage(ctx, dbConn, &msg, createSnippets)
err := HandleIncomingMessage(ctx, dbConn, &msg, createSnippets)
if err != nil {
errLog := logging.ExtractLogger(ctx).Error()
if errors.Is(err, errNotEnoughInfo) {
errLog = logging.ExtractLogger(ctx).Warn()
}
errLog.Err(err).Msg("failed to process Discord message")
}
@ -179,37 +186,3 @@ func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earlies
}
}
}
func handleHistoryMessage(ctx context.Context, dbConn *pgxpool.Pool, msg *Message, createSnippets bool) error {
var tx pgx.Tx
for {
var err error
tx, err = dbConn.Begin(ctx)
if err != nil {
logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to start transaction for message")
time.Sleep(1 * time.Second)
continue
}
break
}
newMsg, err := SaveMessageAndContents(ctx, tx, msg)
if err != nil {
return err
}
if createSnippets {
if doSnippet, err := AllowedToCreateMessageSnippets(ctx, tx, newMsg.UserID); doSnippet && err == nil {
err := CreateMessageSnippets(ctx, tx, msg.ID)
if err != nil {
return err
}
}
}
err = tx.Commit(ctx)
if err != nil {
return err
}
return nil
}

45
src/discord/library.go
View File

@ -1,45 +0,0 @@
package discord
import (
"context"
"git.handmade.network/hmn/hmn/src/oops"
)
func (bot *botInstance) processLibraryMsg(ctx context.Context, msg *Message) error {
switch msg.Type {
case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
default:
return nil
}
if !msg.OriginalHasFields("content") {
return nil
}
if !messageHasLinks(msg.Content) {
err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
if err != nil {
return oops.New(err, "failed to delete message")
}
if !msg.Author.IsBot {
channel, err := CreateDM(ctx, msg.Author.ID)
if err != nil {
return oops.New(err, "failed to create DM channel")
}
err = SendMessages(ctx, bot.dbConn, MessageToSend{
ChannelID: channel.ID,
Req: CreateMessageRequest{
Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.",
},
})
if err != nil {
return oops.New(err, "failed to send showcase warning message")
}
}
}
return nil
}

957
src/discord/message_handling.go Normal file
View File

@ -0,0 +1,957 @@
package discord
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"git.handmade.network/hmn/hmn/src/assets"
"git.handmade.network/hmn/hmn/src/config"
"git.handmade.network/hmn/hmn/src/db"
"git.handmade.network/hmn/hmn/src/hmndata"
"git.handmade.network/hmn/hmn/src/logging"
"git.handmade.network/hmn/hmn/src/models"
"git.handmade.network/hmn/hmn/src/oops"
"git.handmade.network/hmn/hmn/src/parsing"
"github.com/google/uuid"
)
func HandleIncomingMessage(ctx context.Context, dbConn db.ConnOrTx, msg *Message, createSnippets bool) error {
deleted := false
var err error
// NOTE(asaf): All functions called here should verify that the message applies to them.
if !deleted && err == nil {
deleted, err = CleanUpLibrary(ctx, dbConn, msg)
}
if !deleted && err == nil {
deleted, err = CleanUpShowcase(ctx, dbConn, msg)
}
if !deleted && err == nil {
err = MaybeInternMessage(ctx, dbConn, msg)
}
if err == nil {
err = HandleInternedMessage(ctx, dbConn, msg, deleted, createSnippets)
}
return err
}
func CleanUpShowcase(ctx context.Context, dbConn db.ConnOrTx, msg *Message) (bool, error) {
deleted := false
if msg.ChannelID == config.Config.Discord.ShowcaseChannelID {
switch msg.Type {
case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
default:
return deleted, nil
}
hasGoodContent := true
if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) {
hasGoodContent = false
}
hasGoodAttachments := true
if msg.OriginalHasFields("attachments") && len(msg.Attachments) == 0 {
hasGoodAttachments = false
}
if !hasGoodContent && !hasGoodAttachments {
err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
if err != nil {
return deleted, oops.New(err, "failed to delete message")
}
deleted = true
if !msg.Author.IsBot {
channel, err := CreateDM(ctx, msg.Author.ID)
if err != nil {
return deleted, oops.New(err, "failed to create DM channel")
}
err = SendMessages(ctx, dbConn, MessageToSend{
ChannelID: channel.ID,
Req: CreateMessageRequest{
Content: "Posts in #project-showcase are required to have either an image/video or a link. Discuss showcase content in #projects.",
},
})
if err != nil {
return deleted, oops.New(err, "failed to send showcase warning message")
}
}
}
}
return deleted, nil
}
func CleanUpLibrary(ctx context.Context, dbConn db.ConnOrTx, msg *Message) (bool, error) {
deleted := false
if msg.ChannelID == config.Config.Discord.LibraryChannelID {
switch msg.Type {
case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
default:
return deleted, nil
}
if !msg.OriginalHasFields("content") {
return deleted, nil
}
if !messageHasLinks(msg.Content) {
err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
if err != nil {
return deleted, oops.New(err, "failed to delete message")
}
deleted = true
if !msg.Author.IsBot {
channel, err := CreateDM(ctx, msg.Author.ID)
if err != nil {
return deleted, oops.New(err, "failed to create DM channel")
}
err = SendMessages(ctx, dbConn, MessageToSend{
ChannelID: channel.ID,
Req: CreateMessageRequest{
Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.",
},
})
if err != nil {
return deleted, oops.New(err, "failed to send showcase warning message")
}
}
}
}
return deleted, nil
}
func MaybeInternMessage(ctx context.Context, dbConn db.ConnOrTx, msg *Message) error {
if msg.ChannelID == config.Config.Discord.ShowcaseChannelID {
err := InternMessage(ctx, dbConn, msg)
if errors.Is(err, errNotEnoughInfo) {
logging.ExtractLogger(ctx).Warn().
Interface("msg", msg).
Msg("didn't have enough info to intern Discord message")
} else if err != nil {
return err
}
}
return nil
}
/*
Ensures that a Discord message is stored in the database. This function is
idempotent and can be called regardless of whether the item already exists in
the database.
This does not create snippets or save content or do anything besides save the message itself.
*/
var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this")
func InternMessage(
ctx context.Context,
dbConn db.ConnOrTx,
msg *Message,
) error {
_, err := db.QueryOne(ctx, dbConn, models.DiscordMessage{},
`
SELECT $columns
FROM handmade_discordmessage
WHERE id = $1
`,
msg.ID,
)
if errors.Is(err, db.NotFound) {
if !msg.OriginalHasFields("author", "timestamp") {
return errNotEnoughInfo
}
guildID := msg.GuildID
if guildID == nil {
/*
This is weird, but it can happen when we fetch messages from
history instead of receiving it from the gateway. In this case
we just assume it's from the HMN server.
*/
guildID = &config.Config.Discord.GuildID
}
_, err = dbConn.Exec(ctx,
`
INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created)
VALUES ($1, $2, $3, $4, $5, $6, $7)
`,
msg.ID,
msg.ChannelID,
*guildID,
msg.JumpURL(),
msg.Author.ID,
msg.Time(),
false,
)
if err != nil {
return oops.New(err, "failed to save new discord message")
}
} else if err != nil {
return oops.New(err, "failed to check for existing Discord message")
}
return nil
}
type InternedMessage struct {
Message models.DiscordMessage `db:"message"`
MessageContent *models.DiscordMessageContent `db:"content"`
HMNUser *models.User `db:"hmnuser"`
DiscordUser *models.DiscordUser `db:"duser"`
}
func FetchInternedMessage(ctx context.Context, dbConn db.ConnOrTx, msgId string) (*InternedMessage, error) {
result, err := db.QueryOne(ctx, dbConn, InternedMessage{},
`
SELECT $columns
FROM
handmade_discordmessage AS message
LEFT JOIN handmade_discordmessagecontent AS content ON content.message_id = message.id
LEFT JOIN handmade_discorduser AS duser ON duser.userid = message.user_id
LEFT JOIN auth_user AS hmnuser ON hmnuser.id = duser.hmn_user_id
LEFT JOIN handmade_asset AS hmnuser_avatar ON hmnuser_avatar.id = hmnuser.avatar_asset_id
WHERE message.id = $1
`,
msgId,
)
if err != nil {
if errors.Is(err, db.NotFound) {
return nil, nil
} else {
return nil, err
}
}
interned := result.(*InternedMessage)
return interned, nil
}
// Checks if a message is interned and handles it to the extent possible:
// 1. Saves/updates content
// 2. Saves/updates snippet
// 3. Deletes content/snippet
func HandleInternedMessage(ctx context.Context, dbConn db.ConnOrTx, msg *Message, deleted bool, createSnippet bool) error {
tx, err := dbConn.Begin(ctx)
if err != nil {
return oops.New(err, "failed to start transaction")
}
defer tx.Rollback(ctx)
interned, err := FetchInternedMessage(ctx, tx, msg.ID)
if err != nil {
return err
}
if interned != nil {
if !deleted {
err = SaveMessageContents(ctx, tx, interned, msg)
if err != nil {
return err
}
if createSnippet {
err = HandleSnippetForInternedMessage(ctx, tx, interned, false)
if err != nil {
return err
}
}
} else {
err = DeleteInternedMessage(ctx, tx, interned)
if err != nil {
return err
}
}
}
err = tx.Commit(ctx)
if err != nil {
return oops.New(err, "failed to commit Discord message updates")
}
return nil
}
func DeleteInternedMessage(ctx context.Context, dbConn db.ConnOrTx, interned *InternedMessage) error {
isnippet, err := db.QueryOne(ctx, dbConn, models.Snippet{},
`
SELECT $columns
FROM handmade_snippet
WHERE discord_message_id = $1
`,
interned.Message.ID,
)
if err != nil && !errors.Is(err, db.NotFound) {
return oops.New(err, "failed to fetch snippet for discord message")
}
var snippet *models.Snippet
if !errors.Is(err, db.NotFound) {
snippet = isnippet.(*models.Snippet)
}
// NOTE(asaf): Also deletes the following through a db cascade:
// * handmade_discordmessageattachment
// * handmade_discordmessagecontent
// * handmade_discordmessageembed
// DOES NOT DELETE ASSETS FOR CONTENT/EMBEDS
_, err = dbConn.Exec(ctx,
`
DELETE FROM handmade_discordmessage
WHERE id = $1
`,
interned.Message.ID,
)
if snippet != nil {
userApprovesDeletion := interned.HMNUser != nil && snippet.OwnerID == interned.HMNUser.ID && interned.HMNUser.DiscordDeleteSnippetOnMessageDelete
if !snippet.EditedOnWebsite && userApprovesDeletion {
// NOTE(asaf): Does not delete asset!
_, err = dbConn.Exec(ctx,
`
DELETE FROM handmade_snippet
WHERE id = $1
`,
snippet.ID,
)
if err != nil {
return oops.New(err, "failed to delete snippet")
}
}
}
return nil
}
/*
Processes a single Discord message, saving as much of the message's content
and attachments as allowed by our rules and user settings. Does NOT create
snippets.
Idempotent; can be called any time whether the contents exist or not.
NOTE!!: Replaces interned.MessageContent if it was created or updated!!
*/
func SaveMessageContents(
ctx context.Context,
dbConn db.ConnOrTx,
interned *InternedMessage,
msg *Message,
) error {
if interned.DiscordUser != nil {
// We have a linked Discord account, so save the message contents (regardless of
// whether we create a snippet or not).
if msg.OriginalHasFields("content") {
_, err := dbConn.Exec(ctx,
`
INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content)
VALUES ($1, $2, $3)
ON CONFLICT (message_id) DO UPDATE SET
discord_id = EXCLUDED.discord_id,
last_content = EXCLUDED.last_content
`,
interned.Message.ID,
interned.DiscordUser.ID,
CleanUpMarkdown(ctx, msg.Content),
)
if err != nil {
return oops.New(err, "failed to create or update message contents")
}
icontent, err := db.QueryOne(ctx, dbConn, models.DiscordMessageContent{},
`
SELECT $columns
FROM
handmade_discordmessagecontent
WHERE
handmade_discordmessagecontent.message_id = $1
`,
interned.Message.ID,
)
if err != nil {
return oops.New(err, "failed to fetch message contents")
}
interned.MessageContent = icontent.(*models.DiscordMessageContent)
} // TODO(asaf): What happens if we edit the message and delete the content but keep the attachment??
// Save attachments
if msg.OriginalHasFields("attachments") {
for _, attachment := range msg.Attachments {
_, err := saveAttachment(ctx, dbConn, &attachment, interned.DiscordUser.HMNUserId, msg.ID)
if err != nil {
return oops.New(err, "failed to save attachment")
}
}
}
// Save / delete embeds
if msg.OriginalHasFields("embeds") {
numSavedEmbeds, err := db.QueryInt(ctx, dbConn,
`
SELECT COUNT(*)
FROM handmade_discordmessageembed
WHERE message_id = $1
`,
msg.ID,
)
if err != nil {
return oops.New(err, "failed to count existing embeds")
}
if numSavedEmbeds == 0 {
// No embeds yet, so save new ones
for _, embed := range msg.Embeds {
_, err := saveEmbed(ctx, dbConn, &embed, interned.DiscordUser.HMNUserId, msg.ID)
if err != nil {
return oops.New(err, "failed to save embed")
}
}
} else if len(msg.Embeds) > 0 {
// Embeds were removed from the message
_, err := dbConn.Exec(ctx,
`
DELETE FROM handmade_discordmessageembed
WHERE message_id = $1
`,
msg.ID,
)
if err != nil {
return oops.New(err, "failed to delete embeds")
}
}
}
}
return nil
}
var discordDownloadClient = &http.Client{
Timeout: 10 * time.Second,
}
type DiscordResourceBadStatusCode error
func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, "", oops.New(err, "failed to make Discord download request")
}
res, err := discordDownloadClient.Do(req)
if err != nil {
return nil, "", oops.New(err, "failed to fetch Discord resource data")
}
defer res.Body.Close()
if res.StatusCode < 200 || 299 < res.StatusCode {
return nil, "", DiscordResourceBadStatusCode(fmt.Errorf("status code %d from Discord resource: %s", res.StatusCode, url))
}
content, err := io.ReadAll(res.Body)
if err != nil {
panic(err)
}
return content, res.Header.Get("Content-Type"), nil
}
/*
Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment
that already exists
*/
func saveAttachment(
ctx context.Context,
tx db.ConnOrTx,
attachment *Attachment,
hmnUserID int,
discordMessageID string,
) (*models.DiscordMessageAttachment, error) {
iexisting, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
`
SELECT $columns
FROM handmade_discordmessageattachment
WHERE id = $1
`,
attachment.ID,
)
if err == nil {
return iexisting.(*models.DiscordMessageAttachment), nil
} else if errors.Is(err, db.NotFound) {
// this is fine, just create it
} else {
return nil, oops.New(err, "failed to check for existing attachment")
}
width := 0
height := 0
if attachment.Width != nil {
width = *attachment.Width
}
if attachment.Height != nil {
height = *attachment.Height
}
content, _, err := downloadDiscordResource(ctx, attachment.Url)
if err != nil {
return nil, oops.New(err, "failed to download Discord attachment")
}
contentType := "application/octet-stream"
if attachment.ContentType != nil {
contentType = *attachment.ContentType
}
asset, err := assets.Create(ctx, tx, assets.CreateInput{
Content: content,
Filename: attachment.Filename,
ContentType: contentType,
UploaderID: &hmnUserID,
Width: width,
Height: height,
})
if err != nil {
return nil, oops.New(err, "failed to save asset for Discord attachment")
}
// TODO(db): RETURNING plz thanks
_, err = tx.Exec(ctx,
`
INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id)
VALUES ($1, $2, $3)
`,
attachment.ID,
asset.ID,
discordMessageID,
)
if err != nil {
return nil, oops.New(err, "failed to save Discord attachment data")
}
iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
`
SELECT $columns
FROM handmade_discordmessageattachment
WHERE id = $1
`,
attachment.ID,
)
if err != nil {
return nil, oops.New(err, "failed to fetch new Discord attachment data")
}
return iDiscordAttachment.(*models.DiscordMessageAttachment), nil
}
// Saves an embed from Discord. NOTE: This is _not_ idempotent, so only call it
// if you do not have any embeds saved for this message yet.
func saveEmbed(
ctx context.Context,
tx db.ConnOrTx,
embed *Embed,
hmnUserID int,
discordMessageID string,
) (*models.DiscordMessageEmbed, error) {
isOkImageType := func(contentType string) bool {
return strings.HasPrefix(contentType, "image/")
}
isOkVideoType := func(contentType string) bool {
return strings.HasPrefix(contentType, "video/")
}
maybeSaveImageish := func(i EmbedImageish, contentTypeCheck func(string) bool) (*uuid.UUID, error) {
content, contentType, err := downloadDiscordResource(ctx, *i.Url)
if err != nil {
var statusError DiscordResourceBadStatusCode
if errors.As(err, &statusError) {
return nil, nil
} else {
return nil, oops.New(err, "failed to save Discord embed")
}
}
if contentTypeCheck(contentType) {
in := assets.CreateInput{
Content: content,
Filename: "embed",
ContentType: contentType,
UploaderID: &hmnUserID,
}
if i.Width != nil {
in.Width = *i.Width
}
if i.Height != nil {
in.Height = *i.Height
}
asset, err := assets.Create(ctx, tx, in)
if err != nil {
return nil, oops.New(err, "failed to create asset from embed")
}
return &asset.ID, nil
}
return nil, nil
}
var imageAssetId *uuid.UUID
var videoAssetId *uuid.UUID
var err error
if embed.Video != nil && embed.Video.Url != nil {
videoAssetId, err = maybeSaveImageish(embed.Video.EmbedImageish, isOkVideoType)
} else if embed.Image != nil && embed.Image.Url != nil {
imageAssetId, err = maybeSaveImageish(embed.Image.EmbedImageish, isOkImageType)
} else if embed.Thumbnail != nil && embed.Thumbnail.Url != nil {
imageAssetId, err = maybeSaveImageish(embed.Thumbnail.EmbedImageish, isOkImageType)
}
if err != nil {
return nil, err
}
// Save the embed into the db
// TODO(db): Insert, RETURNING
var savedEmbedId int
err = tx.QueryRow(ctx,
`
INSERT INTO handmade_discordmessageembed (title, description, url, message_id, image_id, video_id)
VALUES ($1, $2, $3, $4, $5, $6)
RETURNING id
`,
embed.Title,
embed.Description,
embed.Url,
discordMessageID,
imageAssetId,
videoAssetId,
).Scan(&savedEmbedId)
if err != nil {
return nil, oops.New(err, "failed to insert new embed")
}
iDiscordEmbed, err := db.QueryOne(ctx, tx, models.DiscordMessageEmbed{},
`
SELECT $columns
FROM handmade_discordmessageembed
WHERE id = $1
`,
savedEmbedId,
)
if err != nil {
return nil, oops.New(err, "failed to fetch new Discord embed data")
}
return iDiscordEmbed.(*models.DiscordMessageEmbed), nil
}
func FetchSnippetForMessage(ctx context.Context, dbConn db.ConnOrTx, msgID string) (*models.Snippet, error) {
iresult, err := db.QueryOne(ctx, dbConn, models.Snippet{},
`
SELECT $columns
FROM handmade_snippet
WHERE discord_message_id = $1
`,
msgID,
)
if err != nil {
if errors.Is(err, db.NotFound) {
return nil, nil
} else {
return nil, oops.New(err, "failed to fetch existing snippet for message %s", msgID)
}
}
return iresult.(*models.Snippet), nil
}
/*
Potentially creates or updates a snippet for the given interned message.
It uses the content saved in the database to do this. If we do not have any
content saved, nothing will happen.
If a user does not have their Discord account linked, this function will
naturally do nothing because we have no message content saved.
If forceCreate is true, it does not check any user settings such as automatically creating snippets from
#project-showcase. If we have the content, it will make a snippet for it, no
questions asked. Bear that in mind.
*/
func HandleSnippetForInternedMessage(ctx context.Context, dbConn db.ConnOrTx, interned *InternedMessage, forceCreate bool) error {
if interned.HMNUser == nil {
// NOTE(asaf): Can't handle snippets when there's no linked user
return nil
}
if interned.MessageContent == nil {
// NOTE(asaf): Can't have a snippet without content
// NOTE(asaf): Messages that only have an attachment also have blank content
// TODO(asaf): Do we need to delete existing snippets in this case??? Not entirely sure how to trigger this through discord
return nil
}
tx, err := dbConn.Begin(ctx)
if err != nil {
oops.New(err, "failed to start transaction")
}
defer tx.Rollback(ctx)
existingSnippet, err := FetchSnippetForMessage(ctx, tx, interned.Message.ID)
if err != nil {
return oops.New(err, "failed to check for existing snippet for message %s", interned.Message.ID)
}
if existingSnippet != nil {
// TODO(asaf): We're not handling the case where embeds were removed or modified.
// Also not handling the case where a message had both an attachment and an embed
// and the attachment was removed (leaving only the embed).
LinkedUserIsSnippetOwner := existingSnippet.OwnerID == interned.DiscordUser.HMNUserId
if LinkedUserIsSnippetOwner && !existingSnippet.EditedOnWebsite {
contentMarkdown := interned.MessageContent.LastContent
contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
_, err := tx.Exec(ctx,
`
UPDATE handmade_snippet
SET
description = $1,
_description_html = $2
WHERE id = $3
`,
contentMarkdown,
contentHTML,
existingSnippet.ID,
)
if err != nil {
return oops.New(err, "failed to update content of snippet on message edit")
}
existingSnippet.Description = contentMarkdown
existingSnippet.DescriptionHtml = contentHTML
}
} else {
userAllowsSnippet := interned.HMNUser.DiscordSaveShowcase || forceCreate
shouldCreate := !interned.Message.SnippetCreated && userAllowsSnippet
if shouldCreate {
// Get an asset ID or URL to make a snippet from
assetId, url, err := getSnippetAssetOrUrl(ctx, tx, &interned.Message)
if assetId != nil || url != nil {
contentMarkdown := interned.MessageContent.LastContent
contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
_, err = tx.Exec(ctx,
`
INSERT INTO handmade_snippet (url, "when", description, _description_html, asset_id, discord_message_id, owner_id)
VALUES ($1, $2, $3, $4, $5, $6, $7)
`,
url,
interned.Message.SentAt,
contentMarkdown,
contentHTML,
assetId,
interned.Message.ID,
interned.HMNUser.ID,
)
if err != nil {
return oops.New(err, "failed to create snippet from attachment")
}
existingSnippet, err = FetchSnippetForMessage(ctx, tx, interned.Message.ID)
if err != nil {
return oops.New(err, "failed to fetch newly-created snippet")
}
_, err = tx.Exec(ctx,
`
UPDATE handmade_discordmessage
SET snippet_created = TRUE
WHERE id = $1
`,
interned.Message.ID,
)
if err != nil {
return oops.New(err, "failed to mark message as having snippet")
}
}
}
}
if existingSnippet != nil {
// Update tags
// Try to associate tags in the message with project tags in HMN.
// Match only tags for projects in which the current user is a collaborator.
messageTags := getDiscordTags(existingSnippet.Description)
var desiredTags []int
var allTags []int
if len(messageTags) > 0 {
// Fetch projects so we know what tags the user can apply to their snippet.
projects, err := hmndata.FetchProjects(ctx, tx, interned.HMNUser, hmndata.ProjectsQuery{
OwnerIDs: []int{interned.HMNUser.ID},
})
if err != nil {
return oops.New(err, "failed to look up user projects")
}
projectIDs := make([]int, len(projects))
for i, p := range projects {
projectIDs[i] = p.Project.ID
}
type tagsRow struct {
Tag models.Tag `db:"tags"`
}
iUserTags, err := db.Query(ctx, tx, tagsRow{},
`
SELECT $columns
FROM
tags
JOIN handmade_project AS project ON project.tag = tags.id
WHERE
project.id = ANY ($1)
`,
projectIDs,
)
if err != nil {
return oops.New(err, "failed to fetch tags for user projects")
}
for _, itag := range iUserTags {
tag := itag.(*tagsRow).Tag
for _, messageTag := range messageTags {
allTags = append(allTags, tag.ID)
if strings.EqualFold(tag.Text, messageTag) {
desiredTags = append(desiredTags, tag.ID)
}
}
}
}
_, err = tx.Exec(ctx,
`
DELETE FROM snippet_tags
WHERE
snippet_id = $1
AND tag_id = ANY ($2)
`,
existingSnippet.ID,
allTags,
)
if err != nil {
return oops.New(err, "failed to clear tags from snippet")
}
for _, tagID := range desiredTags {
_, err = tx.Exec(ctx,
`
INSERT INTO snippet_tags (snippet_id, tag_id)
VALUES ($1, $2)
ON CONFLICT DO NOTHING
`,
existingSnippet.ID,
tagID,
)
if err != nil {
return oops.New(err, "failed to associate snippet with tag")
}
}
}
err = tx.Commit(ctx)
if err != nil {
return oops.New(err, "failed to commit transaction")
}
return nil
}
// TODO(asaf): I believe this will also match https://example.com?hello=1&whatever=5
// Probably need to add word boundaries.
var REDiscordTag = regexp.MustCompile(`&([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*)`)
func getDiscordTags(content string) []string {
matches := REDiscordTag.FindAllStringSubmatch(content, -1)
result := make([]string, len(matches))
for i, m := range matches {
result[i] = m[1]
}
return result
}
// NOTE(ben): This is maybe redundant with the regexes we use for markdown. But
// do we actually want to reuse those, or should we keep them separate?
var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`)
func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msg *models.DiscordMessage) (*uuid.UUID, *string, error) {
// Check attachments
attachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{},
`
SELECT $columns
FROM handmade_discordmessageattachment
WHERE message_id = $1
`,
msg.ID,
)
if err != nil {
return nil, nil, oops.New(err, "failed to fetch message attachments")
}
for _, iattachment := range attachments {
attachment := iattachment.(*models.DiscordMessageAttachment)
return &attachment.AssetID, nil, nil
}
// Check embeds
embeds, err := db.Query(ctx, tx, models.DiscordMessageEmbed{},
`
SELECT $columns
FROM handmade_discordmessageembed
WHERE message_id = $1
`,
msg.ID,
)
if err != nil {
return nil, nil, oops.New(err, "failed to fetch discord embeds")
}
for _, iembed := range embeds {
embed := iembed.(*models.DiscordMessageEmbed)
if embed.VideoID != nil {
return embed.VideoID, nil, nil
} else if embed.ImageID != nil {
return embed.ImageID, nil, nil
} else if embed.URL != nil {
if RESnippetableUrl.MatchString(*embed.URL) {
return nil, embed.URL, nil
}
}
}
return nil, nil, nil
}
var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`)
func messageHasLinks(content string) bool {
links := reDiscordMessageLink.FindAllString(content, -1)
for _, link := range links {
_, err := url.Parse(strings.TrimSpace(link))
if err == nil {
return true
}
}
return false
}

873
src/discord/showcase.go
View File

@ -1,873 +0,0 @@
package discord
import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"git.handmade.network/hmn/hmn/src/assets"
"git.handmade.network/hmn/hmn/src/config"
"git.handmade.network/hmn/hmn/src/db"
"git.handmade.network/hmn/hmn/src/hmndata"
"git.handmade.network/hmn/hmn/src/logging"
"git.handmade.network/hmn/hmn/src/models"
"git.handmade.network/hmn/hmn/src/oops"
"git.handmade.network/hmn/hmn/src/parsing"
"github.com/google/uuid"
)
var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`)
var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this")
func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) error {
switch msg.Type {
case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
default:
return nil
}
didDelete, err := bot.maybeDeleteShowcaseMsg(ctx, msg)
if err != nil {
return err
}
if didDelete {
return nil
}
tx, err := bot.dbConn.Begin(ctx)
if err != nil {
panic(err)
}
defer tx.Rollback(ctx)
// save the message, maybe save its contents
newMsg, err := SaveMessageAndContents(ctx, tx, msg)
if errors.Is(err, errNotEnoughInfo) {
logging.ExtractLogger(ctx).Warn().
Interface("msg", msg).
Msg("didn't have enough info to process Discord message")
return nil
} else if err != nil {
return err
}
// ...and maybe make a snippet too, if the user wants us to
duser, err := FetchDiscordUser(ctx, tx, newMsg.UserID)
if err == nil && duser.HMNUser.DiscordSaveShowcase {
err = CreateMessageSnippets(ctx, tx, newMsg.UserID, msg.ID)
if err != nil {
return oops.New(err, "failed to create snippet in gateway")
}
} else {
if err == db.NotFound {
// this is fine, just don't create a snippet
} else {
return err
}
}
err = tx.Commit(ctx)
if err != nil {
return oops.New(err, "failed to commit Discord message updates")
}
return nil
}
func (bot *botInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message) (didDelete bool, err error) {
hasGoodContent := true
if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) {
hasGoodContent = false
}
hasGoodAttachments := true
if msg.OriginalHasFields("attachments") && len(msg.Attachments) == 0 {
hasGoodAttachments = false
}
didDelete = false
if !hasGoodContent && !hasGoodAttachments {
didDelete = true
err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
if err != nil {
return false, oops.New(err, "failed to delete message")
}
if !msg.Author.IsBot {
channel, err := CreateDM(ctx, msg.Author.ID)
if err != nil {
return false, oops.New(err, "failed to create DM channel")
}
err = SendMessages(ctx, bot.dbConn, MessageToSend{
ChannelID: channel.ID,
Req: CreateMessageRequest{
Content: "Posts in #project-showcase are required to have either an image/video or a link. Discuss showcase content in #projects.",
},
})
if err != nil {
return false, oops.New(err, "failed to send showcase warning message")
}
}
}
return didDelete, nil
}
/*
Ensures that a Discord message is stored in the database. This function is
idempotent and can be called regardless of whether the item already exists in
the database.
This does not create snippets or do anything besides save the message itself.
*/
func SaveMessage(
ctx context.Context,
tx db.ConnOrTx,
msg *Message,
) (*models.DiscordMessage, error) {
iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{},
`
SELECT $columns
FROM handmade_discordmessage
WHERE id = $1
`,
msg.ID,
)
if errors.Is(err, db.NotFound) {
if !msg.OriginalHasFields("author", "timestamp") {
return nil, errNotEnoughInfo
}
guildID := msg.GuildID
if guildID == nil {
/*
This is weird, but it can happen when we fetch messages from
history instead of receiving it from the gateway. In this case
we just assume it's from the HMN server.
*/
guildID = &config.Config.Discord.GuildID
}
_, err = tx.Exec(ctx,
`
INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created)
VALUES ($1, $2, $3, $4, $5, $6, $7)
`,
msg.ID,
msg.ChannelID,
*guildID,
msg.JumpURL(),
msg.Author.ID,
msg.Time(),
false,
)
if err != nil {
return nil, oops.New(err, "failed to save new discord message")
}
/*
TODO(db): This is a spot where it would be really nice to be able
to use RETURNING, and avoid this second query.
*/
iDiscordMessage, err = db.QueryOne(ctx, tx, models.DiscordMessage{},
`
SELECT $columns
FROM handmade_discordmessage
WHERE id = $1
`,
msg.ID,
)
if err != nil {
panic(err)
}
} else if err != nil {
return nil, oops.New(err, "failed to check for existing Discord message")
}
return iDiscordMessage.(*models.DiscordMessage), nil
}
/*
Processes a single Discord message, saving as much of the message's content
and attachments as allowed by our rules and user settings. Does NOT create
snippets.
Idempotent; can be called any time whether the message exists or not.
*/
func SaveMessageAndContents(
ctx context.Context,
tx db.ConnOrTx,
msg *Message,
) (*models.DiscordMessage, error) {
newMsg, err := SaveMessage(ctx, tx, msg)
if err != nil {
return nil, err
}
// Check for linked Discord user
iDiscordUser, err := db.QueryOne(ctx, tx, models.DiscordUser{},
`
SELECT $columns
FROM handmade_discorduser
WHERE userid = $1
`,
newMsg.UserID,
)
if errors.Is(err, db.NotFound) {
return newMsg, nil
} else if err != nil {
return nil, oops.New(err, "failed to look up linked Discord user")
}
discordUser := iDiscordUser.(*models.DiscordUser)
// We have a linked Discord account, so save the message contents (regardless of
// whether we create a snippet or not).
if msg.OriginalHasFields("content") {
_, err = tx.Exec(ctx,
`
INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content)
VALUES ($1, $2, $3)
ON CONFLICT (message_id) DO UPDATE SET
discord_id = EXCLUDED.discord_id,
last_content = EXCLUDED.last_content
`,
newMsg.ID,
discordUser.ID,
CleanUpMarkdown(ctx, msg.Content),
)
}
// Save attachments
if msg.OriginalHasFields("attachments") {
for _, attachment := range msg.Attachments {
_, err := saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID)
if err != nil {
return nil, oops.New(err, "failed to save attachment")
}
}
}
// Save / delete embeds
if msg.OriginalHasFields("embeds") {
numSavedEmbeds, err := db.QueryInt(ctx, tx,
`
SELECT COUNT(*)
FROM handmade_discordmessageembed
WHERE message_id = $1
`,
msg.ID,
)
if err != nil {
return nil, oops.New(err, "failed to count existing embeds")
}
if numSavedEmbeds == 0 {
// No embeds yet, so save new ones
for _, embed := range msg.Embeds {
_, err := saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID)
if err != nil {
return nil, oops.New(err, "failed to save embed")
}
}
} else if len(msg.Embeds) > 0 {
// Embeds were removed from the message
_, err := tx.Exec(ctx,
`
DELETE FROM handmade_discordmessageembed
WHERE message_id = $1
`,
msg.ID,
)
if err != nil {
return nil, oops.New(err, "failed to delete embeds")
}
}
}
return newMsg, nil
}
var discordDownloadClient = &http.Client{
Timeout: 10 * time.Second,
}
type DiscordResourceBadStatusCode error
func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, "", oops.New(err, "failed to make Discord download request")
}
res, err := discordDownloadClient.Do(req)
if err != nil {
return nil, "", oops.New(err, "failed to fetch Discord resource data")
}
defer res.Body.Close()
if res.StatusCode < 200 || 299 < res.StatusCode {
return nil, "", DiscordResourceBadStatusCode(fmt.Errorf("status code %d from Discord resource: %s", res.StatusCode, url))
}
content, err := io.ReadAll(res.Body)
if err != nil {
panic(err)
}
return content, res.Header.Get("Content-Type"), nil
}
/*
Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment
that already exists
*/
func saveAttachment(
ctx context.Context,
tx db.ConnOrTx,
attachment *Attachment,
hmnUserID int,
discordMessageID string,
) (*models.DiscordMessageAttachment, error) {
iexisting, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
`
SELECT $columns
FROM handmade_discordmessageattachment
WHERE id = $1
`,
attachment.ID,
)
if err == nil {
return iexisting.(*models.DiscordMessageAttachment), nil
} else if errors.Is(err, db.NotFound) {
// this is fine, just create it
} else {
return nil, oops.New(err, "failed to check for existing attachment")
}
width := 0
height := 0
if attachment.Width != nil {
width = *attachment.Width
}
if attachment.Height != nil {
height = *attachment.Height
}
content, _, err := downloadDiscordResource(ctx, attachment.Url)
if err != nil {
return nil, oops.New(err, "failed to download Discord attachment")
}
contentType := "application/octet-stream"
if attachment.ContentType != nil {
contentType = *attachment.ContentType
}
asset, err := assets.Create(ctx, tx, assets.CreateInput{
Content: content,
Filename: attachment.Filename,
ContentType: contentType,
UploaderID: &hmnUserID,
Width: width,
Height: height,
})
if err != nil {
return nil, oops.New(err, "failed to save asset for Discord attachment")
}
// TODO(db): RETURNING plz thanks
_, err = tx.Exec(ctx,
`
INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id)
VALUES ($1, $2, $3)
`,
attachment.ID,
asset.ID,
discordMessageID,
)
if err != nil {
return nil, oops.New(err, "failed to save Discord attachment data")
}
iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
`
SELECT $columns
FROM handmade_discordmessageattachment
WHERE id = $1
`,
attachment.ID,
)
if err != nil {
return nil, oops.New(err, "failed to fetch new Discord attachment data")
}
return iDiscordAttachment.(*models.DiscordMessageAttachment), nil
}
// Saves an embed from Discord. NOTE: This is _not_ idempotent, so only call it
// if you do not have any embeds saved for this message yet.
func saveEmbed(
ctx context.Context,
tx db.ConnOrTx,
embed *Embed,
hmnUserID int,
discordMessageID string,
) (*models.DiscordMessageEmbed, error) {
isOkImageType := func(contentType string) bool {
return strings.HasPrefix(contentType, "image/")
}
isOkVideoType := func(contentType string) bool {
return strings.HasPrefix(contentType, "video/")
}
maybeSaveImageish := func(i EmbedImageish, contentTypeCheck func(string) bool) (*uuid.UUID, error) {
content, contentType, err := downloadDiscordResource(ctx, *i.Url)
if err != nil {
var statusError DiscordResourceBadStatusCode
if errors.As(err, &statusError) {
return nil, nil
} else {
return nil, oops.New(err, "failed to save Discord embed")
}
}
if contentTypeCheck(contentType) {
in := assets.CreateInput{
Content: content,
Filename: "embed",
ContentType: contentType,
UploaderID: &hmnUserID,
}
if i.Width != nil {
in.Width = *i.Width
}
if i.Height != nil {
in.Height = *i.Height
}
asset, err := assets.Create(ctx, tx, in)
if err != nil {
return nil, oops.New(err, "failed to create asset from embed")
}
return &asset.ID, nil
}
return nil, nil
}
var imageAssetId *uuid.UUID
var videoAssetId *uuid.UUID
var err error
if embed.Video != nil && embed.Video.Url != nil {
videoAssetId, err = maybeSaveImageish(embed.Video.EmbedImageish, isOkVideoType)
} else if embed.Image != nil && embed.Image.Url != nil {
imageAssetId, err = maybeSaveImageish(embed.Image.EmbedImageish, isOkImageType)
} else if embed.Thumbnail != nil && embed.Thumbnail.Url != nil {
imageAssetId, err = maybeSaveImageish(embed.Thumbnail.EmbedImageish, isOkImageType)
}
if err != nil {
return nil, err
}
// Save the embed into the db
// TODO(db): Insert, RETURNING
var savedEmbedId int
err = tx.QueryRow(ctx,
`
INSERT INTO handmade_discordmessageembed (title, description, url, message_id, image_id, video_id)
VALUES ($1, $2, $3, $4, $5, $6)
RETURNING id
`,
embed.Title,
embed.Description,
embed.Url,
discordMessageID,
imageAssetId,
videoAssetId,
).Scan(&savedEmbedId)
if err != nil {
return nil, oops.New(err, "failed to insert new embed")
}
iDiscordEmbed, err := db.QueryOne(ctx, tx, models.DiscordMessageEmbed{},
`
SELECT $columns
FROM handmade_discordmessageembed
WHERE id = $1
`,
savedEmbedId,
)
if err != nil {
return nil, oops.New(err, "failed to fetch new Discord embed data")
}
return iDiscordEmbed.(*models.DiscordMessageEmbed), nil
}
type DiscordUserAndStuff struct {
DiscordUser models.DiscordUser `db:"duser"`
HMNUser models.User `db:"u"`
}
func FetchDiscordUser(ctx context.Context, dbConn db.ConnOrTx, discordUserID string) (*DiscordUserAndStuff, error) {
iuser, err := db.QueryOne(ctx, dbConn, DiscordUserAndStuff{},
`
SELECT $columns
FROM
handmade_discorduser AS duser
JOIN auth_user AS u ON duser.hmn_user_id = u.id
LEFT JOIN handmade_asset AS u_avatar ON u_avatar.id = u.avatar_asset_id
WHERE
duser.userid = $1
`,
discordUserID,
)
if err != nil {
return nil, err
}
return iuser.(*DiscordUserAndStuff), nil
}
/*
Checks settings and permissions to decide whether we are allowed to create
snippets for a user.
*/
func AllowedToCreateMessageSnippets(ctx context.Context, tx db.ConnOrTx, discordUserId string) (bool, error) {
u, err := FetchDiscordUser(ctx, tx, discordUserId)
if errors.Is(err, db.NotFound) {
return false, nil
} else if err != nil {
return false, oops.New(err, "failed to check if we can save Discord message")
}
return u.HMNUser.DiscordSaveShowcase, nil
}
/*
Attempts to create snippets from Discord messages. If a snippet already exists
for any message, no new snippet will be created.
It uses the content saved in the database to do this. If we do not have any
content saved, nothing will happen.
If a user does not have their Discord account linked, this function will
naturally do nothing because we have no message content saved. However, it does
not check any user settings such as automatically creating snippets from
#project-showcase. If we have the content, it will make a snippet for it, no
questions asked. Bear that in mind.
*/
func CreateMessageSnippets(ctx context.Context, dbConn db.ConnOrTx, msgIDs ...string) error {
tx, err := dbConn.Begin(ctx)
if err != nil {
return oops.New(err, "failed to begin transaction")
}
defer tx.Rollback(ctx)
for _, msgID := range msgIDs {
// Check for existing snippet
type existingSnippetResult struct {
Message models.DiscordMessage `db:"msg"`
MessageContent *models.DiscordMessageContent `db:"c"`
Snippet *models.Snippet `db:"snippet"`
DiscordUser *models.DiscordUser `db:"duser"`
}
iexisting, err := db.QueryOne(ctx, tx, existingSnippetResult{},
`
SELECT $columns
FROM
handmade_discordmessage AS msg
LEFT JOIN handmade_discordmessagecontent AS c ON c.message_id = msg.id
LEFT JOIN handmade_snippet AS snippet ON snippet.discord_message_id = msg.id
LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid
WHERE
msg.id = $1
`,
msgID,
)
if err != nil {
return oops.New(err, "failed to check for existing snippet for message %s", msgID)
}
existing := iexisting.(*existingSnippetResult)
if existing.Snippet != nil {
// A snippet already exists - maybe update its content.
if existing.MessageContent != nil && !existing.Snippet.EditedOnWebsite {
contentMarkdown := existing.MessageContent.LastContent
contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
_, err := tx.Exec(ctx,
`
UPDATE handmade_snippet
SET
description = $1,
_description_html = $2
WHERE id = $3
`,
contentMarkdown,
contentHTML,
existing.Snippet.ID,
)
if err != nil {
logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to update content of snippet on message edit")
}
continue
}
}
if existing.Message.SnippetCreated {
// A snippet once existed but no longer does
// (we do not create another one in this case)
return nil
}
if existing.MessageContent == nil || existing.DiscordUser == nil {
return nil
}
// Get an asset ID or URL to make a snippet from
assetId, url, err := getSnippetAssetOrUrl(ctx, tx, &existing.Message)
if assetId == nil && url == nil {
// Nothing to make a snippet from!
return nil
}
contentMarkdown := existing.MessageContent.LastContent
contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
// TODO(db): Insert
_, err = tx.Exec(ctx,
`
INSERT INTO handmade_snippet (url, "when", description, _description_html, asset_id, discord_message_id, owner_id)
VALUES ($1, $2, $3, $4, $5, $6, $7)
`,
url,
existing.Message.SentAt,
contentMarkdown,
contentHTML,
assetId,
msgID,
existing.DiscordUser.HMNUserId,
)
if err != nil {
return oops.New(err, "failed to create snippet from attachment")
}
_, err = tx.Exec(ctx,
`
UPDATE handmade_discordmessage
SET snippet_created = TRUE
WHERE id = $1
`,
msgID,
)
if err != nil {
return oops.New(err, "failed to mark message as having snippet")
}
}
err = tx.Commit(ctx)
if err != nil {
return oops.New(err, "failed to commit transaction")
}
return nil
}
/*
Associates any Discord tags with website tags for projects. Idempotent; will
clear out any existing project tags and then add new ones.
If no Discord user is linked, or no snippet exists, or whatever, this will do
nothing and return no error.
*/
func UpdateSnippetTagsIfAny(ctx context.Context, dbConn db.ConnOrTx, msg *Message) error {
tx, err := dbConn.Begin(ctx)
if err != nil {
return oops.New(err, "failed to start transaction")
}
defer tx.Rollback(ctx)
// Fetch the Discord user; we only process messages for users with linked
// Discord accounts
u, err := FetchDiscordUser(ctx, tx, msg.Author.ID)
if err == db.NotFound {
return nil
} else if err != nil {
return oops.New(err, "failed to look up HMN user information from Discord user")
}
// Fetch the s associated with this Discord message (if any). If the
// s has already been edited on the website we'll skip it.
s, err := hmndata.FetchSnippetForDiscordMessage(ctx, tx, &u.HMNUser, msg.ID, hmndata.SnippetQuery{})
if err == db.NotFound {
return nil
} else if err != nil {
return err
}
// Fetch projects so we know what tags the user can apply to their snippet.
projects, err := hmndata.FetchProjects(ctx, tx, &u.HMNUser, hmndata.ProjectsQuery{
OwnerIDs: []int{u.HMNUser.ID},
})
if err != nil {
return oops.New(err, "failed to look up user projects")
}
projectIDs := make([]int, len(projects))
for i, p := range projects {
projectIDs[i] = p.Project.ID
}
// Delete any existing project tags for this snippet. We don't want to
// delete other tags in case in the future we have manual tagging on the
// website or whatever, and this would clear those out.
_, err = tx.Exec(ctx,
`
DELETE FROM snippet_tags
WHERE
snippet_id = $1
AND tag_id IN (
SELECT tag FROM handmade_project
)
`,
s.Snippet.ID,
)
if err != nil {
return oops.New(err, "failed to delete existing snippet tags")
}
// Try to associate tags in the message with project tags in HMN.
// Match only tags for projects in which the current user is a collaborator.
messageTags := getDiscordTags(s.Snippet.Description)
type tagsRow struct {
Tag models.Tag `db:"tags"`
}
iUserTags, err := db.Query(ctx, tx, tagsRow{},
`
SELECT $columns
FROM
tags
JOIN handmade_project AS project ON project.tag = tags.id
JOIN handmade_user_projects AS user_project ON user_project.project_id = project.id
WHERE
project.id = ANY ($1)
`,
projectIDs,
)
if err != nil {
return oops.New(err, "failed to fetch tags for user projects")
}
var tagIDs []int
for _, itag := range iUserTags {
tag := itag.(*tagsRow).Tag
for _, messageTag := range messageTags {
if tag.Text == messageTag {
tagIDs = append(tagIDs, tag.ID)
}
}
}
for _, tagID := range tagIDs {
_, err = tx.Exec(ctx,
`
INSERT INTO snippet_tags (snippet_id, tag_id)
VALUES ($1, $2)
ON CONFLICT DO NOTHING
`,
s.Snippet.ID,
tagID,
)
if err != nil {
return oops.New(err, "failed to add tag to snippet")
}
}
err = tx.Commit(ctx)
if err != nil {
return oops.New(err, "failed to commit transaction")
}
return nil
}
// NOTE(ben): This is maybe redundant with the regexes we use for markdown. But
// do we actually want to reuse those, or should we keep them separate?
var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`)
func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msg *models.DiscordMessage) (*uuid.UUID, *string, error) {
// Check attachments
attachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{},
`
SELECT $columns
FROM handmade_discordmessageattachment
WHERE message_id = $1
`,
msg.ID,
)
if err != nil {
return nil, nil, oops.New(err, "failed to fetch message attachments")
}
for _, iattachment := range attachments {
attachment := iattachment.(*models.DiscordMessageAttachment)
return &attachment.AssetID, nil, nil
}
// Check embeds
embeds, err := db.Query(ctx, tx, models.DiscordMessageEmbed{},
`
SELECT $columns
FROM handmade_discordmessageembed
WHERE message_id = $1
`,
msg.ID,
)
if err != nil {
return nil, nil, oops.New(err, "failed to fetch discord embeds")
}
for _, iembed := range embeds {
embed := iembed.(*models.DiscordMessageEmbed)
if embed.VideoID != nil {
return embed.VideoID, nil, nil
} else if embed.ImageID != nil {
return embed.ImageID, nil, nil
} else if embed.URL != nil {
if RESnippetableUrl.MatchString(*embed.URL) {
return nil, embed.URL, nil
}
}
}
return nil, nil, nil
}
func messageHasLinks(content string) bool {
links := reDiscordMessageLink.FindAllString(content, -1)
for _, link := range links {
_, err := url.Parse(strings.TrimSpace(link))
if err == nil {
return true
}
}
return false
}
var REDiscordTag = regexp.MustCompile(`&([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*)`)
func getDiscordTags(content string) []string {
matches := REDiscordTag.FindAllStringSubmatch(content, -1)
result := make([]string, len(matches))
for i, m := range matches {
result[i] = strings.ToLower(m[1])
}
return result
}

26
src/website/discord.go
View File

@ -143,17 +143,6 @@ func DiscordShowcaseBacklog(c *RequestContext) ResponseData {
}
duser := iduser.(*models.DiscordUser)
ok, err := discord.AllowedToCreateMessageSnippets(c.Context(), c.Conn, duser.UserID)
if err != nil {
return c.ErrorResponse(http.StatusInternalServerError, err)
}
if !ok {
// Not allowed to do this, bail out
c.Logger.Warn().Msg("was not allowed to save user snippets")
return c.Redirect(hmnurl.BuildUserProfile(c.CurrentUser.Username), http.StatusSeeOther)
}
type messageIdQuery struct {
MessageID string `db:"msg.id"`
}
@ -177,9 +166,18 @@ func DiscordShowcaseBacklog(c *RequestContext) ResponseData {
for _, imsgId := range iMsgIDs {
msgIDs = append(msgIDs, imsgId.(*messageIdQuery).MessageID)
}
err = discord.CreateMessageSnippets(c.Context(), c.Conn, msgIDs...)
if err != nil {
return c.ErrorResponse(http.StatusInternalServerError, err)
for _, msgID := range msgIDs {
interned, err := discord.FetchInternedMessage(c.Context(), c.Conn, msgID)
if err != nil {
return c.ErrorResponse(http.StatusInternalServerError, err)
}
if interned != nil {
// NOTE(asaf): Creating snippet even if the checkbox is off because the user asked us to.
err = discord.HandleSnippetForInternedMessage(c.Context(), c.Conn, interned, true)
if err != nil {
return c.ErrorResponse(http.StatusInternalServerError, err)
}
}
}
return c.Redirect(hmnurl.BuildUserProfile(c.CurrentUser.Username), http.StatusSeeOther)