Watch for snippet updates on all messages

This captures stuff in jam-showcase and ryan's stuff in #projects
This commit is contained in:
Ben Visness 2021-12-13 01:40:54 -06:00
parent 6d609f1fae
commit 3b8b02a856
6 changed files with 230 additions and 151 deletions

View File

@ -6,13 +6,20 @@ import (
"git.handmade.network/hmn/hmn/src/db" "git.handmade.network/hmn/hmn/src/db"
"git.handmade.network/hmn/hmn/src/discord" "git.handmade.network/hmn/hmn/src/discord"
"git.handmade.network/hmn/hmn/src/logging"
"git.handmade.network/hmn/hmn/src/website" "git.handmade.network/hmn/hmn/src/website"
"github.com/spf13/cobra" "github.com/spf13/cobra"
) )
func init() { func init() {
rootCommand := &cobra.Command{
Use: "discord",
Short: "Commands for interacting with Discord",
}
website.WebsiteCommand.AddCommand(rootCommand)
scrapeCommand := &cobra.Command{ scrapeCommand := &cobra.Command{
Use: "discordscrapechannel [<channel id>...]", Use: "scrapechannel [<channel id>...]",
Short: "Scrape the entire history of Discord channels", Short: "Scrape the entire history of Discord channels",
Long: "Scrape the entire history of Discord channels, saving message content (but not creating snippets)", Long: "Scrape the entire history of Discord channels, saving message content (but not creating snippets)",
Run: func(cmd *cobra.Command, args []string) { Run: func(cmd *cobra.Command, args []string) {
@ -25,6 +32,22 @@ func init() {
} }
}, },
} }
rootCommand.AddCommand(scrapeCommand)
website.WebsiteCommand.AddCommand(scrapeCommand) makeSnippetCommand := &cobra.Command{
Use: "makesnippet [<message id>...]",
Short: "Make snippets from saved Discord messages",
Long: "Make snippets from Discord messages whose content we have already saved. Useful for creating snippets from messages in non-showcase channels.",
Run: func(cmd *cobra.Command, args []string) {
ctx := context.Background()
conn := db.NewConnPool(1, 1)
defer conn.Close()
err := discord.CreateMessageSnippets(ctx, conn, args...)
if err != nil {
logging.Error().Err(err).Msg("failed to create snippets")
}
},
}
rootCommand.AddCommand(makeSnippetCommand)
} }

View File

@ -601,15 +601,6 @@ func (bot *botInstance) messageCreateOrUpdate(ctx context.Context, msg *Message)
return nil return nil
} }
// if msg.ChannelID == config.Config.Discord.JamShowcaseChannelID {
// err := bot.processShowcaseMsg(ctx, msg)
// if err != nil {
// logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to process jam showcase message")
// return nil
// }
// return nil
// }
if msg.ChannelID == config.Config.Discord.LibraryChannelID { if msg.ChannelID == config.Config.Discord.LibraryChannelID {
err := bot.processLibraryMsg(ctx, msg) err := bot.processLibraryMsg(ctx, msg)
if err != nil { if err != nil {
@ -619,6 +610,11 @@ func (bot *botInstance) messageCreateOrUpdate(ctx context.Context, msg *Message)
return nil return nil
} }
err := UpdateSnippetTagsIfAny(ctx, bot.dbConn, msg)
if err != nil {
logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to update tags for Discord snippet")
}
return nil return nil
} }

View File

@ -199,13 +199,11 @@ func handleHistoryMessage(ctx context.Context, dbConn *pgxpool.Pool, msg *Messag
return err return err
} }
if createSnippets { if createSnippets {
if doSnippet, err := AllowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil { if doSnippet, err := AllowedToCreateMessageSnippets(ctx, tx, newMsg.UserID); doSnippet && err == nil {
_, err := CreateMessageSnippet(ctx, tx, newMsg.UserID, msg.ID) err := CreateMessageSnippets(ctx, tx, msg.ID)
if err != nil { if err != nil {
return err return err
} }
} else if err != nil {
return err
} }
} }

View File

@ -47,7 +47,7 @@ func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) er
} }
defer tx.Rollback(ctx) defer tx.Rollback(ctx)
// save the message, maybe save its contents, and maybe make a snippet too // save the message, maybe save its contents
newMsg, err := SaveMessageAndContents(ctx, tx, msg) newMsg, err := SaveMessageAndContents(ctx, tx, msg)
if errors.Is(err, errNotEnoughInfo) { if errors.Is(err, errNotEnoughInfo) {
logging.ExtractLogger(ctx).Warn(). logging.ExtractLogger(ctx).Warn().
@ -57,13 +57,20 @@ func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) er
} else if err != nil { } else if err != nil {
return err return err
} }
if doSnippet, err := AllowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil {
_, err := CreateMessageSnippet(ctx, tx, newMsg.UserID, msg.ID) // ...and maybe make a snippet too, if the user wants us to
duser, err := FetchDiscordUser(ctx, tx, newMsg.UserID)
if err == nil && duser.HMNUser.DiscordSaveShowcase {
err = CreateMessageSnippets(ctx, tx, newMsg.UserID, msg.ID)
if err != nil { if err != nil {
return oops.New(err, "failed to create snippet in gateway") return oops.New(err, "failed to create snippet in gateway")
} }
} else if err != nil { } else {
return oops.New(err, "failed to check snippet permissions in gateway") if err == db.NotFound {
// this is fine, just don't create a snippet
} else {
return err
}
} }
err = tx.Commit(ctx) err = tx.Commit(ctx)
@ -534,7 +541,7 @@ func FetchDiscordUser(ctx context.Context, dbConn db.ConnOrTx, discordUserID str
Checks settings and permissions to decide whether we are allowed to create Checks settings and permissions to decide whether we are allowed to create
snippets for a user. snippets for a user.
*/ */
func AllowedToCreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, discordUserId string) (bool, error) { func AllowedToCreateMessageSnippets(ctx context.Context, tx db.ConnOrTx, discordUserId string) (bool, error) {
u, err := FetchDiscordUser(ctx, tx, discordUserId) u, err := FetchDiscordUser(ctx, tx, discordUserId)
if errors.Is(err, db.NotFound) { if errors.Is(err, db.NotFound) {
return false, nil return false, nil
@ -546,23 +553,27 @@ func AllowedToCreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, discordU
} }
/* /*
Attempts to create a snippet from a Discord message. If a snippet already Attempts to create snippets from Discord messages. If a snippet already exists
exists, it will be returned and no new snippets will be created. for any message, no new snippet will be created.
It uses the content saved in the database to do this. If we do not have It uses the content saved in the database to do this. If we do not have any
any content saved, nothing will happen. content saved, nothing will happen.
Does not check user preferences around snippets. If a user does not have their Discord account linked, this function will
naturally do nothing because we have no message content saved. However, it does
not check any user settings such as automatically creating snippets from
#project-showcase. If we have the content, it will make a snippet for it, no
questions asked. Bear that in mind.
*/ */
func CreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, userID, msgID string) (snippet *models.Snippet, err error) { func CreateMessageSnippets(ctx context.Context, dbConn db.ConnOrTx, msgIDs ...string) error {
defer func() { tx, err := dbConn.Begin(ctx)
err := updateSnippetTags(ctx, tx, userID, snippet)
if err != nil { if err != nil {
logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to update tags for Discord snippet") return oops.New(err, "failed to begin transaction")
} }
}() defer tx.Rollback(ctx)
// Check for existing snippet, maybe return it for _, msgID := range msgIDs {
// Check for existing snippet
type existingSnippetResult struct { type existingSnippetResult struct {
Message models.DiscordMessage `db:"msg"` Message models.DiscordMessage `db:"msg"`
MessageContent *models.DiscordMessageContent `db:"c"` MessageContent *models.DiscordMessageContent `db:"c"`
@ -583,24 +594,23 @@ func CreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, userID, msgID str
msgID, msgID,
) )
if err != nil { if err != nil {
return nil, oops.New(err, "failed to check for existing snippet for message %s", msgID) return oops.New(err, "failed to check for existing snippet for message %s", msgID)
} }
existing := iexisting.(*existingSnippetResult) existing := iexisting.(*existingSnippetResult)
if existing.Snippet != nil { if existing.Snippet != nil {
// A snippet already exists - maybe update its content, then return it // A snippet already exists - maybe update its content.
if existing.MessageContent != nil && !existing.Snippet.EditedOnWebsite { if existing.MessageContent != nil && !existing.Snippet.EditedOnWebsite {
contentMarkdown := existing.MessageContent.LastContent contentMarkdown := existing.MessageContent.LastContent
contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown) contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
iSnippet, err := db.QueryOne(ctx, tx, models.Snippet{}, _, err := tx.Exec(ctx,
` `
UPDATE handmade_snippet UPDATE handmade_snippet
SET SET
description = $1, description = $1,
_description_html = $2 _description_html = $2
WHERE id = $3 WHERE id = $3
RETURNING $columns
`, `,
contentMarkdown, contentMarkdown,
contentHTML, contentHTML,
@ -609,38 +619,35 @@ func CreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, userID, msgID str
if err != nil { if err != nil {
logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to update content of snippet on message edit") logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to update content of snippet on message edit")
} }
return iSnippet.(*models.Snippet), nil continue
} else {
return existing.Snippet, nil
} }
} }
if existing.Message.SnippetCreated { if existing.Message.SnippetCreated {
// A snippet once existed but no longer does // A snippet once existed but no longer does
// (we do not create another one in this case) // (we do not create another one in this case)
return nil, nil return nil
} }
if existing.MessageContent == nil || existing.DiscordUser == nil { if existing.MessageContent == nil || existing.DiscordUser == nil {
return nil, nil return nil
} }
// Get an asset ID or URL to make a snippet from // Get an asset ID or URL to make a snippet from
assetId, url, err := getSnippetAssetOrUrl(ctx, tx, &existing.Message) assetId, url, err := getSnippetAssetOrUrl(ctx, tx, &existing.Message)
if assetId == nil && url == nil { if assetId == nil && url == nil {
// Nothing to make a snippet from! // Nothing to make a snippet from!
return nil, nil return nil
} }
contentMarkdown := existing.MessageContent.LastContent contentMarkdown := existing.MessageContent.LastContent
contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown) contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
// TODO(db): Insert // TODO(db): Insert
isnippet, err := db.QueryOne(ctx, tx, models.Snippet{}, _, err = tx.Exec(ctx,
` `
INSERT INTO handmade_snippet (url, "when", description, _description_html, asset_id, discord_message_id, owner_id) INSERT INTO handmade_snippet (url, "when", description, _description_html, asset_id, discord_message_id, owner_id)
VALUES ($1, $2, $3, $4, $5, $6, $7) VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING $columns
`, `,
url, url,
existing.Message.SentAt, existing.Message.SentAt,
@ -651,7 +658,7 @@ func CreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, userID, msgID str
existing.DiscordUser.HMNUserId, existing.DiscordUser.HMNUserId,
) )
if err != nil { if err != nil {
return nil, oops.New(err, "failed to create snippet from attachment") return oops.New(err, "failed to create snippet from attachment")
} }
_, err = tx.Exec(ctx, _, err = tx.Exec(ctx,
` `
@ -662,29 +669,51 @@ func CreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, userID, msgID str
msgID, msgID,
) )
if err != nil { if err != nil {
return nil, oops.New(err, "failed to mark message as having snippet") return oops.New(err, "failed to mark message as having snippet")
}
} }
return isnippet.(*models.Snippet), nil err = tx.Commit(ctx)
if err != nil {
return oops.New(err, "failed to commit transaction")
}
return nil
} }
/* /*
Associates any Discord tags with website tags. Idempotent; will clear Associates any Discord tags with website tags for projects. Idempotent; will
out any existing tags and then add new ones. clear out any existing project tags and then add new ones.
If no Discord user is linked, or no snippet exists, or whatever, this will do
nothing and return no error.
*/ */
func updateSnippetTags(ctx context.Context, dbConn db.ConnOrTx, userID string, snippet *models.Snippet) error { func UpdateSnippetTagsIfAny(ctx context.Context, dbConn db.ConnOrTx, msg *Message) error {
tx, err := dbConn.Begin(ctx) tx, err := dbConn.Begin(ctx)
if err != nil { if err != nil {
return oops.New(err, "failed to start transaction") return oops.New(err, "failed to start transaction")
} }
defer tx.Rollback(ctx) defer tx.Rollback(ctx)
u, err := FetchDiscordUser(ctx, tx, userID) // Fetch the Discord user; we only process messages for users with linked
if err != nil { // Discord accounts
u, err := FetchDiscordUser(ctx, tx, msg.Author.ID)
if err == db.NotFound {
return nil
} else if err != nil {
return oops.New(err, "failed to look up HMN user information from Discord user") return oops.New(err, "failed to look up HMN user information from Discord user")
// we shouldn't see a "not found" here because of the AllowedToBlahBlahBlah check earlier in the process
} }
// Fetch the s associated with this Discord message (if any). If the
// s has already been edited on the website we'll skip it.
s, err := hmndata.FetchSnippetForDiscordMessage(ctx, tx, &u.HMNUser, msg.ID, hmndata.SnippetQuery{})
if err == db.NotFound {
return nil
} else if err != nil {
return err
}
// Fetch projects so we know what tags the user can apply to their snippet.
projects, err := hmndata.FetchProjects(ctx, tx, &u.HMNUser, hmndata.ProjectsQuery{ projects, err := hmndata.FetchProjects(ctx, tx, &u.HMNUser, hmndata.ProjectsQuery{
OwnerIDs: []int{u.HMNUser.ID}, OwnerIDs: []int{u.HMNUser.ID},
}) })
@ -696,13 +725,19 @@ func updateSnippetTags(ctx context.Context, dbConn db.ConnOrTx, userID string, s
projectIDs[i] = p.Project.ID projectIDs[i] = p.Project.ID
} }
// Delete any existing tags for this snippet // Delete any existing project tags for this snippet. We don't want to
// delete other tags in case in the future we have manual tagging on the
// website or whatever, and this would clear those out.
_, err = tx.Exec(ctx, _, err = tx.Exec(ctx,
` `
DELETE FROM snippet_tags DELETE FROM snippet_tags
WHERE snippet_id = $1 WHERE
snippet_id = $1
AND tag_id IN (
SELECT tag FROM handmade_project
)
`, `,
snippet.ID, s.Snippet.ID,
) )
if err != nil { if err != nil {
return oops.New(err, "failed to delete existing snippet tags") return oops.New(err, "failed to delete existing snippet tags")
@ -710,7 +745,7 @@ func updateSnippetTags(ctx context.Context, dbConn db.ConnOrTx, userID string, s
// Try to associate tags in the message with project tags in HMN. // Try to associate tags in the message with project tags in HMN.
// Match only tags for projects in which the current user is a collaborator. // Match only tags for projects in which the current user is a collaborator.
messageTags := getDiscordTags(snippet.Description) messageTags := getDiscordTags(s.Snippet.Description)
type tagsRow struct { type tagsRow struct {
Tag models.Tag `db:"tags"` Tag models.Tag `db:"tags"`
} }
@ -748,7 +783,7 @@ func updateSnippetTags(ctx context.Context, dbConn db.ConnOrTx, userID string, s
VALUES ($1, $2) VALUES ($1, $2)
ON CONFLICT DO NOTHING ON CONFLICT DO NOTHING
`, `,
snippet.ID, s.Snippet.ID,
tagID, tagID,
) )
if err != nil { if err != nil {

View File

@ -13,6 +13,7 @@ type SnippetQuery struct {
IDs []int IDs []int
OwnerIDs []int OwnerIDs []int
Tags []int Tags []int
DiscordMessageIDs []string
Limit, Offset int // if empty, no pagination Limit, Offset int // if empty, no pagination
} }
@ -92,6 +93,9 @@ func FetchSnippets(
if len(q.OwnerIDs) > 0 { if len(q.OwnerIDs) > 0 {
qb.Add(`AND snippet.owner_id = ANY ($?)`, q.OwnerIDs) qb.Add(`AND snippet.owner_id = ANY ($?)`, q.OwnerIDs)
} }
if len(q.DiscordMessageIDs) > 0 {
qb.Add(`AND snippet.discord_message_id = ANY ($?)`, q.DiscordMessageIDs)
}
if currentUser == nil { if currentUser == nil {
qb.Add( qb.Add(
`AND owner.status = $? -- snippet owner is Approved`, `AND owner.status = $? -- snippet owner is Approved`,
@ -204,3 +208,26 @@ func FetchSnippet(
return res[0], nil return res[0], nil
} }
func FetchSnippetForDiscordMessage(
ctx context.Context,
dbConn db.ConnOrTx,
currentUser *models.User,
discordMessageID string,
q SnippetQuery,
) (SnippetAndStuff, error) {
q.DiscordMessageIDs = []string{discordMessageID}
q.Limit = 1
q.Offset = 0
res, err := FetchSnippets(ctx, dbConn, currentUser, q)
if err != nil {
return SnippetAndStuff{}, oops.New(err, "failed to fetch snippet for Discord message")
}
if len(res) == 0 {
return SnippetAndStuff{}, db.NotFound
}
return res[0], nil
}

View File

@ -143,7 +143,7 @@ func DiscordShowcaseBacklog(c *RequestContext) ResponseData {
} }
duser := iduser.(*models.DiscordUser) duser := iduser.(*models.DiscordUser)
ok, err := discord.AllowedToCreateMessageSnippet(c.Context(), c.Conn, duser.UserID) ok, err := discord.AllowedToCreateMessageSnippets(c.Context(), c.Conn, duser.UserID)
if err != nil { if err != nil {
return c.ErrorResponse(http.StatusInternalServerError, err) return c.ErrorResponse(http.StatusInternalServerError, err)
} }
@ -157,7 +157,7 @@ func DiscordShowcaseBacklog(c *RequestContext) ResponseData {
type messageIdQuery struct { type messageIdQuery struct {
MessageID string `db:"msg.id"` MessageID string `db:"msg.id"`
} }
imsgIds, err := db.Query(c.Context(), c.Conn, messageIdQuery{}, itMsgIds, err := db.Query(c.Context(), c.Conn, messageIdQuery{},
` `
SELECT $columns SELECT $columns
FROM FROM
@ -169,15 +169,15 @@ func DiscordShowcaseBacklog(c *RequestContext) ResponseData {
duser.UserID, duser.UserID,
config.Config.Discord.ShowcaseChannelID, config.Config.Discord.ShowcaseChannelID,
) )
msgIds := imsgIds.ToSlice() iMsgIDs := itMsgIds.ToSlice()
for _, imsgId := range msgIds { var msgIDs []string
msgId := imsgId.(*messageIdQuery) for _, imsgId := range iMsgIDs {
_, err := discord.CreateMessageSnippet(c.Context(), c.Conn, duser.UserID, msgId.MessageID) msgIDs = append(msgIDs, imsgId.(*messageIdQuery).MessageID)
if err != nil {
c.Logger.Warn().Err(err).Msg("failed to create snippet from showcase backlog")
continue
} }
err = discord.CreateMessageSnippets(c.Context(), c.Conn, msgIDs...)
if err != nil {
return c.ErrorResponse(http.StatusInternalServerError, err)
} }
return c.Redirect(hmnurl.BuildUserProfile(c.CurrentUser.Username), http.StatusSeeOther) return c.Redirect(hmnurl.BuildUserProfile(c.CurrentUser.Username), http.StatusSeeOther)