Add background features to the Discord bot

This commit is contained in:
Ben Visness 2021-08-26 22:59:12 -05:00
parent 042e9166fd
commit 16ae2188d1
14 changed files with 451 additions and 102 deletions

View File

@ -58,6 +58,7 @@ func typeIsQueryable(t reflect.Type) bool {
// This interface should match both a direct pgx connection or a pgx transaction. // This interface should match both a direct pgx connection or a pgx transaction.
type ConnOrTx interface { type ConnOrTx interface {
Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error) Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error)
QueryRow(ctx context.Context, sql string, args ...interface{}) pgx.Row
Exec(ctx context.Context, sql string, args ...interface{}) (pgconn.CommandTag, error) Exec(ctx context.Context, sql string, args ...interface{}) (pgconn.CommandTag, error)
} }

30
src/discord/cmd/cmd.go Normal file
View File

@ -0,0 +1,30 @@
package cmd
import (
"context"
"time"
"git.handmade.network/hmn/hmn/src/db"
"git.handmade.network/hmn/hmn/src/discord"
"git.handmade.network/hmn/hmn/src/website"
"github.com/spf13/cobra"
)
func init() {
scrapeCommand := &cobra.Command{
Use: "discordscrapechannel [<channel id>...]",
Short: "Scrape the entire history of Discord channels",
Long: "Scrape the entire history of Discord channels, saving message content (but not creating snippets)",
Run: func(cmd *cobra.Command, args []string) {
ctx := context.Background()
conn := db.NewConnPool(1, 1)
defer conn.Close()
for _, channelID := range args {
discord.Scrape(ctx, conn, channelID, time.Time{}, false)
}
},
}
website.WebsiteCommand.AddCommand(scrapeCommand)
}

210
src/discord/history.go Normal file
View File

@ -0,0 +1,210 @@
package discord
import (
"context"
"errors"
"time"
"git.handmade.network/hmn/hmn/src/config"
"git.handmade.network/hmn/hmn/src/db"
"git.handmade.network/hmn/hmn/src/logging"
"git.handmade.network/hmn/hmn/src/models"
"github.com/jackc/pgx/v4"
"github.com/jackc/pgx/v4/pgxpool"
)
func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{} {
log := logging.ExtractLogger(ctx).With().Str("discord goroutine", "history watcher").Logger()
ctx = logging.AttachLoggerToContext(&log, ctx)
done := make(chan struct{})
go func() {
defer func() {
log.Debug().Msg("shut down Discord history watcher")
done <- struct{}{}
}()
backfillInterval := 1 * time.Hour
newUserTicker := time.NewTicker(5 * time.Second)
backfillTicker := time.NewTicker(backfillInterval)
lastBackfillTime := time.Now().Add(-backfillInterval)
for {
select {
case <-ctx.Done():
return
case <-newUserTicker.C:
// Get content for messages when a user links their account (but do not create snippets)
fetchMissingContent(ctx, dbConn)
case <-backfillTicker.C:
// Run a backfill to patch up places where the Discord bot missed (does create snippets)
Scrape(ctx, dbConn,
config.Config.Discord.ShowcaseChannelID,
lastBackfillTime,
true,
)
}
}
}()
return done
}
func fetchMissingContent(ctx context.Context, dbConn *pgxpool.Pool) {
log := logging.ExtractLogger(ctx)
type query struct {
Message models.DiscordMessage `db:"msg"`
}
result, err := db.Query(ctx, dbConn, query{},
`
SELECT $columns
FROM
handmade_discordmessage AS msg
JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid -- only fetch messages for linked discord users
LEFT JOIN handmade_discordmessagecontent AS c ON c.message_id = msg.id
WHERE
c.last_content IS NULL
AND msg.guild_id = $1
`,
config.Config.Discord.GuildID,
)
if err != nil {
log.Error().Err(err).Msg("failed to check for messages without content")
return
}
imessagesWithoutContent := result.ToSlice()
if len(imessagesWithoutContent) > 0 {
log.Info().Msgf("There are %d Discord messages without content, fetching their content now...", len(imessagesWithoutContent))
msgloop:
for _, imsg := range imessagesWithoutContent {
select {
case <-ctx.Done():
log.Info().Msg("Scrape was canceled")
break msgloop
default:
}
msg := imsg.(*query).Message
discordMsg, err := GetChannelMessage(ctx, msg.ChannelID, msg.ID)
if errors.Is(err, NotFound) {
// This message has apparently been deleted; delete it from our database
_, err = dbConn.Exec(ctx,
`
DELETE FROM handmade_discordmessage
WHERE id = $1
`,
msg.ID,
)
if err != nil {
log.Error().Err(err).Msg("failed to delete missing message")
continue
}
log.Info().Str("msg id", msg.ID).Msg("deleted missing Discord message")
continue
} else if err != nil {
log.Error().Err(err).Msg("failed to get message")
continue
}
log.Info().Str("msg", discordMsg.ShortString()).Msg("fetched message for content")
err = handleHistoryMessage(ctx, dbConn, discordMsg, false)
if err != nil {
log.Error().Err(err).Msg("failed to save content for message")
continue
}
}
log.Info().Msgf("Done fetching missing content")
}
}
func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earliestMessageTime time.Time, createSnippets bool) {
log := logging.ExtractLogger(ctx)
log.Info().Msg("Starting scrape")
defer log.Info().Msg("Done with scrape!")
before := ""
for {
msgs, err := GetChannelMessages(ctx, channelID, GetChannelMessagesInput{
Limit: 100,
Before: before,
})
if err != nil {
panic(err) // TODO
}
if len(msgs) == 0 {
logging.Debug().Msg("out of messages, stopping scrape")
return
}
for _, msg := range msgs {
select {
case <-ctx.Done():
log.Info().Msg("Scrape was canceled")
return
default:
}
log.Info().Str("msg", msg.ShortString()).Msg("")
if !earliestMessageTime.IsZero() && msg.Time().Before(earliestMessageTime) {
logging.ExtractLogger(ctx).Info().Time("earliest", earliestMessageTime).Msg("Saw a message before the specified earliest time; exiting")
return
}
err := handleHistoryMessage(ctx, dbConn, &msg, true)
if err != nil {
errLog := logging.ExtractLogger(ctx).Error()
if errors.Is(err, errNotEnoughInfo) {
errLog = logging.ExtractLogger(ctx).Warn()
}
errLog.Err(err).Msg("failed to process Discord message")
}
before = msg.ID
}
}
}
func handleHistoryMessage(ctx context.Context, dbConn *pgxpool.Pool, msg *Message, createSnippets bool) error {
var tx pgx.Tx
for {
var err error
tx, err = dbConn.Begin(ctx)
if err != nil {
logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to start transaction for message")
time.Sleep(1 * time.Second)
continue
}
break
}
newMsg, err := saveMessageAndContents(ctx, tx, msg)
if err != nil {
return err
}
if createSnippets {
if doSnippet, err := allowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil {
_, err := createMessageSnippet(ctx, tx, msg)
if err != nil {
return err
}
} else if err != nil {
return err
}
}
err = tx.Commit(ctx)
if err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,16 @@
package discord
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
)
func TestGetMessage(t *testing.T) {
// t.Skip("this test is only for debugging")
msg, err := GetChannelMessage(context.Background(), "404399251276169217", "764575065772916790")
assert.Nil(t, err)
t.Logf("%+v", msg)
}

View File

@ -252,9 +252,16 @@ func (m *Message) Time() time.Time {
return t return t
} }
func (m *Message) ShortString() string {
return fmt.Sprintf("%s / %s: \"%s\" (%d attachments, %d embeds)", m.Timestamp, m.Author.Username, m.Content, len(m.Attachments), len(m.Embeds))
}
func (m *Message) OriginalHasFields(fields ...string) bool { func (m *Message) OriginalHasFields(fields ...string) bool {
if m.originalMap == nil { if m.originalMap == nil {
return false // If we don't know, we assume the fields are there.
// Usually this is because it came from their API, where we
// always have all fields.
return true
} }
for _, field := range fields { for _, field := range fields {

View File

@ -10,6 +10,7 @@ import (
"net/http" "net/http"
"net/http/httputil" "net/http/httputil"
"net/url" "net/url"
"strconv"
"strings" "strings"
"git.handmade.network/hmn/hmn/src/config" "git.handmade.network/hmn/hmn/src/config"
@ -411,6 +412,93 @@ func RemoveGuildMemberRole(ctx context.Context, userID, roleID string) error {
return nil return nil
} }
func GetChannelMessage(ctx context.Context, channelID, messageID string) (*Message, error) {
const name = "Get Channel Message"
path := fmt.Sprintf("/channels/%s/messages/%s", channelID, messageID)
res, err := doWithRateLimiting(ctx, name, func(ctx context.Context) *http.Request {
return makeRequest(ctx, http.MethodGet, path, nil)
})
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode == http.StatusNotFound {
return nil, NotFound
} else if res.StatusCode >= 400 {
logErrorResponse(ctx, name, res, "")
return nil, oops.New(nil, "received error from Discord")
}
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
panic(err)
}
var msg Message
err = json.Unmarshal(bodyBytes, &msg)
if err != nil {
return nil, oops.New(err, "failed to unmarshal Discord message")
}
return &msg, nil
}
type GetChannelMessagesInput struct {
Around string
Before string
After string
Limit int
}
func GetChannelMessages(ctx context.Context, channelID string, in GetChannelMessagesInput) ([]Message, error) {
const name = "Get Channel Messages"
path := fmt.Sprintf("/channels/%s/messages", channelID)
res, err := doWithRateLimiting(ctx, name, func(ctx context.Context) *http.Request {
req := makeRequest(ctx, http.MethodGet, path, nil)
q := req.URL.Query()
if in.Around != "" {
q.Add("around", in.Around)
}
if in.Before != "" {
q.Add("before", in.Before)
}
if in.After != "" {
q.Add("after", in.After)
}
if in.Limit != 0 {
q.Add("limit", strconv.Itoa(in.Limit))
}
req.URL.RawQuery = q.Encode()
return req
})
if err != nil {
return nil, err
}
defer res.Body.Close()
if res.StatusCode >= 400 {
logErrorResponse(ctx, name, res, "")
return nil, oops.New(nil, "received error from Discord")
}
bodyBytes, err := io.ReadAll(res.Body)
if err != nil {
panic(err)
}
var msgs []Message
err = json.Unmarshal(bodyBytes, &msgs)
if err != nil {
return nil, oops.New(err, "failed to unmarshal Discord message")
}
return msgs, nil
}
func logErrorResponse(ctx context.Context, name string, res *http.Response, msg string) { func logErrorResponse(ctx context.Context, name string, res *http.Response, msg string) {
dump, err := httputil.DumpResponse(res, true) dump, err := httputil.DumpResponse(res, true)
if err != nil { if err != nil {

View File

@ -12,13 +12,13 @@ import (
"time" "time"
"git.handmade.network/hmn/hmn/src/assets" "git.handmade.network/hmn/hmn/src/assets"
"git.handmade.network/hmn/hmn/src/config"
"git.handmade.network/hmn/hmn/src/db" "git.handmade.network/hmn/hmn/src/db"
"git.handmade.network/hmn/hmn/src/logging" "git.handmade.network/hmn/hmn/src/logging"
"git.handmade.network/hmn/hmn/src/models" "git.handmade.network/hmn/hmn/src/models"
"git.handmade.network/hmn/hmn/src/oops" "git.handmade.network/hmn/hmn/src/oops"
"git.handmade.network/hmn/hmn/src/parsing" "git.handmade.network/hmn/hmn/src/parsing"
"github.com/google/uuid" "github.com/google/uuid"
"github.com/jackc/pgx/v4"
) )
var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`) var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`)
@ -47,7 +47,7 @@ func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) er
defer tx.Rollback(ctx) defer tx.Rollback(ctx)
// save the message, maybe save its contents, and maybe make a snippet too // save the message, maybe save its contents, and maybe make a snippet too
newMsg, err := bot.saveMessageAndContents(ctx, tx, msg) newMsg, err := saveMessageAndContents(ctx, tx, msg)
if errors.Is(err, errNotEnoughInfo) { if errors.Is(err, errNotEnoughInfo) {
logging.ExtractLogger(ctx).Warn(). logging.ExtractLogger(ctx).Warn().
Interface("msg", msg). Interface("msg", msg).
@ -56,8 +56,8 @@ func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) er
} else if err != nil { } else if err != nil {
return err return err
} }
if doSnippet, err := bot.allowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil { if doSnippet, err := allowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil {
_, err := bot.createMessageSnippet(ctx, tx, msg) _, err := createMessageSnippet(ctx, tx, msg)
if err != nil { if err != nil {
return oops.New(err, "failed to create snippet in gateway") return oops.New(err, "failed to create snippet in gateway")
} }
@ -120,9 +120,9 @@ the database.
This does not create snippets or do anything besides save the message itself. This does not create snippets or do anything besides save the message itself.
*/ */
func (bot *botInstance) saveMessage( func saveMessage(
ctx context.Context, ctx context.Context,
tx pgx.Tx, tx db.ConnOrTx,
msg *Message, msg *Message,
) (*models.DiscordMessage, error) { ) (*models.DiscordMessage, error) {
iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{}, iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{},
@ -138,6 +138,16 @@ func (bot *botInstance) saveMessage(
return nil, errNotEnoughInfo return nil, errNotEnoughInfo
} }
guildID := msg.GuildID
if guildID == nil {
/*
This is weird, but it can happen when we fetch messages from
history instead of receiving it from the gateway. In this case
we just assume it's from the HMN server.
*/
guildID = &config.Config.Discord.GuildID
}
_, err = tx.Exec(ctx, _, err = tx.Exec(ctx,
` `
INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created) INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created)
@ -145,7 +155,7 @@ func (bot *botInstance) saveMessage(
`, `,
msg.ID, msg.ID,
msg.ChannelID, msg.ChannelID,
*msg.GuildID, *guildID,
msg.JumpURL(), msg.JumpURL(),
msg.Author.ID, msg.Author.ID,
msg.Time(), msg.Time(),
@ -184,12 +194,12 @@ snippets.
Idempotent; can be called any time whether the message exists or not. Idempotent; can be called any time whether the message exists or not.
*/ */
func (bot *botInstance) saveMessageAndContents( func saveMessageAndContents(
ctx context.Context, ctx context.Context,
tx pgx.Tx, tx db.ConnOrTx,
msg *Message, msg *Message,
) (*models.DiscordMessage, error) { ) (*models.DiscordMessage, error) {
newMsg, err := bot.saveMessage(ctx, tx, msg) newMsg, err := saveMessage(ctx, tx, msg)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -231,7 +241,7 @@ func (bot *botInstance) saveMessageAndContents(
// Save attachments // Save attachments
if msg.OriginalHasFields("attachments") { if msg.OriginalHasFields("attachments") {
for _, attachment := range msg.Attachments { for _, attachment := range msg.Attachments {
_, err := bot.saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID) _, err := saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID)
if err != nil { if err != nil {
return nil, oops.New(err, "failed to save attachment") return nil, oops.New(err, "failed to save attachment")
} }
@ -254,7 +264,7 @@ func (bot *botInstance) saveMessageAndContents(
if numSavedEmbeds == 0 { if numSavedEmbeds == 0 {
// No embeds yet, so save new ones // No embeds yet, so save new ones
for _, embed := range msg.Embeds { for _, embed := range msg.Embeds {
_, err := bot.saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID) _, err := saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID)
if err != nil { if err != nil {
return nil, oops.New(err, "failed to save embed") return nil, oops.New(err, "failed to save embed")
} }
@ -310,9 +320,9 @@ func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, e
Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment
that already exists that already exists
*/ */
func (bot *botInstance) saveAttachment( func saveAttachment(
ctx context.Context, ctx context.Context,
tx pgx.Tx, tx db.ConnOrTx,
attachment *Attachment, attachment *Attachment,
hmnUserID int, hmnUserID int,
discordMessageID string, discordMessageID string,
@ -394,9 +404,9 @@ func (bot *botInstance) saveAttachment(
return iDiscordAttachment.(*models.DiscordMessageAttachment), nil return iDiscordAttachment.(*models.DiscordMessageAttachment), nil
} }
func (bot *botInstance) saveEmbed( func saveEmbed(
ctx context.Context, ctx context.Context,
tx pgx.Tx, tx db.ConnOrTx,
embed *Embed, embed *Embed,
hmnUserID int, hmnUserID int,
discordMessageID string, discordMessageID string,
@ -497,8 +507,8 @@ func (bot *botInstance) saveEmbed(
return iDiscordEmbed.(*models.DiscordMessageEmbed), nil return iDiscordEmbed.(*models.DiscordMessageEmbed), nil
} }
func (bot *botInstance) allowedToCreateMessageSnippet(ctx context.Context, tx pgx.Tx, discordUserId string) (bool, error) { func allowedToCreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, discordUserId string) (bool, error) {
canSave, err := db.QueryBool(ctx, bot.dbConn, canSave, err := db.QueryBool(ctx, tx,
` `
SELECT u.discord_save_showcase SELECT u.discord_save_showcase
FROM FROM
@ -518,7 +528,7 @@ func (bot *botInstance) allowedToCreateMessageSnippet(ctx context.Context, tx pg
return canSave, nil return canSave, nil
} }
func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg *Message) (*models.Snippet, error) { func createMessageSnippet(ctx context.Context, tx db.ConnOrTx, msg *Message) (*models.Snippet, error) {
// Check for existing snippet, maybe return it // Check for existing snippet, maybe return it
type existingSnippetResult struct { type existingSnippetResult struct {
Message models.DiscordMessage `db:"msg"` Message models.DiscordMessage `db:"msg"`
@ -548,7 +558,7 @@ func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg
// A snippet already exists - maybe update its content, then return it // A snippet already exists - maybe update its content, then return it
if msg.OriginalHasFields("content") && !existing.Snippet.EditedOnWebsite { if msg.OriginalHasFields("content") && !existing.Snippet.EditedOnWebsite {
contentMarkdown := existing.MessageContent.LastContent contentMarkdown := existing.MessageContent.LastContent
contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.RealMarkdown) contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
_, err := tx.Exec(ctx, _, err := tx.Exec(ctx,
` `
@ -580,14 +590,14 @@ func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg
} }
// Get an asset ID or URL to make a snippet from // Get an asset ID or URL to make a snippet from
assetId, url, err := bot.getSnippetAssetOrUrl(ctx, tx, &existing.Message) assetId, url, err := getSnippetAssetOrUrl(ctx, tx, &existing.Message)
if assetId == nil && url == "" { if assetId == nil && url == nil {
// Nothing to make a snippet from! // Nothing to make a snippet from!
return nil, nil return nil, nil
} }
contentMarkdown := existing.MessageContent.LastContent contentMarkdown := existing.MessageContent.LastContent
contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.RealMarkdown) contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
// TODO(db): Insert // TODO(db): Insert
isnippet, err := db.QueryOne(ctx, tx, models.Snippet{}, isnippet, err := db.QueryOne(ctx, tx, models.Snippet{},
@ -596,7 +606,7 @@ func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg
VALUES ($1, $2, $3, $4, $5, $6, $7) VALUES ($1, $2, $3, $4, $5, $6, $7)
RETURNING $columns RETURNING $columns
`, `,
nil, url,
existing.Message.SentAt, existing.Message.SentAt,
contentMarkdown, contentMarkdown,
contentHTML, contentHTML,
@ -626,7 +636,7 @@ func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg
// do we actually want to reuse those, or should we keep them separate? // do we actually want to reuse those, or should we keep them separate?
var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`) var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`)
func (bot *botInstance) getSnippetAssetOrUrl(ctx context.Context, tx pgx.Tx, msg *models.DiscordMessage) (*uuid.UUID, string, error) { func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msg *models.DiscordMessage) (*uuid.UUID, *string, error) {
// Check attachments // Check attachments
itAttachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{}, itAttachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{},
` `
@ -637,12 +647,12 @@ func (bot *botInstance) getSnippetAssetOrUrl(ctx context.Context, tx pgx.Tx, msg
msg.ID, msg.ID,
) )
if err != nil { if err != nil {
return nil, "", oops.New(err, "failed to fetch message attachments") return nil, nil, oops.New(err, "failed to fetch message attachments")
} }
attachments := itAttachments.ToSlice() attachments := itAttachments.ToSlice()
for _, iattachment := range attachments { for _, iattachment := range attachments {
attachment := iattachment.(*models.DiscordMessageAttachment) attachment := iattachment.(*models.DiscordMessageAttachment)
return &attachment.AssetID, "", nil return &attachment.AssetID, nil, nil
} }
// Check embeds // Check embeds
@ -655,23 +665,23 @@ func (bot *botInstance) getSnippetAssetOrUrl(ctx context.Context, tx pgx.Tx, msg
msg.ID, msg.ID,
) )
if err != nil { if err != nil {
return nil, "", oops.New(err, "failed to fetch discord embeds") return nil, nil, oops.New(err, "failed to fetch discord embeds")
} }
embeds := itEmbeds.ToSlice() embeds := itEmbeds.ToSlice()
for _, iembed := range embeds { for _, iembed := range embeds {
embed := iembed.(*models.DiscordMessageEmbed) embed := iembed.(*models.DiscordMessageEmbed)
if embed.VideoID != nil { if embed.VideoID != nil {
return embed.VideoID, "", nil return embed.VideoID, nil, nil
} else if embed.ImageID != nil { } else if embed.ImageID != nil {
return embed.ImageID, "", nil return embed.ImageID, nil, nil
} else if embed.URL != nil { } else if embed.URL != nil {
if RESnippetableUrl.MatchString(*embed.URL) { if RESnippetableUrl.MatchString(*embed.URL) {
return nil, *embed.URL, nil return nil, embed.URL, nil
} }
} }
} }
return nil, "", nil return nil, nil, nil
} }
func messageHasLinks(content string) bool { func messageHasLinks(content string) bool {

View File

@ -1,48 +0,0 @@
the goal: port the old discord showcase bot
what it does: save #project-showcase posts to your HMN user profile if you have your account linked
stuff we need to worry about:
- old posts from before you linked your account
- posts that come in while the bot is down
- what to do with posts if you unlink your account
- what to do with posts if you re-link your account
✔ - what to do if you edit the original discord message
- what to do if you delete the original discord message
✔ - the user's preferences re: saving content
- we don't want to save content without the user's consent, especially since it may persist after they disable the integration
- manually adding content for various reasons
- maybe a bug prevented something from saving
- ryan used to post everything in #projects for some reason
✔ real-time stuff:
✔ - on new showcase message
- always save the lightweight record
- if we have permission, create a snippet
✔ - on edit
- re-save the lightweight record and content as if it was new
- create snippet, unconditionally???? (bug??)
- update snippet contents if the edit makes sense
✔ - on delete
- delete snippet if the user so desires
- delete the message records
✔ - on bulk delete
- same stuff
background stuff:
- watch mode
- every five seconds
- fetch all HMN users with Discord accounts
- check if we have message records without content
- if so, run a full scrape (no snippets)
- every hour
- run a full scrape, creating snippets
- scrape behavior
- look at every message ever in the channel
- do exactly what the real-time bot does on new messages (although maybe don't do snippets depending on context)
what the heck do we do with discord's markdown
- when we save message contents, we should save both the raw discord markdown and a version with their custom stuff replaced. We do _not_ (yet) need a full markdown parse with HTML tags and stuff. (That arguably doesn't make sense for the handmade_discordmessagecontent record anyway.)
- when we create a snippet, we should store both markdown that makes sense to a user and the rendered version of that HTML. THIS MEANS: The markdown we save is the "clean" version of the Discord markdown.

View File

@ -4,6 +4,7 @@ import (
_ "git.handmade.network/hmn/hmn/src/admintools" _ "git.handmade.network/hmn/hmn/src/admintools"
_ "git.handmade.network/hmn/hmn/src/assets" _ "git.handmade.network/hmn/hmn/src/assets"
_ "git.handmade.network/hmn/hmn/src/buildscss" _ "git.handmade.network/hmn/hmn/src/buildscss"
_ "git.handmade.network/hmn/hmn/src/discord/cmd"
_ "git.handmade.network/hmn/hmn/src/initimage" _ "git.handmade.network/hmn/hmn/src/initimage"
_ "git.handmade.network/hmn/hmn/src/migration" _ "git.handmade.network/hmn/hmn/src/migration"
"git.handmade.network/hmn/hmn/src/website" "git.handmade.network/hmn/hmn/src/website"

View File

@ -10,21 +10,38 @@ import (
) )
// Used for rendering real-time previews of post content. // Used for rendering real-time previews of post content.
var PreviewMarkdown = goldmark.New( var ForumPreviewMarkdown = goldmark.New(
goldmark.WithExtensions(makeGoldmarkExtensions(true)...), goldmark.WithExtensions(makeGoldmarkExtensions(MarkdownOptions{
Previews: true,
Embeds: true,
})...),
) )
// Used for generating the final HTML for a post. // Used for generating the final HTML for a post.
var RealMarkdown = goldmark.New( var ForumRealMarkdown = goldmark.New(
goldmark.WithExtensions(makeGoldmarkExtensions(false)...), goldmark.WithExtensions(makeGoldmarkExtensions(MarkdownOptions{
Previews: false,
Embeds: true,
})...),
) )
// Used for generating plain-text previews of posts. // Used for generating plain-text previews of posts.
var PlaintextMarkdown = goldmark.New( var PlaintextMarkdown = goldmark.New(
goldmark.WithExtensions(makeGoldmarkExtensions(false)...), goldmark.WithExtensions(makeGoldmarkExtensions(MarkdownOptions{
Previews: false,
Embeds: true,
})...),
goldmark.WithRenderer(plaintextRenderer{}), goldmark.WithRenderer(plaintextRenderer{}),
) )
// Used for processing Discord messages
var DiscordMarkdown = goldmark.New(
goldmark.WithExtensions(makeGoldmarkExtensions(MarkdownOptions{
Previews: false,
Embeds: false,
})...),
)
func ParseMarkdown(source string, md goldmark.Markdown) string { func ParseMarkdown(source string, md goldmark.Markdown) string {
var buf bytes.Buffer var buf bytes.Buffer
if err := md.Convert([]byte(source), &buf); err != nil { if err := md.Convert([]byte(source), &buf); err != nil {
@ -34,19 +51,35 @@ func ParseMarkdown(source string, md goldmark.Markdown) string {
return buf.String() return buf.String()
} }
func makeGoldmarkExtensions(preview bool) []goldmark.Extender { type MarkdownOptions struct {
return []goldmark.Extender{ Previews bool
Embeds bool
}
func makeGoldmarkExtensions(opts MarkdownOptions) []goldmark.Extender {
var extenders []goldmark.Extender
extenders = append(extenders,
extension.GFM, extension.GFM,
highlightExtension, highlightExtension,
SpoilerExtension{}, SpoilerExtension{},
)
if opts.Embeds {
extenders = append(extenders,
EmbedExtension{ EmbedExtension{
Preview: preview, Preview: opts.Previews,
}, },
)
}
extenders = append(extenders,
MathjaxExtension{}, MathjaxExtension{},
BBCodeExtension{ BBCodeExtension{
Preview: preview, Preview: opts.Previews,
}, },
} )
return extenders
} }
var highlightExtension = highlighting.NewHighlighting( var highlightExtension = highlighting.NewHighlighting(

View File

@ -10,14 +10,14 @@ import (
func TestMarkdown(t *testing.T) { func TestMarkdown(t *testing.T) {
t.Run("fenced code blocks", func(t *testing.T) { t.Run("fenced code blocks", func(t *testing.T) {
t.Run("multiple lines", func(t *testing.T) { t.Run("multiple lines", func(t *testing.T) {
html := ParseMarkdown("```\nmultiple lines\n\tof code\n```", RealMarkdown) html := ParseMarkdown("```\nmultiple lines\n\tof code\n```", ForumRealMarkdown)
t.Log(html) t.Log(html)
assert.Equal(t, 1, strings.Count(html, "<pre")) assert.Equal(t, 1, strings.Count(html, "<pre"))
assert.Contains(t, html, `class="hmn-code"`) assert.Contains(t, html, `class="hmn-code"`)
assert.Contains(t, html, "multiple lines\n\tof code") assert.Contains(t, html, "multiple lines\n\tof code")
}) })
t.Run("multiple lines with language", func(t *testing.T) { t.Run("multiple lines with language", func(t *testing.T) {
html := ParseMarkdown("```go\nfunc main() {\n\tfmt.Println(\"Hello, world!\")\n}\n```", RealMarkdown) html := ParseMarkdown("```go\nfunc main() {\n\tfmt.Println(\"Hello, world!\")\n}\n```", ForumRealMarkdown)
t.Log(html) t.Log(html)
assert.Equal(t, 1, strings.Count(html, "<pre")) assert.Equal(t, 1, strings.Count(html, "<pre"))
assert.Contains(t, html, `class="hmn-code"`) assert.Contains(t, html, `class="hmn-code"`)
@ -30,7 +30,7 @@ func TestMarkdown(t *testing.T) {
func TestBBCode(t *testing.T) { func TestBBCode(t *testing.T) {
t.Run("[code]", func(t *testing.T) { t.Run("[code]", func(t *testing.T) {
t.Run("one line", func(t *testing.T) { t.Run("one line", func(t *testing.T) {
html := ParseMarkdown("[code]Just some code, you know?[/code]", RealMarkdown) html := ParseMarkdown("[code]Just some code, you know?[/code]", ForumRealMarkdown)
t.Log(html) t.Log(html)
assert.Equal(t, 1, strings.Count(html, "<pre")) assert.Equal(t, 1, strings.Count(html, "<pre"))
assert.Contains(t, html, `class="hmn-code"`) assert.Contains(t, html, `class="hmn-code"`)
@ -41,7 +41,7 @@ func TestBBCode(t *testing.T) {
Multiline code Multiline code
with an indent with an indent
[/code]` [/code]`
html := ParseMarkdown(bbcode, RealMarkdown) html := ParseMarkdown(bbcode, ForumRealMarkdown)
t.Log(html) t.Log(html)
assert.Equal(t, 1, strings.Count(html, "<pre")) assert.Equal(t, 1, strings.Count(html, "<pre"))
assert.Contains(t, html, `class="hmn-code"`) assert.Contains(t, html, `class="hmn-code"`)
@ -54,7 +54,7 @@ func main() {
fmt.Println("Hello, world!") fmt.Println("Hello, world!")
} }
[/code]` [/code]`
html := ParseMarkdown(bbcode, RealMarkdown) html := ParseMarkdown(bbcode, ForumRealMarkdown)
t.Log(html) t.Log(html)
assert.Equal(t, 1, strings.Count(html, "<pre")) assert.Equal(t, 1, strings.Count(html, "<pre"))
assert.Contains(t, html, "Println") assert.Contains(t, html, "Println")
@ -66,7 +66,7 @@ func main() {
func TestSharlock(t *testing.T) { func TestSharlock(t *testing.T) {
t.Skipf("This doesn't pass right now because parts of Sharlock's original source read as indented code blocks, or depend on different line break behavior.") t.Skipf("This doesn't pass right now because parts of Sharlock's original source read as indented code blocks, or depend on different line break behavior.")
t.Run("sanity check", func(t *testing.T) { t.Run("sanity check", func(t *testing.T) {
result := ParseMarkdown(sharlock, RealMarkdown) result := ParseMarkdown(sharlock, ForumRealMarkdown)
for _, line := range strings.Split(result, "\n") { for _, line := range strings.Split(result, "\n") {
assert.NotContains(t, line, "[b]") assert.NotContains(t, line, "[b]")
@ -85,6 +85,6 @@ func TestSharlock(t *testing.T) {
func BenchmarkSharlock(b *testing.B) { func BenchmarkSharlock(b *testing.B) {
for i := 0; i < b.N; i++ { for i := 0; i < b.N; i++ {
ParseMarkdown(sharlock, RealMarkdown) ParseMarkdown(sharlock, ForumRealMarkdown)
} }
} }

View File

@ -474,7 +474,7 @@ func PodcastEpisodeSubmit(c *RequestContext) ResponseData {
} }
c.Perf.StartBlock("MARKDOWN", "Parsing description") c.Perf.StartBlock("MARKDOWN", "Parsing description")
descriptionRendered := parsing.ParseMarkdown(description, parsing.RealMarkdown) descriptionRendered := parsing.ParseMarkdown(description, parsing.ForumRealMarkdown)
c.Perf.EndBlock() c.Perf.EndBlock()
guidStr := "" guidStr := ""

View File

@ -332,7 +332,7 @@ func DeletePost(
} }
func CreatePostVersion(ctx context.Context, tx pgx.Tx, postId int, unparsedContent string, ipString string, editReason string, editorId *int) (versionId int) { func CreatePostVersion(ctx context.Context, tx pgx.Tx, postId int, unparsedContent string, ipString string, editReason string, editorId *int) (versionId int) {
parsed := parsing.ParseMarkdown(unparsedContent, parsing.RealMarkdown) parsed := parsing.ParseMarkdown(unparsedContent, parsing.ForumRealMarkdown)
ip := net.ParseIP(ipString) ip := net.ParseIP(ipString)
const previewMaxLength = 100 const previewMaxLength = 100

View File

@ -45,6 +45,7 @@ var WebsiteCommand = &cobra.Command{
auth.PeriodicallyDeleteInactiveUsers(backgroundJobContext, conn), auth.PeriodicallyDeleteInactiveUsers(backgroundJobContext, conn),
perfCollector.Done, perfCollector.Done,
discord.RunDiscordBot(backgroundJobContext, conn), discord.RunDiscordBot(backgroundJobContext, conn),
discord.RunHistoryWatcher(backgroundJobContext, conn),
) )
signals := make(chan os.Signal, 1) signals := make(chan os.Signal, 1)