Add background features to the Discord bot

src/db/db.go

@@ -58,6 +58,7 @@ func typeIsQueryable(t reflect.Type) bool {
 // This interface should match both a direct pgx connection or a pgx transaction.
 type ConnOrTx interface {
 	Query(ctx context.Context, sql string, args ...interface{}) (pgx.Rows, error)
+	QueryRow(ctx context.Context, sql string, args ...interface{}) pgx.Row
 	Exec(ctx context.Context, sql string, args ...interface{}) (pgconn.CommandTag, error)
 }
 

src/discord/cmd/cmd.go

@@ -0,0 +1,30 @@
+package cmd
+
+import (
+	"context"
+	"time"
+
+	"git.handmade.network/hmn/hmn/src/db"
+	"git.handmade.network/hmn/hmn/src/discord"
+	"git.handmade.network/hmn/hmn/src/website"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	scrapeCommand := &cobra.Command{
+		Use:   "discordscrapechannel [<channel id>...]",
+		Short: "Scrape the entire history of Discord channels",
+		Long:  "Scrape the entire history of Discord channels, saving message content (but not creating snippets)",
+		Run: func(cmd *cobra.Command, args []string) {
+			ctx := context.Background()
+			conn := db.NewConnPool(1, 1)
+			defer conn.Close()
+
+			for _, channelID := range args {
+				discord.Scrape(ctx, conn, channelID, time.Time{}, false)
+			}
+		},
+	}
+
+	website.WebsiteCommand.AddCommand(scrapeCommand)
+}

src/discord/history.go

@@ -0,0 +1,210 @@
+package discord
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"git.handmade.network/hmn/hmn/src/config"
+	"git.handmade.network/hmn/hmn/src/db"
+	"git.handmade.network/hmn/hmn/src/logging"
+	"git.handmade.network/hmn/hmn/src/models"
+	"github.com/jackc/pgx/v4"
+	"github.com/jackc/pgx/v4/pgxpool"
+)
+
+func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{} {
+	log := logging.ExtractLogger(ctx).With().Str("discord goroutine", "history watcher").Logger()
+	ctx = logging.AttachLoggerToContext(&log, ctx)
+
+	done := make(chan struct{})
+
+	go func() {
+		defer func() {
+			log.Debug().Msg("shut down Discord history watcher")
+			done <- struct{}{}
+		}()
+
+		backfillInterval := 1 * time.Hour
+
+		newUserTicker := time.NewTicker(5 * time.Second)
+		backfillTicker := time.NewTicker(backfillInterval)
+
+		lastBackfillTime := time.Now().Add(-backfillInterval)
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case <-newUserTicker.C:
+				// Get content for messages when a user links their account (but do not create snippets)
+				fetchMissingContent(ctx, dbConn)
+			case <-backfillTicker.C:
+				// Run a backfill to patch up places where the Discord bot missed (does create snippets)
+				Scrape(ctx, dbConn,
+					config.Config.Discord.ShowcaseChannelID,
+					lastBackfillTime,
+					true,
+				)
+			}
+		}
+	}()
+
+	return done
+}
+
+func fetchMissingContent(ctx context.Context, dbConn *pgxpool.Pool) {
+	log := logging.ExtractLogger(ctx)
+
+	type query struct {
+		Message models.DiscordMessage `db:"msg"`
+	}
+	result, err := db.Query(ctx, dbConn, query{},
+		`
+		SELECT $columns
+		FROM
+			handmade_discordmessage AS msg
+			JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid -- only fetch messages for linked discord users
+			LEFT JOIN handmade_discordmessagecontent AS c ON c.message_id = msg.id
+		WHERE
+			c.last_content IS NULL
+			AND msg.guild_id = $1
+		`,
+		config.Config.Discord.GuildID,
+	)
+	if err != nil {
+		log.Error().Err(err).Msg("failed to check for messages without content")
+		return
+	}
+	imessagesWithoutContent := result.ToSlice()
+
+	if len(imessagesWithoutContent) > 0 {
+		log.Info().Msgf("There are %d Discord messages without content, fetching their content now...", len(imessagesWithoutContent))
+	msgloop:
+		for _, imsg := range imessagesWithoutContent {
+			select {
+			case <-ctx.Done():
+				log.Info().Msg("Scrape was canceled")
+				break msgloop
+			default:
+			}
+
+			msg := imsg.(*query).Message
+
+			discordMsg, err := GetChannelMessage(ctx, msg.ChannelID, msg.ID)
+			if errors.Is(err, NotFound) {
+				// This message has apparently been deleted; delete it from our database
+				_, err = dbConn.Exec(ctx,
+					`
+					DELETE FROM handmade_discordmessage
+					WHERE id = $1
+					`,
+					msg.ID,
+				)
+				if err != nil {
+					log.Error().Err(err).Msg("failed to delete missing message")
+					continue
+				}
+				log.Info().Str("msg id", msg.ID).Msg("deleted missing Discord message")
+				continue
+			} else if err != nil {
+				log.Error().Err(err).Msg("failed to get message")
+				continue
+			}
+
+			log.Info().Str("msg", discordMsg.ShortString()).Msg("fetched message for content")
+
+			err = handleHistoryMessage(ctx, dbConn, discordMsg, false)
+			if err != nil {
+				log.Error().Err(err).Msg("failed to save content for message")
+				continue
+			}
+		}
+		log.Info().Msgf("Done fetching missing content")
+	}
+}
+
+func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earliestMessageTime time.Time, createSnippets bool) {
+	log := logging.ExtractLogger(ctx)
+
+	log.Info().Msg("Starting scrape")
+	defer log.Info().Msg("Done with scrape!")
+
+	before := ""
+	for {
+		msgs, err := GetChannelMessages(ctx, channelID, GetChannelMessagesInput{
+			Limit:  100,
+			Before: before,
+		})
+		if err != nil {
+			panic(err) // TODO
+		}
+
+		if len(msgs) == 0 {
+			logging.Debug().Msg("out of messages, stopping scrape")
+			return
+		}
+
+		for _, msg := range msgs {
+			select {
+			case <-ctx.Done():
+				log.Info().Msg("Scrape was canceled")
+				return
+			default:
+			}
+
+			log.Info().Str("msg", msg.ShortString()).Msg("")
+
+			if !earliestMessageTime.IsZero() && msg.Time().Before(earliestMessageTime) {
+				logging.ExtractLogger(ctx).Info().Time("earliest", earliestMessageTime).Msg("Saw a message before the specified earliest time; exiting")
+				return
+			}
+
+			err := handleHistoryMessage(ctx, dbConn, &msg, true)
+			if err != nil {
+				errLog := logging.ExtractLogger(ctx).Error()
+				if errors.Is(err, errNotEnoughInfo) {
+					errLog = logging.ExtractLogger(ctx).Warn()
+				}
+				errLog.Err(err).Msg("failed to process Discord message")
+			}
+
+			before = msg.ID
+		}
+	}
+}
+
+func handleHistoryMessage(ctx context.Context, dbConn *pgxpool.Pool, msg *Message, createSnippets bool) error {
+	var tx pgx.Tx
+	for {
+		var err error
+		tx, err = dbConn.Begin(ctx)
+		if err != nil {
+			logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to start transaction for message")
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		break
+	}
+
+	newMsg, err := saveMessageAndContents(ctx, tx, msg)
+	if err != nil {
+		return err
+	}
+	if createSnippets {
+		if doSnippet, err := allowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil {
+			_, err := createMessageSnippet(ctx, tx, msg)
+			if err != nil {
+				return err
+			}
+		} else if err != nil {
+			return err
+		}
+	}
+
+	err = tx.Commit(ctx)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

src/discord/message_test.go

@@ -0,0 +1,16 @@
+package discord
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetMessage(t *testing.T) {
+	// t.Skip("this test is only for debugging")
+
+	msg, err := GetChannelMessage(context.Background(), "404399251276169217", "764575065772916790")
+	assert.Nil(t, err)
+	t.Logf("%+v", msg)
+}

src/discord/payloads.go

@@ -252,9 +252,16 @@ func (m *Message) Time() time.Time {
 	return t
 }
 
+func (m *Message) ShortString() string {
+	return fmt.Sprintf("%s / %s: \"%s\" (%d attachments, %d embeds)", m.Timestamp, m.Author.Username, m.Content, len(m.Attachments), len(m.Embeds))
+}
+
 func (m *Message) OriginalHasFields(fields ...string) bool {
 	if m.originalMap == nil {
-		return false
+		// If we don't know, we assume the fields are there.
+		// Usually this is because it came from their API, where we
+		// always have all fields.
+		return true
 	}
 
 	for _, field := range fields {

src/discord/rest.go

@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"net/http/httputil"
 	"net/url"
+	"strconv"
 	"strings"
 
 	"git.handmade.network/hmn/hmn/src/config"
@@ -411,6 +412,93 @@ func RemoveGuildMemberRole(ctx context.Context, userID, roleID string) error {
 	return nil
 }
 
+func GetChannelMessage(ctx context.Context, channelID, messageID string) (*Message, error) {
+	const name = "Get Channel Message"
+
+	path := fmt.Sprintf("/channels/%s/messages/%s", channelID, messageID)
+	res, err := doWithRateLimiting(ctx, name, func(ctx context.Context) *http.Request {
+		return makeRequest(ctx, http.MethodGet, path, nil)
+	})
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode == http.StatusNotFound {
+		return nil, NotFound
+	} else if res.StatusCode >= 400 {
+		logErrorResponse(ctx, name, res, "")
+		return nil, oops.New(nil, "received error from Discord")
+	}
+
+	bodyBytes, err := io.ReadAll(res.Body)
+	if err != nil {
+		panic(err)
+	}
+
+	var msg Message
+	err = json.Unmarshal(bodyBytes, &msg)
+	if err != nil {
+		return nil, oops.New(err, "failed to unmarshal Discord message")
+	}
+
+	return &msg, nil
+}
+
+type GetChannelMessagesInput struct {
+	Around string
+	Before string
+	After  string
+	Limit  int
+}
+
+func GetChannelMessages(ctx context.Context, channelID string, in GetChannelMessagesInput) ([]Message, error) {
+	const name = "Get Channel Messages"
+
+	path := fmt.Sprintf("/channels/%s/messages", channelID)
+	res, err := doWithRateLimiting(ctx, name, func(ctx context.Context) *http.Request {
+		req := makeRequest(ctx, http.MethodGet, path, nil)
+		q := req.URL.Query()
+		if in.Around != "" {
+			q.Add("around", in.Around)
+		}
+		if in.Before != "" {
+			q.Add("before", in.Before)
+		}
+		if in.After != "" {
+			q.Add("after", in.After)
+		}
+		if in.Limit != 0 {
+			q.Add("limit", strconv.Itoa(in.Limit))
+		}
+		req.URL.RawQuery = q.Encode()
+
+		return req
+	})
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode >= 400 {
+		logErrorResponse(ctx, name, res, "")
+		return nil, oops.New(nil, "received error from Discord")
+	}
+
+	bodyBytes, err := io.ReadAll(res.Body)
+	if err != nil {
+		panic(err)
+	}
+
+	var msgs []Message
+	err = json.Unmarshal(bodyBytes, &msgs)
+	if err != nil {
+		return nil, oops.New(err, "failed to unmarshal Discord message")
+	}
+
+	return msgs, nil
+}
+
 func logErrorResponse(ctx context.Context, name string, res *http.Response, msg string) {
 	dump, err := httputil.DumpResponse(res, true)
 	if err != nil {

src/discord/showcase.go

@@ -12,13 +12,13 @@ import (
 	"time"
 
 	"git.handmade.network/hmn/hmn/src/assets"
+	"git.handmade.network/hmn/hmn/src/config"
 	"git.handmade.network/hmn/hmn/src/db"
 	"git.handmade.network/hmn/hmn/src/logging"
 	"git.handmade.network/hmn/hmn/src/models"
 	"git.handmade.network/hmn/hmn/src/oops"
 	"git.handmade.network/hmn/hmn/src/parsing"
 	"github.com/google/uuid"
-	"github.com/jackc/pgx/v4"
 )
 
 var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`)
@@ -47,7 +47,7 @@ func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) er
 	defer tx.Rollback(ctx)
 
 	// save the message, maybe save its contents, and maybe make a snippet too
-	newMsg, err := bot.saveMessageAndContents(ctx, tx, msg)
+	newMsg, err := saveMessageAndContents(ctx, tx, msg)
 	if errors.Is(err, errNotEnoughInfo) {
 		logging.ExtractLogger(ctx).Warn().
 			Interface("msg", msg).
@@ -56,8 +56,8 @@ func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) er
 	} else if err != nil {
 		return err
 	}
-	if doSnippet, err := bot.allowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil {
+	if doSnippet, err := allowedToCreateMessageSnippet(ctx, tx, newMsg.UserID); doSnippet && err == nil {
-		_, err := bot.createMessageSnippet(ctx, tx, msg)
+		_, err := createMessageSnippet(ctx, tx, msg)
 		if err != nil {
 			return oops.New(err, "failed to create snippet in gateway")
 		}
@@ -120,9 +120,9 @@ the database.
 
 This does not create snippets or do anything besides save the message itself.
 */
-func (bot *botInstance) saveMessage(
+func saveMessage(
 	ctx context.Context,
-	tx pgx.Tx,
+	tx db.ConnOrTx,
 	msg *Message,
 ) (*models.DiscordMessage, error) {
 	iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{},
@@ -138,6 +138,16 @@ func (bot *botInstance) saveMessage(
 			return nil, errNotEnoughInfo
 		}
 
+		guildID := msg.GuildID
+		if guildID == nil {
+			/*
+				This is weird, but it can happen when we fetch messages from
+				history instead of receiving it from the gateway. In this case
+				we just assume it's from the HMN server.
+			*/
+			guildID = &config.Config.Discord.GuildID
+		}
+
 		_, err = tx.Exec(ctx,
 			`
 			INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created)
@@ -145,7 +155,7 @@ func (bot *botInstance) saveMessage(
 			`,
 			msg.ID,
 			msg.ChannelID,
-			*msg.GuildID,
+			*guildID,
 			msg.JumpURL(),
 			msg.Author.ID,
 			msg.Time(),
@@ -184,12 +194,12 @@ snippets.
 
 Idempotent; can be called any time whether the message exists or not.
 */
-func (bot *botInstance) saveMessageAndContents(
+func saveMessageAndContents(
 	ctx context.Context,
-	tx pgx.Tx,
+	tx db.ConnOrTx,
 	msg *Message,
 ) (*models.DiscordMessage, error) {
-	newMsg, err := bot.saveMessage(ctx, tx, msg)
+	newMsg, err := saveMessage(ctx, tx, msg)
 	if err != nil {
 		return nil, err
 	}
@@ -231,7 +241,7 @@ func (bot *botInstance) saveMessageAndContents(
 	// Save attachments
 	if msg.OriginalHasFields("attachments") {
 		for _, attachment := range msg.Attachments {
-			_, err := bot.saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID)
+			_, err := saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID)
 			if err != nil {
 				return nil, oops.New(err, "failed to save attachment")
 			}
@@ -254,7 +264,7 @@ func (bot *botInstance) saveMessageAndContents(
 		if numSavedEmbeds == 0 {
 			// No embeds yet, so save new ones
 			for _, embed := range msg.Embeds {
-				_, err := bot.saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID)
+				_, err := saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID)
 				if err != nil {
 					return nil, oops.New(err, "failed to save embed")
 				}
@@ -310,9 +320,9 @@ func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, e
 Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment
 that already exists
 */
-func (bot *botInstance) saveAttachment(
+func saveAttachment(
 	ctx context.Context,
-	tx pgx.Tx,
+	tx db.ConnOrTx,
 	attachment *Attachment,
 	hmnUserID int,
 	discordMessageID string,
@@ -394,9 +404,9 @@ func (bot *botInstance) saveAttachment(
 	return iDiscordAttachment.(*models.DiscordMessageAttachment), nil
 }
 
-func (bot *botInstance) saveEmbed(
+func saveEmbed(
 	ctx context.Context,
-	tx pgx.Tx,
+	tx db.ConnOrTx,
 	embed *Embed,
 	hmnUserID int,
 	discordMessageID string,
@@ -497,8 +507,8 @@ func (bot *botInstance) saveEmbed(
 	return iDiscordEmbed.(*models.DiscordMessageEmbed), nil
 }
 
-func (bot *botInstance) allowedToCreateMessageSnippet(ctx context.Context, tx pgx.Tx, discordUserId string) (bool, error) {
+func allowedToCreateMessageSnippet(ctx context.Context, tx db.ConnOrTx, discordUserId string) (bool, error) {
-	canSave, err := db.QueryBool(ctx, bot.dbConn,
+	canSave, err := db.QueryBool(ctx, tx,
 		`
 		SELECT u.discord_save_showcase
 		FROM
@@ -518,7 +528,7 @@ func (bot *botInstance) allowedToCreateMessageSnippet(ctx context.Context, tx pg
 	return canSave, nil
 }
 
-func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg *Message) (*models.Snippet, error) {
+func createMessageSnippet(ctx context.Context, tx db.ConnOrTx, msg *Message) (*models.Snippet, error) {
 	// Check for existing snippet, maybe return it
 	type existingSnippetResult struct {
 		Message        models.DiscordMessage         `db:"msg"`
@@ -548,7 +558,7 @@ func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg
 		// A snippet already exists - maybe update its content, then return it
 		if msg.OriginalHasFields("content") && !existing.Snippet.EditedOnWebsite {
 			contentMarkdown := existing.MessageContent.LastContent
-			contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.RealMarkdown)
+			contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
 
 			_, err := tx.Exec(ctx,
 				`
@@ -580,14 +590,14 @@ func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg
 	}
 
 	// Get an asset ID or URL to make a snippet from
-	assetId, url, err := bot.getSnippetAssetOrUrl(ctx, tx, &existing.Message)
+	assetId, url, err := getSnippetAssetOrUrl(ctx, tx, &existing.Message)
-	if assetId == nil && url == "" {
+	if assetId == nil && url == nil {
 		// Nothing to make a snippet from!
 		return nil, nil
 	}
 
 	contentMarkdown := existing.MessageContent.LastContent
-	contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.RealMarkdown)
+	contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
 
 	// TODO(db): Insert
 	isnippet, err := db.QueryOne(ctx, tx, models.Snippet{},
@@ -596,7 +606,7 @@ func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg
 		VALUES ($1, $2, $3, $4, $5, $6, $7)
 		RETURNING $columns
 		`,
-		nil,
+		url,
 		existing.Message.SentAt,
 		contentMarkdown,
 		contentHTML,
@@ -626,7 +636,7 @@ func (bot *botInstance) createMessageSnippet(ctx context.Context, tx pgx.Tx, msg
 // do we actually want to reuse those, or should we keep them separate?
 var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`)
 
-func (bot *botInstance) getSnippetAssetOrUrl(ctx context.Context, tx pgx.Tx, msg *models.DiscordMessage) (*uuid.UUID, string, error) {
+func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msg *models.DiscordMessage) (*uuid.UUID, *string, error) {
 	// Check attachments
 	itAttachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{},
 		`
@@ -637,12 +647,12 @@ func (bot *botInstance) getSnippetAssetOrUrl(ctx context.Context, tx pgx.Tx, msg
 		msg.ID,
 	)
 	if err != nil {
-		return nil, "", oops.New(err, "failed to fetch message attachments")
+		return nil, nil, oops.New(err, "failed to fetch message attachments")
 	}
 	attachments := itAttachments.ToSlice()
 	for _, iattachment := range attachments {
 		attachment := iattachment.(*models.DiscordMessageAttachment)
-		return &attachment.AssetID, "", nil
+		return &attachment.AssetID, nil, nil
 	}
 
 	// Check embeds
@@ -655,23 +665,23 @@ func (bot *botInstance) getSnippetAssetOrUrl(ctx context.Context, tx pgx.Tx, msg
 		msg.ID,
 	)
 	if err != nil {
-		return nil, "", oops.New(err, "failed to fetch discord embeds")
+		return nil, nil, oops.New(err, "failed to fetch discord embeds")
 	}
 	embeds := itEmbeds.ToSlice()
 	for _, iembed := range embeds {
 		embed := iembed.(*models.DiscordMessageEmbed)
 		if embed.VideoID != nil {
-			return embed.VideoID, "", nil
+			return embed.VideoID, nil, nil
 		} else if embed.ImageID != nil {
-			return embed.ImageID, "", nil
+			return embed.ImageID, nil, nil
 		} else if embed.URL != nil {
 			if RESnippetableUrl.MatchString(*embed.URL) {
-				return nil, *embed.URL, nil
+				return nil, embed.URL, nil
 			}
 		}
 	}
 
-	return nil, "", nil
+	return nil, nil, nil
 }
 
 func messageHasLinks(content string) bool {

src/discord/todo.txt

@@ -1,48 +0,0 @@
-the goal: port the old discord showcase bot
-
-what it does: save #project-showcase posts to your HMN user profile if you have your account linked
-
-stuff we need to worry about:
-- old posts from before you linked your account
-- posts that come in while the bot is down
-- what to do with posts if you unlink your account
-- what to do with posts if you re-link your account
-✔ - what to do if you edit the original discord message
-- what to do if you delete the original discord message
-✔ - the user's preferences re: saving content
-    - we don't want to save content without the user's consent, especially since it may persist after they disable the integration
-- manually adding content for various reasons
-    - maybe a bug prevented something from saving
-    - ryan used to post everything in #projects for some reason
-
-
-✔ real-time stuff:
-✔ - on new showcase message
-    - always save the lightweight record
-    - if we have permission, create a snippet
-✔ - on edit
-    - re-save the lightweight record and content as if it was new
-    - create snippet, unconditionally???? (bug??)
-    - update snippet contents if the edit makes sense
-✔ - on delete
-    - delete snippet if the user so desires
-    - delete the message records
-✔ - on bulk delete
-    - same stuff
-
-background stuff:
-- watch mode
-    - every five seconds
-        - fetch all HMN users with Discord accounts
-        - check if we have message records without content
-        - if so, run a full scrape (no snippets)
-    - every hour
-        - run a full scrape, creating snippets
-- scrape behavior
-    - look at every message ever in the channel
-    - do exactly what the real-time bot does on new messages (although maybe don't do snippets depending on context)
-
-
-what the heck do we do with discord's markdown
-- when we save message contents, we should save both the raw discord markdown and a version with their custom stuff replaced. We do _not_ (yet) need a full markdown parse with HTML tags and stuff. (That arguably doesn't make sense for the handmade_discordmessagecontent record anyway.)
-- when we create a snippet, we should store both markdown that makes sense to a user and the rendered version of that HTML. THIS MEANS: The markdown we save is the "clean" version of the Discord markdown.

src/main.go

@@ -4,6 +4,7 @@ import (
 	_ "git.handmade.network/hmn/hmn/src/admintools"
 	_ "git.handmade.network/hmn/hmn/src/assets"
 	_ "git.handmade.network/hmn/hmn/src/buildscss"
+	_ "git.handmade.network/hmn/hmn/src/discord/cmd"
 	_ "git.handmade.network/hmn/hmn/src/initimage"
 	_ "git.handmade.network/hmn/hmn/src/migration"
 	"git.handmade.network/hmn/hmn/src/website"

src/parsing/parsing.go

@@ -10,21 +10,38 @@ import (
 )
 
 // Used for rendering real-time previews of post content.
-var PreviewMarkdown = goldmark.New(
+var ForumPreviewMarkdown = goldmark.New(
-	goldmark.WithExtensions(makeGoldmarkExtensions(true)...),
+	goldmark.WithExtensions(makeGoldmarkExtensions(MarkdownOptions{
+		Previews: true,
+		Embeds:   true,
+	})...),
 )
 
 // Used for generating the final HTML for a post.
-var RealMarkdown = goldmark.New(
+var ForumRealMarkdown = goldmark.New(
-	goldmark.WithExtensions(makeGoldmarkExtensions(false)...),
+	goldmark.WithExtensions(makeGoldmarkExtensions(MarkdownOptions{
+		Previews: false,
+		Embeds:   true,
+	})...),
 )
 
 // Used for generating plain-text previews of posts.
 var PlaintextMarkdown = goldmark.New(
-	goldmark.WithExtensions(makeGoldmarkExtensions(false)...),
+	goldmark.WithExtensions(makeGoldmarkExtensions(MarkdownOptions{
+		Previews: false,
+		Embeds:   true,
+	})...),
 	goldmark.WithRenderer(plaintextRenderer{}),
 )
 
+// Used for processing Discord messages
+var DiscordMarkdown = goldmark.New(
+	goldmark.WithExtensions(makeGoldmarkExtensions(MarkdownOptions{
+		Previews: false,
+		Embeds:   false,
+	})...),
+)
+
 func ParseMarkdown(source string, md goldmark.Markdown) string {
 	var buf bytes.Buffer
 	if err := md.Convert([]byte(source), &buf); err != nil {
@@ -34,19 +51,35 @@ func ParseMarkdown(source string, md goldmark.Markdown) string {
 	return buf.String()
 }
 
-func makeGoldmarkExtensions(preview bool) []goldmark.Extender {
+type MarkdownOptions struct {
-	return []goldmark.Extender{
+	Previews bool
+	Embeds   bool
+}
+
+func makeGoldmarkExtensions(opts MarkdownOptions) []goldmark.Extender {
+	var extenders []goldmark.Extender
+	extenders = append(extenders,
 		extension.GFM,
 		highlightExtension,
 		SpoilerExtension{},
+	)
+
+	if opts.Embeds {
+		extenders = append(extenders,
 			EmbedExtension{
-			Preview: preview,
+				Preview: opts.Previews,
 			},
+		)
+	}
+
+	extenders = append(extenders,
 		MathjaxExtension{},
 		BBCodeExtension{
-			Preview: preview,
+			Preview: opts.Previews,
 		},
-	}
+	)
+
+	return extenders
 }
 
 var highlightExtension = highlighting.NewHighlighting(

src/parsing/parsing_test.go

@@ -10,14 +10,14 @@ import (
 func TestMarkdown(t *testing.T) {
 	t.Run("fenced code blocks", func(t *testing.T) {
 		t.Run("multiple lines", func(t *testing.T) {
-			html := ParseMarkdown("```\nmultiple lines\n\tof code\n```", RealMarkdown)
+			html := ParseMarkdown("```\nmultiple lines\n\tof code\n```", ForumRealMarkdown)
 			t.Log(html)
 			assert.Equal(t, 1, strings.Count(html, "<pre"))
 			assert.Contains(t, html, `class="hmn-code"`)
 			assert.Contains(t, html, "multiple lines\n\tof code")
 		})
 		t.Run("multiple lines with language", func(t *testing.T) {
-			html := ParseMarkdown("```go\nfunc main() {\n\tfmt.Println(\"Hello, world!\")\n}\n```", RealMarkdown)
+			html := ParseMarkdown("```go\nfunc main() {\n\tfmt.Println(\"Hello, world!\")\n}\n```", ForumRealMarkdown)
 			t.Log(html)
 			assert.Equal(t, 1, strings.Count(html, "<pre"))
 			assert.Contains(t, html, `class="hmn-code"`)
@@ -30,7 +30,7 @@ func TestMarkdown(t *testing.T) {
 func TestBBCode(t *testing.T) {
 	t.Run("[code]", func(t *testing.T) {
 		t.Run("one line", func(t *testing.T) {
-			html := ParseMarkdown("[code]Just some code, you know?[/code]", RealMarkdown)
+			html := ParseMarkdown("[code]Just some code, you know?[/code]", ForumRealMarkdown)
 			t.Log(html)
 			assert.Equal(t, 1, strings.Count(html, "<pre"))
 			assert.Contains(t, html, `class="hmn-code"`)
@@ -41,7 +41,7 @@ func TestBBCode(t *testing.T) {
 Multiline code
 	with an indent
 [/code]`
-			html := ParseMarkdown(bbcode, RealMarkdown)
+			html := ParseMarkdown(bbcode, ForumRealMarkdown)
 			t.Log(html)
 			assert.Equal(t, 1, strings.Count(html, "<pre"))
 			assert.Contains(t, html, `class="hmn-code"`)
@@ -54,7 +54,7 @@ func main() {
 	fmt.Println("Hello, world!")
 }
 [/code]`
-			html := ParseMarkdown(bbcode, RealMarkdown)
+			html := ParseMarkdown(bbcode, ForumRealMarkdown)
 			t.Log(html)
 			assert.Equal(t, 1, strings.Count(html, "<pre"))
 			assert.Contains(t, html, "Println")
@@ -66,7 +66,7 @@ func main() {
 func TestSharlock(t *testing.T) {
 	t.Skipf("This doesn't pass right now because parts of Sharlock's original source read as indented code blocks, or depend on different line break behavior.")
 	t.Run("sanity check", func(t *testing.T) {
-		result := ParseMarkdown(sharlock, RealMarkdown)
+		result := ParseMarkdown(sharlock, ForumRealMarkdown)
 
 		for _, line := range strings.Split(result, "\n") {
 			assert.NotContains(t, line, "[b]")
@@ -85,6 +85,6 @@ func TestSharlock(t *testing.T) {
 
 func BenchmarkSharlock(b *testing.B) {
 	for i := 0; i < b.N; i++ {
-		ParseMarkdown(sharlock, RealMarkdown)
+		ParseMarkdown(sharlock, ForumRealMarkdown)
 	}
 }

src/website/podcast.go

@@ -474,7 +474,7 @@ func PodcastEpisodeSubmit(c *RequestContext) ResponseData {
 	}
 
 	c.Perf.StartBlock("MARKDOWN", "Parsing description")
-	descriptionRendered := parsing.ParseMarkdown(description, parsing.RealMarkdown)
+	descriptionRendered := parsing.ParseMarkdown(description, parsing.ForumRealMarkdown)
 	c.Perf.EndBlock()
 
 	guidStr := ""

src/website/threads_and_posts_helper.go

@@ -332,7 +332,7 @@ func DeletePost(
 }
 
 func CreatePostVersion(ctx context.Context, tx pgx.Tx, postId int, unparsedContent string, ipString string, editReason string, editorId *int) (versionId int) {
-	parsed := parsing.ParseMarkdown(unparsedContent, parsing.RealMarkdown)
+	parsed := parsing.ParseMarkdown(unparsedContent, parsing.ForumRealMarkdown)
 	ip := net.ParseIP(ipString)
 
 	const previewMaxLength = 100

src/website/website.go

@@ -45,6 +45,7 @@ var WebsiteCommand = &cobra.Command{
 			auth.PeriodicallyDeleteInactiveUsers(backgroundJobContext, conn),
 			perfCollector.Done,
 			discord.RunDiscordBot(backgroundJobContext, conn),
+			discord.RunHistoryWatcher(backgroundJobContext, conn),
 		)
 
 		signals := make(chan os.Signal, 1)