Code review

This commit is contained in:
Asaf Gartner 2022-02-07 14:21:40 +02:00
parent 92d6a31aa9
commit 43651d98e8
4 changed files with 49 additions and 50 deletions

View File

@ -606,17 +606,17 @@ func (bot *botInstance) messageDelete(ctx context.Context, msgDelete MessageDele
interned, err := FetchInternedMessage(ctx, bot.dbConn, msgDelete.ID) interned, err := FetchInternedMessage(ctx, bot.dbConn, msgDelete.ID)
if err != nil { if err != nil {
if !errors.Is(err, db.NotFound) {
log.Error().Err(err).Msg("failed to fetch interned message") log.Error().Err(err).Msg("failed to fetch interned message")
}
return return
} }
if interned != nil {
err = DeleteInternedMessage(ctx, bot.dbConn, interned) err = DeleteInternedMessage(ctx, bot.dbConn, interned)
if err != nil { if err != nil {
log.Error().Err(err).Msg("failed to delete interned message") log.Error().Err(err).Msg("failed to delete interned message")
return return
} }
} }
}
type MessageToSend struct { type MessageToSend struct {
ChannelID string ChannelID string

View File

@ -30,15 +30,28 @@ func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{
done <- struct{}{} done <- struct{}{}
}() }()
backfillInterval := 1 * time.Hour
newUserTicker := time.NewTicker(5 * time.Second) newUserTicker := time.NewTicker(5 * time.Second)
// NOTE(asaf): 5 seconds to ensure this runs on start, we then reset it to the correct backfillFirstRun := make(chan struct{}, 1)
// interval after the first run. backfillFirstRun <- struct{}{}
backfillTicker := time.NewTicker(5 * time.Second) backfillTicker := time.NewTicker(1 * time.Hour)
lastBackfillTime := time.Now().Add(-3 * time.Hour)
runBackfill := func() {
log.Info().Msg("Running backfill")
// Run a backfill to patch up places where the Discord bot missed (does create snippets)
now := time.Now()
done := Scrape(ctx, dbConn,
config.Config.Discord.ShowcaseChannelID,
lastBackfillTime,
true,
)
if done {
lastBackfillTime = now
}
}
lastBackfillTime := time.Now().Add(-backfillInterval)
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
@ -46,17 +59,10 @@ func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{
case <-newUserTicker.C: case <-newUserTicker.C:
// Get content for messages when a user links their account (but do not create snippets) // Get content for messages when a user links their account (but do not create snippets)
fetchMissingContent(ctx, dbConn) fetchMissingContent(ctx, dbConn)
case <-backfillFirstRun:
runBackfill()
case <-backfillTicker.C: case <-backfillTicker.C:
backfillTicker.Reset(backfillInterval) runBackfill()
// TODO(asaf): Do we need to update lastBackfillTime here? Otherwise we'll be rescraping
// from (start up time - 1 hour) every time.
// Run a backfill to patch up places where the Discord bot missed (does create snippets)
Scrape(ctx, dbConn,
config.Config.Discord.ShowcaseChannelID,
lastBackfillTime,
true,
)
} }
} }
}() }()
@ -107,11 +113,11 @@ func fetchMissingContent(ctx context.Context, dbConn *pgxpool.Pool) {
// This message has apparently been deleted; delete it from our database // This message has apparently been deleted; delete it from our database
interned, err := FetchInternedMessage(ctx, dbConn, msg.ID) interned, err := FetchInternedMessage(ctx, dbConn, msg.ID)
if err != nil { if err != nil {
if !errors.Is(err, db.NotFound) {
log.Error().Str("Message ID", msg.ID).Msg("couldn't find interned message")
} else {
log.Error().Err(err).Msg("failed to fetch interned message") log.Error().Err(err).Msg("failed to fetch interned message")
continue
} }
if interned == nil {
log.Error().Msg("couldn't find interned message")
continue continue
} }
err = DeleteInternedMessage(ctx, dbConn, interned) err = DeleteInternedMessage(ctx, dbConn, interned)
@ -138,7 +144,7 @@ func fetchMissingContent(ctx context.Context, dbConn *pgxpool.Pool) {
} }
} }
func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earliestMessageTime time.Time, createSnippets bool) { func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earliestMessageTime time.Time, createSnippets bool) bool {
log := logging.ExtractLogger(ctx) log := logging.ExtractLogger(ctx)
log.Info().Msg("Starting scrape") log.Info().Msg("Starting scrape")
@ -152,19 +158,19 @@ func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earlies
}) })
if err != nil { if err != nil {
logging.Error().Err(err).Msg("failed to get messages while scraping") logging.Error().Err(err).Msg("failed to get messages while scraping")
return return false
} }
if len(msgs) == 0 { if len(msgs) == 0 {
logging.Debug().Msg("out of messages, stopping scrape") logging.Debug().Msg("out of messages, stopping scrape")
return return true
} }
for _, msg := range msgs { for _, msg := range msgs {
select { select {
case <-ctx.Done(): case <-ctx.Done():
log.Info().Msg("Scrape was canceled") log.Info().Msg("Scrape was canceled")
return return false
default: default:
} }
@ -172,7 +178,7 @@ func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earlies
if !earliestMessageTime.IsZero() && msg.Time().Before(earliestMessageTime) { if !earliestMessageTime.IsZero() && msg.Time().Before(earliestMessageTime) {
logging.ExtractLogger(ctx).Info().Time("earliest", earliestMessageTime).Msg("Saw a message before the specified earliest time; exiting") logging.ExtractLogger(ctx).Info().Time("earliest", earliestMessageTime).Msg("Saw a message before the specified earliest time; exiting")
return return true
} }
err := HandleIncomingMessage(ctx, dbConn, &msg, createSnippets) err := HandleIncomingMessage(ctx, dbConn, &msg, createSnippets)

View File

@ -151,6 +151,8 @@ func MaybeInternMessage(ctx context.Context, dbConn db.ConnOrTx, msg *Message) e
return nil return nil
} }
var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this")
/* /*
Ensures that a Discord message is stored in the database. This function is Ensures that a Discord message is stored in the database. This function is
idempotent and can be called regardless of whether the item already exists in idempotent and can be called regardless of whether the item already exists in
@ -158,8 +160,6 @@ the database.
This does not create snippets or save content or do anything besides save the message itself. This does not create snippets or save content or do anything besides save the message itself.
*/ */
var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this")
func InternMessage( func InternMessage(
ctx context.Context, ctx context.Context,
dbConn db.ConnOrTx, dbConn db.ConnOrTx,
@ -233,12 +233,8 @@ func FetchInternedMessage(ctx context.Context, dbConn db.ConnOrTx, msgId string)
msgId, msgId,
) )
if err != nil { if err != nil {
if errors.Is(err, db.NotFound) {
return nil, nil
} else {
return nil, err return nil, err
} }
}
interned := result.(*InternedMessage) interned := result.(*InternedMessage)
return interned, nil return interned, nil
@ -256,11 +252,9 @@ func HandleInternedMessage(ctx context.Context, dbConn db.ConnOrTx, msg *Message
defer tx.Rollback(ctx) defer tx.Rollback(ctx)
interned, err := FetchInternedMessage(ctx, tx, msg.ID) interned, err := FetchInternedMessage(ctx, tx, msg.ID)
if err != nil { if err != nil && !errors.Is(err, db.NotFound) {
return err return err
} } else if err == nil {
if interned != nil {
if !deleted { if !deleted {
err = SaveMessageContents(ctx, tx, interned, msg) err = SaveMessageContents(ctx, tx, interned, msg)
if err != nil { if err != nil {
@ -717,8 +711,8 @@ func HandleSnippetForInternedMessage(ctx context.Context, dbConn db.ConnOrTx, in
// TODO(asaf): We're not handling the case where embeds were removed or modified. // TODO(asaf): We're not handling the case where embeds were removed or modified.
// Also not handling the case where a message had both an attachment and an embed // Also not handling the case where a message had both an attachment and an embed
// and the attachment was removed (leaving only the embed). // and the attachment was removed (leaving only the embed).
LinkedUserIsSnippetOwner := existingSnippet.OwnerID == interned.DiscordUser.HMNUserId linkedUserIsSnippetOwner := existingSnippet.OwnerID == interned.DiscordUser.HMNUserId
if LinkedUserIsSnippetOwner && !existingSnippet.EditedOnWebsite { if linkedUserIsSnippetOwner && !existingSnippet.EditedOnWebsite {
contentMarkdown := interned.MessageContent.LastContent contentMarkdown := interned.MessageContent.LastContent
contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown) contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)

View File

@ -168,10 +168,9 @@ func DiscordShowcaseBacklog(c *RequestContext) ResponseData {
} }
for _, msgID := range msgIDs { for _, msgID := range msgIDs {
interned, err := discord.FetchInternedMessage(c.Context(), c.Conn, msgID) interned, err := discord.FetchInternedMessage(c.Context(), c.Conn, msgID)
if err != nil { if err != nil && !errors.Is(err, db.NotFound) {
return c.ErrorResponse(http.StatusInternalServerError, err) return c.ErrorResponse(http.StatusInternalServerError, err)
} } else if err == nil {
if interned != nil {
// NOTE(asaf): Creating snippet even if the checkbox is off because the user asked us to. // NOTE(asaf): Creating snippet even if the checkbox is off because the user asked us to.
err = discord.HandleSnippetForInternedMessage(c.Context(), c.Conn, interned, true) err = discord.HandleSnippetForInternedMessage(c.Context(), c.Conn, interned, true)
if err != nil { if err != nil {