hmn
/
hmn
4
Fork 9
Code Issues 59 Pull Requests 1 Releases Activity

Added panic recovery to all of our background jobs 

Fixes issue #32
This commit is contained in:
Asaf Gartner 2022-06-16 00:33:57 +03:00
parent 870a073e22
commit b165bf7c23
5 changed files with 115 additions and 75 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/auth/auth.go
View File

@ -18,6 +18,7 @@ import (
"git.handmade.network/hmn/hmn/src/logging"
"git.handmade.network/hmn/hmn/src/models"
"git.handmade.network/hmn/hmn/src/oops"
"git.handmade.network/hmn/hmn/src/utils"
"github.com/jackc/pgx/v4/pgxpool"
"golang.org/x/crypto/argon2"
@ -244,6 +245,8 @@ func PeriodicallyDeleteInactiveUsers(ctx context.Context, conn *pgxpool.Pool) jo
for {
select {
case <-t.C:
err := func() (err error) {
defer utils.RecoverPanicAsError(&err)
n, err := DeleteInactiveUsers(ctx, conn)
if err == nil {
if n > 0 {
@ -261,6 +264,11 @@ func PeriodicallyDeleteInactiveUsers(ctx context.Context, conn *pgxpool.Pool) jo
} else {
logging.Error().Err(err).Msg("Failed to delete expired password resets")
}
return nil
}()
if err != nil {
logging.Error().Err(err).Msg("Panicked in PeriodicallyDeleteInactiveUsers")
}
case <-ctx.Done():
return
}

8
src/auth/session.go
View File

@ -15,6 +15,7 @@ import (
"git.handmade.network/hmn/hmn/src/logging"
"git.handmade.network/hmn/hmn/src/models"
"git.handmade.network/hmn/hmn/src/oops"
"git.handmade.network/hmn/hmn/src/utils"
"github.com/jackc/pgx/v4/pgxpool"
)
@ -142,6 +143,8 @@ func PeriodicallyDeleteExpiredSessions(ctx context.Context, conn *pgxpool.Pool)
for {
select {
case <-t.C:
err := func() (err error) {
defer utils.RecoverPanicAsError(&err)
n, err := DeleteExpiredSessions(ctx, conn)
if err == nil {
if n > 0 {
@ -150,6 +153,11 @@ func PeriodicallyDeleteExpiredSessions(ctx context.Context, conn *pgxpool.Pool)
} else {
logging.Error().Err(err).Msg("Failed to delete expired sessions")
}
return nil
}()
if err != nil {
logging.Error().Err(err).Msg("Panicked in PeriodicallyDeleteExpiredSessions")
}
case <-ctx.Done():
return
}

7
src/discord/gateway.go
View File

@ -51,7 +51,8 @@ func RunDiscordBot(ctx context.Context, dbConn *pgxpool.Pool) jobs.Job {
default:
}
func() {
err := func() (retErr error) {
defer utils.RecoverPanicAsError(&retErr)
log.Info().Msg("Connecting to the Discord gateway")
bot := newBotInstance(dbConn)
err := bot.Run(ctx)
@ -84,7 +85,11 @@ func RunDiscordBot(ctx context.Context, dbConn *pgxpool.Pool) jobs.Job {
time.Sleep(delay)
boff.Reset()
return nil
}()
if err != nil {
log.Error().Err(err).Msg("Panicked in RunDiscordBot")
}
}
}()
return job

12
src/discord/history.go
View File

@ -10,6 +10,7 @@ import (
"git.handmade.network/hmn/hmn/src/jobs"
"git.handmade.network/hmn/hmn/src/logging"
"git.handmade.network/hmn/hmn/src/models"
"git.handmade.network/hmn/hmn/src/utils"
"github.com/jackc/pgx/v4/pgxpool"
)
@ -52,9 +53,11 @@ func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) jobs.Job {
}
for {
done, err := func() (done bool, err error) {
defer utils.RecoverPanicAsError(&err)
select {
case <-ctx.Done():
return
return true, nil
case <-newUserTicker.C:
// Get content for messages when a user links their account (but do not create snippets)
fetchMissingContent(ctx, dbConn)
@ -63,6 +66,13 @@ func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) jobs.Job {
case <-backfillTicker.C:
runBackfill()
}
return false, nil
}()
if err != nil {
log.Error().Err(err).Msg("Panicked in RunHistoryWatcher")
} else if done {
return
}
}
}()

23
src/twitch/twitch.go
View File

@ -14,6 +14,7 @@ import (
"git.handmade.network/hmn/hmn/src/models"
"git.handmade.network/hmn/hmn/src/oops"
"git.handmade.network/hmn/hmn/src/perf"
"git.handmade.network/hmn/hmn/src/utils"
"github.com/jackc/pgx/v4/pgxpool"
)
@ -45,12 +46,6 @@ func MonitorTwitchSubscriptions(ctx context.Context, dbConn *pgxpool.Pool) jobs.
}()
log.Info().Msg("Running twitch monitor...")
err := refreshAccessToken(ctx)
if err != nil {
log.Error().Err(err).Msg("Failed to fetch refresh token on start")
return
}
monitorTicker := time.NewTicker(2 * time.Hour)
firstRunChannel := make(chan struct{}, 1)
firstRunChannel <- struct{}{}
@ -58,12 +53,14 @@ func MonitorTwitchSubscriptions(ctx context.Context, dbConn *pgxpool.Pool) jobs.
timers := make([]*time.Timer, 0)
expiredTimers := make(chan *time.Timer, 10)
for {
done, err := func() (done bool, retErr error) {
defer utils.RecoverPanicAsError(&retErr)
select {
case <-ctx.Done():
for _, timer := range timers {
timer.Stop()
}
return
return true, nil
case expired := <-expiredTimers:
for idx, timer := range timers {
if timer == expired {
@ -72,6 +69,11 @@ func MonitorTwitchSubscriptions(ctx context.Context, dbConn *pgxpool.Pool) jobs.
}
}
case <-firstRunChannel:
err := refreshAccessToken(ctx)
if err != nil {
log.Error().Err(err).Msg("Failed to fetch refresh token on start")
return true, nil
}
syncWithTwitch(ctx, dbConn, true)
case <-monitorTicker.C:
syncWithTwitch(ctx, dbConn, true)
@ -106,6 +108,13 @@ func MonitorTwitchSubscriptions(ctx context.Context, dbConn *pgxpool.Pool) jobs.
processEventSubNotification(ctx, dbConn, &notification)
}
}
return false, nil
}()
if err != nil {
log.Error().Err(err).Msg("Panicked in MonitorTwitchSubscriptions")
} else if done {
return
}
}
}()