Added panic recovery to all of our background jobs

Fixes issue #32

src/auth/auth.go

@@ -18,6 +18,7 @@ import (
 	"git.handmade.network/hmn/hmn/src/logging"
 	"git.handmade.network/hmn/hmn/src/models"
 	"git.handmade.network/hmn/hmn/src/oops"
+	"git.handmade.network/hmn/hmn/src/utils"
 
 	"github.com/jackc/pgx/v4/pgxpool"
 	"golang.org/x/crypto/argon2"
@@ -244,6 +245,8 @@ func PeriodicallyDeleteInactiveUsers(ctx context.Context, conn *pgxpool.Pool) jo
 		for {
 			select {
 			case <-t.C:
+				err := func() (err error) {
+					defer utils.RecoverPanicAsError(&err)
 					n, err := DeleteInactiveUsers(ctx, conn)
 					if err == nil {
 						if n > 0 {
@@ -261,6 +264,11 @@ func PeriodicallyDeleteInactiveUsers(ctx context.Context, conn *pgxpool.Pool) jo
 					} else {
 						logging.Error().Err(err).Msg("Failed to delete expired password resets")
 					}
+					return nil
+				}()
+				if err != nil {
+					logging.Error().Err(err).Msg("Panicked in PeriodicallyDeleteInactiveUsers")
+				}
 			case <-ctx.Done():
 				return
 			}

src/auth/session.go

@@ -15,6 +15,7 @@ import (
 	"git.handmade.network/hmn/hmn/src/logging"
 	"git.handmade.network/hmn/hmn/src/models"
 	"git.handmade.network/hmn/hmn/src/oops"
+	"git.handmade.network/hmn/hmn/src/utils"
 	"github.com/jackc/pgx/v4/pgxpool"
 )
 
@@ -142,6 +143,8 @@ func PeriodicallyDeleteExpiredSessions(ctx context.Context, conn *pgxpool.Pool)
 		for {
 			select {
 			case <-t.C:
+				err := func() (err error) {
+					defer utils.RecoverPanicAsError(&err)
 					n, err := DeleteExpiredSessions(ctx, conn)
 					if err == nil {
 						if n > 0 {
@@ -150,6 +153,11 @@ func PeriodicallyDeleteExpiredSessions(ctx context.Context, conn *pgxpool.Pool)
 					} else {
 						logging.Error().Err(err).Msg("Failed to delete expired sessions")
 					}
+					return nil
+				}()
+				if err != nil {
+					logging.Error().Err(err).Msg("Panicked in PeriodicallyDeleteExpiredSessions")
+				}
 			case <-ctx.Done():
 				return
 			}

src/discord/gateway.go

@@ -51,7 +51,8 @@ func RunDiscordBot(ctx context.Context, dbConn *pgxpool.Pool) jobs.Job {
 			default:
 			}
 
-			func() {
+			err := func() (retErr error) {
+				defer utils.RecoverPanicAsError(&retErr)
 				log.Info().Msg("Connecting to the Discord gateway")
 				bot := newBotInstance(dbConn)
 				err := bot.Run(ctx)
@@ -84,7 +85,11 @@ func RunDiscordBot(ctx context.Context, dbConn *pgxpool.Pool) jobs.Job {
 				time.Sleep(delay)
 
 				boff.Reset()
+				return nil
 			}()
+			if err != nil {
+				log.Error().Err(err).Msg("Panicked in RunDiscordBot")
+			}
 		}
 	}()
 	return job

src/discord/history.go

@@ -10,6 +10,7 @@ import (
 	"git.handmade.network/hmn/hmn/src/jobs"
 	"git.handmade.network/hmn/hmn/src/logging"
 	"git.handmade.network/hmn/hmn/src/models"
+	"git.handmade.network/hmn/hmn/src/utils"
 	"github.com/jackc/pgx/v4/pgxpool"
 )
 
@@ -52,9 +53,11 @@ func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) jobs.Job {
 		}
 
 		for {
+			done, err := func() (done bool, err error) {
+				defer utils.RecoverPanicAsError(&err)
 				select {
 				case <-ctx.Done():
-				return
+					return true, nil
 				case <-newUserTicker.C:
 					// Get content for messages when a user links their account (but do not create snippets)
 					fetchMissingContent(ctx, dbConn)
@@ -63,6 +66,13 @@ func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) jobs.Job {
 				case <-backfillTicker.C:
 					runBackfill()
 				}
+				return false, nil
+			}()
+			if err != nil {
+				log.Error().Err(err).Msg("Panicked in RunHistoryWatcher")
+			} else if done {
+				return
+			}
 		}
 	}()
 

src/twitch/twitch.go

@@ -14,6 +14,7 @@ import (
 	"git.handmade.network/hmn/hmn/src/models"
 	"git.handmade.network/hmn/hmn/src/oops"
 	"git.handmade.network/hmn/hmn/src/perf"
+	"git.handmade.network/hmn/hmn/src/utils"
 	"github.com/jackc/pgx/v4/pgxpool"
 )
 
@@ -45,12 +46,6 @@ func MonitorTwitchSubscriptions(ctx context.Context, dbConn *pgxpool.Pool) jobs.
 		}()
 		log.Info().Msg("Running twitch monitor...")
 
-		err := refreshAccessToken(ctx)
-		if err != nil {
-			log.Error().Err(err).Msg("Failed to fetch refresh token on start")
-			return
-		}
-
 		monitorTicker := time.NewTicker(2 * time.Hour)
 		firstRunChannel := make(chan struct{}, 1)
 		firstRunChannel <- struct{}{}
@@ -58,12 +53,14 @@ func MonitorTwitchSubscriptions(ctx context.Context, dbConn *pgxpool.Pool) jobs.
 		timers := make([]*time.Timer, 0)
 		expiredTimers := make(chan *time.Timer, 10)
 		for {
+			done, err := func() (done bool, retErr error) {
+				defer utils.RecoverPanicAsError(&retErr)
 				select {
 				case <-ctx.Done():
 					for _, timer := range timers {
 						timer.Stop()
 					}
-				return
+					return true, nil
 				case expired := <-expiredTimers:
 					for idx, timer := range timers {
 						if timer == expired {
@@ -72,6 +69,11 @@ func MonitorTwitchSubscriptions(ctx context.Context, dbConn *pgxpool.Pool) jobs.
 						}
 					}
 				case <-firstRunChannel:
+					err := refreshAccessToken(ctx)
+					if err != nil {
+						log.Error().Err(err).Msg("Failed to fetch refresh token on start")
+						return true, nil
+					}
 					syncWithTwitch(ctx, dbConn, true)
 				case <-monitorTicker.C:
 					syncWithTwitch(ctx, dbConn, true)
@@ -106,6 +108,13 @@ func MonitorTwitchSubscriptions(ctx context.Context, dbConn *pgxpool.Pool) jobs.
 						processEventSubNotification(ctx, dbConn, &notification)
 					}
 				}
+				return false, nil
+			}()
+			if err != nil {
+				log.Error().Err(err).Msg("Panicked in MonitorTwitchSubscriptions")
+			} else if done {
+				return
+			}
 		}
 	}()