Fix zero-items bug with getPageInfo

The code of conduct has historically not been of much use, while the
communication guidelines are more tangible. The two are also just kind
of redundant.

public/js/image_selector.js

@@ -1,4 +1,4 @@
-function ImageSelector(form, maxFileSize, container) {
+function ImageSelector(form, maxFileSize, container, defaultImageUrl) {
 	this.form = form;
 	this.maxFileSize = maxFileSize;
 	this.fileInput = container.querySelector(".image_input");
@@ -8,6 +8,7 @@ function ImageSelector(form, maxFileSize, container) {
 	this.removeLink = container.querySelector(".remove");
 	this.originalImageUrl = this.imageEl.getAttribute("data-original");
 	this.currentImageUrl = this.originalImageUrl;
+	this.defaultImageUrl = defaultImageUrl || "";
 
 	this.fileInput.value = "";
 	this.removeImageInput.value = "";
@@ -45,7 +46,7 @@ ImageSelector.prototype.removeImage = function() {
 	this.updateSizeLimit(0);
 	this.fileInput.value = "";
 	this.removeImageInput.value = "true";
-	this.setImageUrl("");
+	this.setImageUrl(this.defaultImageUrl);
 	this.updateButtons();
 };
 
@@ -82,13 +83,15 @@ ImageSelector.prototype.setImageUrl = function(url) {
 };
 
 ImageSelector.prototype.updateButtons = function() {
-	if (this.originalImageUrl.length > 0 && this.currentImageUrl != this.originalImageUrl) {
+	if ((this.originalImageUrl.length > 0 && this.originalImageUrl != this.defaultImageUrl)
+		&& this.currentImageUrl != this.originalImageUrl) {
+
 		this.resetLink.style.display = "inline-block";
 	} else {
 		this.resetLink.style.display = "none";
 	}
 
-	if (!this.fileInput.required && this.currentImageUrl != "") {
+	if (!this.fileInput.required && this.currentImageUrl != this.defaultImageUrl) {
 		this.removeLink.style.display = "inline-block";
 	} else {
 		this.removeLink.style.display = "none";

public/js/showcase.js

@@ -1,4 +1,5 @@
 const TimelineMediaTypes = {
+	UNKNOWN: 0,
     IMAGE: 1,
     VIDEO: 2,
     AUDIO: 3,

public/style.css

@@ -7793,12 +7793,8 @@ article code {
       vertical-align: top;
       width: 90%; }
 
-.site-search {
-  width: 100%; }
-  .site-search:focus {
-    width: 200%; }
-  .site-search[type=text].lite {
-    transition: border-bottom-color 60ms ease-in-out, width 300ms ease; }
+.site-search[type=text].lite {
+  transition: border-bottom-color 60ms ease-in-out, width 300ms ease; }
 
 #search_button_homepage {
   margin: 0px;
@@ -7941,6 +7937,9 @@ pre {
   margin-top: 0.5em;
   margin-bottom: 0.5em; }
 
+.post-content li:not(:last-child) {
+  margin-bottom: 0.2em; }
+
 .post-content img {
   max-width: 100%; }
 

src/admintools/adminproject.go

@@ -163,7 +163,7 @@ func addProjectTagCommand(projectCommand *cobra.Command) {
 			conn := db.NewConnPool(1, 1)
 			defer conn.Close()
 
-			resultTag, err := hmndata.SetProjectTag(ctx, conn, projectID, tag)
+			resultTag, err := hmndata.SetProjectTag(ctx, conn, nil, projectID, tag)
 			if err != nil {
 				panic(err)
 			}

src/admintools/admintools.go

@@ -1,23 +1,16 @@
 package admintools
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
-	"image"
-	"net/http"
 	"os"
-	"path"
 	"strconv"
-	"strings"
 	"time"
 
-	"git.handmade.network/hmn/hmn/src/assets"
 	"git.handmade.network/hmn/hmn/src/auth"
 	"git.handmade.network/hmn/hmn/src/db"
 	"git.handmade.network/hmn/hmn/src/email"
-	"git.handmade.network/hmn/hmn/src/hmndata"
 	"git.handmade.network/hmn/hmn/src/logging"
 	"git.handmade.network/hmn/hmn/src/models"
 	"git.handmade.network/hmn/hmn/src/oops"
@@ -360,135 +353,5 @@ func init() {
 	moveThreadsToSubforumCommand.MarkFlagRequired("subforum_slug")
 	adminCommand.AddCommand(moveThreadsToSubforumCommand)
 
-	uploadProjectLogos := &cobra.Command{
-		Use:   "uploadprojectlogos",
-		Short: "Uploads project imagefiles to S3 and replaces them with assets",
-		Run: func(cmd *cobra.Command, args []string) {
-			ctx := context.Background()
-			conn := db.NewConnPool(1, 1)
-			defer conn.Close()
-
-			allProjects, err := db.Query(ctx, conn, models.Project{}, `SELECT $columns FROM handmade_project`)
-			if err != nil {
-				panic(oops.New(err, "Failed to fetch projects from db"))
-			}
-
-			var fixupProjects []*models.Project
-			numImages := 0
-			for _, project := range allProjects {
-				p := project.(*models.Project)
-				if p.LogoLight != "" || p.LogoDark != "" {
-					fixupProjects = append(fixupProjects, p)
-				}
-
-				if p.LogoLight != "" {
-					numImages += 1
-				}
-				if p.LogoDark != "" {
-					numImages += 1
-				}
-			}
-
-			fmt.Printf("%d images to upload\n", numImages)
-
-			uploadImage := func(ctx context.Context, conn db.ConnOrTx, filepath string, owner *models.User) *models.Asset {
-				filepath = "./public/media/" + filepath
-				contents, err := os.ReadFile(filepath)
-				if err != nil {
-					panic(oops.New(err, fmt.Sprintf("Failed to read file: %s", filepath)))
-				}
-				width := 0
-				height := 0
-				fileExtensionOverrides := []string{".svg"}
-				fileExt := strings.ToLower(path.Ext(filepath))
-				tryDecode := true
-				for _, ext := range fileExtensionOverrides {
-					if fileExt == ext {
-						tryDecode = false
-					}
-				}
-				if tryDecode {
-					config, _, err := image.DecodeConfig(bytes.NewReader(contents))
-					if err != nil {
-						panic(oops.New(err, fmt.Sprintf("Failed to decode file: %s", filepath)))
-					}
-					width = config.Width
-					height = config.Height
-				}
-
-				mime := http.DetectContentType(contents)
-				filename := path.Base(filepath)
-
-				asset, err := assets.Create(ctx, conn, assets.CreateInput{
-					Content:     contents,
-					Filename:    filename,
-					ContentType: mime,
-					UploaderID:  &owner.ID,
-					Width:       width,
-					Height:      height,
-				})
-				if err != nil {
-					panic(oops.New(err, "Failed to create asset"))
-				}
-
-				return asset
-			}
-
-			for _, p := range fixupProjects {
-				owners, err := hmndata.FetchProjectOwners(ctx, conn, p.ID)
-				if err != nil {
-					panic(oops.New(err, "Failed to fetch project owners"))
-				}
-				if len(owners) == 0 {
-					fmt.Printf("PROBLEM!! Project %d (%s) doesn't have owners!!\n", p.ID, p.Name)
-					continue
-				}
-				if p.LogoLight != "" {
-					lightAsset := uploadImage(ctx, conn, p.LogoLight, owners[0])
-					_, err := conn.Exec(ctx,
-						`
-						UPDATE handmade_project
-						SET
-							logolight_asset_id = $2,
-							logolight = NULL
-						WHERE
-							id = $1
-						`,
-						p.ID,
-						lightAsset.ID,
-					)
-					if err != nil {
-						panic(oops.New(err, "Failed to update project"))
-					}
-					numImages -= 1
-					fmt.Printf(".")
-				}
-				if p.LogoDark != "" {
-					darkAsset := uploadImage(ctx, conn, p.LogoDark, owners[0])
-					_, err := conn.Exec(ctx,
-						`
-						UPDATE handmade_project
-						SET
-							logodark_asset_id = $2,
-							logodark = NULL
-						WHERE
-							id = $1
-						`,
-						p.ID,
-						darkAsset.ID,
-					)
-					if err != nil {
-						panic(oops.New(err, "Failed to update project"))
-					}
-					numImages -= 1
-					fmt.Printf(".")
-				}
-			}
-
-			fmt.Printf("\nDone! %d images not patched for some reason.\n\n", numImages)
-		},
-	}
-	adminCommand.AddCommand(uploadProjectLogos)
-
 	addProjectCommands(adminCommand)
 }

src/config/config.go.example

@@ -57,6 +57,14 @@ var Config = HMNConfig{
 		MemberRoleID:      "",
 		ShowcaseChannelID: "",
 		LibraryChannelID:  "",
+		StreamsChannelID:  "",
+	},
+	Twitch: TwitchConfig{
+		ClientID:       "",
+		ClientSecret:   "",
+		EventSubSecret: "",
+		BaseUrl:        "https://api.twitch.tv/helix",
+		BaseIDUrl:      "https://id.twitch.tv/oauth2",
 	},
 	EpisodeGuide: EpisodeGuide{
 		CineraOutputPath: "./annotations/",

src/config/types.go

@@ -27,6 +27,7 @@ type HMNConfig struct {
 	Email        EmailConfig
 	DigitalOcean DigitalOceanConfig
 	Discord      DiscordConfig
+	Twitch       TwitchConfig
 	EpisodeGuide EpisodeGuide
 }
 
@@ -76,9 +77,18 @@ type DiscordConfig struct {
 	MemberRoleID         string
 	ShowcaseChannelID    string
 	LibraryChannelID     string
+	StreamsChannelID     string
 	JamShowcaseChannelID string
 }
 
+type TwitchConfig struct {
+	ClientID       string
+	ClientSecret   string
+	EventSubSecret string // NOTE(asaf): Between 10-100 chars long. Anything will do.
+	BaseUrl        string
+	BaseIDUrl      string
+}
+
 type EpisodeGuide struct {
 	CineraOutputPath string
 	Projects         map[string]string // NOTE(asaf): Maps from slugs to default episode guide topic

src/db/db.go

@@ -242,12 +242,22 @@ func Query(ctx context.Context, conn ConnOrTx, destExample interface{}, query st
 
 func QueryIterator(ctx context.Context, conn ConnOrTx, destExample interface{}, query string, args ...interface{}) (*StructQueryIterator, error) {
 	destType := reflect.TypeOf(destExample)
-	columnNames, fieldPaths, err := getColumnNamesAndPaths(destType, nil, "")
+	columnNames, fieldPaths, err := getColumnNamesAndPaths(destType, nil, nil)
 	if err != nil {
 		return nil, oops.New(err, "failed to generate column names")
 	}
 
-	columnNamesString := strings.Join(columnNames, ", ")
+	columns := make([]string, 0, len(columnNames))
+	for _, strSlice := range columnNames {
+		tableName := strings.Join(strSlice[0:len(strSlice)-1], "_")
+		fullName := strSlice[len(strSlice)-1]
+		if tableName != "" {
+			fullName = tableName + "." + fullName
+		}
+		columns = append(columns, fullName)
+	}
+
+	columnNamesString := strings.Join(columns, ", ")
 	query = strings.Replace(query, "$columns", columnNamesString, -1)
 
 	rows, err := conn.Query(ctx, query, args...)
@@ -282,8 +292,8 @@ func QueryIterator(ctx context.Context, conn ConnOrTx, destExample interface{},
 	return it, nil
 }
 
-func getColumnNamesAndPaths(destType reflect.Type, pathSoFar []int, prefix string) (names []string, paths [][]int, err error) {
-	var columnNames []string
+func getColumnNamesAndPaths(destType reflect.Type, pathSoFar []int, prefix []string) (names [][]string, paths [][]int, err error) {
+	var columnNames [][]string
 	var fieldPaths [][]int
 
 	if destType.Kind() == reflect.Ptr {
@@ -301,7 +311,10 @@ func getColumnNamesAndPaths(destType reflect.Type, pathSoFar []int, prefix strin
 	var anonPrefixes []AnonPrefix
 
 	for _, field := range reflect.VisibleFields(destType) {
-		path := append(pathSoFar, field.Index...)
+		path := make([]int, len(pathSoFar))
+		copy(path, pathSoFar)
+		path = append(path, field.Index...)
+		fieldColumnNames := prefix[:]
 
 		if columnName := field.Tag.Get("db"); columnName != "" {
 			if field.Anonymous {
@@ -318,7 +331,7 @@ func getColumnNamesAndPaths(destType reflect.Type, pathSoFar []int, prefix strin
 							}
 						}
 						if equal {
-							columnName = anonPrefix.Prefix + "." + columnName
+							fieldColumnNames = append(fieldColumnNames, anonPrefix.Prefix)
 							break
 						}
 					}
@@ -329,11 +342,13 @@ func getColumnNamesAndPaths(destType reflect.Type, pathSoFar []int, prefix strin
 				fieldType = fieldType.Elem()
 			}
 
+			fieldColumnNames = append(fieldColumnNames, columnName)
+
 			if typeIsQueryable(fieldType) {
-				columnNames = append(columnNames, prefix+columnName)
+				columnNames = append(columnNames, fieldColumnNames)
 				fieldPaths = append(fieldPaths, path)
 			} else if fieldType.Kind() == reflect.Struct {
-				subCols, subPaths, err := getColumnNamesAndPaths(fieldType, path, columnName+".")
+				subCols, subPaths, err := getColumnNamesAndPaths(fieldType, path, fieldColumnNames)
 				if err != nil {
 					return nil, nil, err
 				}

src/discord/cmd/cmd.go

@@ -2,6 +2,9 @@ package cmd
 
 import (
 	"context"
+	"errors"
+	"fmt"
+	"os"
 	"time"
 
 	"git.handmade.network/hmn/hmn/src/db"
@@ -35,18 +38,45 @@ func init() {
 	rootCommand.AddCommand(scrapeCommand)
 
 	makeSnippetCommand := &cobra.Command{
-		Use:   "makesnippet [<message id>...]",
-		Short: "Make snippets from saved Discord messages",
-		Long:  "Make snippets from Discord messages whose content we have already saved. Useful for creating snippets from messages in non-showcase channels.",
+		Use:   "makesnippet <channel id> [<message id>...]",
+		Short: "Make snippets from Discord messages",
+		Long:  "Creates snippets from the specified messages in the specified channel. Will create a snippet as long as the poster of the message linked their account regardless of user settings.",
 		Run: func(cmd *cobra.Command, args []string) {
+			if len(args) < 2 {
+				cmd.Usage()
+				os.Exit(1)
+			}
 			ctx := context.Background()
 			conn := db.NewConnPool(1, 1)
 			defer conn.Close()
 
-			err := discord.CreateMessageSnippets(ctx, conn, args...)
-			if err != nil {
-				logging.Error().Err(err).Msg("failed to create snippets")
+			chanID := args[0]
+
+			count := 0
+
+			for _, msgID := range args[1:] {
+				message, err := discord.GetChannelMessage(ctx, chanID, msgID)
+				if errors.Is(err, discord.NotFound) {
+					logging.Warn().Msg(fmt.Sprintf("no message found on discord for id %s", msgID))
+					continue
+				} else if err != nil {
+					logging.Error().Msg(fmt.Sprintf("failed to fetch discord message id %s", msgID))
+					continue
+				}
+				err = discord.InternMessage(ctx, conn, message)
+				if err != nil {
+					logging.Error().Msg(fmt.Sprintf("failed to intern discord message id %s", msgID))
+					continue
+				}
+				err = discord.HandleInternedMessage(ctx, conn, message, false, true)
+				if err != nil {
+					logging.Error().Msg(fmt.Sprintf("failed to handle interned message id %s", msgID))
+					continue
+				}
+				count += 1
 			}
+
+			logging.Info().Msg(fmt.Sprintf("Handled %d messages", count))
 		},
 	}
 	rootCommand.AddCommand(makeSnippetCommand)

src/discord/commands.go

@@ -5,6 +5,8 @@ import (
 	"errors"
 	"fmt"
 
+	"git.handmade.network/hmn/hmn/src/hmndata"
+
 	"git.handmade.network/hmn/hmn/src/db"
 	"git.handmade.network/hmn/hmn/src/hmnurl"
 	"git.handmade.network/hmn/hmn/src/logging"
@@ -97,6 +99,7 @@ func (bot *botInstance) handleProfileCommand(ctx context.Context, i *Interaction
 		FROM
 			handmade_discorduser AS duser
 			JOIN auth_user ON duser.hmn_user_id = auth_user.id
+			LEFT JOIN handmade_asset AS auth_user_avatar ON auth_user_avatar.id = auth_user.avatar_asset_id
 		WHERE
 			duser.userid = $1
 		`,
@@ -121,11 +124,31 @@ func (bot *botInstance) handleProfileCommand(ctx context.Context, i *Interaction
 	}
 	res := ires.(*profileResult)
 
+	projectsAndStuff, err := hmndata.FetchProjects(ctx, bot.dbConn, nil, hmndata.ProjectsQuery{
+		OwnerIDs: []int{res.HMNUser.ID},
+	})
+	if err != nil {
+		logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to fetch user projects")
+	}
+
 	url := hmnurl.BuildUserProfile(res.HMNUser.Username)
+	msg := fmt.Sprintf("<@%s>'s profile can be viewed at %s.", member.User.ID, url)
+	if len(projectsAndStuff) > 0 {
+		projectNoun := "projects"
+		if len(projectsAndStuff) == 1 {
+			projectNoun = "project"
+		}
+		msg += fmt.Sprintf(" They have %d %s:\n", len(projectsAndStuff), projectNoun)
+
+		for _, p := range projectsAndStuff {
+			msg += fmt.Sprintf("- %s: %s\n", p.Project.Name, hmndata.UrlContextForProject(&p.Project).BuildHomepage())
+		}
+	}
+
 	err = CreateInteractionResponse(ctx, i.ID, i.Token, InteractionResponse{
 		Type: InteractionCallbackTypeChannelMessageWithSource,
 		Data: &InteractionCallbackData{
-			Content: fmt.Sprintf("<@%s>'s profile can be viewed at %s.", member.User.ID, url),
+			Content: msg,
 			Flags:   FlagEphemeral,
 		},
 	})

src/discord/gateway.go

@@ -591,100 +591,29 @@ func (bot *botInstance) messageCreateOrUpdate(ctx context.Context, msg *Message)
 		return nil
 	}
 
-	if msg.ChannelID == config.Config.Discord.ShowcaseChannelID {
-		err := bot.processShowcaseMsg(ctx, msg)
-		if err != nil {
-			logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to process showcase message")
-			return nil
-		}
-		return nil
-	}
-
-	if msg.ChannelID == config.Config.Discord.LibraryChannelID {
-		err := bot.processLibraryMsg(ctx, msg)
-		if err != nil {
-			logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to process library message")
-			return nil
-		}
-		return nil
-	}
-
-	err := UpdateSnippetTagsIfAny(ctx, bot.dbConn, msg)
+	err := HandleIncomingMessage(ctx, bot.dbConn, msg, true)
 	if err != nil {
-		logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to update tags for Discord snippet")
+		logging.ExtractLogger(ctx).Error().Err(err).Msg("failed to handle incoming message")
 	}
 
+	// NOTE(asaf): Since any error from HandleIncomingMessage is an internal error and not a discord
+	//             error, we only want to log it and not restart the bot. So we're not returning the error.
 	return nil
 }
 
 func (bot *botInstance) messageDelete(ctx context.Context, msgDelete MessageDelete) {
 	log := logging.ExtractLogger(ctx)
 
-	tx, err := bot.dbConn.Begin(ctx)
+	interned, err := FetchInternedMessage(ctx, bot.dbConn, msgDelete.ID)
 	if err != nil {
-		log.Error().Err(err).Msg("failed to start transaction")
-		return
-	}
-	defer tx.Rollback(ctx)
-
-	type deleteMessageQuery struct {
-		Message     models.DiscordMessage `db:"msg"`
-		DiscordUser *models.DiscordUser   `db:"duser"`
-		HMNUser     *models.User          `db:"hmnuser"`
-		SnippetID   *int                  `db:"snippet.id"`
-	}
-	iresult, err := db.QueryOne(ctx, tx, deleteMessageQuery{},
-		`
-		SELECT $columns
-		FROM
-			handmade_discordmessage AS msg
-			LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid
-			LEFT JOIN auth_user AS hmnuser ON duser.hmn_user_id = hmnuser.id
-			LEFT JOIN handmade_snippet AS snippet ON snippet.discord_message_id = msg.id
-		WHERE msg.id = $1 AND msg.channel_id = $2
-		`,
-		msgDelete.ID, msgDelete.ChannelID,
-	)
-	if errors.Is(err, db.NotFound) {
-		return
-	} else if err != nil {
-		log.Error().Err(err).Msg("failed to check for message to delete")
-		return
-	}
-	result := iresult.(*deleteMessageQuery)
-
-	log.Debug().Msg("deleting Discord message")
-	_, err = tx.Exec(ctx,
-		`
-		DELETE FROM handmade_discordmessage
-		WHERE id = $1 AND channel_id = $2
-		`,
-		msgDelete.ID,
-		msgDelete.ChannelID,
-	)
-
-	shouldDeleteSnippet := result.HMNUser != nil && result.HMNUser.DiscordDeleteSnippetOnMessageDelete
-	if result.SnippetID != nil && shouldDeleteSnippet {
-		log.Debug().
-			Int("snippet_id", *result.SnippetID).
-			Int("user_id", result.HMNUser.ID).
-			Msg("deleting snippet from Discord message")
-		_, err = tx.Exec(ctx,
-			`
-			DELETE FROM handmade_snippet
-			WHERE id = $1
-			`,
-			result.SnippetID,
-		)
-		if err != nil {
-			log.Error().Err(err).Msg("failed to delete snippet")
-			return
+		if !errors.Is(err, db.NotFound) {
+			log.Error().Err(err).Msg("failed to fetch interned message")
 		}
+		return
 	}
-
-	err = tx.Commit(ctx)
+	err = DeleteInternedMessage(ctx, bot.dbConn, interned)
 	if err != nil {
-		log.Error().Err(err).Msg("failed to delete Discord message")
+		log.Error().Err(err).Msg("failed to delete interned message")
 		return
 	}
 }
@@ -697,7 +626,7 @@ type MessageToSend struct {
 
 func SendMessages(
 	ctx context.Context,
-	conn *pgxpool.Pool,
+	conn db.ConnOrTx,
 	msgs ...MessageToSend,
 ) error {
 	tx, err := conn.Begin(ctx)

src/discord/history.go

@@ -9,7 +9,6 @@ import (
 	"git.handmade.network/hmn/hmn/src/db"
 	"git.handmade.network/hmn/hmn/src/logging"
 	"git.handmade.network/hmn/hmn/src/models"
-	"github.com/jackc/pgx/v4"
 	"github.com/jackc/pgx/v4/pgxpool"
 )
 
@@ -31,12 +30,28 @@ func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{
 			done <- struct{}{}
 		}()
 
-		backfillInterval := 1 * time.Hour
-
 		newUserTicker := time.NewTicker(5 * time.Second)
-		backfillTicker := time.NewTicker(backfillInterval)
 
-		lastBackfillTime := time.Now().Add(-backfillInterval)
+		backfillFirstRun := make(chan struct{}, 1)
+		backfillFirstRun <- struct{}{}
+		backfillTicker := time.NewTicker(1 * time.Hour)
+
+		lastBackfillTime := time.Now().Add(-3 * time.Hour)
+
+		runBackfill := func() {
+			log.Info().Msg("Running backfill")
+			// Run a backfill to patch up places where the Discord bot missed (does create snippets)
+			now := time.Now()
+			done := Scrape(ctx, dbConn,
+				config.Config.Discord.ShowcaseChannelID,
+				lastBackfillTime,
+				true,
+			)
+			if done {
+				lastBackfillTime = now
+			}
+		}
+
 		for {
 			select {
 			case <-ctx.Done():
@@ -44,13 +59,10 @@ func RunHistoryWatcher(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{
 			case <-newUserTicker.C:
 				// Get content for messages when a user links their account (but do not create snippets)
 				fetchMissingContent(ctx, dbConn)
+			case <-backfillFirstRun:
+				runBackfill()
 			case <-backfillTicker.C:
-				// Run a backfill to patch up places where the Discord bot missed (does create snippets)
-				Scrape(ctx, dbConn,
-					config.Config.Discord.ShowcaseChannelID,
-					lastBackfillTime,
-					true,
-				)
+				runBackfill()
 			}
 		}
 	}()
@@ -99,13 +111,16 @@ func fetchMissingContent(ctx context.Context, dbConn *pgxpool.Pool) {
 			discordMsg, err := GetChannelMessage(ctx, msg.ChannelID, msg.ID)
 			if errors.Is(err, NotFound) {
 				// This message has apparently been deleted; delete it from our database
-				_, err = dbConn.Exec(ctx,
-					`
-					DELETE FROM handmade_discordmessage
-					WHERE id = $1
-					`,
-					msg.ID,
-				)
+				interned, err := FetchInternedMessage(ctx, dbConn, msg.ID)
+				if err != nil {
+					if !errors.Is(err, db.NotFound) {
+						log.Error().Str("Message ID", msg.ID).Msg("couldn't find interned message")
+					} else {
+						log.Error().Err(err).Msg("failed to fetch interned message")
+					}
+					continue
+				}
+				err = DeleteInternedMessage(ctx, dbConn, interned)
 				if err != nil {
 					log.Error().Err(err).Msg("failed to delete missing message")
 					continue
@@ -119,7 +134,7 @@ func fetchMissingContent(ctx context.Context, dbConn *pgxpool.Pool) {
 
 			log.Info().Str("msg", discordMsg.ShortString()).Msg("fetched message for content")
 
-			err = handleHistoryMessage(ctx, dbConn, discordMsg, false)
+			err = HandleInternedMessage(ctx, dbConn, discordMsg, false, false)
 			if err != nil {
 				log.Error().Err(err).Msg("failed to save content for message")
 				continue
@@ -129,7 +144,7 @@ func fetchMissingContent(ctx context.Context, dbConn *pgxpool.Pool) {
 	}
 }
 
-func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earliestMessageTime time.Time, createSnippets bool) {
+func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earliestMessageTime time.Time, createSnippets bool) bool {
 	log := logging.ExtractLogger(ctx)
 
 	log.Info().Msg("Starting scrape")
@@ -143,19 +158,19 @@ func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earlies
 		})
 		if err != nil {
 			logging.Error().Err(err).Msg("failed to get messages while scraping")
-			return
+			return false
 		}
 
 		if len(msgs) == 0 {
 			logging.Debug().Msg("out of messages, stopping scrape")
-			return
+			return true
 		}
 
 		for _, msg := range msgs {
 			select {
 			case <-ctx.Done():
 				log.Info().Msg("Scrape was canceled")
-				return
+				return false
 			default:
 			}
 
@@ -163,15 +178,13 @@ func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earlies
 
 			if !earliestMessageTime.IsZero() && msg.Time().Before(earliestMessageTime) {
 				logging.ExtractLogger(ctx).Info().Time("earliest", earliestMessageTime).Msg("Saw a message before the specified earliest time; exiting")
-				return
+				return true
 			}
 
-			err := handleHistoryMessage(ctx, dbConn, &msg, createSnippets)
+			err := HandleIncomingMessage(ctx, dbConn, &msg, createSnippets)
+
 			if err != nil {
 				errLog := logging.ExtractLogger(ctx).Error()
-				if errors.Is(err, errNotEnoughInfo) {
-					errLog = logging.ExtractLogger(ctx).Warn()
-				}
 				errLog.Err(err).Msg("failed to process Discord message")
 			}
 
@@ -179,37 +192,3 @@ func Scrape(ctx context.Context, dbConn *pgxpool.Pool, channelID string, earlies
 		}
 	}
 }
-
-func handleHistoryMessage(ctx context.Context, dbConn *pgxpool.Pool, msg *Message, createSnippets bool) error {
-	var tx pgx.Tx
-	for {
-		var err error
-		tx, err = dbConn.Begin(ctx)
-		if err != nil {
-			logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to start transaction for message")
-			time.Sleep(1 * time.Second)
-			continue
-		}
-		break
-	}
-
-	newMsg, err := SaveMessageAndContents(ctx, tx, msg)
-	if err != nil {
-		return err
-	}
-	if createSnippets {
-		if doSnippet, err := AllowedToCreateMessageSnippets(ctx, tx, newMsg.UserID); doSnippet && err == nil {
-			err := CreateMessageSnippets(ctx, tx, msg.ID)
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	err = tx.Commit(ctx)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}

src/discord/library.go

@@ -1,45 +0,0 @@
-package discord
-
-import (
-	"context"
-
-	"git.handmade.network/hmn/hmn/src/oops"
-)
-
-func (bot *botInstance) processLibraryMsg(ctx context.Context, msg *Message) error {
-	switch msg.Type {
-	case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
-	default:
-		return nil
-	}
-
-	if !msg.OriginalHasFields("content") {
-		return nil
-	}
-
-	if !messageHasLinks(msg.Content) {
-		err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
-		if err != nil {
-			return oops.New(err, "failed to delete message")
-		}
-
-		if !msg.Author.IsBot {
-			channel, err := CreateDM(ctx, msg.Author.ID)
-			if err != nil {
-				return oops.New(err, "failed to create DM channel")
-			}
-
-			err = SendMessages(ctx, bot.dbConn, MessageToSend{
-				ChannelID: channel.ID,
-				Req: CreateMessageRequest{
-					Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.",
-				},
-			})
-			if err != nil {
-				return oops.New(err, "failed to send showcase warning message")
-			}
-		}
-	}
-
-	return nil
-}

src/discord/message_handling.go

@@ -0,0 +1,949 @@
+package discord
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"time"
+
+	"git.handmade.network/hmn/hmn/src/assets"
+	"git.handmade.network/hmn/hmn/src/config"
+	"git.handmade.network/hmn/hmn/src/db"
+	"git.handmade.network/hmn/hmn/src/hmndata"
+	"git.handmade.network/hmn/hmn/src/logging"
+	"git.handmade.network/hmn/hmn/src/models"
+	"git.handmade.network/hmn/hmn/src/oops"
+	"git.handmade.network/hmn/hmn/src/parsing"
+	"github.com/google/uuid"
+)
+
+func HandleIncomingMessage(ctx context.Context, dbConn db.ConnOrTx, msg *Message, createSnippets bool) error {
+	deleted := false
+	var err error
+
+	// NOTE(asaf): All functions called here should verify that the message applies to them.
+
+	if !deleted && err == nil {
+		deleted, err = CleanUpLibrary(ctx, dbConn, msg)
+	}
+
+	if !deleted && err == nil {
+		deleted, err = CleanUpShowcase(ctx, dbConn, msg)
+	}
+
+	if !deleted && err == nil {
+		err = MaybeInternMessage(ctx, dbConn, msg)
+	}
+
+	if err == nil {
+		err = HandleInternedMessage(ctx, dbConn, msg, deleted, createSnippets)
+	}
+
+	return err
+}
+
+func CleanUpShowcase(ctx context.Context, dbConn db.ConnOrTx, msg *Message) (bool, error) {
+	deleted := false
+	if msg.ChannelID == config.Config.Discord.ShowcaseChannelID {
+		switch msg.Type {
+		case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
+		default:
+			return deleted, nil
+		}
+
+		hasGoodContent := true
+		if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) {
+			hasGoodContent = false
+		}
+
+		hasGoodAttachments := true
+		if msg.OriginalHasFields("attachments") && len(msg.Attachments) == 0 {
+			hasGoodAttachments = false
+		}
+
+		if !hasGoodContent && !hasGoodAttachments {
+			err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
+			if err != nil {
+				return deleted, oops.New(err, "failed to delete message")
+			}
+			deleted = true
+
+			if !msg.Author.IsBot {
+				channel, err := CreateDM(ctx, msg.Author.ID)
+				if err != nil {
+					return deleted, oops.New(err, "failed to create DM channel")
+				}
+
+				err = SendMessages(ctx, dbConn, MessageToSend{
+					ChannelID: channel.ID,
+					Req: CreateMessageRequest{
+						Content: "Posts in #project-showcase are required to have either an image/video or a link. Discuss showcase content in #projects.",
+					},
+				})
+				if err != nil {
+					return deleted, oops.New(err, "failed to send showcase warning message")
+				}
+			}
+		}
+	}
+
+	return deleted, nil
+}
+
+func CleanUpLibrary(ctx context.Context, dbConn db.ConnOrTx, msg *Message) (bool, error) {
+	deleted := false
+	if msg.ChannelID == config.Config.Discord.LibraryChannelID {
+		switch msg.Type {
+		case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
+		default:
+			return deleted, nil
+		}
+
+		if !msg.OriginalHasFields("content") {
+			return deleted, nil
+		}
+
+		if !messageHasLinks(msg.Content) {
+			err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
+			if err != nil {
+				return deleted, oops.New(err, "failed to delete message")
+			}
+			deleted = true
+
+			if !msg.Author.IsBot {
+				channel, err := CreateDM(ctx, msg.Author.ID)
+				if err != nil {
+					return deleted, oops.New(err, "failed to create DM channel")
+				}
+
+				err = SendMessages(ctx, dbConn, MessageToSend{
+					ChannelID: channel.ID,
+					Req: CreateMessageRequest{
+						Content: "Posts in #the-library are required to have a link. Discuss library content in other relevant channels.",
+					},
+				})
+				if err != nil {
+					return deleted, oops.New(err, "failed to send showcase warning message")
+				}
+			}
+		}
+	}
+
+	return deleted, nil
+}
+
+func MaybeInternMessage(ctx context.Context, dbConn db.ConnOrTx, msg *Message) error {
+	if msg.ChannelID == config.Config.Discord.ShowcaseChannelID {
+		err := InternMessage(ctx, dbConn, msg)
+		if errors.Is(err, errNotEnoughInfo) {
+			logging.ExtractLogger(ctx).Warn().
+				Interface("msg", msg).
+				Msg("didn't have enough info to intern Discord message")
+		} else if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this")
+
+/*
+Ensures that a Discord message is stored in the database. This function is
+idempotent and can be called regardless of whether the item already exists in
+the database.
+
+This does not create snippets or save content or do anything besides save the message itself.
+*/
+func InternMessage(
+	ctx context.Context,
+	dbConn db.ConnOrTx,
+	msg *Message,
+) error {
+	_, err := db.QueryOne(ctx, dbConn, models.DiscordMessage{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessage
+		WHERE id = $1
+		`,
+		msg.ID,
+	)
+	if errors.Is(err, db.NotFound) {
+		if !msg.OriginalHasFields("author", "timestamp") {
+			return errNotEnoughInfo
+		}
+
+		guildID := msg.GuildID
+		if guildID == nil {
+			/*
+				This is weird, but it can happen when we fetch messages from
+				history instead of receiving it from the gateway. In this case
+				we just assume it's from the HMN server.
+			*/
+			guildID = &config.Config.Discord.GuildID
+		}
+
+		_, err = dbConn.Exec(ctx,
+			`
+			INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created)
+			VALUES ($1, $2, $3, $4, $5, $6, $7)
+			`,
+			msg.ID,
+			msg.ChannelID,
+			*guildID,
+			msg.JumpURL(),
+			msg.Author.ID,
+			msg.Time(),
+			false,
+		)
+		if err != nil {
+			return oops.New(err, "failed to save new discord message")
+		}
+	} else if err != nil {
+		return oops.New(err, "failed to check for existing Discord message")
+	}
+
+	return nil
+}
+
+type InternedMessage struct {
+	Message        models.DiscordMessage         `db:"message"`
+	MessageContent *models.DiscordMessageContent `db:"content"`
+	HMNUser        *models.User                  `db:"hmnuser"`
+	DiscordUser    *models.DiscordUser           `db:"duser"`
+}
+
+func FetchInternedMessage(ctx context.Context, dbConn db.ConnOrTx, msgId string) (*InternedMessage, error) {
+	result, err := db.QueryOne(ctx, dbConn, InternedMessage{},
+		`
+		SELECT $columns
+		FROM
+			handmade_discordmessage AS message
+			LEFT JOIN handmade_discordmessagecontent AS content ON content.message_id = message.id
+			LEFT JOIN handmade_discorduser AS duser ON duser.userid = message.user_id
+			LEFT JOIN auth_user AS hmnuser ON hmnuser.id = duser.hmn_user_id
+			LEFT JOIN handmade_asset AS hmnuser_avatar ON hmnuser_avatar.id = hmnuser.avatar_asset_id
+		WHERE message.id = $1
+		`,
+		msgId,
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	interned := result.(*InternedMessage)
+	return interned, nil
+}
+
+// Checks if a message is interned and handles it to the extent possible:
+// 1. Saves/updates content
+// 2. Saves/updates snippet
+// 3. Deletes content/snippet
+func HandleInternedMessage(ctx context.Context, dbConn db.ConnOrTx, msg *Message, deleted bool, createSnippet bool) error {
+	tx, err := dbConn.Begin(ctx)
+	if err != nil {
+		return oops.New(err, "failed to start transaction")
+	}
+	defer tx.Rollback(ctx)
+
+	interned, err := FetchInternedMessage(ctx, tx, msg.ID)
+	if err != nil && !errors.Is(err, db.NotFound) {
+		return err
+	} else if err == nil {
+		if !deleted {
+			err = SaveMessageContents(ctx, tx, interned, msg)
+			if err != nil {
+				return err
+
+			}
+			if createSnippet {
+				err = HandleSnippetForInternedMessage(ctx, tx, interned, false)
+				if err != nil {
+					return err
+				}
+			}
+		} else {
+			err = DeleteInternedMessage(ctx, tx, interned)
+			if err != nil {
+				return err
+			}
+		}
+	}
+	err = tx.Commit(ctx)
+	if err != nil {
+		return oops.New(err, "failed to commit Discord message updates")
+	}
+
+	return nil
+}
+
+func DeleteInternedMessage(ctx context.Context, dbConn db.ConnOrTx, interned *InternedMessage) error {
+	isnippet, err := db.QueryOne(ctx, dbConn, models.Snippet{},
+		`
+		SELECT $columns
+		FROM handmade_snippet
+		WHERE discord_message_id = $1
+		`,
+		interned.Message.ID,
+	)
+	if err != nil && !errors.Is(err, db.NotFound) {
+		return oops.New(err, "failed to fetch snippet for discord message")
+	}
+	var snippet *models.Snippet
+	if !errors.Is(err, db.NotFound) {
+		snippet = isnippet.(*models.Snippet)
+	}
+
+	// NOTE(asaf): Also deletes the following through a db cascade:
+	//			   * handmade_discordmessageattachment
+	//			   * handmade_discordmessagecontent
+	//			   * handmade_discordmessageembed
+	//             DOES NOT DELETE ASSETS FOR CONTENT/EMBEDS
+	_, err = dbConn.Exec(ctx,
+		`
+		DELETE FROM handmade_discordmessage
+		WHERE id = $1
+		`,
+		interned.Message.ID,
+	)
+
+	if snippet != nil {
+		userApprovesDeletion := interned.HMNUser != nil && snippet.OwnerID == interned.HMNUser.ID && interned.HMNUser.DiscordDeleteSnippetOnMessageDelete
+		if !snippet.EditedOnWebsite && userApprovesDeletion {
+			// NOTE(asaf): Does not delete asset!
+			_, err = dbConn.Exec(ctx,
+				`
+				DELETE FROM handmade_snippet
+				WHERE id = $1
+				`,
+				snippet.ID,
+			)
+			if err != nil {
+				return oops.New(err, "failed to delete snippet")
+			}
+		}
+	}
+
+	return nil
+}
+
+/*
+Processes a single Discord message, saving as much of the message's content
+and attachments as allowed by our rules and user settings. Does NOT create
+snippets.
+
+Idempotent; can be called any time whether the contents exist or not.
+
+NOTE!!: Replaces interned.MessageContent if it was created or updated!!
+*/
+func SaveMessageContents(
+	ctx context.Context,
+	dbConn db.ConnOrTx,
+	interned *InternedMessage,
+	msg *Message,
+) error {
+	if interned.DiscordUser != nil {
+		// We have a linked Discord account, so save the message contents (regardless of
+		// whether we create a snippet or not).
+		if msg.OriginalHasFields("content") {
+			_, err := dbConn.Exec(ctx,
+				`
+				INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content)
+				VALUES ($1, $2, $3)
+				ON CONFLICT (message_id) DO UPDATE SET
+					discord_id = EXCLUDED.discord_id,
+					last_content = EXCLUDED.last_content
+				`,
+				interned.Message.ID,
+				interned.DiscordUser.ID,
+				CleanUpMarkdown(ctx, msg.Content),
+			)
+			if err != nil {
+				return oops.New(err, "failed to create or update message contents")
+			}
+
+			icontent, err := db.QueryOne(ctx, dbConn, models.DiscordMessageContent{},
+				`
+				SELECT $columns
+				FROM
+					handmade_discordmessagecontent
+				WHERE
+					handmade_discordmessagecontent.message_id = $1
+				`,
+				interned.Message.ID,
+			)
+			if err != nil {
+				return oops.New(err, "failed to fetch message contents")
+			}
+			interned.MessageContent = icontent.(*models.DiscordMessageContent)
+		}
+
+		// Save attachments
+		if msg.OriginalHasFields("attachments") {
+			for _, attachment := range msg.Attachments {
+				_, err := saveAttachment(ctx, dbConn, &attachment, interned.DiscordUser.HMNUserId, msg.ID)
+				if err != nil {
+					return oops.New(err, "failed to save attachment")
+				}
+			}
+		}
+
+		// Save / delete embeds
+		if msg.OriginalHasFields("embeds") {
+			numSavedEmbeds, err := db.QueryInt(ctx, dbConn,
+				`
+			SELECT COUNT(*)
+			FROM handmade_discordmessageembed
+			WHERE message_id = $1
+			`,
+				msg.ID,
+			)
+			if err != nil {
+				return oops.New(err, "failed to count existing embeds")
+			}
+			if numSavedEmbeds == 0 {
+				// No embeds yet, so save new ones
+				for _, embed := range msg.Embeds {
+					_, err := saveEmbed(ctx, dbConn, &embed, interned.DiscordUser.HMNUserId, msg.ID)
+					if err != nil {
+						return oops.New(err, "failed to save embed")
+					}
+				}
+			} else if len(msg.Embeds) > 0 {
+				// Embeds were removed from the message
+				_, err := dbConn.Exec(ctx,
+					`
+				DELETE FROM handmade_discordmessageembed
+				WHERE message_id = $1
+				`,
+					msg.ID,
+				)
+				if err != nil {
+					return oops.New(err, "failed to delete embeds")
+				}
+			}
+		}
+	}
+	return nil
+}
+
+var discordDownloadClient = &http.Client{
+	Timeout: 10 * time.Second,
+}
+
+type DiscordResourceBadStatusCode error
+
+func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, "", oops.New(err, "failed to make Discord download request")
+	}
+	res, err := discordDownloadClient.Do(req)
+	if err != nil {
+		return nil, "", oops.New(err, "failed to fetch Discord resource data")
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode < 200 || 299 < res.StatusCode {
+		return nil, "", DiscordResourceBadStatusCode(fmt.Errorf("status code %d from Discord resource: %s", res.StatusCode, url))
+	}
+
+	content, err := io.ReadAll(res.Body)
+	if err != nil {
+		panic(err)
+	}
+
+	return content, res.Header.Get("Content-Type"), nil
+}
+
+/*
+Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment
+that already exists
+*/
+func saveAttachment(
+	ctx context.Context,
+	tx db.ConnOrTx,
+	attachment *Attachment,
+	hmnUserID int,
+	discordMessageID string,
+) (*models.DiscordMessageAttachment, error) {
+	iexisting, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessageattachment
+		WHERE id = $1
+		`,
+		attachment.ID,
+	)
+	if err == nil {
+		return iexisting.(*models.DiscordMessageAttachment), nil
+	} else if errors.Is(err, db.NotFound) {
+		// this is fine, just create it
+	} else {
+		return nil, oops.New(err, "failed to check for existing attachment")
+	}
+
+	width := 0
+	height := 0
+	if attachment.Width != nil {
+		width = *attachment.Width
+	}
+	if attachment.Height != nil {
+		height = *attachment.Height
+	}
+
+	content, _, err := downloadDiscordResource(ctx, attachment.Url)
+	if err != nil {
+		return nil, oops.New(err, "failed to download Discord attachment")
+	}
+
+	contentType := "application/octet-stream"
+	if attachment.ContentType != nil {
+		contentType = *attachment.ContentType
+	}
+
+	asset, err := assets.Create(ctx, tx, assets.CreateInput{
+		Content:     content,
+		Filename:    attachment.Filename,
+		ContentType: contentType,
+
+		UploaderID: &hmnUserID,
+		Width:      width,
+		Height:     height,
+	})
+	if err != nil {
+		return nil, oops.New(err, "failed to save asset for Discord attachment")
+	}
+
+	// TODO(db): RETURNING plz thanks
+	_, err = tx.Exec(ctx,
+		`
+		INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id)
+		VALUES ($1, $2, $3)
+		`,
+		attachment.ID,
+		asset.ID,
+		discordMessageID,
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to save Discord attachment data")
+	}
+
+	iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessageattachment
+		WHERE id = $1
+		`,
+		attachment.ID,
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to fetch new Discord attachment data")
+	}
+
+	return iDiscordAttachment.(*models.DiscordMessageAttachment), nil
+}
+
+// Saves an embed from Discord. NOTE: This is _not_ idempotent, so only call it
+// if you do not have any embeds saved for this message yet.
+func saveEmbed(
+	ctx context.Context,
+	tx db.ConnOrTx,
+	embed *Embed,
+	hmnUserID int,
+	discordMessageID string,
+) (*models.DiscordMessageEmbed, error) {
+	isOkImageType := func(contentType string) bool {
+		return strings.HasPrefix(contentType, "image/")
+	}
+
+	isOkVideoType := func(contentType string) bool {
+		return strings.HasPrefix(contentType, "video/")
+	}
+
+	maybeSaveImageish := func(i EmbedImageish, contentTypeCheck func(string) bool) (*uuid.UUID, error) {
+		content, contentType, err := downloadDiscordResource(ctx, *i.Url)
+		if err != nil {
+			var statusError DiscordResourceBadStatusCode
+			if errors.As(err, &statusError) {
+				return nil, nil
+			} else {
+				return nil, oops.New(err, "failed to save Discord embed")
+			}
+		}
+		if contentTypeCheck(contentType) {
+			in := assets.CreateInput{
+				Content:     content,
+				Filename:    "embed",
+				ContentType: contentType,
+				UploaderID:  &hmnUserID,
+			}
+
+			if i.Width != nil {
+				in.Width = *i.Width
+			}
+			if i.Height != nil {
+				in.Height = *i.Height
+			}
+
+			asset, err := assets.Create(ctx, tx, in)
+			if err != nil {
+				return nil, oops.New(err, "failed to create asset from embed")
+			}
+			return &asset.ID, nil
+		}
+
+		return nil, nil
+	}
+
+	var imageAssetId *uuid.UUID
+	var videoAssetId *uuid.UUID
+	var err error
+
+	if embed.Video != nil && embed.Video.Url != nil {
+		videoAssetId, err = maybeSaveImageish(embed.Video.EmbedImageish, isOkVideoType)
+	} else if embed.Image != nil && embed.Image.Url != nil {
+		imageAssetId, err = maybeSaveImageish(embed.Image.EmbedImageish, isOkImageType)
+	} else if embed.Thumbnail != nil && embed.Thumbnail.Url != nil {
+		imageAssetId, err = maybeSaveImageish(embed.Thumbnail.EmbedImageish, isOkImageType)
+	}
+	if err != nil {
+		return nil, err
+	}
+
+	// Save the embed into the db
+	// TODO(db): Insert, RETURNING
+	var savedEmbedId int
+	err = tx.QueryRow(ctx,
+		`
+		INSERT INTO handmade_discordmessageembed (title, description, url, message_id, image_id, video_id)
+		VALUES ($1, $2, $3, $4, $5, $6)
+		RETURNING id
+		`,
+		embed.Title,
+		embed.Description,
+		embed.Url,
+		discordMessageID,
+		imageAssetId,
+		videoAssetId,
+	).Scan(&savedEmbedId)
+	if err != nil {
+		return nil, oops.New(err, "failed to insert new embed")
+	}
+
+	iDiscordEmbed, err := db.QueryOne(ctx, tx, models.DiscordMessageEmbed{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessageembed
+		WHERE id = $1
+		`,
+		savedEmbedId,
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to fetch new Discord embed data")
+	}
+
+	return iDiscordEmbed.(*models.DiscordMessageEmbed), nil
+}
+
+func FetchSnippetForMessage(ctx context.Context, dbConn db.ConnOrTx, msgID string) (*models.Snippet, error) {
+	iresult, err := db.QueryOne(ctx, dbConn, models.Snippet{},
+		`
+		SELECT $columns
+		FROM handmade_snippet
+		WHERE discord_message_id = $1
+		`,
+		msgID,
+	)
+
+	if err != nil {
+		if errors.Is(err, db.NotFound) {
+			return nil, nil
+		} else {
+			return nil, oops.New(err, "failed to fetch existing snippet for message %s", msgID)
+		}
+	}
+
+	return iresult.(*models.Snippet), nil
+}
+
+/*
+Potentially creates or updates a snippet for the given interned message.
+It uses the content saved in the database to do this. If we do not have any
+content saved, nothing will happen.
+
+If a user does not have their Discord account linked, this function will
+naturally do nothing because we have no message content saved.
+If forceCreate is true, it does not check any user settings such as automatically creating snippets from
+#project-showcase. If we have the content, it will make a snippet for it, no
+questions asked. Bear that in mind.
+*/
+func HandleSnippetForInternedMessage(ctx context.Context, dbConn db.ConnOrTx, interned *InternedMessage, forceCreate bool) error {
+	if interned.HMNUser == nil {
+		// NOTE(asaf): Can't handle snippets when there's no linked user
+		return nil
+	}
+
+	if interned.MessageContent == nil {
+		// NOTE(asaf): Can't have a snippet without content
+		// NOTE(asaf): Messages that only have an attachment also have blank content
+		// TODO(asaf): Do we need to delete existing snippets in this case??? Not entirely sure how to trigger this through discord
+		return nil
+	}
+
+	tx, err := dbConn.Begin(ctx)
+	if err != nil {
+		oops.New(err, "failed to start transaction")
+	}
+	defer tx.Rollback(ctx)
+
+	existingSnippet, err := FetchSnippetForMessage(ctx, tx, interned.Message.ID)
+	if err != nil {
+		return oops.New(err, "failed to check for existing snippet for message %s", interned.Message.ID)
+	}
+
+	if existingSnippet != nil {
+		// TODO(asaf): We're not handling the case where embeds were removed or modified.
+		//             Also not handling the case where a message had both an attachment and an embed
+		//             and the attachment was removed (leaving only the embed).
+		linkedUserIsSnippetOwner := existingSnippet.OwnerID == interned.DiscordUser.HMNUserId
+		if linkedUserIsSnippetOwner && !existingSnippet.EditedOnWebsite {
+			contentMarkdown := interned.MessageContent.LastContent
+			contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
+
+			_, err := tx.Exec(ctx,
+				`
+				UPDATE handmade_snippet
+				SET
+					description = $1,
+					_description_html = $2
+				WHERE id = $3
+				`,
+				contentMarkdown,
+				contentHTML,
+				existingSnippet.ID,
+			)
+			if err != nil {
+				return oops.New(err, "failed to update content of snippet on message edit")
+			}
+			existingSnippet.Description = contentMarkdown
+			existingSnippet.DescriptionHtml = contentHTML
+		}
+	} else {
+		userAllowsSnippet := interned.HMNUser.DiscordSaveShowcase || forceCreate
+		shouldCreate := !interned.Message.SnippetCreated && userAllowsSnippet
+
+		if shouldCreate {
+			// Get an asset ID or URL to make a snippet from
+			assetId, url, err := getSnippetAssetOrUrl(ctx, tx, &interned.Message)
+			if assetId != nil || url != nil {
+				contentMarkdown := interned.MessageContent.LastContent
+				contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
+
+				_, err = tx.Exec(ctx,
+					`
+					INSERT INTO handmade_snippet (url, "when", description, _description_html, asset_id, discord_message_id, owner_id)
+					VALUES ($1, $2, $3, $4, $5, $6, $7)
+					`,
+					url,
+					interned.Message.SentAt,
+					contentMarkdown,
+					contentHTML,
+					assetId,
+					interned.Message.ID,
+					interned.HMNUser.ID,
+				)
+				if err != nil {
+					return oops.New(err, "failed to create snippet from attachment")
+				}
+
+				existingSnippet, err = FetchSnippetForMessage(ctx, tx, interned.Message.ID)
+				if err != nil {
+					return oops.New(err, "failed to fetch newly-created snippet")
+				}
+
+				_, err = tx.Exec(ctx,
+					`
+					UPDATE handmade_discordmessage
+					SET snippet_created = TRUE
+					WHERE id = $1
+					`,
+					interned.Message.ID,
+				)
+				if err != nil {
+					return oops.New(err, "failed to mark message as having snippet")
+				}
+			}
+		}
+	}
+
+	if existingSnippet != nil {
+		// Update tags
+
+		// Try to associate tags in the message with project tags in HMN.
+		// Match only tags for projects in which the current user is a collaborator.
+		messageTags := getDiscordTags(existingSnippet.Description)
+
+		var desiredTags []int
+		var allTags []int
+
+		// Fetch projects so we know what tags the user can apply to their snippet.
+		projects, err := hmndata.FetchProjects(ctx, tx, interned.HMNUser, hmndata.ProjectsQuery{
+			OwnerIDs: []int{interned.HMNUser.ID},
+		})
+		if err != nil {
+			return oops.New(err, "failed to look up user projects")
+		}
+
+		projectIDs := make([]int, len(projects))
+		for i, p := range projects {
+			projectIDs[i] = p.Project.ID
+		}
+
+		type tagsRow struct {
+			Tag models.Tag `db:"tags"`
+		}
+		iUserTags, err := db.Query(ctx, tx, tagsRow{},
+			`
+			SELECT $columns
+			FROM
+				tags
+				JOIN handmade_project AS project ON project.tag = tags.id
+			WHERE
+				project.id = ANY ($1)
+			`,
+			projectIDs,
+		)
+		if err != nil {
+			return oops.New(err, "failed to fetch tags for user projects")
+		}
+
+		for _, itag := range iUserTags {
+			tag := itag.(*tagsRow).Tag
+			allTags = append(allTags, tag.ID)
+			for _, messageTag := range messageTags {
+				if strings.EqualFold(tag.Text, messageTag) {
+					desiredTags = append(desiredTags, tag.ID)
+				}
+			}
+		}
+
+		_, err = tx.Exec(ctx,
+			`
+			DELETE FROM snippet_tags
+			WHERE
+				snippet_id = $1
+				AND tag_id = ANY ($2)
+			`,
+			existingSnippet.ID,
+			allTags,
+		)
+		if err != nil {
+			return oops.New(err, "failed to clear tags from snippet")
+		}
+
+		for _, tagID := range desiredTags {
+			_, err = tx.Exec(ctx,
+				`
+				INSERT INTO snippet_tags (snippet_id, tag_id)
+				VALUES ($1, $2)
+				ON CONFLICT DO NOTHING
+				`,
+				existingSnippet.ID,
+				tagID,
+			)
+			if err != nil {
+				return oops.New(err, "failed to associate snippet with tag")
+			}
+		}
+	}
+
+	err = tx.Commit(ctx)
+	if err != nil {
+		return oops.New(err, "failed to commit transaction")
+	}
+
+	return nil
+}
+
+// TODO(asaf): I believe this will also match https://example.com?hello=1&whatever=5
+//             Probably need to add word boundaries.
+var REDiscordTag = regexp.MustCompile(`&([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*)`)
+
+func getDiscordTags(content string) []string {
+	matches := REDiscordTag.FindAllStringSubmatch(content, -1)
+	result := make([]string, len(matches))
+	for i, m := range matches {
+		result[i] = m[1]
+	}
+	return result
+}
+
+// NOTE(ben): This is maybe redundant with the regexes we use for markdown. But
+// do we actually want to reuse those, or should we keep them separate?
+var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`)
+
+func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msg *models.DiscordMessage) (*uuid.UUID, *string, error) {
+	// Check attachments
+	attachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessageattachment
+		WHERE message_id = $1
+		`,
+		msg.ID,
+	)
+	if err != nil {
+		return nil, nil, oops.New(err, "failed to fetch message attachments")
+	}
+	for _, iattachment := range attachments {
+		attachment := iattachment.(*models.DiscordMessageAttachment)
+		return &attachment.AssetID, nil, nil
+	}
+
+	// Check embeds
+	embeds, err := db.Query(ctx, tx, models.DiscordMessageEmbed{},
+		`
+		SELECT $columns
+		FROM handmade_discordmessageembed
+		WHERE message_id = $1
+		`,
+		msg.ID,
+	)
+	if err != nil {
+		return nil, nil, oops.New(err, "failed to fetch discord embeds")
+	}
+	for _, iembed := range embeds {
+		embed := iembed.(*models.DiscordMessageEmbed)
+		if embed.VideoID != nil {
+			return embed.VideoID, nil, nil
+		} else if embed.ImageID != nil {
+			return embed.ImageID, nil, nil
+		} else if embed.URL != nil {
+			if RESnippetableUrl.MatchString(*embed.URL) {
+				return nil, embed.URL, nil
+			}
+		}
+	}
+
+	return nil, nil, nil
+}
+
+var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`)
+
+func messageHasLinks(content string) bool {
+	links := reDiscordMessageLink.FindAllString(content, -1)
+	for _, link := range links {
+		_, err := url.Parse(strings.TrimSpace(link))
+		if err == nil {
+			return true
+		}
+	}
+
+	return false
+}

src/discord/showcase.go

@@ -1,872 +0,0 @@
-package discord
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"regexp"
-	"strings"
-	"time"
-
-	"git.handmade.network/hmn/hmn/src/assets"
-	"git.handmade.network/hmn/hmn/src/config"
-	"git.handmade.network/hmn/hmn/src/db"
-	"git.handmade.network/hmn/hmn/src/hmndata"
-	"git.handmade.network/hmn/hmn/src/logging"
-	"git.handmade.network/hmn/hmn/src/models"
-	"git.handmade.network/hmn/hmn/src/oops"
-	"git.handmade.network/hmn/hmn/src/parsing"
-	"github.com/google/uuid"
-)
-
-var reDiscordMessageLink = regexp.MustCompile(`https?://.+?(\s|$)`)
-
-var errNotEnoughInfo = errors.New("Discord didn't send enough info in this event for us to do this")
-
-func (bot *botInstance) processShowcaseMsg(ctx context.Context, msg *Message) error {
-	switch msg.Type {
-	case MessageTypeDefault, MessageTypeReply, MessageTypeApplicationCommand:
-	default:
-		return nil
-	}
-
-	didDelete, err := bot.maybeDeleteShowcaseMsg(ctx, msg)
-	if err != nil {
-		return err
-	}
-	if didDelete {
-		return nil
-	}
-
-	tx, err := bot.dbConn.Begin(ctx)
-	if err != nil {
-		panic(err)
-	}
-	defer tx.Rollback(ctx)
-
-	// save the message, maybe save its contents
-	newMsg, err := SaveMessageAndContents(ctx, tx, msg)
-	if errors.Is(err, errNotEnoughInfo) {
-		logging.ExtractLogger(ctx).Warn().
-			Interface("msg", msg).
-			Msg("didn't have enough info to process Discord message")
-		return nil
-	} else if err != nil {
-		return err
-	}
-
-	// ...and maybe make a snippet too, if the user wants us to
-	duser, err := FetchDiscordUser(ctx, tx, newMsg.UserID)
-	if err == nil && duser.HMNUser.DiscordSaveShowcase {
-		err = CreateMessageSnippets(ctx, tx, newMsg.UserID, msg.ID)
-		if err != nil {
-			return oops.New(err, "failed to create snippet in gateway")
-		}
-	} else {
-		if err == db.NotFound {
-			// this is fine, just don't create a snippet
-		} else {
-			return err
-		}
-	}
-
-	err = tx.Commit(ctx)
-	if err != nil {
-		return oops.New(err, "failed to commit Discord message updates")
-	}
-
-	return nil
-}
-
-func (bot *botInstance) maybeDeleteShowcaseMsg(ctx context.Context, msg *Message) (didDelete bool, err error) {
-	hasGoodContent := true
-	if msg.OriginalHasFields("content") && !messageHasLinks(msg.Content) {
-		hasGoodContent = false
-	}
-
-	hasGoodAttachments := true
-	if msg.OriginalHasFields("attachments") && len(msg.Attachments) == 0 {
-		hasGoodAttachments = false
-	}
-
-	didDelete = false
-	if !hasGoodContent && !hasGoodAttachments {
-		didDelete = true
-		err := DeleteMessage(ctx, msg.ChannelID, msg.ID)
-		if err != nil {
-			return false, oops.New(err, "failed to delete message")
-		}
-
-		if !msg.Author.IsBot {
-			channel, err := CreateDM(ctx, msg.Author.ID)
-			if err != nil {
-				return false, oops.New(err, "failed to create DM channel")
-			}
-
-			err = SendMessages(ctx, bot.dbConn, MessageToSend{
-				ChannelID: channel.ID,
-				Req: CreateMessageRequest{
-					Content: "Posts in #project-showcase are required to have either an image/video or a link. Discuss showcase content in #projects.",
-				},
-			})
-			if err != nil {
-				return false, oops.New(err, "failed to send showcase warning message")
-			}
-		}
-	}
-
-	return didDelete, nil
-}
-
-/*
-Ensures that a Discord message is stored in the database. This function is
-idempotent and can be called regardless of whether the item already exists in
-the database.
-
-This does not create snippets or do anything besides save the message itself.
-*/
-func SaveMessage(
-	ctx context.Context,
-	tx db.ConnOrTx,
-	msg *Message,
-) (*models.DiscordMessage, error) {
-	iDiscordMessage, err := db.QueryOne(ctx, tx, models.DiscordMessage{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessage
-		WHERE id = $1
-		`,
-		msg.ID,
-	)
-	if errors.Is(err, db.NotFound) {
-		if !msg.OriginalHasFields("author", "timestamp") {
-			return nil, errNotEnoughInfo
-		}
-
-		guildID := msg.GuildID
-		if guildID == nil {
-			/*
-				This is weird, but it can happen when we fetch messages from
-				history instead of receiving it from the gateway. In this case
-				we just assume it's from the HMN server.
-			*/
-			guildID = &config.Config.Discord.GuildID
-		}
-
-		_, err = tx.Exec(ctx,
-			`
-			INSERT INTO handmade_discordmessage (id, channel_id, guild_id, url, user_id, sent_at, snippet_created)
-			VALUES ($1, $2, $3, $4, $5, $6, $7)
-			`,
-			msg.ID,
-			msg.ChannelID,
-			*guildID,
-			msg.JumpURL(),
-			msg.Author.ID,
-			msg.Time(),
-			false,
-		)
-		if err != nil {
-			return nil, oops.New(err, "failed to save new discord message")
-		}
-
-		/*
-			TODO(db): This is a spot where it would be really nice to be able
-			to use RETURNING, and avoid this second query.
-		*/
-		iDiscordMessage, err = db.QueryOne(ctx, tx, models.DiscordMessage{},
-			`
-			SELECT $columns
-			FROM handmade_discordmessage
-			WHERE id = $1
-			`,
-			msg.ID,
-		)
-		if err != nil {
-			panic(err)
-		}
-	} else if err != nil {
-		return nil, oops.New(err, "failed to check for existing Discord message")
-	}
-
-	return iDiscordMessage.(*models.DiscordMessage), nil
-}
-
-/*
-Processes a single Discord message, saving as much of the message's content
-and attachments as allowed by our rules and user settings. Does NOT create
-snippets.
-
-Idempotent; can be called any time whether the message exists or not.
-*/
-func SaveMessageAndContents(
-	ctx context.Context,
-	tx db.ConnOrTx,
-	msg *Message,
-) (*models.DiscordMessage, error) {
-	newMsg, err := SaveMessage(ctx, tx, msg)
-	if err != nil {
-		return nil, err
-	}
-
-	// Check for linked Discord user
-	iDiscordUser, err := db.QueryOne(ctx, tx, models.DiscordUser{},
-		`
-		SELECT $columns
-		FROM handmade_discorduser
-		WHERE userid = $1
-		`,
-		newMsg.UserID,
-	)
-	if errors.Is(err, db.NotFound) {
-		return newMsg, nil
-	} else if err != nil {
-		return nil, oops.New(err, "failed to look up linked Discord user")
-	}
-	discordUser := iDiscordUser.(*models.DiscordUser)
-
-	// We have a linked Discord account, so save the message contents (regardless of
-	// whether we create a snippet or not).
-
-	if msg.OriginalHasFields("content") {
-		_, err = tx.Exec(ctx,
-			`
-			INSERT INTO handmade_discordmessagecontent (message_id, discord_id, last_content)
-			VALUES ($1, $2, $3)
-			ON CONFLICT (message_id) DO UPDATE SET
-				discord_id = EXCLUDED.discord_id,
-				last_content = EXCLUDED.last_content
-			`,
-			newMsg.ID,
-			discordUser.ID,
-			CleanUpMarkdown(ctx, msg.Content),
-		)
-	}
-
-	// Save attachments
-	if msg.OriginalHasFields("attachments") {
-		for _, attachment := range msg.Attachments {
-			_, err := saveAttachment(ctx, tx, &attachment, discordUser.HMNUserId, msg.ID)
-			if err != nil {
-				return nil, oops.New(err, "failed to save attachment")
-			}
-		}
-	}
-
-	// Save / delete embeds
-	if msg.OriginalHasFields("embeds") {
-		numSavedEmbeds, err := db.QueryInt(ctx, tx,
-			`
-			SELECT COUNT(*)
-			FROM handmade_discordmessageembed
-			WHERE message_id = $1
-			`,
-			msg.ID,
-		)
-		if err != nil {
-			return nil, oops.New(err, "failed to count existing embeds")
-		}
-		if numSavedEmbeds == 0 {
-			// No embeds yet, so save new ones
-			for _, embed := range msg.Embeds {
-				_, err := saveEmbed(ctx, tx, &embed, discordUser.HMNUserId, msg.ID)
-				if err != nil {
-					return nil, oops.New(err, "failed to save embed")
-				}
-			}
-		} else if len(msg.Embeds) > 0 {
-			// Embeds were removed from the message
-			_, err := tx.Exec(ctx,
-				`
-				DELETE FROM handmade_discordmessageembed
-				WHERE message_id = $1
-				`,
-				msg.ID,
-			)
-			if err != nil {
-				return nil, oops.New(err, "failed to delete embeds")
-			}
-		}
-	}
-
-	return newMsg, nil
-}
-
-var discordDownloadClient = &http.Client{
-	Timeout: 10 * time.Second,
-}
-
-type DiscordResourceBadStatusCode error
-
-func downloadDiscordResource(ctx context.Context, url string) ([]byte, string, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
-	if err != nil {
-		return nil, "", oops.New(err, "failed to make Discord download request")
-	}
-	res, err := discordDownloadClient.Do(req)
-	if err != nil {
-		return nil, "", oops.New(err, "failed to fetch Discord resource data")
-	}
-	defer res.Body.Close()
-
-	if res.StatusCode < 200 || 299 < res.StatusCode {
-		return nil, "", DiscordResourceBadStatusCode(fmt.Errorf("status code %d from Discord resource: %s", res.StatusCode, url))
-	}
-
-	content, err := io.ReadAll(res.Body)
-	if err != nil {
-		panic(err)
-	}
-
-	return content, res.Header.Get("Content-Type"), nil
-}
-
-/*
-Saves a Discord attachment as an HMN asset. Idempotent; will not create an attachment
-that already exists
-*/
-func saveAttachment(
-	ctx context.Context,
-	tx db.ConnOrTx,
-	attachment *Attachment,
-	hmnUserID int,
-	discordMessageID string,
-) (*models.DiscordMessageAttachment, error) {
-	iexisting, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessageattachment
-		WHERE id = $1
-		`,
-		attachment.ID,
-	)
-	if err == nil {
-		return iexisting.(*models.DiscordMessageAttachment), nil
-	} else if errors.Is(err, db.NotFound) {
-		// this is fine, just create it
-	} else {
-		return nil, oops.New(err, "failed to check for existing attachment")
-	}
-
-	width := 0
-	height := 0
-	if attachment.Width != nil {
-		width = *attachment.Width
-	}
-	if attachment.Height != nil {
-		height = *attachment.Height
-	}
-
-	content, _, err := downloadDiscordResource(ctx, attachment.Url)
-	if err != nil {
-		return nil, oops.New(err, "failed to download Discord attachment")
-	}
-
-	contentType := "application/octet-stream"
-	if attachment.ContentType != nil {
-		contentType = *attachment.ContentType
-	}
-
-	asset, err := assets.Create(ctx, tx, assets.CreateInput{
-		Content:     content,
-		Filename:    attachment.Filename,
-		ContentType: contentType,
-
-		UploaderID: &hmnUserID,
-		Width:      width,
-		Height:     height,
-	})
-	if err != nil {
-		return nil, oops.New(err, "failed to save asset for Discord attachment")
-	}
-
-	// TODO(db): RETURNING plz thanks
-	_, err = tx.Exec(ctx,
-		`
-		INSERT INTO handmade_discordmessageattachment (id, asset_id, message_id)
-		VALUES ($1, $2, $3)
-		`,
-		attachment.ID,
-		asset.ID,
-		discordMessageID,
-	)
-	if err != nil {
-		return nil, oops.New(err, "failed to save Discord attachment data")
-	}
-
-	iDiscordAttachment, err := db.QueryOne(ctx, tx, models.DiscordMessageAttachment{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessageattachment
-		WHERE id = $1
-		`,
-		attachment.ID,
-	)
-	if err != nil {
-		return nil, oops.New(err, "failed to fetch new Discord attachment data")
-	}
-
-	return iDiscordAttachment.(*models.DiscordMessageAttachment), nil
-}
-
-// Saves an embed from Discord. NOTE: This is _not_ idempotent, so only call it
-// if you do not have any embeds saved for this message yet.
-func saveEmbed(
-	ctx context.Context,
-	tx db.ConnOrTx,
-	embed *Embed,
-	hmnUserID int,
-	discordMessageID string,
-) (*models.DiscordMessageEmbed, error) {
-	isOkImageType := func(contentType string) bool {
-		return strings.HasPrefix(contentType, "image/")
-	}
-
-	isOkVideoType := func(contentType string) bool {
-		return strings.HasPrefix(contentType, "video/")
-	}
-
-	maybeSaveImageish := func(i EmbedImageish, contentTypeCheck func(string) bool) (*uuid.UUID, error) {
-		content, contentType, err := downloadDiscordResource(ctx, *i.Url)
-		if err != nil {
-			var statusError DiscordResourceBadStatusCode
-			if errors.As(err, &statusError) {
-				return nil, nil
-			} else {
-				return nil, oops.New(err, "failed to save Discord embed")
-			}
-		}
-		if contentTypeCheck(contentType) {
-			in := assets.CreateInput{
-				Content:     content,
-				Filename:    "embed",
-				ContentType: contentType,
-				UploaderID:  &hmnUserID,
-			}
-
-			if i.Width != nil {
-				in.Width = *i.Width
-			}
-			if i.Height != nil {
-				in.Height = *i.Height
-			}
-
-			asset, err := assets.Create(ctx, tx, in)
-			if err != nil {
-				return nil, oops.New(err, "failed to create asset from embed")
-			}
-			return &asset.ID, nil
-		}
-
-		return nil, nil
-	}
-
-	var imageAssetId *uuid.UUID
-	var videoAssetId *uuid.UUID
-	var err error
-
-	if embed.Video != nil && embed.Video.Url != nil {
-		videoAssetId, err = maybeSaveImageish(embed.Video.EmbedImageish, isOkVideoType)
-	} else if embed.Image != nil && embed.Image.Url != nil {
-		imageAssetId, err = maybeSaveImageish(embed.Image.EmbedImageish, isOkImageType)
-	} else if embed.Thumbnail != nil && embed.Thumbnail.Url != nil {
-		imageAssetId, err = maybeSaveImageish(embed.Thumbnail.EmbedImageish, isOkImageType)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	// Save the embed into the db
-	// TODO(db): Insert, RETURNING
-	var savedEmbedId int
-	err = tx.QueryRow(ctx,
-		`
-		INSERT INTO handmade_discordmessageembed (title, description, url, message_id, image_id, video_id)
-		VALUES ($1, $2, $3, $4, $5, $6)
-		RETURNING id
-		`,
-		embed.Title,
-		embed.Description,
-		embed.Url,
-		discordMessageID,
-		imageAssetId,
-		videoAssetId,
-	).Scan(&savedEmbedId)
-	if err != nil {
-		return nil, oops.New(err, "failed to insert new embed")
-	}
-
-	iDiscordEmbed, err := db.QueryOne(ctx, tx, models.DiscordMessageEmbed{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessageembed
-		WHERE id = $1
-		`,
-		savedEmbedId,
-	)
-	if err != nil {
-		return nil, oops.New(err, "failed to fetch new Discord embed data")
-	}
-
-	return iDiscordEmbed.(*models.DiscordMessageEmbed), nil
-}
-
-type DiscordUserAndStuff struct {
-	DiscordUser models.DiscordUser `db:"duser"`
-	HMNUser     models.User        `db:"u"`
-}
-
-func FetchDiscordUser(ctx context.Context, dbConn db.ConnOrTx, discordUserID string) (*DiscordUserAndStuff, error) {
-	iuser, err := db.QueryOne(ctx, dbConn, DiscordUserAndStuff{},
-		`
-		SELECT $columns
-		FROM
-			handmade_discorduser AS duser
-			JOIN auth_user AS u ON duser.hmn_user_id = u.id
-		WHERE
-			duser.userid = $1
-		`,
-		discordUserID,
-	)
-	if err != nil {
-		return nil, err
-	}
-	return iuser.(*DiscordUserAndStuff), nil
-}
-
-/*
-Checks settings and permissions to decide whether we are allowed to create
-snippets for a user.
-*/
-func AllowedToCreateMessageSnippets(ctx context.Context, tx db.ConnOrTx, discordUserId string) (bool, error) {
-	u, err := FetchDiscordUser(ctx, tx, discordUserId)
-	if errors.Is(err, db.NotFound) {
-		return false, nil
-	} else if err != nil {
-		return false, oops.New(err, "failed to check if we can save Discord message")
-	}
-
-	return u.HMNUser.DiscordSaveShowcase, nil
-}
-
-/*
-Attempts to create snippets from Discord messages. If a snippet already exists
-for any message, no new snippet will be created.
-
-It uses the content saved in the database to do this. If we do not have any
-content saved, nothing will happen.
-
-If a user does not have their Discord account linked, this function will
-naturally do nothing because we have no message content saved. However, it does
-not check any user settings such as automatically creating snippets from
-#project-showcase. If we have the content, it will make a snippet for it, no
-questions asked. Bear that in mind.
-*/
-func CreateMessageSnippets(ctx context.Context, dbConn db.ConnOrTx, msgIDs ...string) error {
-	tx, err := dbConn.Begin(ctx)
-	if err != nil {
-		return oops.New(err, "failed to begin transaction")
-	}
-	defer tx.Rollback(ctx)
-
-	for _, msgID := range msgIDs {
-		// Check for existing snippet
-		type existingSnippetResult struct {
-			Message        models.DiscordMessage         `db:"msg"`
-			MessageContent *models.DiscordMessageContent `db:"c"`
-			Snippet        *models.Snippet               `db:"snippet"`
-			DiscordUser    *models.DiscordUser           `db:"duser"`
-		}
-		iexisting, err := db.QueryOne(ctx, tx, existingSnippetResult{},
-			`
-			SELECT $columns
-			FROM
-				handmade_discordmessage AS msg
-				LEFT JOIN handmade_discordmessagecontent AS c ON c.message_id = msg.id
-				LEFT JOIN handmade_snippet AS snippet ON snippet.discord_message_id = msg.id
-				LEFT JOIN handmade_discorduser AS duser ON msg.user_id = duser.userid
-			WHERE
-				msg.id = $1
-			`,
-			msgID,
-		)
-		if err != nil {
-			return oops.New(err, "failed to check for existing snippet for message %s", msgID)
-		}
-		existing := iexisting.(*existingSnippetResult)
-
-		if existing.Snippet != nil {
-			// A snippet already exists - maybe update its content.
-			if existing.MessageContent != nil && !existing.Snippet.EditedOnWebsite {
-				contentMarkdown := existing.MessageContent.LastContent
-				contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
-
-				_, err := tx.Exec(ctx,
-					`
-					UPDATE handmade_snippet
-					SET
-						description = $1,
-						_description_html = $2
-					WHERE id = $3
-					`,
-					contentMarkdown,
-					contentHTML,
-					existing.Snippet.ID,
-				)
-				if err != nil {
-					logging.ExtractLogger(ctx).Warn().Err(err).Msg("failed to update content of snippet on message edit")
-				}
-				continue
-			}
-		}
-
-		if existing.Message.SnippetCreated {
-			// A snippet once existed but no longer does
-			// (we do not create another one in this case)
-			return nil
-		}
-
-		if existing.MessageContent == nil || existing.DiscordUser == nil {
-			return nil
-		}
-
-		// Get an asset ID or URL to make a snippet from
-		assetId, url, err := getSnippetAssetOrUrl(ctx, tx, &existing.Message)
-		if assetId == nil && url == nil {
-			// Nothing to make a snippet from!
-			return nil
-		}
-
-		contentMarkdown := existing.MessageContent.LastContent
-		contentHTML := parsing.ParseMarkdown(contentMarkdown, parsing.DiscordMarkdown)
-
-		// TODO(db): Insert
-		_, err = tx.Exec(ctx,
-			`
-			INSERT INTO handmade_snippet (url, "when", description, _description_html, asset_id, discord_message_id, owner_id)
-			VALUES ($1, $2, $3, $4, $5, $6, $7)
-			`,
-			url,
-			existing.Message.SentAt,
-			contentMarkdown,
-			contentHTML,
-			assetId,
-			msgID,
-			existing.DiscordUser.HMNUserId,
-		)
-		if err != nil {
-			return oops.New(err, "failed to create snippet from attachment")
-		}
-		_, err = tx.Exec(ctx,
-			`
-			UPDATE handmade_discordmessage
-			SET snippet_created = TRUE
-			WHERE id = $1
-			`,
-			msgID,
-		)
-		if err != nil {
-			return oops.New(err, "failed to mark message as having snippet")
-		}
-	}
-
-	err = tx.Commit(ctx)
-	if err != nil {
-		return oops.New(err, "failed to commit transaction")
-	}
-
-	return nil
-}
-
-/*
-Associates any Discord tags with website tags for projects. Idempotent; will
-clear out any existing project tags and then add new ones.
-
-If no Discord user is linked, or no snippet exists, or whatever, this will do
-nothing and return no error.
-*/
-func UpdateSnippetTagsIfAny(ctx context.Context, dbConn db.ConnOrTx, msg *Message) error {
-	tx, err := dbConn.Begin(ctx)
-	if err != nil {
-		return oops.New(err, "failed to start transaction")
-	}
-	defer tx.Rollback(ctx)
-
-	// Fetch the Discord user; we only process messages for users with linked
-	// Discord accounts
-	u, err := FetchDiscordUser(ctx, tx, msg.Author.ID)
-	if err == db.NotFound {
-		return nil
-	} else if err != nil {
-		return oops.New(err, "failed to look up HMN user information from Discord user")
-	}
-
-	// Fetch the s associated with this Discord message (if any). If the
-	// s has already been edited on the website we'll skip it.
-	s, err := hmndata.FetchSnippetForDiscordMessage(ctx, tx, &u.HMNUser, msg.ID, hmndata.SnippetQuery{})
-	if err == db.NotFound {
-		return nil
-	} else if err != nil {
-		return err
-	}
-
-	// Fetch projects so we know what tags the user can apply to their snippet.
-	projects, err := hmndata.FetchProjects(ctx, tx, &u.HMNUser, hmndata.ProjectsQuery{
-		OwnerIDs: []int{u.HMNUser.ID},
-	})
-	if err != nil {
-		return oops.New(err, "failed to look up user projects")
-	}
-	projectIDs := make([]int, len(projects))
-	for i, p := range projects {
-		projectIDs[i] = p.Project.ID
-	}
-
-	// Delete any existing project tags for this snippet. We don't want to
-	// delete other tags in case in the future we have manual tagging on the
-	// website or whatever, and this would clear those out.
-	_, err = tx.Exec(ctx,
-		`
-		DELETE FROM snippet_tags
-		WHERE
-			snippet_id = $1
-			AND tag_id IN (
-				SELECT tag FROM handmade_project
-			)
-		`,
-		s.Snippet.ID,
-	)
-	if err != nil {
-		return oops.New(err, "failed to delete existing snippet tags")
-	}
-
-	// Try to associate tags in the message with project tags in HMN.
-	// Match only tags for projects in which the current user is a collaborator.
-	messageTags := getDiscordTags(s.Snippet.Description)
-	type tagsRow struct {
-		Tag models.Tag `db:"tags"`
-	}
-	iUserTags, err := db.Query(ctx, tx, tagsRow{},
-		`
-		SELECT $columns
-		FROM
-			tags
-			JOIN handmade_project AS project ON project.tag = tags.id
-			JOIN handmade_user_projects AS user_project ON user_project.project_id = project.id
-		WHERE
-			project.id = ANY ($1)
-		`,
-		projectIDs,
-	)
-	if err != nil {
-		return oops.New(err, "failed to fetch tags for user projects")
-	}
-
-	var tagIDs []int
-	for _, itag := range iUserTags {
-		tag := itag.(*tagsRow).Tag
-		for _, messageTag := range messageTags {
-			if tag.Text == messageTag {
-				tagIDs = append(tagIDs, tag.ID)
-			}
-		}
-	}
-
-	for _, tagID := range tagIDs {
-		_, err = tx.Exec(ctx,
-			`
-			INSERT INTO snippet_tags (snippet_id, tag_id)
-			VALUES ($1, $2)
-			ON CONFLICT DO NOTHING
-			`,
-			s.Snippet.ID,
-			tagID,
-		)
-		if err != nil {
-			return oops.New(err, "failed to add tag to snippet")
-		}
-	}
-
-	err = tx.Commit(ctx)
-	if err != nil {
-		return oops.New(err, "failed to commit transaction")
-	}
-
-	return nil
-}
-
-// NOTE(ben): This is maybe redundant with the regexes we use for markdown. But
-// do we actually want to reuse those, or should we keep them separate?
-var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`)
-
-func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msg *models.DiscordMessage) (*uuid.UUID, *string, error) {
-	// Check attachments
-	attachments, err := db.Query(ctx, tx, models.DiscordMessageAttachment{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessageattachment
-		WHERE message_id = $1
-		`,
-		msg.ID,
-	)
-	if err != nil {
-		return nil, nil, oops.New(err, "failed to fetch message attachments")
-	}
-	for _, iattachment := range attachments {
-		attachment := iattachment.(*models.DiscordMessageAttachment)
-		return &attachment.AssetID, nil, nil
-	}
-
-	// Check embeds
-	embeds, err := db.Query(ctx, tx, models.DiscordMessageEmbed{},
-		`
-		SELECT $columns
-		FROM handmade_discordmessageembed
-		WHERE message_id = $1
-		`,
-		msg.ID,
-	)
-	if err != nil {
-		return nil, nil, oops.New(err, "failed to fetch discord embeds")
-	}
-	for _, iembed := range embeds {
-		embed := iembed.(*models.DiscordMessageEmbed)
-		if embed.VideoID != nil {
-			return embed.VideoID, nil, nil
-		} else if embed.ImageID != nil {
-			return embed.ImageID, nil, nil
-		} else if embed.URL != nil {
-			if RESnippetableUrl.MatchString(*embed.URL) {
-				return nil, embed.URL, nil
-			}
-		}
-	}
-
-	return nil, nil, nil
-}
-
-func messageHasLinks(content string) bool {
-	links := reDiscordMessageLink.FindAllString(content, -1)
-	for _, link := range links {
-		_, err := url.Parse(strings.TrimSpace(link))
-		if err == nil {
-			return true
-		}
-	}
-
-	return false
-}
-
-var REDiscordTag = regexp.MustCompile(`&([a-zA-Z0-9]+(-[a-zA-Z0-9]+)*)`)
-
-func getDiscordTags(content string) []string {
-	matches := REDiscordTag.FindAllStringSubmatch(content, -1)
-	result := make([]string, len(matches))
-	for i, m := range matches {
-		result[i] = strings.ToLower(m[1])
-	}
-	return result
-}

src/hmndata/project_helper.go

@@ -9,7 +9,6 @@ import (
 	"git.handmade.network/hmn/hmn/src/models"
 	"git.handmade.network/hmn/hmn/src/oops"
 	"git.handmade.network/hmn/hmn/src/perf"
-	"git.handmade.network/hmn/hmn/src/templates"
 )
 
 type ProjectTypeQuery int
@@ -52,10 +51,6 @@ func (p *ProjectAndStuff) TagText() string {
 	}
 }
 
-func (p *ProjectAndStuff) LogoURL(theme string) string {
-	return templates.ProjectLogoUrl(&p.Project, p.LogoLightAsset, p.LogoDarkAsset, theme)
-}
-
 func FetchProjects(
 	ctx context.Context,
 	dbConn db.ConnOrTx,
@@ -366,12 +361,16 @@ func FetchMultipleProjectsOwners(
 			userIds = append(userIds, userProject.UserID)
 		}
 	}
-	iusers, err := db.Query(ctx, tx, models.User{},
+	type userQuery struct {
+		User models.User `db:"auth_user"`
+	}
+	iusers, err := db.Query(ctx, tx, userQuery{},
 		`
 		SELECT $columns
 		FROM auth_user
+		LEFT JOIN handmade_asset AS auth_user_avatar ON auth_user_avatar.id = auth_user.avatar_asset_id
 		WHERE
-			id = ANY($1)
+			auth_user.id = ANY($1)
 		`,
 		userIds,
 	)
@@ -398,9 +397,9 @@ func FetchMultipleProjectsOwners(
 		// Get the full user record we fetched
 		var user *models.User
 		for _, iuser := range iusers {
-			u := iuser.(*models.User)
+			u := iuser.(*userQuery).User
 			if u.ID == userProject.UserID {
-				user = u
+				user = &u
 			}
 		}
 		if user == nil {
@@ -450,6 +449,7 @@ func UrlContextForProject(p *models.Project) *hmnurl.UrlContext {
 func SetProjectTag(
 	ctx context.Context,
 	dbConn db.ConnOrTx,
+	currentUser *models.User,
 	projectID int,
 	tagText string,
 ) (*models.Tag, error) {
@@ -459,12 +459,12 @@ func SetProjectTag(
 	}
 	defer tx.Rollback(ctx)
 
-	p, err := FetchProject(ctx, tx, nil, projectID, ProjectsQuery{
+	p, err := FetchProject(ctx, tx, currentUser, projectID, ProjectsQuery{
 		Lifecycles:    models.AllProjectLifecycles,
 		IncludeHidden: true,
 	})
 	if err != nil {
-		return nil, err
+		return nil, oops.New(err, "Failed to fetch project")
 	}
 
 	var resultTag *models.Tag

src/hmndata/snippet_helper.go

@@ -80,6 +80,7 @@ func FetchSnippets(
 		FROM
 			handmade_snippet AS snippet
 			LEFT JOIN auth_user AS owner ON snippet.owner_id = owner.id
+			LEFT JOIN handmade_asset AS owner_avatar ON owner_avatar.id = owner.avatar_asset_id
 			LEFT JOIN handmade_asset AS asset ON snippet.asset_id = asset.id
 			LEFT JOIN handmade_discordmessage AS discord_message ON snippet.discord_message_id = discord_message.id
 		WHERE

src/hmndata/threads_and_posts_helper.go

@@ -78,7 +78,9 @@ func FetchThreads(
 			JOIN handmade_postversion AS first_version ON first_version.id = first_post.current_id
 			JOIN handmade_postversion AS last_version ON last_version.id = last_post.current_id
 			LEFT JOIN auth_user AS first_author ON first_author.id = first_post.author_id
+			LEFT JOIN handmade_asset AS first_author_avatar ON first_author_avatar.id = first_author.avatar_asset_id
 			LEFT JOIN auth_user AS last_author ON last_author.id = last_post.author_id
+			LEFT JOIN handmade_asset AS last_author_avatar ON last_author_avatar.id = last_author.avatar_asset_id
 			LEFT JOIN handmade_threadlastreadinfo AS tlri ON (
 				tlri.thread_id = thread.id
 				AND tlri.user_id = $?
@@ -153,7 +155,9 @@ func FetchThreads(
 		row := *iresult.(*resultRow)
 
 		hasRead := false
-		if row.ThreadLastReadTime != nil && row.ThreadLastReadTime.After(row.LastPost.PostDate) {
+		if currentUser != nil && currentUser.MarkedAllReadAt.After(row.LastPost.PostDate) {
+			hasRead = true
+		} else if row.ThreadLastReadTime != nil && row.ThreadLastReadTime.After(row.LastPost.PostDate) {
 			hasRead = true
 		} else if row.ForumLastReadTime != nil && row.ForumLastReadTime.After(row.LastPost.PostDate) {
 			hasRead = true
@@ -221,6 +225,7 @@ func CountThreads(
 			JOIN handmade_project AS project ON thread.project_id = project.id
 			JOIN handmade_post AS first_post ON first_post.id = thread.first_id
 			LEFT JOIN auth_user AS first_author ON first_author.id = first_post.author_id
+			LEFT JOIN handmade_asset AS first_author_avatar ON first_author_avatar.id = first_author.avatar_asset_id
 		WHERE
 			NOT thread.deleted
 			AND ( -- project has valid lifecycle
@@ -330,7 +335,9 @@ func FetchPosts(
 			JOIN handmade_project AS project ON post.project_id = project.id
 			JOIN handmade_postversion AS ver ON ver.id = post.current_id
 			LEFT JOIN auth_user AS author ON author.id = post.author_id
+			LEFT JOIN handmade_asset AS author_avatar ON author_avatar.id = author.avatar_asset_id
 			LEFT JOIN auth_user AS editor ON ver.editor_id = editor.id
+			LEFT JOIN handmade_asset AS editor_avatar ON editor_avatar.id = editor.avatar_asset_id
 			LEFT JOIN handmade_threadlastreadinfo AS tlri ON (
 				tlri.thread_id = thread.id
 				AND tlri.user_id = $?
@@ -344,6 +351,7 @@ func FetchPosts(
 			-- check fails.
 			LEFT JOIN handmade_post AS reply_post ON reply_post.id = post.reply_id
 			LEFT JOIN auth_user AS reply_author ON reply_post.author_id = reply_author.id
+			LEFT JOIN handmade_asset AS reply_author_avatar ON reply_author_avatar.id = reply_author.avatar_asset_id
 		WHERE
 			NOT thread.deleted
 			AND NOT post.deleted
@@ -407,7 +415,9 @@ func FetchPosts(
 		row := *iresult.(*resultRow)
 
 		hasRead := false
-		if row.ThreadLastReadTime != nil && row.ThreadLastReadTime.After(row.Post.PostDate) {
+		if currentUser != nil && currentUser.MarkedAllReadAt.After(row.Post.PostDate) {
+			hasRead = true
+		} else if row.ThreadLastReadTime != nil && row.ThreadLastReadTime.After(row.Post.PostDate) {
 			hasRead = true
 		} else if row.ForumLastReadTime != nil && row.ForumLastReadTime.After(row.Post.PostDate) {
 			hasRead = true
@@ -543,6 +553,7 @@ func CountPosts(
 			JOIN handmade_thread AS thread ON post.thread_id = thread.id
 			JOIN handmade_project AS project ON post.project_id = project.id
 			LEFT JOIN auth_user AS author ON author.id = post.author_id
+			LEFT JOIN handmade_asset AS author_avatar ON author_avatar.id = author.avatar_asset_id
 		WHERE
 			NOT thread.deleted
 			AND NOT post.deleted

src/hmndata/twitch.go

@@ -0,0 +1,111 @@
+package hmndata
+
+import (
+	"context"
+	"regexp"
+	"strings"
+
+	"git.handmade.network/hmn/hmn/src/db"
+	"git.handmade.network/hmn/hmn/src/models"
+	"git.handmade.network/hmn/hmn/src/oops"
+)
+
+const InvalidUserTwitchID = "INVALID_USER"
+
+type TwitchStreamer struct {
+	TwitchID    string
+	TwitchLogin string
+	UserID      *int
+	ProjectID   *int
+}
+
+var twitchRegex = regexp.MustCompile(`twitch\.tv/(?P<login>[^/]+)$`)
+
+func FetchTwitchStreamers(ctx context.Context, dbConn db.ConnOrTx) ([]TwitchStreamer, error) {
+	type linkResult struct {
+		Link models.Link `db:"link"`
+	}
+	streamers, err := db.Query(ctx, dbConn, linkResult{},
+		`
+		SELECT $columns
+		FROM
+			handmade_links AS link
+			LEFT JOIN auth_user AS link_owner ON link_owner.id = link.user_id
+		WHERE
+			url ~* 'twitch\.tv/([^/]+)$' AND
+			((link.user_id IS NOT NULL AND link_owner.status = $1) OR (link.project_id IS NOT NULL AND
+			(SELECT COUNT(*)
+			FROM
+				handmade_user_projects AS hup
+				JOIN auth_user AS project_owner ON project_owner.id = hup.user_id
+			WHERE
+				hup.project_id = link.project_id AND
+				project_owner.status != $1
+			) = 0))
+		`,
+		models.UserStatusApproved,
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to fetch twitch links")
+	}
+
+	result := make([]TwitchStreamer, 0, len(streamers))
+	for _, s := range streamers {
+		dbStreamer := s.(*linkResult).Link
+
+		streamer := TwitchStreamer{
+			UserID:    dbStreamer.UserID,
+			ProjectID: dbStreamer.ProjectID,
+		}
+
+		match := twitchRegex.FindStringSubmatch(dbStreamer.URL)
+		if match != nil {
+			login := strings.ToLower(match[twitchRegex.SubexpIndex("login")])
+			streamer.TwitchLogin = login
+		}
+		if len(streamer.TwitchLogin) > 0 {
+			duplicate := false
+			for _, r := range result {
+				if r.TwitchLogin == streamer.TwitchLogin {
+					duplicate = true
+					break
+				}
+			}
+			if !duplicate {
+				result = append(result, streamer)
+			}
+		}
+	}
+
+	return result, nil
+}
+
+func FetchTwitchLoginsForUserOrProject(ctx context.Context, dbConn db.ConnOrTx, userId *int, projectId *int) ([]string, error) {
+	links, err := db.Query(ctx, dbConn, models.Link{},
+		`
+		SELECT $columns
+		FROM
+			handmade_links AS link
+		WHERE
+			url ~* 'twitch\.tv/([^/]+)$'
+			AND ((user_id = $1 AND project_id IS NULL) OR (user_id IS NULL AND project_id = $2))
+		ORDER BY url ASC
+		`,
+		userId,
+		projectId,
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to fetch twitch links")
+	}
+	result := make([]string, 0, len(links))
+
+	for _, l := range links {
+		url := l.(*models.Link).URL
+		match := twitchRegex.FindStringSubmatch(url)
+		if match != nil {
+			login := strings.ToLower(match[twitchRegex.SubexpIndex("login")])
+			result = append(result, login)
+		}
+	}
+	return result, nil
+}

src/hmnurl/urls.go

@@ -143,13 +143,6 @@ func BuildAbout() string {
 	return Url("/about", nil)
 }
 
-var RegexCodeOfConduct = regexp.MustCompile("^/code-of-conduct$")
-
-func BuildCodeOfConduct() string {
-	defer CatchPanic()
-	return Url("/code-of-conduct", nil)
-}
-
 var RegexCommunicationGuidelines = regexp.MustCompile("^/communication-guidelines$")
 
 func BuildCommunicationGuidelines() string {
@@ -694,6 +687,18 @@ func BuildAPICheckUsername() string {
 	return Url("/api/check_username", nil)
 }
 
+/*
+* Twitch stuff
+ */
+
+var RegexTwitchEventSubCallback = regexp.MustCompile("^/twitch_eventsub$")
+
+func BuildTwitchEventSubCallback() string {
+	return Url("/twitch_eventsub", nil)
+}
+
+var RegexTwitchDebugPage = regexp.MustCompile("^/twitch_debug$")
+
 /*
 * User assets
  */

src/migration/migrations/2021-12-26T101633Z_UserAvatarAssetId.go

@@ -0,0 +1,47 @@
+package migrations
+
+import (
+	"context"
+	"time"
+
+	"git.handmade.network/hmn/hmn/src/migration/types"
+	"github.com/jackc/pgx/v4"
+)
+
+func init() {
+	registerMigration(UserAvatarAssetId{})
+}
+
+type UserAvatarAssetId struct{}
+
+func (m UserAvatarAssetId) Version() types.MigrationVersion {
+	return types.MigrationVersion(time.Date(2021, 12, 26, 10, 16, 33, 0, time.UTC))
+}
+
+func (m UserAvatarAssetId) Name() string {
+	return "UserAvatarAssetId"
+}
+
+func (m UserAvatarAssetId) Description() string {
+	return "Add avatar_asset_id to users"
+}
+
+func (m UserAvatarAssetId) Up(ctx context.Context, tx pgx.Tx) error {
+	_, err := tx.Exec(ctx,
+		`
+		ALTER TABLE auth_user
+			ADD COLUMN avatar_asset_id UUID REFERENCES handmade_asset (id) ON DELETE SET NULL;
+		`,
+	)
+	return err
+}
+
+func (m UserAvatarAssetId) Down(ctx context.Context, tx pgx.Tx) error {
+	_, err := tx.Exec(ctx,
+		`
+		ALTER TABLE auth_user
+			DROP COLUMN avatar_asset_id;
+		`,
+	)
+	return err
+}

src/migration/migrations/2022-02-13T132653Z_RemoveUserAvatarUrl.go

@@ -0,0 +1,47 @@
+package migrations
+
+import (
+	"context"
+	"time"
+
+	"git.handmade.network/hmn/hmn/src/migration/types"
+	"github.com/jackc/pgx/v4"
+)
+
+func init() {
+	registerMigration(RemoveUserAvatarUrl{})
+}
+
+type RemoveUserAvatarUrl struct{}
+
+func (m RemoveUserAvatarUrl) Version() types.MigrationVersion {
+	return types.MigrationVersion(time.Date(2022, 2, 13, 13, 26, 53, 0, time.UTC))
+}
+
+func (m RemoveUserAvatarUrl) Name() string {
+	return "RemoveUserAvatarUrl"
+}
+
+func (m RemoveUserAvatarUrl) Description() string {
+	return "Remove avatar url field from users as we're using assets now"
+}
+
+func (m RemoveUserAvatarUrl) Up(ctx context.Context, tx pgx.Tx) error {
+	_, err := tx.Exec(ctx,
+		`
+		ALTER TABLE auth_user
+			DROP COLUMN avatar;
+		`,
+	)
+	return err
+}
+
+func (m RemoveUserAvatarUrl) Down(ctx context.Context, tx pgx.Tx) error {
+	_, err := tx.Exec(ctx,
+		`
+		ALTER TABLE auth_user
+			ADD COLUMN avatar character varying(100);
+		`,
+	)
+	return err
+}

src/migration/migrations/2022-02-13T200155Z_RemoveProjectLogoUrls.go

@@ -0,0 +1,49 @@
+package migrations
+
+import (
+	"context"
+	"time"
+
+	"git.handmade.network/hmn/hmn/src/migration/types"
+	"github.com/jackc/pgx/v4"
+)
+
+func init() {
+	registerMigration(RemoveProjectLogoUrls{})
+}
+
+type RemoveProjectLogoUrls struct{}
+
+func (m RemoveProjectLogoUrls) Version() types.MigrationVersion {
+	return types.MigrationVersion(time.Date(2022, 2, 13, 20, 1, 55, 0, time.UTC))
+}
+
+func (m RemoveProjectLogoUrls) Name() string {
+	return "RemoveProjectLogoUrls"
+}
+
+func (m RemoveProjectLogoUrls) Description() string {
+	return "Remove project logo url fields as we're now using assets"
+}
+
+func (m RemoveProjectLogoUrls) Up(ctx context.Context, tx pgx.Tx) error {
+	_, err := tx.Exec(ctx,
+		`
+		ALTER TABLE handmade_project
+			DROP COLUMN logolight,
+			DROP COLUMN logodark;
+		`,
+	)
+	return err
+}
+
+func (m RemoveProjectLogoUrls) Down(ctx context.Context, tx pgx.Tx) error {
+	_, err := tx.Exec(ctx,
+		`
+		ALTER TABLE handmade_project
+			ADD COLUMN logolight character varying(100),
+			ADD COLUMN logodark character varying(100);
+		`,
+	)
+	return err
+}

src/migration/migrations/2022-03-15T012144Z_TwitchTables.go

@@ -0,0 +1,60 @@
+package migrations
+
+import (
+	"context"
+	"time"
+
+	"git.handmade.network/hmn/hmn/src/migration/types"
+	"git.handmade.network/hmn/hmn/src/oops"
+	"github.com/jackc/pgx/v4"
+)
+
+func init() {
+	registerMigration(TwitchTables{})
+}
+
+type TwitchTables struct{}
+
+func (m TwitchTables) Version() types.MigrationVersion {
+	return types.MigrationVersion(time.Date(2022, 3, 15, 1, 21, 44, 0, time.UTC))
+}
+
+func (m TwitchTables) Name() string {
+	return "TwitchTables"
+}
+
+func (m TwitchTables) Description() string {
+	return "Create tables for live twitch streams and twitch ID cache"
+}
+
+func (m TwitchTables) Up(ctx context.Context, tx pgx.Tx) error {
+	_, err := tx.Exec(ctx,
+		`
+		CREATE TABLE twitch_streams (
+			twitch_id VARCHAR(255) NOT NULL,
+			twitch_login VARCHAR(255) NOT NULL,
+			title VARCHAR(255) NOT NULL,
+			started_at TIMESTAMP WITH TIME ZONE
+		);
+		`,
+	)
+
+	if err != nil {
+		return oops.New(err, "failed to create twitch tables")
+	}
+	return err
+}
+
+func (m TwitchTables) Down(ctx context.Context, tx pgx.Tx) error {
+	_, err := tx.Exec(ctx,
+		`
+		DROP TABLE twitch_ids;
+		DROP TABLE twitch_streams;
+		`,
+	)
+
+	if err != nil {
+		return oops.New(err, "failed to create twitch tables")
+	}
+	return err
+}

src/migration/migrations/2022-03-15T063506Z_AddIndexOnTwitchStreams.go

@@ -0,0 +1,45 @@
+package migrations
+
+import (
+	"context"
+	"time"
+
+	"git.handmade.network/hmn/hmn/src/migration/types"
+	"github.com/jackc/pgx/v4"
+)
+
+func init() {
+	registerMigration(AddIndexOnTwitchStreams{})
+}
+
+type AddIndexOnTwitchStreams struct{}
+
+func (m AddIndexOnTwitchStreams) Version() types.MigrationVersion {
+	return types.MigrationVersion(time.Date(2022, 3, 15, 6, 35, 6, 0, time.UTC))
+}
+
+func (m AddIndexOnTwitchStreams) Name() string {
+	return "AddIndexOnTwitchStreams"
+}
+
+func (m AddIndexOnTwitchStreams) Description() string {
+	return "Add unique index on twitch streams"
+}
+
+func (m AddIndexOnTwitchStreams) Up(ctx context.Context, tx pgx.Tx) error {
+	_, err := tx.Exec(ctx,
+		`
+		CREATE UNIQUE INDEX twitch_streams_twitch_id ON twitch_streams (twitch_id);
+		`,
+	)
+	return err
+}
+
+func (m AddIndexOnTwitchStreams) Down(ctx context.Context, tx pgx.Tx) error {
+	_, err := tx.Exec(ctx,
+		`
+		DROP INDEX twitch_streams_twitch_id;
+		`,
+	)
+	return err
+}

src/models/project.go

@@ -72,13 +72,11 @@ type Project struct {
 	Color1 string `db:"color_1"`
 	Color2 string `db:"color_2"`
 
-	LogoLight string `db:"logolight"`
-	LogoDark  string `db:"logodark"`
-
 	Personal              bool      `db:"personal"`
 	Hidden                bool      `db:"hidden"`
 	Featured              bool      `db:"featured"`
 	DateApproved          time.Time `db:"date_approved"`
+	DateCreated           time.Time `db:"date_created"`
 	AllLastUpdated        time.Time `db:"all_last_updated"`
 	ForumLastUpdated      time.Time `db:"forum_last_updated"`
 	BlogLastUpdated       time.Time `db:"blog_last_updated"`

src/models/twitch.go

@@ -0,0 +1,15 @@
+package models
+
+import "time"
+
+type TwitchID struct {
+	ID    string `db:"id"`
+	Login string `db:"login"`
+}
+
+type TwitchStream struct {
+	ID        string    `db:"twitch_id"`
+	Login     string    `db:"twitch_login"`
+	Title     string    `db:"title"`
+	StartedAt time.Time `db:"started_at"`
+}

src/models/user.go

@@ -3,6 +3,8 @@ package models
 import (
 	"reflect"
 	"time"
+
+	"github.com/google/uuid"
 )
 
 var UserType = reflect.TypeOf(User{})
@@ -29,11 +31,12 @@ type User struct {
 	IsStaff bool       `db:"is_staff"`
 	Status  UserStatus `db:"status"`
 
-	Name      string  `db:"name"`
-	Bio       string  `db:"bio"`
-	Blurb     string  `db:"blurb"`
-	Signature string  `db:"signature"`
-	Avatar    *string `db:"avatar"`
+	Name          string     `db:"name"`
+	Bio           string     `db:"bio"`
+	Blurb         string     `db:"blurb"`
+	Signature     string     `db:"signature"`
+	AvatarAssetID *uuid.UUID `db:"avatar_asset_id"`
+	AvatarAsset   *Asset     `db:"avatar"`
 
 	DarkTheme bool   `db:"darktheme"`
 	Timezone  string `db:"timezone"`

src/parsing/renderer.go

@@ -2,6 +2,7 @@ package parsing
 
 import (
 	"io"
+	"regexp"
 
 	"github.com/yuin/goldmark/ast"
 	"github.com/yuin/goldmark/renderer"
@@ -11,6 +12,8 @@ type plaintextRenderer struct{}
 
 var _ renderer.Renderer = plaintextRenderer{}
 
+var backslashRegex = regexp.MustCompile("\\\\(?P<char>[\\\\\\x60!\"#$%&'()*+,-./:;<=>?@\\[\\]^_{|}~])")
+
 func (r plaintextRenderer) Render(w io.Writer, source []byte, n ast.Node) error {
 	return ast.Walk(n, func(n ast.Node, entering bool) (ast.WalkStatus, error) {
 		if !entering {
@@ -20,8 +23,7 @@ func (r plaintextRenderer) Render(w io.Writer, source []byte, n ast.Node) error
 		switch n.Kind() {
 		case ast.KindText:
 			n := n.(*ast.Text)
-
-			_, err := w.Write(n.Text(source))
+			_, err := w.Write(backslashRegex.ReplaceAll(n.Text(source), []byte("$1")))
 			if err != nil {
 				return ast.WalkContinue, err
 			}

src/rawdata/scss/_content.scss

@@ -109,6 +109,10 @@ pre {
     margin-bottom: 0.5em;
   }
 
+  li:not(:last-child) {
+    margin-bottom: 0.2em;
+  }
+
   img {
     max-width: 100%;
   }

src/rawdata/scss/_core.scss

@@ -720,12 +720,6 @@ footer {
 }
 
 .site-search {
-  width: 100%;
-
-  &:focus {
-    width: 200%;
-  }
-
   &[type=text].lite {
     // wow CSS selector priority sucks
 

src/templates/mapping.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"strings"
 
+	"git.handmade.network/hmn/hmn/src/hmndata"
 	"git.handmade.network/hmn/hmn/src/hmnurl"
 	"git.handmade.network/hmn/hmn/src/models"
 )
@@ -63,16 +64,13 @@ func ProjectLogoUrl(p *models.Project, lightAsset *models.Asset, darkAsset *mode
 	if theme == "dark" {
 		if darkAsset != nil {
 			return hmnurl.BuildS3Asset(darkAsset.S3Key)
-		} else {
-			return hmnurl.BuildUserFile(p.LogoDark)
 		}
 	} else {
 		if lightAsset != nil {
 			return hmnurl.BuildS3Asset(lightAsset.S3Key)
-		} else {
-			return hmnurl.BuildUserFile(p.LogoLight)
 		}
 	}
+	return ""
 }
 
 func ProjectToTemplate(
@@ -100,8 +98,10 @@ func ProjectToTemplate(
 	}
 }
 
-func (p *Project) AddLogo(logoUrl string) {
-	p.Logo = logoUrl
+func ProjectAndStuffToTemplate(p *hmndata.ProjectAndStuff, url string, theme string) Project {
+	res := ProjectToTemplate(&p.Project, url)
+	res.Logo = ProjectLogoUrl(&p.Project, p.LogoLightAsset, p.LogoDarkAsset, theme)
+	return res
 }
 
 var ProjectLifecycleValues = map[models.ProjectLifecycle]string{
@@ -161,15 +161,19 @@ func ThreadToTemplate(t *models.Thread) Thread {
 	}
 }
 
+func UserAvatarDefaultUrl(currentTheme string) string {
+	return hmnurl.BuildTheme("empty-avatar.svg", currentTheme, true)
+}
+
 func UserAvatarUrl(u *models.User, currentTheme string) string {
 	if currentTheme == "" {
 		currentTheme = "light"
 	}
 	avatar := ""
-	if u != nil && u.Avatar != nil && len(*u.Avatar) > 0 {
-		avatar = hmnurl.BuildUserFile(*u.Avatar)
+	if u != nil && u.AvatarAsset != nil {
+		avatar = hmnurl.BuildS3Asset(u.AvatarAsset.S3Key)
 	} else {
-		avatar = hmnurl.BuildTheme("empty-avatar.svg", currentTheme, true)
+		avatar = UserAvatarDefaultUrl(currentTheme)
 	}
 	return avatar
 }
@@ -178,7 +182,7 @@ func UserToTemplate(u *models.User, currentTheme string) User {
 	if u == nil {
 		return User{
 			Name:      "Deleted user",
-			AvatarUrl: UserAvatarUrl(u, currentTheme),
+			AvatarUrl: UserAvatarUrl(nil, currentTheme),
 		}
 	}
 
@@ -200,8 +204,8 @@ func UserToTemplate(u *models.User, currentTheme string) User {
 		Blurb:      u.Blurb,
 		Signature:  u.Signature,
 		DateJoined: u.DateJoined,
-		AvatarUrl:  UserAvatarUrl(u, currentTheme),
 		ProfileUrl: hmnurl.BuildUserProfile(u.Username),
+		AvatarUrl:  UserAvatarUrl(u, currentTheme),
 
 		DarkTheme: u.DarkTheme,
 		Timezone:  u.Timezone,

src/templates/src/admin_approval_queue.html

@@ -2,74 +2,105 @@
 
 {{ define "content" }}
 <div class="content-block">
-    {{ range .Posts }}
-        <div class="post background-even pa3">
-            <div class="fl w-100 w-25-l pt3 pa3-l flex tc-l">
-                <div class="fl w-20 mw3 dn-l w3">
-                    <!-- Mobile avatar -->
-                    <div class="aspect-ratio--1x1 contain bg-center" style="background-image:url('{{ .Author.AvatarUrl }}');"></div>
-                </div>
-                <div class="w-100-l pl3 pl0-l flex flex-column items-center-l">
-                    <div>
-                        <a class="username" href="{{ .Author.ProfileUrl }}" target="_blank">{{ .Author.Username }}</a>
-                    </div>
-                    {{ if and .Author.Name (ne .Author.Name .Author.Username) }}
-                        <div class="c--dim f7"> {{ .Author.Name }} </div>
-                    {{ end }}
-                    <!-- Large avatar -->
-                    <div class="dn db-l w-60 pv2">
-                        <div class="aspect-ratio--1x1 contain bg-center" style="background-image:url('{{ .Author.AvatarUrl }}');"></div>
-                    </div>
-                    <div class="i c--dim f7">
-                        {{ if .Author.Blurb }}
-                            {{ .Author.Blurb }} {{/* TODO: Linebreaks? */}}
-                        {{ else if .Author.Bio }}
-                            {{ .Author.Bio }}
-                        {{ end }}
-                    </div>
-                </div>
-            </div>
-            <div class="fl w-100 w-75-l pv3 pa3-l">
-                <div class="w-100 flex-l flex-row-reverse-l">
-                    <div class="inline-flex flex-row-reverse pl3-l pb3 items-center">
-                        <div class="postid">
-                            <a name="{{ .ID }}" href="{{ .Url }}">#{{ .ID }}</a>
-                        </div>
-						<div class="flex pr3">
-							<form method="POST" class="mr4" action="{{ $.SubmitUrl }}">
-								{{ csrftoken $.Session }}
-								<input type="hidden" name="action" value="{{ $.ApprovalAction }}" />
-								<input type="hidden" name="user_id" value="{{ .Author.ID }}" />
-								<input type="submit" value="Approve User" />
-							</form>
-							<form method="POST" action="{{ $.SubmitUrl }}">
-								{{ csrftoken $.Session }}
-								<input type="hidden" name="action" value="{{ $.SpammerAction }}" />
-								<input type="hidden" name="user_id" value="{{ .Author.ID }}" />
-								<input type="submit" value="Mark as spammer" />
-							</form>
-						</div>
-                    </div>
-                    <div class="w-100 pb3">
-                        <div class="b" role="heading" aria-level="2">{{ .Title }}</div>
-                        {{ timehtml (relativedate .PostDate) .PostDate }}
-						{{ if and $.User.IsStaff .IP }}
-							<span>[{{ .IP }}]</span>
+	{{ range .UnapprovedUsers }}
+		<div class="flex flex-row bg--card mb3 pa2">
+			<div class="
+				sidebar flex-shrink-0 
+				flex flex-column items-stretch-l
+				overflow-hidden
+			" style="width: 200px;">
+				<a class="db" href="{{ .User.ProfileUrl }}">{{ .User.Username }}</a>
+				<div>{{ .User.Name }}</div>
+				<div class="w-100 flex-shrink-0 flex justify-center">
+					<img class="br3" alt="{{ .User.Name }}'s Avatar" src="{{ .User.AvatarUrl }}">
+				</div>
+				<div class="mt3 mt0-ns mt3-l ml3-ns ml0-l flex flex-column items-start overflow-hidden">
+					{{ with or .User.Bio .User.Blurb }}
+						<div class="mb3">{{ . }}</div>
+					{{ end }}
+					<div class="w-100 w-auto-ns w-100-l">
+						{{ if .User.Email }}
+							<div class="pair flex">
+								<div class="key flex-auto flex-shrink-0 mr2">Email</div>
+								<div class="value projectlink truncate">{{ .User.Email }}</div>
+							</div>
 						{{ end }}
-                    </div>
-                </div>
-                <div class="post-content overflow-x-auto">
-                    {{ .Content }}
-                </div>
-                {{/* {% if post.author.signature|length %}
-                    <div class="signature"><hr />
-                        {{ post.author.signature|bbdecode|safe }}
-                    </div>
-                {% endif %} */}}
-            </div>
-            <div class="cb"></div>
-        </div>
-    {{ end }}
+						{{ range .UserLinks }}
+							<div class="pair flex">
+								<div class="key flex-auto flex-shrink-0 mr2">{{ .Name }}</div>
+								<div class="value projectlink truncate"><a class="external" href="{{ .Url }}" ><span class="icon-{{ .Icon }}"></span> {{ .LinkText }}</a></div>
+							</div>
+						{{ end }}
+
+						<div>{{ absoluteshortdate .User.DateJoined }}</div>
+					</div>
+				</div>
+				<div>
+					<form method="POST" class="mb2" action="{{ $.SubmitUrl }}">
+						{{ csrftoken $.Session }}
+						<input type="hidden" name="action" value="{{ $.ApprovalAction }}" />
+						<input type="hidden" name="user_id" value="{{ .User.ID }}" />
+						<input type="submit" class="w-100" value="Approve User" />
+					</form>
+					<form method="POST" action="{{ $.SubmitUrl }}">
+						{{ csrftoken $.Session }}
+						<input type="hidden" name="action" value="{{ $.SpammerAction }}" />
+						<input type="hidden" name="user_id" value="{{ .User.ID }}" />
+						<input type="submit" class="w-100" value="Mark as spammer" />
+					</form>
+				</div>
+			</div>
+			<div class="flex-grow-1 flex flex-column ml3">
+				{{ range .ProjectsWithLinks }}
+					<div class="project-card flex br2 overflow-hidden items-center relative mv3 w-100">
+						{{ with .Project.Logo }}
+							<div class="image-container flex-shrink-0">
+								<div class="image bg-center cover" style="background-image:url({{ . }})"></div>
+							</div>
+						{{ end }}
+						<div class="details pa3 flex-grow-1">
+							<h3 class="mb1"><a href="{{ .Project.Url }}">{{ .Project.Name }}</a></h3>
+							<div class="blurb">{{ .Project.Blurb }}</div>
+							<div class="badges mt2">
+								{{ if .Project.LifecycleString }}
+									<span class="badge {{ .Project.LifecycleBadgeClass }}">{{ .Project.LifecycleString }}</span>
+								{{ end }}
+							</div>
+							{{ range .Links }}
+								<div class="pair flex">
+									<div class="key flex-auto flex-shrink-0 mr2">{{ .Name }}</div>
+									<div class="value projectlink truncate"><a class="external" href="{{ .Url }}" ><span class="icon-{{ .Icon }}"></span> {{ .LinkText }}</a></div>
+								</div>
+							{{ end }}
+						</div>
+					</div>
+				{{ end }}
+				{{ range .Posts }}
+					<div class="post background-even pa3">
+						<div class="fl w-100 pv3 pa3-l">
+							<div class="w-100 flex-l flex-row-reverse-l">
+								<div class="inline-flex flex-row-reverse pl3-l pb3 items-center">
+									<div class="postid">
+										<a name="{{ .ID }}" href="{{ .Url }}">#{{ .ID }}</a>
+									</div>
+								</div>
+								<div class="w-100 pb3">
+									<div class="b" role="heading" aria-level="2">{{ .Title }}</div>
+									{{ timehtml (relativedate .PostDate) .PostDate }}
+									{{ if and $.User.IsStaff .IP }}
+										<span>[{{ .IP }}]</span>
+									{{ end }}
+								</div>
+							</div>
+							<div class="post-content overflow-x-auto">
+								{{ .Content }}
+							</div>
+						</div>
+					</div>
+				{{ end }}
+			</div>
+		</div>
+	{{ end }}
 </div>
 {{ end }}
 

src/templates/src/code_of_conduct.html

@@ -1,49 +0,0 @@
-{{ template "base.html" . }}
-
-{{ define "content" }}
-<div class="center-layout mw7 mv3 ph3 ph0-ns post-content">
-	<h1>Handmade Code of Conduct v1.0</h1>
-	<p>The Handmade community is an international community of programmers, designers, artists, musicians, mathematicians, and other creatives dedicated to building and improving high quality software.</p>
-	<p>Outlined herein are the guidelines we pledge to uphold to maintain a healthy community, stay true to the ideas first explored in our <a href="/manifesto">Manifesto</a> and refined by valuable feedback, and ensure we mature into a functional, inclusive, and innovative network.</p>
-	<h2>COMMUNITY</h2>
-	<h3>OPEN-MINDED</h3>
-	<p>The Handmade community strives to be unprejudiced—we welcome unusual ideas, encourage different points of view, and consider their effectiveness in reality.</p>
-	<p>The Handmade community does not waste time and alienate others by engaging in flame-wars, drawing out pointless arguments, singling out developers, or making attacks on the hard work of others; instead, we learn from exercising the methods that we believe to be reasonable, and offer suggestions on how to improve ourselves and others.</p>
-	<h3>DOWN-TO-EARTH</h3>
-	<p>Handmade favors the languages and tools that first serve the users of our software by not wasting their time and resources, and second those who develop it by making meaningful abstractions oriented to the task at hand.</p>
-	<p>We try to minimize the emergent complexity of tightly coupled systems, avoid the over-complication by refusing to blindly apply accepted strategies without clear understanding of their costs, and we prefer that which is simple to that which is easy. When uncertain, we make measurements and follow the data.</p>
-	<h3>DIVERSE</h3>
-	<p>Handmade encourages participation by everyone. We will do everything in our power to ensure everyone feels accepted and respected in their interactions with our community.</p>
-	<p>If someone has been harmed or offended, it is our responsibility to listen carefully and respectfully, and do our best to right the wrong. We accept that jokes and trolling can be taken too far and are not valid excuses for the alienation of any person.</p>
-	<p>Although this list cannot be exhaustive, we explicitly honour diversity in age, culture, ethnicity, family background, gender identity or expression, language, national origin, neurotype, phenotype, political beliefs, profession, race, religion, sexual orientation, socio-economic status, membership in other communities, and technical ability.</p>
-	<p>Some of the ideas and wording for this statement were based on diversity statements from <a class="external" href="https://www.ubuntu.com/about/about-ubuntu/diversity">Ubuntu Diversity Page</a>, which is in turn based on the Python community and Dreamwidth Studios (CC-BY-SA 3.0).</p>
-	<p>To see how we encourage participation by everyone, see our <a href="/communication-guidelines">Handmade Guide to Community Interaction</a>.</p>
-	<h2>INDIVIDUAL</h2>
-	<h3>INQUISITIVE</h3>
-	<p>The Handmade developer strives to understand their creations on a technical level. They will take the time on their personal software projects to meet their goals to the best of their abilities, without taking shortcuts that diminish the value of their work.</p>
-	<p>They include technical understanding and user experience as important metrics for the quality of their creations, and minimize trade-offs which impact these negatively.</p>
-	<p>They will ask questions exhaustively, and always re-evaluate what they consider to be a “good solution” in light of new evidence. Their curiosity is one of the primary driving forces in their work.</p>
-	<h3>CONSIDERATE</h3>
-	<p>The developer practices empathy; they try to understand how those they interact with and those they create for see the world in order to better understand each other.</p>
-	<p>They realize that their actions in both professional and community contexts can have consequences for other people that they may not immediately understand, and will do their best to correct themselves when they make a mistake.</p>
-	<p>They will, to the best of their abilities, keep those who enjoy their ongoing personal software projects up to date on the state of development and be honest about their progress and achievement of their stated goals.</p>
-	<h3>WILLING TO SHARE</h3>
-	<p>Handmade community members share their knowledge and expertise unflinchingly. They will not hesitate to lend a hand if the opportunity to improve the software development space arises.</p>
-	<p>They realize that their fellow community member not knowing something is an opportunity rather than a character flaw. They also realize that their knowledge and experience, however deep and long-collected, is not absolute, and accept that the experiences of others may differ.</p>
-	<h2>LEADERS</h2>
-	<h3>TIGHT-KNIT</h3>
-	<p>The Handmade Dev Team acts as one unit.</p>
-	<p>The staff are open with each other, make decisions unanimously, and perform their roles admirably for the benefit of the community.</p>
-	<h3>RECEPTIVE</h3>
-	<p>The leaders are receptive to the state of the community.</p>
-	<p>They will listen to everyone's concerns and make careful, considered judgment calls to move forward or solve a problem.</p>
-	<p>They will never place personal benefit over the well-being of the community, and only act against the community's immediate short-term interests if it’s for the long-term benefit of everyone involved.</p>
-	<h3>SERIOUS</h3>
-	<p>The leaders are serious about their roles.</p>
-	<p>They will uphold the ideas explored in the <a href="/manifesto">Manifesto</a>, setting the prime example of Handmade values in their development, behavior, and character.</p>
-	<p>They agree to enforce the code of conduct as written and accepted by the community, understanding that there are times when enforcement involves consequences for those shown to be in repeated and flagrant violation thereof.</p>
-	<p>They also agree to ensure all future revisions to such are accepted by and in the best interest of the community.</p>
-	<p>This code of conduct is released under a <a class="external" href="https://creativecommons.org/licenses/by-sa/4.0">Creative Commons Attribution-ShareAlike 4.0 International License</a>.</p>
-	<p class="c--dim i">Written by Jeroen van Rijn</p>
-</div>
-{{ end }}

src/templates/src/communication_guidelines.html

@@ -2,25 +2,25 @@
 
 {{ define "content" }}
 <div class="center-layout mw7 mv3 ph3 ph0-ns post-content">
-	<h1>Handmade Guide to Community Interaction v1.0</h1>
+	<h1>Handmade Guide to Community Interaction</h1>
 	<h2>Our Philosophy</h2>
-	<p>The Handmade community strives to create an environment conducive to innovation, education, and constructive discussion. To that end, we expect members of the site to respect the following set of principles to maintain civil discourse and create an inclusive environment.</p>
+	<p>The Handmade community strives to create an environment conducive to innovation, education, and constructive discussion. To that end, we expect members of the community to respect the following set of principles to maintain civil discourse and create an inclusive environment.</p>
 	<h2>Discourse</h2>
 	<ul>
 		<li>The community is mostly business, with a bit of fun. Discussion should primarily be about software development, and in particular about the relevant project or topic at hand.</li>
 		<li>Support arguments with evidence and rational discussion. Don't resort to ad hominem attacks or personality judgments.</li>
 		<li>Differing opinions are valuable and should be respected. Not everyone sees eye-to-eye on every matter, but we're all trying to write useful software.</li>
-		<li>Language has meaning, and can be used in destructive ways. Be aware that what you say may not seem injurious to you, but might make someone else's experience on the site tangibly worse. If you are asked to reconsider your conduct, please exercise some introspection. We want everyone to feel welcome here. This includes people both inside and outside the Handmade community.</li>
+		<li>Language has meaning, and can be used in destructive ways. Be aware that what you say may not seem injurious to you, but might make someone else's experience tangibly worse. If you are asked to reconsider your conduct, please exercise some introspection. We want everyone to feel welcome here. This includes people both inside and outside the Handmade community.</li>
 		<li>Be aware of differences in English comprehension and culture. What you say may be misinterpreted by a more fluent or less fluent speaker, or someone from a different culture. Misunderstandings should be resolved with the help of a third party, preferably a staff member.</li>
 	</ul>
 	<h2>Inclusiveness</h2>
 	<p>The Handmade community will encourage participation of underrepresented groups. In order to promote diversity among the community, a Handmade community member shall not:</p>
 	<ul>
-		<li>Sexualize or objectify other members, even if you think it’s a compliment.</li>
+		<li>Sexualize or objectify others, even if you think it's a compliment.</li>
 		<li>Use pejoratives or excessively rely on foul language, especially if used to attack an individual.</li>
 		<li>Denigrate, belittle, defame, or speak ill of any individual.</li>
 		<li>Leave unsubstantiated or unjust criticism on someone's work, or disparage a person while critiquing their work.</li>
-		<li>Dismiss other people’s contributions, opinions, or concerns based on their personal attributes, especially those described in the Diversity section of the code of conduct.</li>
+		<li>Dismiss other people's contributions, opinions, or concerns based on their personal attributes.</li>
 		<li>Be rude. Disagreement is not an excuse to be flippant or inconsiderate.</li>
 		<li>Antagonize others, particularly if a disagreement has occurred. Instead, a peaceful resolution should be sought, with an impartial third party if necessary.</li>
 		<li>Assume a motive for uncharacteristic behavior. Everyone has a bad day.</li>
@@ -39,6 +39,6 @@
 	<p>It is up to the discretion of the moderators to determine when these guidelines are being met, and they are permitted to act accordingly.</p>
 	<p>If you feel an action has been made against you unreasonably, please contact the <a href="/contact">Handmade Network Team</a> privately with your grievance and we will review the decision.</p>
 	<p>This code of conduct is released under a <a class="external" href="https://creativecommons.org/licenses/by-sa/4.0">Creative Commons Attribution-ShareAlike 4.0 International License</a>.</p>
-	<p class="c--dim i">Written by Jeroen van Rijn</p>
+	<p class="c--dim i">Last updated by Ben Visness. Original by Abner Coimbre.</p>
 </div>
 {{ end }}

src/templates/src/include/footer.html

@@ -19,9 +19,6 @@
         <li class="{{ $footerClasses }}">
             <a href="{{ .Footer.ProjectIndexUrl }}">Projects</a>
         </li>
-        <li class="{{ $footerClasses }}">
-            <a href="{{ .Footer.CodeOfConductUrl }}">Code of Conduct</a>
-        </li>
         <li class="{{ $footerClasses }}">
             <a href="{{ .Footer.CommunicationGuidelinesUrl }}">Communication Guidelines</a>
         </li>
@@ -29,4 +26,9 @@
             <a href="{{ .Footer.ContactUrl }}">Contact</a>
         </li>
     </ul>
+	<form onsubmit="this.querySelector('input[name=q]').value = this.querySelector('#searchstring').value + ' site:handmade.network';" class="ma0 mt3 bg--card pa1 br2 dib" method="GET" action="{{ .Footer.SearchActionUrl }}" target="_blank">
+		<input type="hidden" name="q" />
+		<input class="site-search bn lite pa2 fira" type="text" id="searchstring" value="" placeholder="Search with DuckDuckGo" size="18" />
+		<input id="search_button_homepage" type="submit" value="Go"/>
+	</form>
 </footer>

src/templates/src/include/timeline_item.html

@@ -56,6 +56,10 @@
 				<audio src="{{ .AssetUrl }}" controls>
 			{{ else if eq .Type mediaembed }}
 				{{ .EmbedHTML }}
+			{{ else }}
+				<div class="project-card br2 pv1 ph2">
+					<a href="{{ .AssetUrl }}" target="_blank">{{ .Filename }} ({{ filesize .FileSize }})</a>
+				</div>
 			{{ end }}
 		</div>
 	{{ end }}

src/templates/src/landing.html

@@ -5,6 +5,7 @@
 {{ end }}
 
 {{ define "content" }}
+{{/*
 <div class="mb3 ph3 ph0-ns">
     <style>
         #hms-banner {
@@ -99,6 +100,7 @@
         </div>
     </a>
 </div>
+*/}}
 
 {{ if not .User }}
 	<div class="mb3 ph3 ph0-ns">
@@ -119,6 +121,7 @@
 			#welcome-logo svg {
 				height: 100%;
 				fill: currentColor;
+				opacity: 0.9;
 			}
 			
 			#welcome-content a {

src/templates/src/manifesto.html

@@ -2,22 +2,21 @@
 
 {{ define "content" }}
 <div class="center-layout mw7 mv3 ph3 ph0-ns post-content">
-	<p>Modern computer hardware is amazing. Manufacturers have orchestrated billions of pieces of silicon into terrifyingly complex and efficient structures that sweep electrons through innumerable tangled paths, branchings, and reunions with the sole purpose of performing computations at more than a billion times per second. This awe-inspiring piece of computational wizardry has at its disposal multiple billions of uniquely addressible silicon plates where it can store the results of millions of computations in an array of several vanishingly small chips. All of this hardware, though each component often sits no further than 7 or 8 centimeters away from the others, cycles so fast that the speed of light, a physical law of the universe, limits the rate at which they communicate with each other.</p>
-	<h2>So why is software still slow?</h2>
-	<p>Why does it take your operating system 10 seconds, 30 seconds, a minute to boot up? Why does your word processor freeze when you save a document on the cloud? Why does your web browser take 3, 4, 10 seconds to load a web page? Why does your phone struggle to keep more than a few apps open at a time? And why does each update somehow make the problem worse?</p>
-	<h2>We made it slow.</h2>
-	<p>Not necessarily you, not necessarily me, not necessarily any single person in particular. But we, the software development community, made it slow by ignoring the fundamental reality of our occupation. We write code, code that runs on computers. Real computers, with central processing units and random access memory and hard disk drives and display buffers. Real computers, with integer and bitwise math and floating point units and L2 caches, with threads and cores and a tenuous little network connection to a million billion other computers. Real computers not built for ease of human understanding but for blindingly, incomprehensibly fast speed.</p>
-	<h2>A lot of us have forgotten that.</h2>
-	<p>In our haste to get our products, our projects, the works of our hands and minds, to as many people as possible, we take shortcuts. We make assumptions. We generalize, and abstract, and assume that just because these problems have been solved before that they never need to be solved again. We build abstraction layers, then forget we built them and build more on top.</p>
-	<p>And it's true that many of us think we do not have the time, the money, the mental bandwidth to always consider these things in detail. The deadline is approaching or the rent is due or we have taxes to fill out and a manager on our back and someone asking us why we always spend so much time at the office, and we just have to stick the library or virtual machine or garbage collector in there to cover up the places we can't think through right now.</p>
-	<p>Others of us were never taught to think about the computer itself. We learned about objects and classes and templates and how to make our code clean and pretty. We learned how to write code to make the client or the manager or the teacher happy, but made the processor churn. And because we did, that amazing speed we'd been granted was wasted, by us, in a death by a thousand abstraction layers.</p>
-	<h2>But some of us aren't satisfied with that.</h2>
-	<p>Some of us take a few extra steps into the covered territory, the wheels sitting, motionless, in a pile behind us, examine their designs and decide there is a better way. The more experienced among us remember how software used to be, the potential that we know exists for computer programs to be useful, general, and efficient. Others of us got fed up with the tools we were expected to use without complaint, but which failed us time and time again. Some of us are just curious and don't know what's good for us. Don't trust what we've been told is good for us.</p>
-	<p>We sat down and looked at our hardware, and examined our data, and thought about how to use the one to transform the other. We tinkered, and measured, and read, and compared, and wrote, and refined, and modified, and measured again, over and over, until we found we had built the same thing, but 10 times faster and incomparably more useful to the people we designed it for. And we had built it by hand.</p>
-	<p>That is what Handmade means. It's not a technique or a language or a management strategy, it isn't a formula or a library or an abstraction. It's an idea. The idea that we can build software that works with the computer, not against it. The idea that sometimes an individual programmer can be more productive than a large team, that a small group can do more than an army of software engineers and *do it better*. The idea that programming is about transforming data and we wield the code, the tool we use to bend that data to our will.</p>
-	<p>It doesn't require a degree, or a dissertation, or a decade of experience. You don't need an expensive computer or a certificate or even prior knowledge. All you need is an open mind and a sense of curiosity. We'll help you with the rest.</p>
+	<h2>Computers are amazing.</h2>
+	<p>Computers have changed our lives for the better. They allow us to learn, connect with each other, and express ourselves in amazing new ways. And every year computers get more powerful, less expensive, and more accessible - computers today can do things we hardly dreamed of twenty years ago.</p>
+	<h2>So why is software so terrible?</h2>
+	<p>Why do web pages take ten seconds to load? Why do apps mess up scrolling? Why does your phone battery still die so quickly? And why does each update somehow make the problem worse?</p>
+	<p>And why do we all use huge frameworks that no one understands? Why do our projects take several minutes to compile? Why do we have to restart our language servers every twenty minutes? And why does everyone think this is fine?</p>
+	<h2>We made it terrible.</h2>
+	<p>Not necessarily you or me, not necessarily anyone in particular. But we, the software development community, made it terrible through our thoughtless behavior. We ignored the hardware. We glued together libraries so we didn't have to learn. We built layers on top of layers, until no one knew how anything worked.</p>
+	<p>But worst of all: we put our own desires above the user's.</p>
+	<p>You may have learned that programming is about classes, monads, or type systems. You may have been taught to keep your code clean and pretty, abstract and future-proof. None of that matters when the end result is garbage.</p>
+	<h2>But there is another way.</h2>
+	<p>Some of us aren't satisfied with the current state of software. We think that wheels need to be reinvented. We like looking under the hood, understanding what others take for granted. We remember how software used to be, and know how much potential there is to make it better. We fight against the status quo, because we know how things <em>could</em> be.</p>
+	<p>This is what Handmade means. It's not a technique or a language or a management strategy. It's not a library or a framework or a paradigm. It's an idea. The idea that we can build software that works with the computer, not against it. The idea that the user matters more than the programmer. The idea that sometimes a small group can do more than an army of software engineers, and <em>do it better</em>.</p>
+	<p>You don't need a degree, a dissertation, or a decade of experience. You don't need an expensive computer or a certificate. All you need is an open mind and a sense of curiosity. We'll help you with the rest.</p>
 	<h2>Will you join us?</h2>
 	<p>Will you build your software by hand?</p>
-	<p class="c--dim i">Written by Andrew Chronister</p>
+	<p class="c--dim i">Written by Ben Visness and the Handmade community. Original by Andrew Chronister.</p>
 </div>
 {{ end }}

src/templates/src/project_edit.html

@@ -137,6 +137,19 @@
 						<div class="c--dim f7">Plaintext only. No links or markdown.</div>
 					</div>
 				</div>
+				<div class="edit-form-row">
+					<div class="pt-input-ns">Project links:</div>
+					<div>
+						<textarea class="links" name="links" id="links" maxlength="2048" data-max-chars="2048">
+							{{- .ProjectSettings.LinksText -}}
+						</textarea>
+						<div class="c--dim f7">
+							<div>Relevant links to put on the project page.</div>
+							<div>Format: url [Title] (e.g. <code>http://example.com/ Example Site</code>)</div>
+							<div>(1 per line, 10 max)</div>
+						</div>
+					</div>
+				</div>
 				<div class="edit-form-row">
 					<div class="pt-input-ns">Full description:</div>
 					<div>

src/templates/src/snippet.html

@@ -21,6 +21,10 @@
                     <div class="mb3 aspect-ratio aspect-ratio--16x9">
                         {{ .EmbedHTML }}
                     </div>
+				{{ else }}
+					<div class="project-card br2 pv1 ph2">
+						<a href="{{ .AssetUrl }}" target="_blank">{{ .Filename }} ({{ filesize .FileSize }})</a>
+					</div>
                 {{ end }}
             {{ end }}
         </div>

src/templates/src/user_settings.html

@@ -2,6 +2,7 @@
 
 {{ define "extrahead" }}
 	<script src="{{ static "js/tabs.js" }}"></script>
+	<script src="{{ static "js/image_selector.js" }}"></script>
 {{ end }}
 
 {{ define "content" }}
@@ -41,12 +42,19 @@
         </div>
         <div class="edit-form-row">
             <div>Avatar:</div>
-            <div>
-                <input type="file" name="avatar" id="avatar">
-                <div class="avatar-preview"><img src="{{ .User.AvatarUrl }}" width="200px">
-                </div>
+            <div class="user_avatar">
+				{{ template "image_selector.html" imageselectordata "avatar" .User.AvatarUrl false }}
                 <div class="c--dim f7">(Allowed image types: PNG, JPEG and GIF. Avatars may weigh up to 1MB and will be resized if larger than 400x400 pixels)</div>
             </div>
+			<script>
+				let avatarMaxFileSize = {{ .AvatarMaxFileSize }};
+				let avatarSelector = new ImageSelector(
+					document.querySelector("#user_form"),
+					avatarMaxFileSize,
+					document.querySelector(".user_avatar"),
+					{{ .DefaultAvatarUrl }}
+				);
+			</script>
         </div>
         <div class="edit-form-row">
             <div class="pt-input-ns">Short bio:</div>
@@ -198,4 +206,4 @@
         }
     </script>
 </form>
-{{ end }}
+{{ end }}

src/templates/svg/hmn_circuit.svg

@@ -1,102 +1,83 @@
 <?xml version="1.0" encoding="UTF-8" standalone="no"?>
 <!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
-<svg viewBox="0 0 3706 1082" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" xmlns:serif="http://www.serif.com/" style="fill-rule:evenodd;clip-rule:evenodd;stroke-linejoin:round;stroke-miterlimit:2;">
-    <g id="Banner-Logo" serif:id="Banner Logo" transform="matrix(48.7805,0,0,48.7805,-9370.23,-9931.81)">
-        <g transform="matrix(1,0,0,1,-0.497771,-0.118656)">
-            <g transform="matrix(1,0,0,-1,-0.808418,430.006)">
-                <path d="M193.396,215.074L201.366,215.074" style="fill:none;"/>
-            </g>
-            <g transform="matrix(1,0,0,-1,-0.808418,430.006)">
-                <path d="M194.555,216.234C193.915,216.234 193.396,215.714 193.396,215.074C193.396,214.434 193.915,213.915 194.555,213.915C195.195,213.915 195.715,214.434 195.715,215.074C195.715,215.714 195.195,216.234 194.555,216.234ZM194.555,215.461C194.768,215.461 194.942,215.288 194.942,215.074C194.942,214.861 194.768,214.688 194.555,214.688C194.342,214.688 194.169,214.861 194.169,215.074C194.169,215.288 194.342,215.461 194.555,215.461Z" style=""/>
-            </g>
-            <g transform="matrix(1,0,0,-1,-0.808418,430.006)">
-                <path d="M195.328,214.688L201.366,214.688C201.579,214.688 201.753,214.861 201.753,215.074C201.753,215.288 201.579,215.461 201.366,215.461L195.328,215.461C195.115,215.461 194.942,215.288 194.942,215.074C194.942,214.861 195.115,214.688 195.328,214.688Z" style=""/>
-            </g>
-            <g transform="matrix(-1,0,0,-1,461.953,430.006)">
-                <path d="M193.396,215.074L201.366,215.074" style="fill:none;"/>
-            </g>
-            <g transform="matrix(-1,0,0,-1,461.953,430.006)">
-                <path d="M194.555,213.915C193.915,213.915 193.396,214.434 193.396,215.074C193.396,215.714 193.915,216.234 194.555,216.234C195.195,216.234 195.715,215.714 195.715,215.074C195.715,214.434 195.195,213.915 194.555,213.915ZM194.555,214.688C194.768,214.688 194.942,214.861 194.942,215.074C194.942,215.288 194.768,215.461 194.555,215.461C194.342,215.461 194.169,215.288 194.169,215.074C194.169,214.861 194.342,214.688 194.555,214.688Z" style=""/>
-            </g>
-            <g transform="matrix(-1,0,0,-1,461.953,430.006)">
-                <path d="M195.328,215.461L201.366,215.461C201.579,215.461 201.753,215.288 201.753,215.074C201.753,214.861 201.579,214.688 201.366,214.688L195.328,214.688C195.115,214.688 194.942,214.861 194.942,215.074C194.942,215.288 195.115,215.461 195.328,215.461Z" style=""/>
-            </g>
-            <g transform="matrix(1,0,0,-1,0.0431238,430.006)">
-                <path d="M200.515,215.074L201.656,213.933" style="fill:none;"/>
-            </g>
-            <g transform="matrix(1,0,0,-1,0.0431238,430.006)">
-                <path d="M200.241,214.801L201.383,213.66L201.929,214.206L200.788,215.348L200.241,214.801Z" style=""/>
-            </g>
-            <g transform="matrix(1,0,0,-1,58.9314,428.865)">
-                <path d="M200.515,215.074L201.656,213.933" style="fill:none;"/>
-            </g>
-            <g transform="matrix(1,0,0,-1,58.9314,428.865)">
-                <path d="M200.241,214.801L201.383,213.66L201.929,214.206L200.788,215.348L200.241,214.801Z" style=""/>
-            </g>
-            <g transform="matrix(1,0,0,-1,1.04312,431.006)">
-                <path d="M205.833,209.755L207.639,207.95" style="fill:none;"/>
-            </g>
-            <g transform="matrix(1,0,0,-1,1.04312,431.006)">
-                <path d="M205.56,209.482L207.365,207.677L207.912,208.223L206.107,210.029L205.56,209.482Z" style=""/>
-            </g>
-            <g transform="matrix(1,0,0,-1,1.04312,431.006)">
-                <path d="M229.737,206.754C229.436,207.055 228.946,207.055 228.644,206.754C228.342,206.452 228.342,205.962 228.644,205.66C228.946,205.359 229.436,205.359 229.737,205.66C230.039,205.962 230.039,206.452 229.737,206.754ZM229.191,206.207C229.191,206.207 229.191,206.207 229.191,206.207Z" style=""/>
-            </g>
-            <g transform="matrix(1,0,0,-1,1.04312,431.006)">
-                <path d="M207.639,207.564L227.288,207.564C227.288,207.564 228.644,206.207 228.644,206.207C228.795,206.056 229.04,206.056 229.191,206.207C229.342,206.358 229.342,206.603 229.191,206.754C228.534,207.411 227.721,208.223 227.721,208.223C227.648,208.296 227.55,208.337 227.448,208.337L207.639,208.337C207.425,208.337 207.252,208.163 207.252,207.95C207.252,207.737 207.425,207.564 207.639,207.564Z" style=""/>
-            </g>
-            <g transform="matrix(1,0,0,-1,1.04312,431.006)">
-                <path d="M230.737,207.564L251.13,207.564C251.233,207.564 251.331,207.604 251.403,207.677L253.209,209.482L252.662,210.029L250.97,208.337C250.97,208.337 230.737,208.337 230.737,208.337L230.737,207.564Z" style=""/>
-            </g>
-            <g transform="matrix(-1,0,0,1,459.812,-1.40971)">
-                <path d="M205.833,209.755L207.639,207.95" style="fill:none;"/>
-            </g>
-            <g transform="matrix(-1,0,0,1,459.812,-1.40971)">
-                <path d="M205.56,209.482L207.365,207.677L207.912,208.223L206.107,210.029L205.56,209.482Z" style=""/>
-            </g>
-            <g transform="matrix(-1,0,0,1,459.812,-1.40971)">
-                <path d="M229.737,206.754C229.436,207.055 228.946,207.055 228.644,206.754C228.342,206.452 228.342,205.962 228.644,205.66C228.946,205.359 229.436,205.359 229.737,205.66C230.039,205.962 230.039,206.452 229.737,206.754Z" style=""/>
-            </g>
-            <g transform="matrix(-1,0,0,1,459.812,-1.40971)">
-                <path d="M207.639,207.564L227.288,207.564C227.288,207.564 228.644,206.207 228.644,206.207C228.795,206.056 229.04,206.056 229.191,206.207C229.342,206.358 229.342,206.603 229.191,206.754C228.534,207.411 227.721,208.223 227.721,208.223C227.648,208.296 227.55,208.337 227.448,208.337L207.639,208.337C207.425,208.337 207.252,208.163 207.252,207.95C207.252,207.737 207.425,207.564 207.639,207.564Z" style=""/>
-            </g>
-            <g transform="matrix(-1,0,0,1,459.812,-1.40971)">
-                <path d="M230.737,207.564L251.13,207.564C251.233,207.564 251.331,207.604 251.403,207.677L253.209,209.482L252.662,210.029L250.97,208.337C250.97,208.337 230.737,208.337 230.737,208.337L230.737,207.564Z" style=""/>
-            </g>
-            <g transform="matrix(-1,0,0,-1,469.812,412.749)">
-                <path d="M240.737,208.337L250.97,208.337C250.97,208.337 251.662,209.029 251.662,209.029L252.209,208.482L251.403,207.677C251.331,207.604 251.233,207.564 251.13,207.564L240.737,207.564L240.737,208.337Z" style=""/>
-            </g>
-            <g transform="matrix(1,0,0,-1,-36.8607,412.749)">
-                <path d="M243.737,207.564L251.13,207.564C251.233,207.564 251.331,207.604 251.403,207.677L252.209,208.482L251.662,209.029L250.97,208.337C250.97,208.337 243.737,208.337 243.737,208.337L243.737,207.564Z" style=""/>
-            </g>
-            <g transform="matrix(1,0,0,-1,-31.3696,432.749)">
-                <path d="M242.737,207.564L251.13,207.564C251.233,207.564 251.331,207.604 251.403,207.677L253.209,209.482L252.662,210.029L250.97,208.337C250.97,208.337 242.737,208.337 242.737,208.337L242.737,207.564Z" style=""/>
-            </g>
-            <g transform="matrix(-1,0,0,-1,487.321,432.821)">
-                <path d="M242.737,208.337L250.97,208.337C250.97,208.337 252.662,210.029 252.662,210.029L253.209,209.482L251.403,207.677C251.331,207.604 251.233,207.564 251.13,207.564L242.737,207.564L242.737,208.337Z" style=""/>
-            </g>
-            <g transform="matrix(-1,0,0,1,492.867,-3.15102)">
-                <path d="M242.737,207.564L251.13,207.564C251.233,207.564 251.331,207.604 251.403,207.677L253.209,209.482L252.662,210.029L250.97,208.337C249.81,208.337 242.737,208.337 242.737,208.337L242.737,207.564Z" style=""/>
-            </g>
-            <g transform="matrix(1,0,0,1,-9.29379,0)">
-                <path d="M217.976,223.056L215.686,225.346" style="fill:none;"/>
-            </g>
-            <g transform="matrix(1,0,0,1,-9.29379,0)">
-                <path d="M216.233,225.892L217.326,224.799L216.233,223.706L215.139,224.799L216.233,225.892Z" style=""/>
-            </g>
-            <g transform="matrix(1,0,0,1,-9.29379,0)">
-                <path d="M217.702,222.783C217.702,222.783 216.89,223.595 216.233,224.252L216.779,224.799C217.436,224.142 218.249,223.329 218.249,223.329L217.702,222.783Z" style=""/>
-            </g>
+<svg width="100%" height="100%" viewBox="0 0 2000 629" version="1.1" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" xml:space="preserve" xmlns:serif="http://www.serif.com/" style="fill-rule:evenodd;clip-rule:evenodd;stroke-linejoin:round;stroke-miterlimit:2;">
+    <g id="Wires" transform="matrix(26.3259,0,0,26.3259,-5070.03,-5340.32)">
+        <g transform="matrix(1,0,0,-1,-0.808418,430.006)">
+            <path d="M194.555,216.234C193.915,216.234 193.396,215.714 193.396,215.074C193.396,214.434 193.915,213.915 194.555,213.915C195.195,213.915 195.715,214.434 195.715,215.074C195.715,215.714 195.195,216.234 194.555,216.234ZM194.555,215.461C194.768,215.461 194.942,215.288 194.942,215.074C194.942,214.861 194.768,214.688 194.555,214.688C194.342,214.688 194.169,214.861 194.169,215.074C194.169,215.288 194.342,215.461 194.555,215.461Z"/>
         </g>
-        <g id="Name-3" serif:id="Name 3" transform="matrix(1,0,0,1,-4.57363,22.1201)">
-            <path d="M213.538,196.869L213.538,188.32L214.496,188.32L214.496,192.663L216.491,192.663L216.491,188.32L217.448,188.32L217.448,196.869L216.491,196.869L216.491,193.62L214.496,193.62L214.496,196.869L213.538,196.869Z" style="fill-opacity:0.85;fill-rule:nonzero;"/>
-            <path d="M221.267,188.32L223.079,196.869L222.087,196.869L221.757,195.216L219.751,195.216L219.409,196.869L218.417,196.869L220.241,188.32L221.267,188.32ZM219.922,194.258L221.586,194.258L220.754,190.03L219.922,194.258Z" style="fill-opacity:0.85;fill-rule:nonzero;"/>
-            <path d="M227.331,188.32L228.242,188.32L228.242,196.869L227.433,196.869L225.131,190.816L225.131,196.869L224.219,196.869L224.219,188.32L225.028,188.32L227.331,194.372L227.331,188.32Z" style="fill-opacity:0.85;fill-rule:nonzero;"/>
-            <path d="M229.496,196.869L229.496,188.32L231.457,188.32C232.072,188.32 232.566,188.514 232.939,188.901C233.311,189.289 233.497,189.798 233.497,190.429L233.497,194.76C233.497,195.391 233.311,195.9 232.939,196.287C232.566,196.675 232.072,196.869 231.457,196.869L229.496,196.869ZM230.454,195.911L231.457,195.911C231.784,195.911 232.046,195.807 232.243,195.598C232.441,195.389 232.54,195.11 232.54,194.76L232.54,190.429C232.54,190.079 232.441,189.8 232.243,189.591C232.046,189.382 231.784,189.277 231.457,189.277L230.454,189.277L230.454,195.911Z" style="fill-opacity:0.85;fill-rule:nonzero;"/>
-            <path d="M239.265,188.32L240.222,188.32L240.222,196.869L239.31,196.869L239.31,191.306L237.84,196.869L237.122,196.869L235.663,191.306L235.663,196.869L234.751,196.869L234.751,188.32L235.708,188.32L237.487,195.113L239.265,188.32Z" style="fill-opacity:0.85;fill-rule:nonzero;"/>
-            <path d="M244.212,188.32L246.024,196.869L245.032,196.869L244.702,195.216L242.696,195.216L242.354,196.869L241.362,196.869L243.186,188.32L244.212,188.32ZM242.867,194.258L244.531,194.258L243.699,190.03L242.867,194.258Z" style="fill-opacity:0.85;fill-rule:nonzero;"/>
-            <path d="M247.05,196.869L247.05,188.32L249.01,188.32C249.626,188.32 250.12,188.514 250.492,188.901C250.864,189.289 251.051,189.798 251.051,190.429L251.051,194.76C251.051,195.391 250.864,195.9 250.492,196.287C250.12,196.675 249.626,196.869 249.01,196.869L247.05,196.869ZM248.007,195.911L249.01,195.911C249.337,195.911 249.599,195.807 249.797,195.598C249.994,195.389 250.093,195.11 250.093,194.76L250.093,190.429C250.093,190.079 249.994,189.8 249.797,189.591C249.599,189.382 249.337,189.277 249.01,189.277L248.007,189.277L248.007,195.911Z" style="fill-opacity:0.85;fill-rule:nonzero;"/>
-            <path d="M255.758,188.32L255.758,189.277L253.262,189.277L253.262,192.081L255.45,192.081L255.45,193.039L253.262,193.039L253.262,195.911L255.758,195.911L255.758,196.869L252.304,196.869L252.304,188.32L255.758,188.32Z" style="fill-opacity:0.85;fill-rule:nonzero;"/>
+        <g transform="matrix(1,0,0,-1,-0.808418,430.006)">
+            <path d="M195.328,214.688L201.366,214.688C201.579,214.688 201.753,214.861 201.753,215.074C201.753,215.288 201.579,215.461 201.366,215.461L195.328,215.461C195.115,215.461 194.942,215.288 194.942,215.074C194.942,214.861 195.115,214.688 195.328,214.688Z"/>
+        </g>
+        <g transform="matrix(-1,0,0,-1,461.953,430.006)">
+            <path d="M194.555,213.915C193.915,213.915 193.396,214.434 193.396,215.074C193.396,215.714 193.915,216.234 194.555,216.234C195.195,216.234 195.715,215.714 195.715,215.074C195.715,214.434 195.195,213.915 194.555,213.915ZM194.555,214.688C194.768,214.688 194.942,214.861 194.942,215.074C194.942,215.288 194.768,215.461 194.555,215.461C194.342,215.461 194.169,215.288 194.169,215.074C194.169,214.861 194.342,214.688 194.555,214.688Z"/>
+        </g>
+        <g transform="matrix(-1,0,0,-1,461.953,430.006)">
+            <path d="M195.328,215.461L201.366,215.461C201.579,215.461 201.753,215.288 201.753,215.074C201.753,214.861 201.579,214.688 201.366,214.688L195.328,214.688C195.115,214.688 194.942,214.861 194.942,215.074C194.942,215.288 195.115,215.461 195.328,215.461Z"/>
+        </g>
+        <g transform="matrix(1,0,0,-1,0.0431238,430.006)">
+            <path d="M200.241,214.801L201.383,213.66L201.929,214.206L200.788,215.348L200.241,214.801Z"/>
+        </g>
+        <g transform="matrix(1,0,0,-1,58.9314,428.865)">
+            <path d="M200.241,214.801L201.383,213.66L201.929,214.206L200.788,215.348L200.241,214.801Z"/>
+        </g>
+        <g transform="matrix(1,0,0,-1,1.04312,431.006)">
+            <path d="M205.56,209.482L207.365,207.677L207.912,208.223L206.107,210.029L205.56,209.482Z"/>
+        </g>
+        <g transform="matrix(1,0,0,-1,1.04312,431.006)">
+            <path d="M229.737,206.754C229.436,207.055 228.946,207.055 228.644,206.754C228.342,206.452 228.342,205.962 228.644,205.66C228.946,205.359 229.436,205.359 229.737,205.66C230.039,205.962 230.039,206.452 229.737,206.754ZM229.192,206.208L229.191,206.207L229.192,206.208Z"/>
+        </g>
+        <g transform="matrix(1,0,0,-1,1.04312,431.006)">
+            <path d="M207.639,207.564L227.288,207.564C227.288,207.564 228.644,206.207 228.644,206.207C228.795,206.056 229.04,206.056 229.191,206.207C229.342,206.358 229.342,206.603 229.191,206.754C228.534,207.411 227.721,208.223 227.721,208.223C227.648,208.296 227.55,208.337 227.448,208.337L207.639,208.337C207.425,208.337 207.252,208.163 207.252,207.95C207.252,207.737 207.425,207.564 207.639,207.564Z"/>
+        </g>
+        <g transform="matrix(1,0,0,-1,1.04312,431.006)">
+            <path d="M230.737,207.564L251.13,207.564C251.233,207.564 251.331,207.604 251.403,207.677L253.209,209.482L252.662,210.029L250.97,208.337C250.97,208.337 230.737,208.337 230.737,208.337L230.737,207.564Z"/>
+        </g>
+        <g transform="matrix(-1,0,0,1,459.812,-1.40971)">
+            <path d="M205.56,209.482L207.365,207.677L207.912,208.223L206.107,210.029L205.56,209.482Z"/>
+        </g>
+        <g transform="matrix(-1,0,0,1,459.812,-1.40971)">
+            <path d="M229.737,206.754C229.436,207.055 228.946,207.055 228.644,206.754C228.342,206.452 228.342,205.962 228.644,205.66C228.946,205.359 229.436,205.359 229.737,205.66C230.039,205.962 230.039,206.452 229.737,206.754L229.737,206.754Z"/>
+        </g>
+        <g transform="matrix(-1,0,0,1,459.812,-1.40971)">
+            <path d="M207.639,207.564L227.288,207.564C227.288,207.564 228.644,206.207 228.644,206.207C228.795,206.056 229.04,206.056 229.191,206.207C229.342,206.358 229.342,206.603 229.191,206.754C228.534,207.411 227.721,208.223 227.721,208.223C227.648,208.296 227.55,208.337 227.448,208.337L207.639,208.337C207.425,208.337 207.252,208.163 207.252,207.95C207.252,207.737 207.425,207.564 207.639,207.564Z"/>
+        </g>
+        <g transform="matrix(-1,0,0,1,459.812,-1.40971)">
+            <path d="M230.737,207.564L251.13,207.564C251.233,207.564 251.331,207.604 251.403,207.677L253.209,209.482L252.662,210.029L250.97,208.337C250.97,208.337 230.737,208.337 230.737,208.337L230.737,207.564Z"/>
+        </g>
+        <g transform="matrix(-1,0,0,-1,469.812,412.749)">
+            <path d="M240.737,208.337L250.97,208.337C250.97,208.337 251.662,209.029 251.662,209.029L252.209,208.482L251.403,207.677C251.331,207.604 251.233,207.564 251.13,207.564L240.737,207.564L240.737,208.337Z"/>
+        </g>
+        <g transform="matrix(1,0,0,-1,-36.8607,412.749)">
+            <path d="M243.737,207.564L251.13,207.564C251.233,207.564 251.331,207.604 251.403,207.677L252.209,208.482L251.662,209.029L250.97,208.337C250.97,208.337 243.737,208.337 243.737,208.337L243.737,207.564Z"/>
+        </g>
+        <g transform="matrix(1,0,0,-1,-31.3696,432.749)">
+            <path d="M242.737,207.564L251.13,207.564C251.233,207.564 251.331,207.604 251.403,207.677L253.209,209.482L252.662,210.029L250.97,208.337C250.97,208.337 242.737,208.337 242.737,208.337L242.737,207.564Z"/>
+        </g>
+        <g transform="matrix(-1,0,0,-1,487.321,432.821)">
+            <path d="M242.737,208.337L250.97,208.337C250.97,208.337 252.662,210.029 252.662,210.029L253.209,209.482L251.403,207.677C251.331,207.604 251.233,207.564 251.13,207.564L242.737,207.564L242.737,208.337Z"/>
+        </g>
+        <g transform="matrix(-1,0,0,1,492.867,-3.15102)">
+            <path d="M242.737,207.564L251.13,207.564C251.233,207.564 251.331,207.604 251.403,207.677L253.209,209.482L252.662,210.029L250.97,208.337C249.81,208.337 242.737,208.337 242.737,208.337L242.737,207.564Z"/>
+        </g>
+        <g transform="matrix(1,0,0,1,-9.29379,0)">
+            <path d="M216.233,225.892L217.326,224.799L216.233,223.706L215.139,224.799L216.233,225.892Z"/>
+        </g>
+        <g transform="matrix(1,0,0,1,-9.29379,0)">
+            <path d="M217.702,222.783C217.702,222.783 216.89,223.595 216.233,224.252L216.779,224.799C217.436,224.142 218.249,223.329 218.249,223.329L217.702,222.783Z"/>
+        </g>
+    </g>
+    <g id="Name" transform="matrix(26.3259,0,0,26.3259,-5177.33,-4754.86)">
+        <path d="M213.538,196.869L213.538,188.32L214.496,188.32L214.496,192.663L216.491,192.663L216.491,188.32L217.448,188.32L217.448,196.869L216.491,196.869L216.491,193.62L214.496,193.62L214.496,196.869L213.538,196.869Z" style="fill-rule:nonzero;"/>
+        <path d="M221.267,188.32L223.079,196.869L222.087,196.869L221.757,195.216L219.751,195.216L219.409,196.869L218.417,196.869L220.241,188.32L221.267,188.32ZM219.922,194.258L221.586,194.258L220.754,190.03L219.922,194.258Z" style="fill-rule:nonzero;"/>
+        <path d="M227.331,188.32L228.242,188.32L228.242,196.869L227.433,196.869L225.131,190.816L225.131,196.869L224.219,196.869L224.219,188.32L225.028,188.32L227.331,194.372L227.331,188.32Z" style="fill-rule:nonzero;"/>
+        <path d="M229.496,196.869L229.496,188.32L231.457,188.32C232.072,188.32 232.566,188.514 232.939,188.901C233.311,189.289 233.497,189.798 233.497,190.429L233.497,194.76C233.497,195.391 233.311,195.9 232.939,196.287C232.566,196.675 232.072,196.869 231.457,196.869L229.496,196.869ZM230.454,195.911L231.457,195.911C231.784,195.911 232.046,195.807 232.243,195.598C232.441,195.389 232.54,195.11 232.54,194.76L232.54,190.429C232.54,190.079 232.441,189.8 232.243,189.591C232.046,189.382 231.784,189.277 231.457,189.277L230.454,189.277L230.454,195.911Z" style="fill-rule:nonzero;"/>
+        <g transform="matrix(1,0,0,1,0,0.0379854)">
+            <path d="M239.265,188.32L240.222,188.32L240.222,196.869L239.31,196.869L239.31,191.306L237.84,196.869L237.122,196.869L235.663,191.306L235.663,196.869L234.751,196.869L234.751,188.32L235.708,188.32L237.487,195.113L239.265,188.32Z" style="fill-rule:nonzero;"/>
+        </g>
+        <path d="M244.212,188.32L246.024,196.869L245.032,196.869L244.702,195.216L242.696,195.216L242.354,196.869L241.362,196.869L243.186,188.32L244.212,188.32ZM242.867,194.258L244.531,194.258L243.699,190.03L242.867,194.258Z" style="fill-rule:nonzero;"/>
+        <path d="M247.05,196.869L247.05,188.32L249.01,188.32C249.626,188.32 250.12,188.514 250.492,188.901C250.864,189.289 251.051,189.798 251.051,190.429L251.051,194.76C251.051,195.391 250.864,195.9 250.492,196.287C250.12,196.675 249.626,196.869 249.01,196.869L247.05,196.869ZM248.007,195.911L249.01,195.911C249.337,195.911 249.599,195.807 249.797,195.598C249.994,195.389 250.093,195.11 250.093,194.76L250.093,190.429C250.093,190.079 249.994,189.8 249.797,189.591C249.599,189.382 249.337,189.277 249.01,189.277L248.007,189.277L248.007,195.911Z" style="fill-rule:nonzero;"/>
+        <g transform="matrix(1,0,0,1,0,0.0379854)">
+            <path d="M255.758,188.32L255.758,189.277L253.262,189.277L253.262,192.081L255.45,192.081L255.45,193.039L253.262,193.039L253.262,195.911L255.758,195.911L255.758,196.869L252.304,196.869L252.304,188.32L255.758,188.32Z" style="fill-rule:nonzero;"/>
         </g>
     </g>
 </svg>

src/templates/templates.go

@@ -193,6 +193,25 @@ var HMNTemplateFuncs = template.FuncMap{
 	"noescape": func(str string) template.HTML {
 		return template.HTML(str)
 	},
+	"filesize": func(numBytes int) string {
+		scales := []string{
+			" bytes",
+			"kb",
+			"mb",
+			"gb",
+		}
+		num := float64(numBytes)
+		scale := 0
+		for num > 1024 && scale < len(scales)-1 {
+			num /= 1024
+			scale += 1
+		}
+		precision := 0
+		if scale > 0 {
+			precision = 2
+		}
+		return fmt.Sprintf("%.*f%s", precision, num, scales[scale])
+	},
 
 	// NOTE(asaf): Template specific functions:
 	"projectcarddata": func(project Project, classes string) ProjectCardData {

src/templates/types.go

@@ -75,6 +75,7 @@ type Footer struct {
 	ProjectIndexUrl            string
 	ForumsUrl                  string
 	ContactUrl                 string
+	SearchActionUrl            string
 }
 
 type Thread struct {
@@ -146,6 +147,7 @@ type ProjectSettings struct {
 
 	Blurb       string
 	Description string
+	LinksText   string
 	Owners      []User
 
 	LightLogo string
@@ -303,7 +305,8 @@ type TimelineItem struct {
 type TimelineItemMediaType int
 
 const (
-	TimelineItemMediaTypeImage TimelineItemMediaType = iota + 1
+	TimelineItemMediaTypeUnknown TimelineItemMediaType = iota
+	TimelineItemMediaTypeImage
 	TimelineItemMediaTypeVideo
 	TimelineItemMediaTypeAudio
 	TimelineItemMediaTypeEmbed
@@ -316,6 +319,8 @@ type TimelineItemMedia struct {
 	ThumbnailUrl        string
 	MimeType            string
 	Width, Height       int
+	Filename            string
+	FileSize            int
 	ExtraOpenGraphItems []OpenGraphItem
 }
 

src/twitch/rest.go

@@ -0,0 +1,450 @@
+package twitch
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"git.handmade.network/hmn/hmn/src/config"
+	"git.handmade.network/hmn/hmn/src/hmnurl"
+	"git.handmade.network/hmn/hmn/src/logging"
+	"git.handmade.network/hmn/hmn/src/oops"
+	"git.handmade.network/hmn/hmn/src/utils"
+)
+
+var twitchAPIBaseUrl = config.Config.Twitch.BaseUrl
+
+var HitRateLimit = errors.New("hit rate limit")
+var MaxRetries = errors.New("hit max retries")
+
+var httpClient = &http.Client{}
+
+// NOTE(asaf): Access token is not thread-safe right now.
+//             All twitch requests are made through the goroutine in MonitorTwitchSubscriptions.
+var activeAccessToken string
+var rateLimitReset time.Time
+
+type twitchUser struct {
+	TwitchID    string
+	TwitchLogin string
+}
+
+func getTwitchUsersByLogin(ctx context.Context, logins []string) ([]twitchUser, error) {
+	result := make([]twitchUser, 0, len(logins))
+	numChunks := len(logins)/100 + 1
+	for i := 0; i < numChunks; i++ {
+		query := url.Values{}
+		query.Add("first", "100")
+		for _, login := range logins[i*100 : utils.IntMin((i+1)*100, len(logins))] {
+			query.Add("login", login)
+		}
+		req, err := http.NewRequestWithContext(ctx, "GET", buildUrl("/users", query.Encode()), nil)
+		if err != nil {
+			return nil, oops.New(err, "failed to create requset")
+		}
+		res, err := doRequest(ctx, true, req)
+		if err != nil {
+			return nil, oops.New(err, "failed to fetch twitch users")
+		}
+
+		type user struct {
+			ID    string `json:"id"`
+			Login string `json:"login"`
+		}
+
+		type twitchResponse struct {
+			Data []user `json:"data"`
+		}
+
+		body, err := io.ReadAll(res.Body)
+		res.Body.Close()
+		if err != nil {
+			return nil, oops.New(err, "failed to read response body while fetching twitch users")
+		}
+
+		var userResponse twitchResponse
+		err = json.Unmarshal(body, &userResponse)
+		if err != nil {
+			return nil, oops.New(err, "failed to parse twitch response while fetching twitch users")
+		}
+
+		for _, u := range userResponse.Data {
+			result = append(result, twitchUser{
+				TwitchID:    u.ID,
+				TwitchLogin: u.Login,
+			})
+		}
+	}
+
+	return result, nil
+}
+
+type streamStatus struct {
+	TwitchID    string
+	TwitchLogin string
+	Live        bool
+	Title       string
+	StartedAt   time.Time
+	Category    string
+	Tags        []string
+}
+
+func getStreamStatus(ctx context.Context, twitchIDs []string) ([]streamStatus, error) {
+	result := make([]streamStatus, 0, len(twitchIDs))
+	numChunks := len(twitchIDs)/100 + 1
+	for i := 0; i < numChunks; i++ {
+		query := url.Values{}
+		query.Add("first", "100")
+		for _, tid := range twitchIDs[i*100 : utils.IntMin((i+1)*100, len(twitchIDs))] {
+			query.Add("user_id", tid)
+		}
+		req, err := http.NewRequestWithContext(ctx, "GET", buildUrl("/streams", query.Encode()), nil)
+		if err != nil {
+			return nil, oops.New(err, "failed to create request")
+		}
+		res, err := doRequest(ctx, true, req)
+		if err != nil {
+			return nil, oops.New(err, "failed to fetch stream statuses")
+		}
+
+		type twitchStatus struct {
+			TwitchID    string   `json:"user_id"`
+			TwitchLogin string   `json:"user_login"`
+			GameID      string   `json:"game_id"`
+			Type        string   `json:"type"`
+			Title       string   `json:"title"`
+			StartedAt   string   `json:"started_at"`
+			Thumbnail   string   `json:"thumbnail_url"`
+			Tags        []string `json:"tag_ids"`
+		}
+
+		type twitchResponse struct {
+			Data []twitchStatus `json:"data"`
+		}
+		body, err := io.ReadAll(res.Body)
+		res.Body.Close()
+		if err != nil {
+			return nil, oops.New(err, "failed to read response body while processing stream statuses")
+		}
+
+		var streamResponse twitchResponse
+		err = json.Unmarshal(body, &streamResponse)
+		if err != nil {
+			return nil, oops.New(err, "failed to parse twitch response while processing stream statuses")
+		}
+
+		for _, d := range streamResponse.Data {
+			started, err := time.Parse(time.RFC3339, d.StartedAt)
+			if err != nil {
+				logging.ExtractLogger(ctx).Warn().Str("Time string", d.StartedAt).Msg("Failed to parse twitch timestamp")
+				started = time.Now()
+			}
+			status := streamStatus{
+				TwitchID:    d.TwitchID,
+				TwitchLogin: d.TwitchLogin,
+				Live:        d.Type == "live",
+				Title:       d.Title,
+				StartedAt:   started,
+				Category:    d.GameID,
+				Tags:        d.Tags,
+			}
+			result = append(result, status)
+		}
+	}
+
+	return result, nil
+}
+
+type twitchEventSub struct {
+	EventID    string
+	TwitchID   string
+	Type       string
+	GoodStatus bool
+}
+
+func getEventSubscriptions(ctx context.Context) ([]twitchEventSub, error) {
+	result := make([]twitchEventSub, 0)
+	after := ""
+	for {
+		query := url.Values{}
+		if len(after) > 0 {
+			query.Add("after", after)
+		}
+		req, err := http.NewRequestWithContext(ctx, "GET", buildUrl("/eventsub/subscriptions", query.Encode()), nil)
+		if err != nil {
+			return nil, oops.New(err, "failed to create request")
+		}
+		res, err := doRequest(ctx, true, req)
+		if err != nil {
+			return nil, oops.New(err, "failed to fetch twitch event subscriptions")
+		}
+
+		type eventSub struct {
+			ID        string `json:"id"`
+			Status    string `json:"status"`
+			Type      string `json:"type"`
+			Condition struct {
+				TwitchID string `json:"broadcaster_user_id"`
+			} `json:"condition"`
+		}
+
+		type twitchResponse struct {
+			Data       []eventSub `json:"data"`
+			Pagination *struct {
+				After string `json:"cursor"`
+			} `json:"pagination"`
+		}
+
+		body, err := io.ReadAll(res.Body)
+		res.Body.Close()
+		if err != nil {
+			return nil, oops.New(err, "failed to read response body while fetching twitch eventsubs")
+		}
+
+		var eventSubResponse twitchResponse
+		err = json.Unmarshal(body, &eventSubResponse)
+		if err != nil {
+			return nil, oops.New(err, "failed to parse twitch response while fetching twitch eventsubs")
+		}
+
+		for _, es := range eventSubResponse.Data {
+			result = append(result, twitchEventSub{
+				EventID:    es.ID,
+				TwitchID:   es.Condition.TwitchID,
+				Type:       es.Type,
+				GoodStatus: es.Status == "enabled" || es.Status == "webhook_callback_verification_pending",
+			})
+		}
+
+		if eventSubResponse.Pagination == nil || eventSubResponse.Pagination.After == "" {
+			return result, nil
+		} else {
+			after = eventSubResponse.Pagination.After
+		}
+	}
+}
+
+func subscribeToEvent(ctx context.Context, eventType string, twitchID string) error {
+	type eventBody struct {
+		Type      string `json:"type"`
+		Version   string `json:"version"`
+		Condition struct {
+			TwitchID string `json:"broadcaster_user_id"`
+		} `json:"condition"`
+		Transport struct {
+			Method   string `json:"method"`
+			Callback string `json:"callback"`
+			Secret   string `json:"secret"`
+		} `json:"transport"`
+	}
+
+	ev := eventBody{
+		Type:    eventType,
+		Version: "1",
+	}
+	ev.Condition.TwitchID = twitchID
+	ev.Transport.Method = "webhook"
+	// NOTE(asaf): Twitch has special treatment for localhost. We can keep this around for live/beta because it just won't replace anything.
+	ev.Transport.Callback = strings.ReplaceAll(hmnurl.BuildTwitchEventSubCallback(), "handmade.local:9001", "localhost")
+	ev.Transport.Secret = config.Config.Twitch.EventSubSecret
+
+	evJson, err := json.Marshal(ev)
+	if err != nil {
+		return oops.New(err, "failed to marshal event sub data")
+	}
+	req, err := http.NewRequestWithContext(ctx, "POST", buildUrl("/eventsub/subscriptions", ""), bytes.NewReader(evJson))
+	req.Header.Set("Content-Type", "application/json")
+	if err != nil {
+		return oops.New(err, "failed to create request")
+	}
+	res, err := doRequest(ctx, true, req)
+	if err != nil {
+		return oops.New(err, "failed to create new event subscription")
+	}
+	defer readAndClose(res)
+
+	if res.StatusCode >= 300 {
+		body, err := io.ReadAll(res.Body)
+		res.Body.Close()
+		if err != nil {
+			return oops.New(err, "failed to read response body while creating twitch eventsubs")
+		}
+		logging.ExtractLogger(ctx).Error().Interface("Headers", res.Header).Int("Status code", res.StatusCode).Str("Body", string(body[:])).Msg("Failed to create twitch event sub")
+		return oops.New(nil, "failed to create new event subscription")
+	}
+	return nil
+}
+
+func unsubscribeFromEvent(ctx context.Context, eventID string) error {
+	query := url.Values{}
+	query.Add("id", eventID)
+	req, err := http.NewRequestWithContext(ctx, "DELETE", buildUrl("/eventsub/subscriptions", query.Encode()), nil)
+	if err != nil {
+		return oops.New(err, "failed to create request")
+	}
+	res, err := doRequest(ctx, true, req)
+	if err != nil {
+		return oops.New(err, "failed to delete new event subscription")
+	}
+	defer readAndClose(res)
+
+	if res.StatusCode > 300 {
+		body, err := io.ReadAll(res.Body)
+		res.Body.Close()
+		if err != nil {
+			return oops.New(err, "failed to read response body while deleting twitch eventsubs")
+		}
+		logging.ExtractLogger(ctx).Error().Interface("Headers", res.Header).Int("Status code", res.StatusCode).Str("Body", string(body[:])).Msg("Failed to delete twitch event sub")
+		return oops.New(nil, "failed to delete new event subscription")
+	}
+	return nil
+}
+
+func doRequest(ctx context.Context, waitOnRateLimit bool, req *http.Request) (*http.Response, error) {
+	serviceUnavailable := false
+	numRetries := 5
+
+	for {
+		if numRetries == 0 {
+			return nil, MaxRetries
+		}
+		numRetries -= 1
+
+		now := time.Now()
+		if rateLimitReset.After(now) {
+			if waitOnRateLimit {
+				timer := time.NewTimer(rateLimitReset.Sub(now))
+				select {
+				case <-timer.C:
+				case <-ctx.Done():
+					return nil, errors.New("request interrupted during rate limiting")
+				}
+			} else {
+				return nil, HitRateLimit
+			}
+		}
+
+		req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", activeAccessToken))
+		req.Header.Set("Client-Id", config.Config.Twitch.ClientID)
+		res, err := httpClient.Do(req)
+		if err != nil {
+			return nil, oops.New(err, "twitch request failed")
+		}
+
+		if res.StatusCode != 503 {
+			serviceUnavailable = false
+		}
+
+		if res.StatusCode >= 200 && res.StatusCode < 300 {
+			return res, nil
+		} else if res.StatusCode == 503 {
+			readAndClose(res)
+			if serviceUnavailable {
+				// NOTE(asaf): The docs say we should retry once if we receive 503
+				return nil, oops.New(nil, "got 503 Service Unavailable twice in a row")
+			} else {
+				serviceUnavailable = true
+			}
+		} else if res.StatusCode == 429 {
+			logging.ExtractLogger(ctx).Warn().Interface("Headers", res.Header).Msg("Hit Twitch rate limit")
+			err = updateRateLimitReset(res)
+			if err != nil {
+				return nil, err
+			}
+		} else if res.StatusCode == 401 {
+			logging.ExtractLogger(ctx).Warn().Msg("Twitch refresh token is invalid. Renewing...")
+			readAndClose(res)
+			err = refreshAccessToken(ctx)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			body, err := io.ReadAll(res.Body)
+			if err != nil {
+				return nil, oops.New(err, "failed to read response body")
+			}
+			logging.ExtractLogger(ctx).Warn().Interface("Headers", res.Header).Int("Status code", res.StatusCode).Str("Body", string(body[:])).Msg("Unexpected status code from twitch")
+			res.Body.Close()
+			return res, oops.New(nil, "got an unexpected status code from twitch")
+		}
+	}
+}
+
+func updateRateLimitReset(res *http.Response) error {
+	defer readAndClose(res)
+
+	resetStr := res.Header.Get("Ratelimit-Reset")
+	if len(resetStr) == 0 {
+		return oops.New(nil, "no ratelimit data on response")
+	}
+
+	resetUnix, err := strconv.Atoi(resetStr)
+	if err != nil {
+		return oops.New(err, "failed to parse reset time")
+	}
+
+	rateLimitReset = time.Unix(int64(resetUnix), 0)
+	return nil
+}
+
+type AccessTokenResponse struct {
+	AccessToken string `json:"access_token"`
+}
+
+func refreshAccessToken(ctx context.Context) error {
+	logging.ExtractLogger(ctx).Info().Msg("Refreshing twitch token")
+	query := url.Values{}
+	query.Add("client_id", config.Config.Twitch.ClientID)
+	query.Add("client_secret", config.Config.Twitch.ClientSecret)
+	query.Add("grant_type", "client_credentials")
+	url := fmt.Sprintf("%s/token?%s", config.Config.Twitch.BaseIDUrl, query.Encode())
+	req, err := http.NewRequestWithContext(ctx, "POST", url, nil)
+	if err != nil {
+		return oops.New(err, "failed to create request")
+	}
+
+	res, err := httpClient.Do(req)
+	if err != nil {
+		return oops.New(err, "failed to request new access token")
+	}
+	defer readAndClose(res)
+
+	if res.StatusCode >= 400 {
+		// NOTE(asaf): The docs don't specify the error cases for this call.
+		// NOTE(asaf): According to the docs rate limiting is per-token, and we don't use a token for this call,
+		//             so who knows how rate limiting works here.
+		body, _ := io.ReadAll(res.Body)
+		logging.ExtractLogger(ctx).Error().Interface("Headers", res.Header).Int("Status code", res.StatusCode).Str("body", string(body[:])).Msg("Got bad status code from twitch access token refresh")
+		return oops.New(nil, "received unexpected status code from twitch access token refresh")
+	}
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return oops.New(err, "failed to read response body")
+	}
+	var accessTokenResponse AccessTokenResponse
+	err = json.Unmarshal(body, &accessTokenResponse)
+	if err != nil {
+		return oops.New(err, "failed to unmarshal access token response")
+	}
+
+	activeAccessToken = accessTokenResponse.AccessToken
+	return nil
+}
+
+func readAndClose(res *http.Response) {
+	io.ReadAll(res.Body)
+	res.Body.Close()
+}
+
+func buildUrl(path string, queryParams string) string {
+	return fmt.Sprintf("%s%s?%s", config.Config.Twitch.BaseUrl, path, queryParams)
+}

src/twitch/twitch.go

@@ -0,0 +1,514 @@
+package twitch
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"git.handmade.network/hmn/hmn/src/config"
+	"git.handmade.network/hmn/hmn/src/db"
+	"git.handmade.network/hmn/hmn/src/discord"
+	"git.handmade.network/hmn/hmn/src/hmndata"
+	"git.handmade.network/hmn/hmn/src/logging"
+	"git.handmade.network/hmn/hmn/src/models"
+	"git.handmade.network/hmn/hmn/src/oops"
+	"git.handmade.network/hmn/hmn/src/perf"
+	"github.com/jackc/pgx/v4/pgxpool"
+)
+
+type twitchNotification struct {
+	TwitchID string
+	Type     twitchNotificationType
+}
+
+var twitchNotificationChannel chan twitchNotification
+var linksChangedChannel chan struct{}
+
+func MonitorTwitchSubscriptions(ctx context.Context, dbConn *pgxpool.Pool) <-chan struct{} {
+	log := logging.ExtractLogger(ctx).With().Str("twitch goroutine", "stream monitor").Logger()
+	ctx = logging.AttachLoggerToContext(&log, ctx)
+
+	if config.Config.Twitch.ClientID == "" {
+		log.Warn().Msg("No twitch config provided.")
+		done := make(chan struct{}, 1)
+		done <- struct{}{}
+		return done
+	}
+
+	twitchNotificationChannel = make(chan twitchNotification, 100)
+	linksChangedChannel = make(chan struct{}, 10)
+	done := make(chan struct{})
+
+	go func() {
+		defer func() {
+			log.Info().Msg("Shutting down twitch monitor")
+			done <- struct{}{}
+		}()
+		log.Info().Msg("Running twitch monitor...")
+
+		err := refreshAccessToken(ctx)
+		if err != nil {
+			log.Error().Err(err).Msg("Failed to fetch refresh token on start")
+			return
+		}
+
+		monitorTicker := time.NewTicker(2 * time.Hour)
+		firstRunChannel := make(chan struct{}, 1)
+		firstRunChannel <- struct{}{}
+
+		timers := make([]*time.Timer, 0)
+		expiredTimers := make(chan *time.Timer, 10)
+		for {
+			select {
+			case <-ctx.Done():
+				for _, timer := range timers {
+					timer.Stop()
+				}
+				return
+			case expired := <-expiredTimers:
+				for idx, timer := range timers {
+					if timer == expired {
+						timers = append(timers[:idx], timers[idx+1:]...)
+						break
+					}
+				}
+			case <-firstRunChannel:
+				syncWithTwitch(ctx, dbConn, true)
+			case <-monitorTicker.C:
+				syncWithTwitch(ctx, dbConn, true)
+			case <-linksChangedChannel:
+				// NOTE(asaf): Since we update links inside transactions for users/projects
+				//             we won't see the updated list of links until the transaction is committed.
+				//             Waiting 5 seconds is just a quick workaround for that. It's not
+				//             convenient to only trigger this after the transaction is committed.
+				var timer *time.Timer
+				t := time.AfterFunc(5*time.Second, func() {
+					expiredTimers <- timer
+					syncWithTwitch(ctx, dbConn, false)
+				})
+				timer = t
+				timers = append(timers, t)
+			case notification := <-twitchNotificationChannel:
+				if notification.Type == notificationTypeRevocation {
+					syncWithTwitch(ctx, dbConn, false)
+				} else {
+					if notification.Type == notificationTypeChannelUpdate {
+						// NOTE(asaf): The twitch API (getStreamStatus) lags behind the notification and
+						//             would return old data if we called it immediately, so we have to
+						//             wait a bit before we process the notification. We can get the
+						//             category from the notification, but not the tags (or the up-to-date title),
+						//             so we can't really skip this.
+						var timer *time.Timer
+						t := time.AfterFunc(3*time.Minute, func() {
+							expiredTimers <- timer
+							processEventSubNotification(ctx, dbConn, &notification)
+						})
+						timer = t
+						timers = append(timers, t)
+					} else {
+						processEventSubNotification(ctx, dbConn, &notification)
+					}
+				}
+			}
+		}
+	}()
+
+	return done
+}
+
+type twitchNotificationType int
+
+const (
+	notificationTypeNone          twitchNotificationType = 0
+	notificationTypeOnline                               = 1
+	notificationTypeOffline                              = 2
+	notificationTypeChannelUpdate                        = 3
+
+	notificationTypeRevocation = 4
+)
+
+func QueueTwitchNotification(messageType string, body []byte) error {
+	var notification twitchNotification
+	if messageType == "notification" {
+		type notificationJson struct {
+			Subscription struct {
+				Type string `json:"type"`
+			} `json:"subscription"`
+			Event struct {
+				BroadcasterUserID    string `json:"broadcaster_user_id"`
+				BroadcasterUserLogin string `json:"broadcaster_user_login"`
+			} `json:"event"`
+		}
+		var incoming notificationJson
+		err := json.Unmarshal(body, &incoming)
+		if err != nil {
+			return oops.New(err, "failed to parse notification body")
+		}
+
+		notification.TwitchID = incoming.Event.BroadcasterUserID
+		switch incoming.Subscription.Type {
+		case "stream.online":
+			notification.Type = notificationTypeOnline
+		case "stream.offline":
+			notification.Type = notificationTypeOffline
+		case "channel.update":
+			notification.Type = notificationTypeChannelUpdate
+		default:
+			return oops.New(nil, "unknown subscription type received")
+		}
+	} else if messageType == "revocation" {
+		notification.Type = notificationTypeRevocation
+	}
+
+	if twitchNotificationChannel != nil && notification.Type != notificationTypeNone {
+		select {
+		case twitchNotificationChannel <- notification:
+		default:
+			return oops.New(nil, "twitch notification channel is full")
+		}
+	}
+	return nil
+}
+
+func UserOrProjectLinksUpdated(twitchLoginsPreChange, twitchLoginsPostChange []string) {
+	if linksChangedChannel != nil {
+		twitchChanged := (len(twitchLoginsPreChange) != len(twitchLoginsPostChange))
+		if !twitchChanged {
+			for idx, _ := range twitchLoginsPreChange {
+				if twitchLoginsPreChange[idx] != twitchLoginsPostChange[idx] {
+					twitchChanged = true
+					break
+				}
+			}
+		}
+		select {
+		case linksChangedChannel <- struct{}{}:
+		default:
+		}
+	}
+}
+
+func syncWithTwitch(ctx context.Context, dbConn *pgxpool.Pool, updateAll bool) {
+	log := logging.ExtractLogger(ctx)
+	log.Info().Msg("Running twitch sync")
+	p := perf.MakeNewRequestPerf("Background job", "", "syncWithTwitch")
+	defer func() {
+		p.EndRequest()
+		perf.LogPerf(p, log.Info())
+	}()
+
+	type twitchSyncStats struct {
+		NumSubbed         int
+		NumUnsubbed       int
+		NumStreamsChecked int
+	}
+	var stats twitchSyncStats
+
+	p.StartBlock("SQL", "Fetch list of streamers")
+	streamers, err := hmndata.FetchTwitchStreamers(ctx, dbConn)
+	if err != nil {
+		log.Error().Err(err).Msg("Error while monitoring twitch")
+		return
+	}
+	p.EndBlock()
+
+	needID := make([]string, 0)
+	streamerMap := make(map[string]*hmndata.TwitchStreamer)
+	for idx, streamer := range streamers {
+		needID = append(needID, streamer.TwitchLogin)
+		streamerMap[streamer.TwitchLogin] = &streamers[idx]
+	}
+
+	p.StartBlock("TwitchAPI", "Fetch twitch user info")
+	twitchUsers, err := getTwitchUsersByLogin(ctx, needID)
+	if err != nil {
+		log.Error().Err(err).Msg("Error while monitoring twitch")
+		return
+	}
+	p.EndBlock()
+
+	for _, tu := range twitchUsers {
+		streamerMap[tu.TwitchLogin].TwitchID = tu.TwitchID
+	}
+
+	validStreamers := make([]hmndata.TwitchStreamer, 0, len(streamers))
+	for _, streamer := range streamers {
+		if len(streamer.TwitchID) > 0 {
+			validStreamers = append(validStreamers, streamer)
+		}
+	}
+
+	p.StartBlock("TwitchAPI", "Fetch event subscriptions")
+	subscriptions, err := getEventSubscriptions(ctx)
+	if err != nil {
+		log.Error().Err(err).Msg("Error while monitoring twitch")
+		return
+	}
+	p.EndBlock()
+
+	const (
+		EventSubNone    = 0 // No event of this type found
+		EventSubRefresh = 1 // Event found, but bad status. Need to unsubscribe and resubscribe.
+		EventSubGood    = 2 // All is well.
+	)
+
+	type isSubbedByType map[string]bool
+
+	streamerEventSubs := make(map[string]isSubbedByType)
+	for _, streamer := range validStreamers {
+		streamerEventSubs[streamer.TwitchID] = make(isSubbedByType)
+		streamerEventSubs[streamer.TwitchID]["channel.update"] = false
+		streamerEventSubs[streamer.TwitchID]["stream.online"] = false
+		streamerEventSubs[streamer.TwitchID]["stream.offline"] = false
+	}
+
+	type unsubEvent struct {
+		TwitchID string
+		EventID  string
+	}
+
+	toUnsub := make([]unsubEvent, 0)
+
+	for _, sub := range subscriptions {
+		handled := false
+		if eventSubs, ok := streamerEventSubs[sub.TwitchID]; ok {
+			if _, ok := eventSubs[sub.Type]; ok { // Make sure it's a known type
+				if !sub.GoodStatus {
+					log.Debug().Str("TwitchID", sub.TwitchID).Str("Event Type", sub.Type).Msg("Twitch doesn't like our sub")
+					toUnsub = append(toUnsub, unsubEvent{TwitchID: sub.TwitchID, EventID: sub.EventID})
+				} else {
+					streamerEventSubs[sub.TwitchID][sub.Type] = true
+				}
+				handled = true
+			}
+		}
+		if !handled {
+			// NOTE(asaf): Found an unknown type or an event subscription that we don't have a matching user for.
+			//             Make sure we unsubscribe.
+			toUnsub = append(toUnsub, unsubEvent{TwitchID: sub.TwitchID, EventID: sub.EventID})
+		}
+	}
+
+	if config.Config.Env != config.Dev { // NOTE(asaf): Can't subscribe to events from dev. We need a non-localhost callback url.
+		p.StartBlock("TwitchAPI", "Sync subscriptions with twitch")
+		for _, ev := range toUnsub {
+			err = unsubscribeFromEvent(ctx, ev.EventID)
+			if err != nil {
+				log.Error().Err(err).Msg("Error while unsubscribing events")
+				// NOTE(asaf): Soft error. Don't care if it fails.
+			}
+			stats.NumUnsubbed += 1
+		}
+
+		for twitchID, evStatuses := range streamerEventSubs {
+			for evType, isSubbed := range evStatuses {
+				if !isSubbed {
+					err = subscribeToEvent(ctx, evType, twitchID)
+					if err != nil {
+						log.Error().Err(err).Msg("Error while monitoring twitch")
+						return
+					}
+					stats.NumSubbed += 1
+				}
+			}
+		}
+		p.EndBlock()
+	}
+
+	tx, err := dbConn.Begin(ctx)
+	if err != nil {
+		log.Error().Err(err).Msg("failed to start transaction")
+	}
+	defer tx.Rollback(ctx)
+
+	allIDs := make([]string, 0, len(validStreamers))
+	for _, streamer := range validStreamers {
+		allIDs = append(allIDs, streamer.TwitchID)
+	}
+	p.StartBlock("SQL", "Remove untracked streamers")
+	_, err = tx.Exec(ctx,
+		`DELETE FROM twitch_streams WHERE twitch_id != ANY($1)`,
+		allIDs,
+	)
+	if err != nil {
+		log.Error().Err(err).Msg("Failed to remove untracked twitch ids from streamer list in db")
+		return
+	}
+	p.EndBlock()
+
+	usersToUpdate := make([]string, 0)
+	if updateAll {
+		usersToUpdate = allIDs
+	} else {
+		// NOTE(asaf): Twitch can revoke our subscriptions, so we need to
+		//             update users whose subs were revoked or missing since last time we checked.
+		for twitchID, evStatuses := range streamerEventSubs {
+			for _, isSubbed := range evStatuses {
+				if !isSubbed {
+					usersToUpdate = append(usersToUpdate, twitchID)
+					break
+				}
+			}
+		}
+	}
+
+	p.StartBlock("TwitchAPI", "Fetch twitch stream statuses")
+	statuses, err := getStreamStatus(ctx, usersToUpdate)
+	if err != nil {
+		log.Error().Err(err).Msg("failed to fetch stream statuses")
+		return
+	}
+	p.EndBlock()
+	p.StartBlock("SQL", "Update stream statuses in db")
+	for _, status := range statuses {
+		log.Debug().Interface("Status", status).Msg("Got streamer")
+		_, err = updateStreamStatusInDB(ctx, tx, &status)
+		if err != nil {
+			log.Error().Err(err).Msg("failed to update twitch stream status")
+		}
+	}
+	p.EndBlock()
+	err = tx.Commit(ctx)
+	if err != nil {
+		log.Error().Err(err).Msg("failed to commit transaction")
+	}
+	stats.NumStreamsChecked += len(usersToUpdate)
+	log.Info().Interface("Stats", stats).Msg("Twitch sync done")
+}
+
+func notifyDiscordOfLiveStream(ctx context.Context, dbConn db.ConnOrTx, twitchLogin string, title string) error {
+	var err error
+	if config.Config.Discord.StreamsChannelID != "" {
+		err = discord.SendMessages(ctx, dbConn, discord.MessageToSend{
+			ChannelID: config.Config.Discord.StreamsChannelID,
+			Req: discord.CreateMessageRequest{
+				Content: fmt.Sprintf("%s is live: https://twitch.tv/%s\n> %s", twitchLogin, twitchLogin, title),
+			},
+		})
+	}
+	return err
+}
+
+func processEventSubNotification(ctx context.Context, dbConn db.ConnOrTx, notification *twitchNotification) {
+	log := logging.ExtractLogger(ctx)
+	log.Debug().Interface("Notification", notification).Msg("Processing twitch notification")
+	if notification.Type == notificationTypeNone {
+		return
+	}
+
+	status := streamStatus{
+		TwitchID: notification.TwitchID,
+		Live:     false,
+	}
+	var err error
+	if notification.Type == notificationTypeChannelUpdate || notification.Type == notificationTypeOnline {
+		result, err := getStreamStatus(ctx, []string{notification.TwitchID})
+		if err != nil || len(result) == 0 {
+			log.Error().Str("TwitchID", notification.TwitchID).Err(err).Msg("failed to fetch stream status")
+			return
+		}
+		allStreamers, err := hmndata.FetchTwitchStreamers(ctx, dbConn)
+		if err != nil {
+			log.Error().Err(err).Msg("failed to fetch hmn streamers")
+			return
+		}
+		for _, streamer := range allStreamers {
+			if streamer.TwitchLogin == result[0].TwitchLogin {
+				status = result[0]
+				break
+			}
+		}
+	}
+
+	log.Debug().Interface("Status", status).Msg("Updating status")
+	inserted, err := updateStreamStatusInDB(ctx, dbConn, &status)
+	if err != nil {
+		log.Error().Err(err).Msg("failed to update twitch stream status")
+	}
+	if inserted {
+		log.Debug().Msg("Notifying discord")
+		err = notifyDiscordOfLiveStream(ctx, dbConn, status.TwitchLogin, status.Title)
+		if err != nil {
+			log.Error().Err(err).Msg("failed to notify discord")
+		}
+	}
+}
+
+func updateStreamStatusInDB(ctx context.Context, conn db.ConnOrTx, status *streamStatus) (bool, error) {
+	log := logging.ExtractLogger(ctx)
+	inserted := false
+	if isStatusRelevant(status) {
+		log.Debug().Msg("Status relevant")
+		_, err := db.QueryOne(ctx, conn, models.TwitchStream{},
+			`
+			SELECT $columns
+			FROM twitch_streams
+			WHERE twitch_id = $1
+			`,
+			status.TwitchID,
+		)
+		if err == db.NotFound {
+			log.Debug().Msg("Inserting new stream")
+			inserted = true
+		} else if err != nil {
+			return false, oops.New(err, "failed to query existing stream")
+		}
+		_, err = conn.Exec(ctx,
+			`
+			INSERT INTO twitch_streams (twitch_id, twitch_login, title, started_at)
+				VALUES ($1, $2, $3, $4)
+			ON CONFLICT (twitch_id) DO UPDATE SET
+				title = EXCLUDED.title,
+				started_at = EXCLUDED.started_at
+			`,
+			status.TwitchID,
+			status.TwitchLogin,
+			status.Title,
+			status.StartedAt,
+		)
+		if err != nil {
+			return false, oops.New(err, "failed to insert twitch streamer into db")
+		}
+	} else {
+		log.Debug().Msg("Stream not relevant")
+		_, err := conn.Exec(ctx,
+			`
+			DELETE FROM twitch_streams WHERE twitch_id = $1
+			`,
+			status.TwitchID,
+		)
+		if err != nil {
+			return false, oops.New(err, "failed to remove twitch streamer from db")
+		}
+	}
+	return inserted, nil
+}
+
+var RelevantCategories = []string{
+	"1469308723", // Software and Game Development
+}
+
+var RelevantTags = []string{
+	"a59f1e4e-257b-4bd0-90c7-189c3efbf917", // Programming
+	"6f86127d-6051-4a38-94bb-f7b475dde109", // Software Development
+}
+
+func isStatusRelevant(status *streamStatus) bool {
+	if status.Live {
+		for _, cat := range RelevantCategories {
+			if status.Category == cat {
+				return true
+			}
+		}
+
+		for _, tag := range RelevantTags {
+			for _, streamTag := range status.Tags {
+				if tag == streamTag {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}

src/website/admin.go

@@ -6,6 +6,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"sort"
 	"strconv"
 	"time"
 
@@ -112,10 +113,23 @@ type postWithTitle struct {
 type adminApprovalQueueData struct {
 	templates.BaseData
 
-	Posts          []postWithTitle
-	SubmitUrl      string
-	ApprovalAction string
-	SpammerAction  string
+	UnapprovedUsers []*unapprovedUserData
+	SubmitUrl       string
+	ApprovalAction  string
+	SpammerAction   string
+}
+
+type projectWithLinks struct {
+	Project templates.Project
+	Links   []templates.Link
+}
+
+type unapprovedUserData struct {
+	User              templates.User
+	Date              time.Time
+	UserLinks         []templates.Link
+	Posts             []postWithTitle
+	ProjectsWithLinks []projectWithLinks
 }
 
 func AdminApprovalQueue(c *RequestContext) ResponseData {
@@ -129,22 +143,103 @@ func AdminApprovalQueue(c *RequestContext) ResponseData {
 		return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to fetch unapproved posts"))
 	}
 
-	data := adminApprovalQueueData{
-		BaseData:       getBaseDataAutocrumb(c, "Admin approval queue"),
-		SubmitUrl:      hmnurl.BuildAdminApprovalQueue(),
-		ApprovalAction: ApprovalQueueActionApprove,
-		SpammerAction:  ApprovalQueueActionSpammer,
+	projects, err := fetchUnapprovedProjects(c)
+	if err != nil {
+		return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to fetch unapproved projects"))
 	}
+
+	unapprovedUsers := make([]*unapprovedUserData, 0)
+	userIDToDataIdx := make(map[int]int)
+
 	for _, p := range posts {
+		var userData *unapprovedUserData
+		if idx, ok := userIDToDataIdx[p.Author.ID]; ok {
+			userData = unapprovedUsers[idx]
+		} else {
+			userData = &unapprovedUserData{
+				User:      templates.UserToTemplate(&p.Author, c.Theme),
+				UserLinks: make([]templates.Link, 0, 10),
+			}
+			unapprovedUsers = append(unapprovedUsers, userData)
+			userIDToDataIdx[p.Author.ID] = len(unapprovedUsers) - 1
+		}
+
+		if p.Post.PostDate.After(userData.Date) {
+			userData.Date = p.Post.PostDate
+		}
 		post := templates.PostToTemplate(&p.Post, &p.Author, c.Theme)
 		post.AddContentVersion(p.CurrentVersion, &p.Author) // NOTE(asaf): Don't care about editors here
 		post.Url = UrlForGenericPost(hmndata.UrlContextForProject(&p.Project), &p.Thread, &p.Post, lineageBuilder)
-		data.Posts = append(data.Posts, postWithTitle{
+		userData.Posts = append(userData.Posts, postWithTitle{
 			Post:  post,
 			Title: p.Thread.Title,
 		})
 	}
 
+	for _, p := range projects {
+		var userData *unapprovedUserData
+		if idx, ok := userIDToDataIdx[p.User.ID]; ok {
+			userData = unapprovedUsers[idx]
+		} else {
+			userData = &unapprovedUserData{
+				User:      templates.UserToTemplate(p.User, c.Theme),
+				UserLinks: make([]templates.Link, 0, 10),
+			}
+			unapprovedUsers = append(unapprovedUsers, userData)
+			userIDToDataIdx[p.User.ID] = len(unapprovedUsers) - 1
+		}
+
+		projectLinks := make([]templates.Link, 0, len(p.ProjectLinks))
+		for _, l := range p.ProjectLinks {
+			projectLinks = append(projectLinks, templates.LinkToTemplate(l))
+		}
+		if p.ProjectAndStuff.Project.DateCreated.After(userData.Date) {
+			userData.Date = p.ProjectAndStuff.Project.DateCreated
+		}
+		userData.ProjectsWithLinks = append(userData.ProjectsWithLinks, projectWithLinks{
+			Project: templates.ProjectAndStuffToTemplate(p.ProjectAndStuff, hmndata.UrlContextForProject(&p.ProjectAndStuff.Project).BuildHomepage(), c.Theme),
+			Links:   projectLinks,
+		})
+	}
+
+	userIds := make([]int, 0, len(unapprovedUsers))
+	for _, u := range unapprovedUsers {
+		userIds = append(userIds, u.User.ID)
+	}
+
+	userLinks, err := db.Query(c.Context(), c.Conn, models.Link{},
+		`
+		SELECT $columns
+		FROM
+			handmade_links
+		WHERE
+			user_id = ANY($1)
+		ORDER BY ordering ASC
+		`,
+		userIds,
+	)
+	if err != nil {
+		return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to fetch user links"))
+	}
+
+	for _, ul := range userLinks {
+		link := ul.(*models.Link)
+		userData := unapprovedUsers[userIDToDataIdx[*link.UserID]]
+		userData.UserLinks = append(userData.UserLinks, templates.LinkToTemplate(link))
+	}
+
+	sort.Slice(unapprovedUsers, func(a, b int) bool {
+		return unapprovedUsers[a].Date.After(unapprovedUsers[b].Date)
+	})
+
+	data := adminApprovalQueueData{
+		BaseData:        getBaseDataAutocrumb(c, "Admin approval queue"),
+		UnapprovedUsers: unapprovedUsers,
+		SubmitUrl:       hmnurl.BuildAdminApprovalQueue(),
+		ApprovalAction:  ApprovalQueueActionApprove,
+		SpammerAction:   ApprovalQueueActionSpammer,
+	}
+
 	var res ResponseData
 	res.MustWriteTemplate("admin_approval_queue.html", data, c.Perf)
 	return res
@@ -162,9 +257,15 @@ func AdminApprovalQueueSubmit(c *RequestContext) ResponseData {
 		return RejectRequest(c, "User id can't be parsed")
 	}
 
-	u, err := db.QueryOne(c.Context(), c.Conn, models.User{},
+	type userQuery struct {
+		User models.User `db:"auth_user"`
+	}
+	u, err := db.QueryOne(c.Context(), c.Conn, userQuery{},
 		`
-		SELECT $columns FROM auth_user WHERE id = $1
+		SELECT $columns
+			FROM auth_user
+			LEFT JOIN handmade_asset AS auth_user_avatar ON auth_user_avatar.id = auth_user.avatar_asset_id
+		WHERE auth_user.id = $1
 		`,
 		userId,
 	)
@@ -175,7 +276,7 @@ func AdminApprovalQueueSubmit(c *RequestContext) ResponseData {
 			return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to fetch user"))
 		}
 	}
-	user := u.(*models.User)
+	user := u.(*userQuery).User
 
 	whatHappened := ""
 	if action == ApprovalQueueActionApprove {
@@ -213,6 +314,10 @@ func AdminApprovalQueueSubmit(c *RequestContext) ResponseData {
 		if err != nil {
 			return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to delete spammer's posts"))
 		}
+		err = deleteAllProjectsForUser(c.Context(), c.Conn, user.ID)
+		if err != nil {
+			return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to delete spammer's projects"))
+		}
 		whatHappened = fmt.Sprintf("%s banned successfully", user.Username)
 	} else {
 		whatHappened = fmt.Sprintf("Unrecognized action: %s", action)
@@ -241,6 +346,7 @@ func fetchUnapprovedPosts(c *RequestContext) ([]*UnapprovedPost, error) {
 			JOIN handmade_thread AS thread ON post.thread_id = thread.id
 			JOIN handmade_postversion AS ver ON ver.id = post.current_id
 			JOIN auth_user AS author ON author.id = post.author_id
+			LEFT JOIN handmade_asset AS author_avatar ON author_avatar.id = author.avatar_asset_id
 		WHERE
 			NOT thread.deleted
 			AND NOT post.deleted
@@ -259,6 +365,86 @@ func fetchUnapprovedPosts(c *RequestContext) ([]*UnapprovedPost, error) {
 	return res, nil
 }
 
+type UnapprovedProject struct {
+	User            *models.User
+	ProjectAndStuff *hmndata.ProjectAndStuff
+	ProjectLinks    []*models.Link
+}
+
+func fetchUnapprovedProjects(c *RequestContext) ([]UnapprovedProject, error) {
+	type unapprovedUser struct {
+		ID int `db:"id"`
+	}
+	it, err := db.Query(c.Context(), c.Conn, unapprovedUser{},
+		`
+		SELECT $columns
+		FROM
+			auth_user AS u
+		WHERE
+			u.status = ANY($1)
+		`,
+		[]models.UserStatus{models.UserStatusConfirmed},
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to fetch unapproved users")
+	}
+	ownerIDs := make([]int, 0, len(it))
+	for _, uid := range it {
+		ownerIDs = append(ownerIDs, uid.(*unapprovedUser).ID)
+	}
+
+	projects, err := hmndata.FetchProjects(c.Context(), c.Conn, c.CurrentUser, hmndata.ProjectsQuery{
+		OwnerIDs:      ownerIDs,
+		IncludeHidden: true,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	projectIDs := make([]int, 0, len(projects))
+	for _, p := range projects {
+		projectIDs = append(projectIDs, p.Project.ID)
+	}
+
+	projectLinks, err := db.Query(c.Context(), c.Conn, models.Link{},
+		`
+		SELECT $columns
+		FROM
+			handmade_links AS link
+		WHERE
+			link.project_id = ANY($1)
+		ORDER BY link.ordering ASC
+		`,
+		projectIDs,
+	)
+	if err != nil {
+		return nil, oops.New(err, "failed to fetch links for projects")
+	}
+
+	var result []UnapprovedProject
+
+	for idx, proj := range projects {
+		links := make([]*models.Link, 0, 10) // NOTE(asaf): 10 should be enough for most projects.
+		for _, l := range projectLinks {
+			link := l.(*models.Link)
+			if *link.ProjectID == proj.Project.ID {
+				links = append(links, link)
+			}
+		}
+		for _, u := range proj.Owners {
+			if u.Status == models.UserStatusConfirmed {
+				result = append(result, UnapprovedProject{
+					User:            u,
+					ProjectAndStuff: &projects[idx],
+					ProjectLinks:    links,
+				})
+			}
+		}
+	}
+
+	return result, nil
+}
+
 func deleteAllPostsForUser(ctx context.Context, conn *pgxpool.Pool, userId int) error {
 	tx, err := conn.Begin(ctx)
 	if err != nil {
@@ -295,3 +481,50 @@ func deleteAllPostsForUser(ctx context.Context, conn *pgxpool.Pool, userId int)
 	}
 	return nil
 }
+
+func deleteAllProjectsForUser(ctx context.Context, conn *pgxpool.Pool, userId int) error {
+	tx, err := conn.Begin(ctx)
+	if err != nil {
+		return oops.New(err, "failed to start transaction")
+	}
+	defer tx.Rollback(ctx)
+
+	toDelete, err := db.Query(ctx, tx, models.Project{},
+		`
+		SELECT $columns
+		FROM
+			handmade_project AS project
+			JOIN handmade_user_projects AS up ON up.project_id = project.id
+		WHERE
+			up.user_id = $1
+		`,
+		userId,
+	)
+	if err != nil {
+		return oops.New(err, "failed to fetch user's projects")
+	}
+
+	var projectIds []int
+	for _, p := range toDelete {
+		projectIds = append(projectIds, p.(*models.Project).ID)
+	}
+
+	if len(projectIds) > 0 {
+		_, err = tx.Exec(ctx,
+			`
+			DELETE FROM handmade_project WHERE id = ANY($1)
+			`,
+			projectIds,
+		)
+		if err != nil {
+			return oops.New(err, "failed to delete user's projects")
+		}
+	}
+
+	err = tx.Commit(ctx)
+	if err != nil {
+		return oops.New(err, "failed to commit transaction")
+	}
+
+	return nil
+}

src/website/api.go

@@ -19,11 +19,15 @@ func APICheckUsername(c *RequestContext) ResponseData {
 		requestedUsername := usernameArgs[0]
 		found = true
 		c.Perf.StartBlock("SQL", "Fetch user")
-		userResult, err := db.QueryOne(c.Context(), c.Conn, models.User{},
+		type userQuery struct {
+			User models.User `db:"auth_user"`
+		}
+		userResult, err := db.QueryOne(c.Context(), c.Conn, userQuery{},
 			`
 			SELECT $columns
 			FROM
 				auth_user
+				LEFT JOIN handmade_asset AS auth_user_avatar ON auth_user_avatar.id = auth_user.avatar_asset_id
 			WHERE
 				LOWER(auth_user.username) = LOWER($1)
 				AND status = ANY ($2)
@@ -39,7 +43,7 @@ func APICheckUsername(c *RequestContext) ResponseData {
 				return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to fetch user: %s", requestedUsername))
 			}
 		} else {
-			canonicalUsername = userResult.(*models.User).Username
+			canonicalUsername = userResult.(*userQuery).User.Username
 		}
 	}
 

src/website/auth.go

@@ -75,7 +75,18 @@ func Login(c *RequestContext) ResponseData {
 		return res
 	}
 
-	userRow, err := db.QueryOne(c.Context(), c.Conn, models.User{}, "SELECT $columns FROM auth_user WHERE LOWER(username) = LOWER($1)", username)
+	type userQuery struct {
+		User models.User `db:"auth_user"`
+	}
+	userRow, err := db.QueryOne(c.Context(), c.Conn, userQuery{},
+		`
+		SELECT $columns
+			FROM auth_user
+			LEFT JOIN handmade_asset AS auth_user_avatar ON auth_user_avatar.id = auth_user.avatar_asset_id
+		WHERE LOWER(username) = LOWER($1)
+		`,
+		username,
+	)
 	if err != nil {
 		if errors.Is(err, db.NotFound) {
 			return showLoginWithFailure(c, redirect)
@@ -83,7 +94,7 @@ func Login(c *RequestContext) ResponseData {
 			return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to look up user by username"))
 		}
 	}
-	user := userRow.(*models.User)
+	user := &userRow.(*userQuery).User
 
 	success, err := tryLogin(c, user, password)
 
@@ -446,10 +457,14 @@ func RequestPasswordResetSubmit(c *RequestContext) ResponseData {
 	var user *models.User
 
 	c.Perf.StartBlock("SQL", "Fetching user")
-	userRow, err := db.QueryOne(c.Context(), c.Conn, models.User{},
+	type userQuery struct {
+		User models.User `db:"auth_user"`
+	}
+	userRow, err := db.QueryOne(c.Context(), c.Conn, userQuery{},
 		`
 		SELECT $columns
 		FROM auth_user
+		LEFT JOIN handmade_asset AS auth_user_avatar ON auth_user_avatar.id = auth_user.avatar_asset_id
 		WHERE
 			LOWER(username) = LOWER($1)
 			AND LOWER(email) = LOWER($2)
@@ -464,7 +479,7 @@ func RequestPasswordResetSubmit(c *RequestContext) ResponseData {
 		}
 	}
 	if userRow != nil {
-		user = userRow.(*models.User)
+		user = &userRow.(*userQuery).User
 	}
 
 	if user != nil {
@@ -776,6 +791,7 @@ func validateUsernameAndToken(c *RequestContext, username string, token string,
 		`
 		SELECT $columns
 		FROM auth_user
+		LEFT JOIN handmade_asset AS auth_user_avatar ON auth_user_avatar.id = auth_user.avatar_asset_id
 		LEFT JOIN handmade_onetimetoken AS onetimetoken ON onetimetoken.owner_id = auth_user.id
 		WHERE
 			LOWER(auth_user.username) = LOWER($1)

src/website/base_data.go

@@ -58,7 +58,7 @@ func getBaseData(c *RequestContext, title string, breadcrumbs []templates.Breadc
 
 		ReportIssueMailto: "team@handmade.network",
 
-		OpenGraphItems: buildDefaultOpenGraphItems(&project, title),
+		OpenGraphItems: buildDefaultOpenGraphItems(&project, c.CurrentProjectLogoUrl, title),
 
 		IsProjectPage: !project.IsHMN(),
 		Header: templates.Header{
@@ -79,11 +79,11 @@ func getBaseData(c *RequestContext, title string, breadcrumbs []templates.Breadc
 			HomepageUrl:                hmnurl.BuildHomepage(),
 			AboutUrl:                   hmnurl.BuildAbout(),
 			ManifestoUrl:               hmnurl.BuildManifesto(),
-			CodeOfConductUrl:           hmnurl.BuildCodeOfConduct(),
 			CommunicationGuidelinesUrl: hmnurl.BuildCommunicationGuidelines(),
 			ProjectIndexUrl:            hmnurl.BuildProjectIndex(1),
 			ForumsUrl:                  hmnurl.HMNProjectContext.BuildForum(nil, 1),
 			ContactUrl:                 hmnurl.BuildContactPage(),
+			SearchActionUrl:            "https://duckduckgo.com",
 		},
 	}
 
@@ -113,14 +113,14 @@ func getBaseData(c *RequestContext, title string, breadcrumbs []templates.Breadc
 	return baseData
 }
 
-func buildDefaultOpenGraphItems(project *models.Project, title string) []templates.OpenGraphItem {
+func buildDefaultOpenGraphItems(project *models.Project, projectLogoUrl string, title string) []templates.OpenGraphItem {
 	if title == "" {
 		title = "Handmade Network"
 	}
 
 	image := hmnurl.BuildPublic("logo.png", false)
 	if !project.IsHMN() {
-		image = hmnurl.BuildUserFile(project.LogoLight)
+		image = projectLogoUrl
 	}
 
 	return []templates.OpenGraphItem{

src/website/discord.go

@@ -78,6 +78,22 @@ func DiscordOAuthCallback(c *RequestContext) ResponseData {
 		return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to save new Discord user info"))
 	}
 
+	if c.CurrentUser.Status == models.UserStatusConfirmed {
+		_, err = c.Conn.Exec(c.Context(),
+			`
+			UPDATE auth_user
+			SET status = $1
+			WHERE id = $2
+			`,
+			models.UserStatusApproved,
+			c.CurrentUser.ID,
+		)
+		if err != nil {
+			c.Logger.Error().Err(err).Msg("failed to set user status to approved after linking discord account")
+			// NOTE(asaf): It's not worth failing the request over this, so we're not returning an error to the user.
+		}
+	}
+
 	return c.Redirect(hmnurl.BuildUserSettings("discord"), http.StatusSeeOther)
 }
 
@@ -143,17 +159,6 @@ func DiscordShowcaseBacklog(c *RequestContext) ResponseData {
 	}
 	duser := iduser.(*models.DiscordUser)
 
-	ok, err := discord.AllowedToCreateMessageSnippets(c.Context(), c.Conn, duser.UserID)
-	if err != nil {
-		return c.ErrorResponse(http.StatusInternalServerError, err)
-	}
-
-	if !ok {
-		// Not allowed to do this, bail out
-		c.Logger.Warn().Msg("was not allowed to save user snippets")
-		return c.Redirect(hmnurl.BuildUserProfile(c.CurrentUser.Username), http.StatusSeeOther)
-	}
-
 	type messageIdQuery struct {
 		MessageID string `db:"msg.id"`
 	}
@@ -177,9 +182,17 @@ func DiscordShowcaseBacklog(c *RequestContext) ResponseData {
 	for _, imsgId := range iMsgIDs {
 		msgIDs = append(msgIDs, imsgId.(*messageIdQuery).MessageID)
 	}
-	err = discord.CreateMessageSnippets(c.Context(), c.Conn, msgIDs...)
-	if err != nil {
-		return c.ErrorResponse(http.StatusInternalServerError, err)
+	for _, msgID := range msgIDs {
+		interned, err := discord.FetchInternedMessage(c.Context(), c.Conn, msgID)
+		if err != nil && !errors.Is(err, db.NotFound) {
+			return c.ErrorResponse(http.StatusInternalServerError, err)
+		} else if err == nil {
+			// NOTE(asaf): Creating snippet even if the checkbox is off because the user asked us to.
+			err = discord.HandleSnippetForInternedMessage(c.Context(), c.Conn, interned, true)
+			if err != nil {
+				return c.ErrorResponse(http.StatusInternalServerError, err)
+			}
+		}
 	}
 
 	return c.Redirect(hmnurl.BuildUserProfile(c.CurrentUser.Username), http.StatusSeeOther)

src/website/links_helper.go

@@ -0,0 +1,40 @@
+package website
+
+import (
+	"fmt"
+	"strings"
+
+	"git.handmade.network/hmn/hmn/src/models"
+)
+
+type ParsedLink struct {
+	Name string
+	Url  string
+}
+
+func ParseLinks(text string) []ParsedLink {
+	lines := strings.Split(text, "\n")
+	res := make([]ParsedLink, 0, len(lines))
+	for _, line := range lines {
+		linkParts := strings.SplitN(line, " ", 2)
+		url := strings.TrimSpace(linkParts[0])
+		name := ""
+		if len(linkParts) > 1 {
+			name = strings.TrimSpace(linkParts[1])
+		}
+		if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
+			continue
+		}
+		res = append(res, ParsedLink{Name: name, Url: url})
+	}
+	return res
+}
+
+func LinksToText(links []interface{}) string {
+	linksText := ""
+	for _, l := range links {
+		link := l.(*models.Link)
+		linksText += fmt.Sprintf("%s %s\n", link.URL, link.Name)
+	}
+	return linksText
+}

src/website/pagination.go

@@ -3,6 +3,8 @@ package website
 import (
 	"math"
 	"strconv"
+
+	"git.handmade.network/hmn/hmn/src/utils"
 )
 
 func getPageInfo(
@@ -14,7 +16,7 @@ func getPageInfo(
 	totalPages int,
 	ok bool,
 ) {
-	totalPages = int(math.Ceil(float64(totalItems) / float64(itemsPerPage)))
+	totalPages = utils.IntMax(1, int(math.Ceil(float64(totalItems)/float64(itemsPerPage))))
 	ok = true
 
 	page = 1

src/website/pagination_test.go

@@ -0,0 +1,36 @@
+package website
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetPageInfo(t *testing.T) {
+	items := []struct {
+		name                string
+		pageParam           string
+		totalItems, perPage int
+		page, totalPages    int
+		ok                  bool
+	}{
+		{"good, no param", "", 85, 10, 1, 9, true},
+		{"good", "2", 85, 10, 2, 9, true},
+		{"too big", "10", 85, 10, 0, 0, false},
+		{"too small", "0", 85, 10, 0, 0, false},
+		{"pizza", "pizza", 85, 10, 0, 0, false},
+		{"zero items, no param", "", 0, 10, 1, 1, true}, // should go to page 1
+		{"zero items, page 1", "1", 0, 10, 1, 1, true},
+		{"zero items, too big", "2", 0, 10, 0, 0, false},
+		{"zero items, too small", "0", 0, 10, 0, 0, false},
+	}
+
+	for _, item := range items {
+		t.Run(item.name, func(t *testing.T) {
+			page, totalPages, ok := getPageInfo(item.pageParam, item.totalItems, item.perPage)
+			assert.Equal(t, item.page, page)
+			assert.Equal(t, item.totalPages, totalPages)
+			assert.Equal(t, item.ok, ok)
+		})
+	}
+}

src/website/projects.go

@@ -9,6 +9,7 @@ import (
 	"math"
 	"math/rand"
 	"net/http"
+	"path"
 	"sort"
 	"strings"
 	"time"
@@ -21,6 +22,7 @@ import (
 	"git.handmade.network/hmn/hmn/src/oops"
 	"git.handmade.network/hmn/hmn/src/parsing"
 	"git.handmade.network/hmn/hmn/src/templates"
+	"git.handmade.network/hmn/hmn/src/twitch"
 	"git.handmade.network/hmn/hmn/src/utils"
 	"github.com/google/uuid"
 	"github.com/jackc/pgx/v4"
@@ -76,8 +78,7 @@ func ProjectIndex(c *RequestContext) ResponseData {
 	var restProjects []templates.Project
 	now := time.Now()
 	for _, p := range officialProjects {
-		templateProject := templates.ProjectToTemplate(&p.Project, hmndata.UrlContextForProject(&p.Project).BuildHomepage())
-		templateProject.AddLogo(p.LogoURL(c.Theme))
+		templateProject := templates.ProjectAndStuffToTemplate(&p, hmndata.UrlContextForProject(&p.Project).BuildHomepage(), c.Theme)
 
 		if p.Project.Slug == "hero" {
 			// NOTE(asaf): Handmade Hero gets special treatment. Must always be first in the list.
@@ -139,8 +140,7 @@ func ProjectIndex(c *RequestContext) ResponseData {
 			if i >= maxPersonalProjects {
 				break
 			}
-			templateProject := templates.ProjectToTemplate(&p.Project, hmndata.UrlContextForProject(&p.Project).BuildHomepage())
-			templateProject.AddLogo(p.LogoURL(c.Theme))
+			templateProject := templates.ProjectAndStuffToTemplate(&p, hmndata.UrlContextForProject(&p.Project).BuildHomepage(), c.Theme)
 			personalProjects = append(personalProjects, templateProject)
 		}
 	}
@@ -244,6 +244,7 @@ func ProjectHomepage(c *RequestContext) ResponseData {
 			handmade_post AS post
 			INNER JOIN handmade_thread AS thread ON thread.id = post.thread_id
 			INNER JOIN auth_user AS author ON author.id = post.author_id
+			LEFT JOIN handmade_asset AS author_avatar ON author_avatar.id = author.avatar_asset_id
 		WHERE
 			post.project_id = $1
 		ORDER BY post.postdate DESC
@@ -272,8 +273,7 @@ func ProjectHomepage(c *RequestContext) ResponseData {
 	if err != nil {
 		return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to fetch project details"))
 	}
-	templateData.Project = templates.ProjectToTemplate(c.CurrentProject, c.UrlContext.BuildHomepage())
-	templateData.Project.AddLogo(p.LogoURL(c.Theme))
+	templateData.Project = templates.ProjectAndStuffToTemplate(&p, c.UrlContext.BuildHomepage(), c.Theme)
 	for _, owner := range owners {
 		templateData.Owners = append(templateData.Owners, templates.UserToTemplate(owner, c.Theme))
 	}
@@ -496,14 +496,37 @@ func ProjectEdit(c *RequestContext) ResponseData {
 	if err != nil {
 		return c.ErrorResponse(http.StatusInternalServerError, err)
 	}
+
+	c.Perf.StartBlock("SQL", "Fetching project links")
+	projectLinkResult, err := db.Query(c.Context(), c.Conn, models.Link{},
+		`
+		SELECT $columns
+		FROM
+			handmade_links as link
+		WHERE
+			link.project_id = $1
+		ORDER BY link.ordering ASC
+		`,
+		p.Project.ID,
+	)
+	if err != nil {
+		return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to fetch project links"))
+	}
+	c.Perf.EndBlock()
+
+	lightLogoUrl := templates.ProjectLogoUrl(&p.Project, p.LogoLightAsset, p.LogoDarkAsset, "light")
+	darkLogoUrl := templates.ProjectLogoUrl(&p.Project, p.LogoLightAsset, p.LogoDarkAsset, "dark")
+
 	projectSettings := templates.ProjectToProjectSettings(
 		&p.Project,
 		p.Owners,
 		p.TagText(),
-		p.LogoURL("light"), p.LogoURL("dark"),
+		lightLogoUrl, darkLogoUrl,
 		c.Theme,
 	)
 
+	projectSettings.LinksText = LinksToText(projectLinkResult)
+
 	var res ResponseData
 	res.MustWriteTemplate("project_edit.html", ProjectEditData{
 		BaseData:        getBaseDataAutocrumb(c, "Edit Project"),
@@ -558,6 +581,7 @@ type ProjectPayload struct {
 	ProjectID         int
 	Name              string
 	Blurb             string
+	Links             []ParsedLink
 	Description       string
 	ParsedDescription string
 	Lifecycle         models.ProjectLifecycle
@@ -600,6 +624,7 @@ func ParseProjectEditForm(c *RequestContext) ProjectEditFormResult {
 		res.RejectionReason = "Projects must have a short description"
 		return res
 	}
+	links := ParseLinks(c.Req.Form.Get("links"))
 	description := c.Req.Form.Get("description")
 	parsedDescription := parsing.ParseMarkdown(description, parsing.ForumRealMarkdown)
 
@@ -650,6 +675,7 @@ func ParseProjectEditForm(c *RequestContext) ProjectEditFormResult {
 	res.Payload = ProjectPayload{
 		Name:              projectName,
 		Blurb:             shortDesc,
+		Links:             links,
 		Description:       description,
 		ParsedDescription: parsedDescription,
 		Lifecycle:         lifecycle,
@@ -714,33 +740,31 @@ func updateProject(ctx context.Context, tx pgx.Tx, user *models.User, payload *P
 		payload.OwnerUsernames = append(payload.OwnerUsernames, selfUsername)
 	}
 
-	var qb db.QueryBuilder
-	qb.Add(
+	_, err := tx.Exec(ctx,
 		`
 		UPDATE handmade_project SET
-			name = $?,
-			blurb = $?,
-			description = $?,
-			descparsed = $?,
-			lifecycle = $?
+			name = $2,
+			blurb = $3,
+			description = $4,
+			descparsed = $5,
+			lifecycle = $6
+		WHERE id = $1
 		`,
+		payload.ProjectID,
 		payload.Name,
 		payload.Blurb,
 		payload.Description,
 		payload.ParsedDescription,
 		payload.Lifecycle,
 	)
-	if user.IsStaff {
-		qb.Add(`, hidden = $?`, payload.Hidden)
-	}
-	qb.Add(`WHERE id = $?`, payload.ProjectID)
-
-	_, err := tx.Exec(ctx, qb.String(), qb.Args()...)
 	if err != nil {
 		return oops.New(err, "Failed to update project")
 	}
 
-	hmndata.SetProjectTag(ctx, tx, payload.ProjectID, payload.Tag)
+	_, err = hmndata.SetProjectTag(ctx, tx, user, payload.ProjectID, payload.Tag)
+	if err != nil {
+		return err
+	}
 
 	if user.IsStaff {
 		_, err = tx.Exec(ctx,
@@ -748,7 +772,8 @@ func updateProject(ctx context.Context, tx pgx.Tx, user *models.User, payload *P
 			UPDATE handmade_project SET
 				slug = $2,
 				featured = $3,
-				personal = $4
+				personal = $4,
+				hidden = $5
 			WHERE
 				id = $1
 			`,
@@ -756,6 +781,7 @@ func updateProject(ctx context.Context, tx pgx.Tx, user *models.User, payload *P
 			payload.Slug,
 			payload.Featured,
 			payload.Personal,
+			payload.Hidden,
 		)
 		if err != nil {
 			return oops.New(err, "Failed to update project with admin fields")
@@ -796,10 +822,14 @@ func updateProject(ctx context.Context, tx pgx.Tx, user *models.User, payload *P
 		}
 	}
 
-	ownerRows, err := db.Query(ctx, tx, models.User{},
+	type userQuery struct {
+		User models.User `db:"auth_user"`
+	}
+	ownerRows, err := db.Query(ctx, tx, userQuery{},
 		`
 		SELECT $columns
 		FROM auth_user
+		LEFT JOIN handmade_asset AS auth_user_avatar ON auth_user_avatar.id = auth_user.avatar_asset_id
 		WHERE LOWER(username) = ANY ($1)
 		`,
 		payload.OwnerUsernames,
@@ -827,7 +857,7 @@ func updateProject(ctx context.Context, tx pgx.Tx, user *models.User, payload *P
 			VALUES
 				($1,      $2)
 			`,
-			ownerRow.(*models.User).ID,
+			ownerRow.(*userQuery).User.ID,
 			payload.ProjectID,
 		)
 		if err != nil {
@@ -835,6 +865,31 @@ func updateProject(ctx context.Context, tx pgx.Tx, user *models.User, payload *P
 		}
 	}
 
+	twitchLoginsPreChange, preErr := hmndata.FetchTwitchLoginsForUserOrProject(ctx, tx, nil, &payload.ProjectID)
+	_, err = tx.Exec(ctx, `DELETE FROM handmade_links WHERE project_id = $1`, payload.ProjectID)
+	if err != nil {
+		return oops.New(err, "Failed to delete project links")
+	}
+	for i, link := range payload.Links {
+		_, err = tx.Exec(ctx,
+			`
+			INSERT INTO handmade_links (name, url, ordering, project_id)
+			VALUES ($1, $2, $3, $4)
+			`,
+			link.Name,
+			link.Url,
+			i,
+			payload.ProjectID,
+		)
+		if err != nil {
+			return oops.New(err, "Failed to insert new project link")
+		}
+	}
+	twitchLoginsPostChange, postErr := hmndata.FetchTwitchLoginsForUserOrProject(ctx, tx, nil, &payload.ProjectID)
+	if preErr == nil && postErr == nil {
+		twitch.UserOrProjectLinksUpdated(twitchLoginsPreChange, twitchLoginsPostChange)
+	}
+
 	return nil
 }
 
@@ -874,13 +929,28 @@ func GetFormImage(c *RequestContext, fieldName string) (FormImage, error) {
 		img.Read(res.Content)
 		img.Seek(0, io.SeekStart)
 
-		config, _, err := image.DecodeConfig(img)
-		if err != nil {
-			return FormImage{}, err
+		fileExtensionOverrides := []string{".svg"}
+		fileExt := strings.ToLower(path.Ext(res.Filename))
+		tryDecode := true
+		for _, ext := range fileExtensionOverrides {
+			if fileExt == ext {
+				tryDecode = false
+			}
+		}
+
+		if tryDecode {
+			config, _, err := image.DecodeConfig(img)
+			if err != nil {
+				return FormImage{}, err
+			}
+			res.Width = config.Width
+			res.Height = config.Height
+			res.Mime = http.DetectContentType(res.Content)
+		} else {
+			if fileExt == ".svg" {
+				res.Mime = "image/svg+xml"
+			}
 		}
-		res.Width = config.Width
-		res.Height = config.Height
-		res.Mime = http.DetectContentType(res.Content)
 	}
 
 	return res, nil

src/website/requesthandling.go

@@ -159,12 +159,13 @@ type RequestContext struct {
 	//             We sometimes need the original response object so that some functions of the http package can set connection-management flags on it.
 	Res http.ResponseWriter
 
-	Conn           *pgxpool.Pool
-	CurrentProject *models.Project
-	CurrentUser    *models.User
-	CurrentSession *models.Session
-	Theme          string
-	UrlContext     *hmnurl.UrlContext
+	Conn                  *pgxpool.Pool
+	CurrentProject        *models.Project
+	CurrentProjectLogoUrl string
+	CurrentUser           *models.User
+	CurrentSession        *models.Session
+	Theme                 string
+	UrlContext            *hmnurl.UrlContext
 
 	CurrentUserCanEditCurrentProject bool
 

src/website/routes.go

@@ -160,7 +160,6 @@ func NewWebsiteRoutes(longRequestContext context.Context, conn *pgxpool.Pool) ht
 	// NOTE(asaf): HMN-only routes:
 	hmnOnly.GET(hmnurl.RegexManifesto, Manifesto)
 	hmnOnly.GET(hmnurl.RegexAbout, About)
-	hmnOnly.GET(hmnurl.RegexCodeOfConduct, CodeOfConduct)
 	hmnOnly.GET(hmnurl.RegexCommunicationGuidelines, CommunicationGuidelines)
 	hmnOnly.GET(hmnurl.RegexContactPage, ContactPage)
 	hmnOnly.GET(hmnurl.RegexMonthlyUpdatePolicy, MonthlyUpdatePolicy)
@@ -206,6 +205,9 @@ func NewWebsiteRoutes(longRequestContext context.Context, conn *pgxpool.Pool) ht
 	hmnOnly.POST(hmnurl.RegexDiscordUnlink, authMiddleware(csrfMiddleware(DiscordUnlink)))
 	hmnOnly.POST(hmnurl.RegexDiscordShowcaseBacklog, authMiddleware(csrfMiddleware(DiscordShowcaseBacklog)))
 
+	hmnOnly.POST(hmnurl.RegexTwitchEventSubCallback, TwitchEventSubCallback)
+	hmnOnly.GET(hmnurl.RegexTwitchDebugPage, TwitchDebugPage)
+
 	hmnOnly.GET(hmnurl.RegexUserProfile, UserProfile)
 	hmnOnly.GET(hmnurl.RegexUserSettings, authMiddleware(UserSettings))
 	hmnOnly.POST(hmnurl.RegexUserSettings, authMiddleware(csrfMiddleware(UserSettingsSave)))
@@ -476,6 +478,7 @@ func LoadCommonWebsiteData(c *RequestContext) (bool, ResponseData) {
 			})
 			if err == nil {
 				c.CurrentProject = &dbProject.Project
+				c.CurrentProjectLogoUrl = templates.ProjectLogoUrl(&dbProject.Project, dbProject.LogoLightAsset, dbProject.LogoDarkAsset, c.Theme)
 				owners = dbProject.Owners
 			} else {
 				if errors.Is(err, db.NotFound) {
@@ -495,26 +498,14 @@ func LoadCommonWebsiteData(c *RequestContext) (bool, ResponseData) {
 				panic(oops.New(err, "failed to fetch HMN project"))
 			}
 			c.CurrentProject = &dbProject.Project
+			c.CurrentProjectLogoUrl = templates.ProjectLogoUrl(&dbProject.Project, dbProject.LogoLightAsset, dbProject.LogoDarkAsset, c.Theme)
 		}
 
 		if c.CurrentProject == nil {
 			panic("failed to load project data")
 		}
 
-		canEditProject := false
-		if c.CurrentUser != nil {
-			if c.CurrentUser.IsStaff {
-				canEditProject = true
-			} else {
-				for _, o := range owners {
-					if o.ID == c.CurrentUser.ID {
-						canEditProject = true
-						break
-					}
-				}
-			}
-		}
-		c.CurrentUserCanEditCurrentProject = canEditProject
+		c.CurrentUserCanEditCurrentProject = CanEditProject(c.CurrentUser, owners)
 
 		c.UrlContext = hmndata.UrlContextForProject(c.CurrentProject)
 	}
@@ -557,7 +548,18 @@ func getCurrentUserAndSession(c *RequestContext, sessionId string) (*models.User
 		}
 	}
 
-	userRow, err := db.QueryOne(c.Context(), c.Conn, models.User{}, "SELECT $columns FROM auth_user WHERE username = $1", session.Username)
+	type userQuery struct {
+		User models.User `db:"auth_user"`
+	}
+	userRow, err := db.QueryOne(c.Context(), c.Conn, userQuery{},
+		`
+		SELECT $columns
+			FROM auth_user
+			LEFT JOIN handmade_asset AS auth_user_avatar ON auth_user_avatar.id = auth_user.avatar_asset_id
+		WHERE username = $1
+		`,
+		session.Username,
+	)
 	if err != nil {
 		if errors.Is(err, db.NotFound) {
 			logging.Debug().Str("username", session.Username).Msg("returning no current user for this request because the user for the session couldn't be found")
@@ -566,7 +568,7 @@ func getCurrentUserAndSession(c *RequestContext, sessionId string) (*models.User
 			return nil, nil, oops.New(err, "failed to get user for session")
 		}
 	}
-	user := userRow.(*models.User)
+	user := &userRow.(*userQuery).User
 
 	return user, session, nil
 }

src/website/staticpages.go

@@ -6,7 +6,7 @@ func Manifesto(c *RequestContext) ResponseData {
 	baseData := getBaseDataAutocrumb(c, "Handmade Manifesto")
 	baseData.OpenGraphItems = append(baseData.OpenGraphItems, templates.OpenGraphItem{
 		Property: "og:description",
-		Value:    "Modern computer hardware is amazing. Manufacturers have orchestrated billions of pieces of silicon into terrifyingly complex and efficient structures…",
+		Value:    "Computers are amazing. So why is software so terrible?",
 	})
 
 	var res ResponseData
@@ -20,18 +20,6 @@ func About(c *RequestContext) ResponseData {
 	return res
 }
 
-func CodeOfConduct(c *RequestContext) ResponseData {
-	baseData := getBaseDataAutocrumb(c, "Code of Conduct")
-	baseData.OpenGraphItems = append(baseData.OpenGraphItems, templates.OpenGraphItem{
-		Property: "og:description",
-		Value:    "The Handmade community is an international community of creatives dedicated to building and improving high quality software. These are the guidelines we pledge to uphold to maintain a healthy community.",
-	})
-
-	var res ResponseData
-	res.MustWriteTemplate("code_of_conduct.html", baseData, c.Perf)
-	return res
-}
-
 func CommunicationGuidelines(c *RequestContext) ResponseData {
 	baseData := getBaseDataAutocrumb(c, "Communication Guidelines")
 	baseData.OpenGraphItems = append(baseData.OpenGraphItems, templates.OpenGraphItem{

src/website/timeline_helper.go

@@ -89,6 +89,8 @@ func SnippetToTimelineItem(
 			item.EmbedMedia = append(item.EmbedMedia, videoMediaItem(asset))
 		} else if strings.HasPrefix(asset.MimeType, "audio/") {
 			item.EmbedMedia = append(item.EmbedMedia, audioMediaItem(asset))
+		} else {
+			item.EmbedMedia = append(item.EmbedMedia, unknownMediaItem(asset))
 		}
 	}
 
@@ -100,7 +102,8 @@ func SnippetToTimelineItem(
 		}
 	}
 
-	if len(item.EmbedMedia) > 0 && (item.EmbedMedia[0].Width == 0 || item.EmbedMedia[0].Height == 0) {
+	if len(item.EmbedMedia) == 0 ||
+		(len(item.EmbedMedia) > 0 && (item.EmbedMedia[0].Width == 0 || item.EmbedMedia[0].Height == 0)) {
 		item.CanShowcase = false
 	}
 
@@ -185,3 +188,15 @@ func youtubeMediaItem(videoId string) templates.TimelineItemMedia {
 		},
 	}
 }
+
+func unknownMediaItem(asset *models.Asset) templates.TimelineItemMedia {
+	assetUrl := hmnurl.BuildS3Asset(asset.S3Key)
+
+	return templates.TimelineItemMedia{
+		Type:     templates.TimelineItemMediaTypeUnknown,
+		AssetUrl: assetUrl,
+		MimeType: asset.MimeType,
+		Filename: asset.Filename,
+		FileSize: asset.Size,
+	}
+}

src/website/twitch.go

@@ -0,0 +1,94 @@
+package website
+
+import (
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+
+	"git.handmade.network/hmn/hmn/src/config"
+	"git.handmade.network/hmn/hmn/src/db"
+	"git.handmade.network/hmn/hmn/src/models"
+	"git.handmade.network/hmn/hmn/src/oops"
+	"git.handmade.network/hmn/hmn/src/twitch"
+)
+
+func TwitchEventSubCallback(c *RequestContext) ResponseData {
+	secret := config.Config.Twitch.EventSubSecret
+	messageId := c.Req.Header.Get("Twitch-Eventsub-Message-Id")
+	timestamp := c.Req.Header.Get("Twitch-Eventsub-Message-Timestamp")
+	signature := c.Req.Header.Get("Twitch-Eventsub-Message-Signature")
+	messageType := c.Req.Header.Get("Twitch-Eventsub-Message-Type")
+
+	body, err := io.ReadAll(c.Req.Body)
+	if err != nil {
+		return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to read request body"))
+	}
+
+	hmacMessage := fmt.Sprintf("%s%s%s", messageId, timestamp, string(body[:]))
+	hmac := hmac.New(sha256.New, []byte(secret))
+	hmac.Write([]byte(hmacMessage))
+	hash := hmac.Sum(nil)
+	hmacStr := "sha256=" + hex.EncodeToString(hash)
+
+	if hmacStr != signature {
+		var res ResponseData
+		res.StatusCode = 403
+		return res
+	}
+
+	c.Logger.Debug().Str("Body", string(body[:])).Str("Type", messageType).Msg("Got twitch webhook")
+
+	if messageType == "webhook_callback_verification" {
+		type challengeReq struct {
+			Challenge string `json:"challenge"`
+		}
+		var data challengeReq
+		err = json.Unmarshal(body, &data)
+		if err != nil {
+			return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to unmarshal twitch verification"))
+		}
+		var res ResponseData
+		res.StatusCode = 200
+
+		res.Header().Set("Content-Type", "text/plain") // NOTE(asaf): No idea why, but the twitch-cli fails when we don't set this.
+		res.Write([]byte(data.Challenge))
+		return res
+	} else {
+		err := twitch.QueueTwitchNotification(messageType, body)
+		if err != nil {
+			c.Logger.Error().Err(err).Msg("Failed to process twitch callback")
+			// NOTE(asaf): Returning 200 either way here
+		}
+		var res ResponseData
+		res.StatusCode = 200
+		return res
+	}
+}
+
+func TwitchDebugPage(c *RequestContext) ResponseData {
+	streams, err := db.Query(c.Context(), c.Conn, models.TwitchStream{},
+		`
+		SELECT $columns
+		FROM
+			twitch_streams
+		ORDER BY started_at DESC
+		`,
+	)
+	if err != nil {
+		return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to fetch twitch streams"))
+	}
+
+	html := ""
+	for _, stream := range streams {
+		s := stream.(*models.TwitchStream)
+		html += fmt.Sprintf(`<a href="https://twitch.tv/%s">%s</a>%s<br />`, s.Login, s.Login, s.Title)
+	}
+	var res ResponseData
+	res.StatusCode = 200
+	res.Write([]byte(html))
+	return res
+}

src/website/user.go

@@ -2,13 +2,12 @@ package website
 
 import (
 	"errors"
-	"fmt"
 	"net/http"
 	"sort"
 	"strconv"
 	"strings"
-	"time"
 
+	"git.handmade.network/hmn/hmn/src/assets"
 	"git.handmade.network/hmn/hmn/src/auth"
 	"git.handmade.network/hmn/hmn/src/config"
 	"git.handmade.network/hmn/hmn/src/db"
@@ -19,6 +18,8 @@ import (
 	"git.handmade.network/hmn/hmn/src/models"
 	"git.handmade.network/hmn/hmn/src/oops"
 	"git.handmade.network/hmn/hmn/src/templates"
+	"git.handmade.network/hmn/hmn/src/twitch"
+	"github.com/google/uuid"
 	"github.com/jackc/pgx/v4"
 )
 
@@ -52,11 +53,15 @@ func UserProfile(c *RequestContext) ResponseData {
 		profileUser = c.CurrentUser
 	} else {
 		c.Perf.StartBlock("SQL", "Fetch user")
-		userResult, err := db.QueryOne(c.Context(), c.Conn, models.User{},
+		type userQuery struct {
+			User models.User `db:"auth_user"`
+		}
+		userResult, err := db.QueryOne(c.Context(), c.Conn, userQuery{},
 			`
 			SELECT $columns
 			FROM
 				auth_user
+				LEFT JOIN handmade_asset AS auth_user_avatar ON auth_user_avatar.id = auth_user.avatar_asset_id
 			WHERE
 				LOWER(auth_user.username) = $1
 			`,
@@ -70,7 +75,7 @@ func UserProfile(c *RequestContext) ResponseData {
 				return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to fetch user: %s", username))
 			}
 		}
-		profileUser = userResult.(*models.User)
+		profileUser = &userResult.(*userQuery).User
 	}
 
 	{
@@ -114,8 +119,7 @@ func UserProfile(c *RequestContext) ResponseData {
 	templateProjects := make([]templates.Project, 0, len(projectsAndStuff))
 	numPersonalProjects := 0
 	for _, p := range projectsAndStuff {
-		templateProject := templates.ProjectToTemplate(&p.Project, hmndata.UrlContextForProject(&p.Project).BuildHomepage())
-		templateProject.AddLogo(p.LogoURL(c.Theme))
+		templateProject := templates.ProjectAndStuffToTemplate(&p, hmndata.UrlContextForProject(&p.Project).BuildHomepage(), c.Theme)
 		templateProjects = append(templateProjects, templateProject)
 
 		if p.Project.Personal {
@@ -201,12 +205,17 @@ func UserProfile(c *RequestContext) ResponseData {
 	return res
 }
 
+var UserAvatarMaxFileSize = 1 * 1024 * 1024
+
 func UserSettings(c *RequestContext) ResponseData {
 	var res ResponseData
 
 	type UserSettingsTemplateData struct {
 		templates.BaseData
 
+		AvatarMaxFileSize int
+		DefaultAvatarUrl  string
+
 		User      templates.User
 		Email     string // these fields are handled specially on templates.User
 		ShowEmail bool
@@ -235,11 +244,7 @@ func UserSettings(c *RequestContext) ResponseData {
 		return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to fetch user links"))
 	}
 
-	linksText := ""
-	for _, ilink := range links {
-		link := ilink.(*models.Link)
-		linksText += fmt.Sprintf("%s %s\n", link.URL, link.Name)
-	}
+	linksText := LinksToText(links)
 
 	var tduser *templates.DiscordUser
 	var numUnsavedMessages int
@@ -284,11 +289,13 @@ func UserSettings(c *RequestContext) ResponseData {
 	baseData := getBaseDataAutocrumb(c, templateUser.Name)
 
 	res.MustWriteTemplate("user_settings.html", UserSettingsTemplateData{
-		BaseData:  baseData,
-		User:      templateUser,
-		Email:     c.CurrentUser.Email,
-		ShowEmail: c.CurrentUser.ShowEmail,
-		LinksText: linksText,
+		BaseData:          baseData,
+		AvatarMaxFileSize: UserAvatarMaxFileSize,
+		DefaultAvatarUrl:  templates.UserAvatarDefaultUrl(c.Theme),
+		User:              templateUser,
+		Email:             c.CurrentUser.Email,
+		ShowEmail:         c.CurrentUser.ShowEmail,
+		LinksText:         linksText,
 
 		SubmitUrl:  hmnurl.BuildUserSettings(""),
 		ContactUrl: hmnurl.BuildContactPage(),
@@ -303,6 +310,14 @@ func UserSettings(c *RequestContext) ResponseData {
 }
 
 func UserSettingsSave(c *RequestContext) ResponseData {
+	maxBodySize := int64(UserAvatarMaxFileSize + 2*1024*1024)
+	c.Req.Body = http.MaxBytesReader(c.Res, c.Req.Body, maxBodySize)
+	err := c.Req.ParseMultipartForm(maxBodySize)
+	if err != nil {
+		// NOTE(asaf): The error for exceeding the max filesize doesn't have a special type, so we can't easily detect it here.
+		return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to parse form"))
+	}
+
 	tx, err := c.Conn.Begin(c.Context())
 	if err != nil {
 		panic(err)
@@ -364,32 +379,21 @@ func UserSettingsSave(c *RequestContext) ResponseData {
 	}
 
 	// Process links
+	twitchLoginsPreChange, preErr := hmndata.FetchTwitchLoginsForUserOrProject(c.Context(), tx, &c.CurrentUser.ID, nil)
 	linksText := form.Get("links")
-	links := strings.Split(linksText, "\n")
+	links := ParseLinks(linksText)
 	_, err = tx.Exec(c.Context(), `DELETE FROM handmade_links WHERE user_id = $1`, c.CurrentUser.ID)
 	if err != nil {
 		c.Logger.Warn().Err(err).Msg("failed to delete old links")
 	} else {
 		for i, link := range links {
-			link = strings.TrimSpace(link)
-			linkParts := strings.SplitN(link, " ", 2)
-			url := strings.TrimSpace(linkParts[0])
-			name := ""
-			if len(linkParts) > 1 {
-				name = strings.TrimSpace(linkParts[1])
-			}
-
-			if !strings.HasPrefix(url, "http://") && !strings.HasPrefix(url, "https://") {
-				continue
-			}
-
 			_, err := tx.Exec(c.Context(),
 				`
 				INSERT INTO handmade_links (name, url, ordering, user_id)
 				VALUES ($1, $2, $3, $4)
 				`,
-				name,
-				url,
+				link.Name,
+				link.Url,
 				i,
 				c.CurrentUser.ID,
 			)
@@ -399,6 +403,10 @@ func UserSettingsSave(c *RequestContext) ResponseData {
 			}
 		}
 	}
+	twitchLoginsPostChange, postErr := hmndata.FetchTwitchLoginsForUserOrProject(c.Context(), tx, &c.CurrentUser.ID, nil)
+	if preErr == nil && postErr == nil {
+		twitch.UserOrProjectLinksUpdated(twitchLoginsPreChange, twitchLoginsPostChange)
+	}
 
 	// Update password
 	oldPassword := form.Get("old_password")
@@ -412,25 +420,39 @@ func UserSettingsSave(c *RequestContext) ResponseData {
 	}
 
 	// Update avatar
-	imageSaveResult := SaveImageFile(c, tx, "avatar", 1*1024*1024, fmt.Sprintf("members/avatars/%s-%d", c.CurrentUser.Username, time.Now().UTC().Unix()))
-	if imageSaveResult.ValidationError != "" {
-		return RejectRequest(c, imageSaveResult.ValidationError)
-	} else if imageSaveResult.FatalError != nil {
-		return c.ErrorResponse(http.StatusInternalServerError, oops.New(imageSaveResult.FatalError, "failed to save new avatar"))
-	} else if imageSaveResult.ImageFile != nil {
-		_, err = tx.Exec(c.Context(),
+	newAvatar, err := GetFormImage(c, "avatar")
+	if err != nil {
+		return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to read image from form"))
+	}
+	var avatarUUID *uuid.UUID
+	if newAvatar.Exists {
+		avatarAsset, err := assets.Create(c.Context(), tx, assets.CreateInput{
+			Content:     newAvatar.Content,
+			Filename:    newAvatar.Filename,
+			ContentType: newAvatar.Mime,
+			UploaderID:  &c.CurrentUser.ID,
+			Width:       newAvatar.Width,
+			Height:      newAvatar.Height,
+		})
+		if err != nil {
+			return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to upload avatar"))
+		}
+		avatarUUID = &avatarAsset.ID
+	}
+	if newAvatar.Exists || newAvatar.Remove {
+		_, err := tx.Exec(c.Context(),
 			`
 			UPDATE auth_user
 			SET
-				avatar = $2
+				avatar_asset_id = $2
 			WHERE
 				id = $1
 			`,
 			c.CurrentUser.ID,
-			imageSaveResult.ImageFile.File,
+			avatarUUID,
 		)
 		if err != nil {
-			return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to update user"))
+			return c.ErrorResponse(http.StatusInternalServerError, oops.New(err, "failed to update user's avatar"))
 		}
 	}
 

src/website/website.go

@@ -17,6 +17,7 @@ import (
 	"git.handmade.network/hmn/hmn/src/logging"
 	"git.handmade.network/hmn/hmn/src/perf"
 	"git.handmade.network/hmn/hmn/src/templates"
+	"git.handmade.network/hmn/hmn/src/twitch"
 	"github.com/spf13/cobra"
 )
 
@@ -46,6 +47,7 @@ var WebsiteCommand = &cobra.Command{
 			perfCollector.Done,
 			discord.RunDiscordBot(backgroundJobContext, conn),
 			discord.RunHistoryWatcher(backgroundJobContext, conn),
+			twitch.MonitorTwitchSubscriptions(backgroundJobContext, conn),
 		)
 
 		signals := make(chan os.Signal, 1)