nick12
/
hmn
forked from hmn/hmn
1
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Unified timeline query

This commit is contained in:
Asaf Gartner 2024-06-27 22:45:23 +03:00
parent 410c94bb51
commit f9ada49278
8 changed files with 479 additions and 150 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

289
src/hmndata/timeline_helper.go Normal file
View File

@ -0,0 +1,289 @@
package hmndata
import (
"context"
"strings"
"git.handmade.network/hmn/hmn/src/db"
"git.handmade.network/hmn/hmn/src/models"
"git.handmade.network/hmn/hmn/src/oops"
"git.handmade.network/hmn/hmn/src/perf"
)
/*
WITH snippet_item AS (
SELECT id,
"when",
'snippet' AS timeline_type,
owner_id,
'' AS title,
_description_html AS parsed_desc,
description AS raw_desc,
asset_id,
discord_message_id,
url,
0 AS project_id,
0 AS thread_id,
0 AS subforum_id,
0 AS thread_type,
false AS stream_ended,
NOW() AS stream_end_time,
'' AS twitch_login,
'' AS stream_id
FROM snippet
),
post_item AS (
SELECT post.id,
postdate AS "when",
'post' AS timeline_type,
author_id AS owner_id,
thread.title AS title,
post_version.text_parsed AS parsed_desc,
post_version.text_raw AS raw_desc,
NULL::uuid AS asset_id,
NULL AS discord_message_id,
NULL AS url,
post.project_id,
thread_id,
subforum_id,
0 AS thread_type,
false AS stream_ended,
NOW() AS stream_end_time,
'' AS twitch_login,
'' AS stream_id
FROM post
JOIN thread ON thread.id = post.thread_id
JOIN post_version ON post_version.id = post.current_id
WHERE post.deleted = false AND thread.deleted = false
)
SELECT * from snippet_item
UNION ALL
SELECT * from post_item
ORDER BY "when" DESC LIMIT 100;
*/
type TimelineQuery struct {
OwnerIDs []int
ProjectIDs []int
SkipSnippets bool
SkipPosts bool
Limit, Offset int
}
type TimelineItemAndStuff struct {
Item models.TimelineItem `db:"item"`
Owner *models.User `db:"owner"`
AvatarAsset *models.Asset `db:"avatar"`
Asset *models.Asset `db:"asset"`
DiscordMessage *models.DiscordMessage `db:"discord_message"`
Projects []*ProjectAndStuff
}
func FetchTimeline(
ctx context.Context,
dbConn db.ConnOrTx,
currentUser *models.User,
q TimelineQuery,
) ([]*TimelineItemAndStuff, error) {
perf := perf.ExtractPerf(ctx)
perf.StartBlock("SQL", "Fetch timeline")
defer perf.EndBlock()
var qb db.QueryBuilder
itemSelects := []string{}
if !q.SkipSnippets {
itemSelects = append(itemSelects, "SELECT * from snippet_item")
}
if !q.SkipPosts {
itemSelects = append(itemSelects, "SELECT * from post_item")
}
if len(itemSelects) == 0 {
return nil, nil
}
itemSelect := strings.Join(itemSelects, " UNION ALL ")
qb.Add(
`
WITH snippet_item AS (
SELECT id,
"when",
'snippet' AS timeline_type,
owner_id,
'' AS title,
_description_html AS parsed_desc,
description AS raw_desc,
asset_id,
discord_message_id,
url,
0 AS project_id,
0 AS thread_id,
0 AS subforum_id,
0 AS thread_type,
TRUE AS first_post
FROM snippet
WHERE TRUE
`,
)
if len(q.ProjectIDs) > 0 {
qb.Add(
`
AND (
SELECT count(*)
FROM snippet_project
WHERE
snippet_project.snippet_id = snippet.id
AND
snippet_project.project_id = ANY($?)
) > 0
`,
q.ProjectIDs,
)
}
qb.Add(
`
),
post_item AS (
SELECT post.id,
postdate AS "when",
'post' AS timeline_type,
author_id AS owner_id,
thread.title AS title,
post_version.text_parsed AS parsed_desc,
post_version.text_raw AS raw_desc,
NULL::uuid AS asset_id,
NULL AS discord_message_id,
NULL AS url,
post.project_id,
thread_id,
subforum_id,
thread_type,
(post.id = thread.first_id) AS first_post
FROM post
JOIN thread ON thread.id = thread_id
JOIN post_version ON post_version.id = current_id
WHERE post.deleted = false AND thread.deleted = false
`,
)
if len(q.ProjectIDs) > 0 {
qb.Add(`AND post.project_id = ANY($?)`, q.ProjectIDs)
}
qb.Add(
`
),
item AS (
`,
)
qb.Add(itemSelect)
qb.Add(
`
)
SELECT $columns FROM item
LEFT JOIN thread ON thread.id = thread_id
LEFT JOIN hmn_user AS owner ON owner_id = owner.id
LEFT JOIN asset AS avatar ON avatar.id = owner.avatar_asset_id
LEFT JOIN asset ON asset_id = asset.id
LEFT JOIN discord_message ON discord_message_id = discord_message.id
WHERE TRUE
`,
)
if len(q.OwnerIDs) > 0 {
qb.Add(`AND owner_id = ANY($?)`, q.OwnerIDs)
}
if currentUser == nil {
qb.Add(
`AND owner.status = $? -- snippet owner is Approved`,
models.UserStatusApproved,
)
} else if !currentUser.IsStaff {
qb.Add(
`
AND (
owner.status = $? -- snippet owner is Approved
OR owner.id = $? -- current user is the snippet owner
)
`,
models.UserStatusApproved,
currentUser.ID,
)
}
qb.Add(
`
ORDER BY "when" DESC
`,
)
if q.Limit > 0 {
qb.Add(`LIMIT $? OFFSET $?`, q.Limit, q.Offset)
}
results, err := db.Query[TimelineItemAndStuff](ctx, dbConn, qb.String(), qb.Args()...)
if err != nil {
return nil, oops.New(err, "failed to fetch timeline items")
}
for idx := range results {
if results[idx].Owner != nil {
results[idx].Owner.AvatarAsset = results[idx].AvatarAsset
}
}
var projectIds []int
var snippetIds []int
projectTargets := make(map[int][]*TimelineItemAndStuff)
snippetItems := make(map[int]*TimelineItemAndStuff)
for _, r := range results {
if r.Item.ProjectID != 0 {
projectIds = append(projectIds, r.Item.ProjectID)
projectTargets[r.Item.ProjectID] = append(projectTargets[r.Item.ProjectID], r)
}
if r.Item.Type == models.TimelineItemTypeSnippet {
snippetIds = append(snippetIds, r.Item.ID)
snippetItems[r.Item.ID] = r
}
}
type snippetProjectRow struct {
SnippetID int `db:"snippet_id"`
ProjectID int `db:"project_id"`
}
snippetProjects, err := db.Query[snippetProjectRow](ctx, dbConn,
`
SELECT $columns
FROM snippet_project
WHERE snippet_id = ANY($1)
`,
snippetIds,
)
if err != nil {
return nil, oops.New(err, "failed to fetch project ids for timeline")
}
for _, sp := range snippetProjects {
projectIds = append(projectIds, sp.ProjectID)
projectTargets[sp.ProjectID] = append(projectTargets[sp.ProjectID], snippetItems[sp.SnippetID])
}
projects, err := FetchProjects(ctx, dbConn, currentUser, ProjectsQuery{
ProjectIDs: projectIds,
IncludeHidden: true,
Lifecycles: models.AllProjectLifecycles,
})
if err != nil {
return nil, oops.New(err, "failed to fetch projects for timeline")
}
for pIdx := range projects {
targets := projectTargets[projects[pIdx].Project.ID]
for _, t := range targets {
t.Projects = append(t.Projects, &projects[pIdx])
}
}
return results, nil
}

38
src/models/timeline_item.go Normal file
View File

@ -0,0 +1,38 @@
package models
import (
"time"
"github.com/google/uuid"
)
const (
TimelineItemTypeSnippet = "snippet"
TimelineItemTypePost = "post"
TimelineItemTypeStream = "stream" // NOTE(asaf): Not currently supported
)
// NOTE(asaf): This is a virtual model made up of several different tables
type TimelineItem struct {
// Common
// NOTE(asaf): Several different items can have the same ID because we're merging several tables
ID int `db:"id"`
Date time.Time `db:"\"when\""`
Type string `db:"timeline_type"`
OwnerID int `db:"owner_id"`
Title string `db:"title"`
ParsedDescription string `db:"parsed_desc"`
RawDescription string `db:"raw_desc"`
// Snippet
AssetID *uuid.UUID `db:"asset_id"`
DiscordMessageID *string `db:"discord_message_id"`
ExternalUrl *string `db:"url"`
// Post
ProjectID int `db:"project_id"`
ThreadID int `db:"thread_id"`
SubforumID int `db:"subforum_id"`
ThreadType ThreadType `db:"thread_type"`
FirstPost bool `db:"first_post"`
}

4
src/templates/src/user_profile.html
View File

@ -1,4 +1,4 @@
{{ template "base.html" . }}
{{ template "base-2024.html" . }}
{{ define "extrahead" }}
<style>
@ -30,7 +30,7 @@
{{ end }}
{{ define "content" }}
<div class="flex flex-column flex-row-l">
<div class="mw-site pa3 center flex g3">
<div class="
flex-shrink-0 self-start-l
flex flex-column flex-row-ns items-start-ns flex-column-l items-stretch-l

8
src/website/following.go
View File

@ -5,12 +5,18 @@ import (
"strconv"
"git.handmade.network/hmn/hmn/src/logging"
"git.handmade.network/hmn/hmn/src/models"
"git.handmade.network/hmn/hmn/src/oops"
"git.handmade.network/hmn/hmn/src/templates"
)
func FollowingTest(c *RequestContext) ResponseData {
timelineItems, err := FetchFollowTimelineForUser(c, c.Conn, c.CurrentUser)
c.Perf.StartBlock("SQL", "Fetch subforum tree")
subforumTree := models.GetFullSubforumTree(c, c.Conn)
lineageBuilder := models.MakeSubforumLineageBuilder(subforumTree)
c.Perf.EndBlock()
timelineItems, err := FetchFollowTimelineForUser(c, c.Conn, c.CurrentUser, lineageBuilder)
if err != nil {
return c.ErrorResponse(http.StatusInternalServerError, err)
}

6
src/website/landing.go
View File

@ -49,7 +49,7 @@ func Index(c *RequestContext) ResponseData {
var newsItems []templates.TimelineItem
if c.CurrentUser != nil {
followingItems, err = FetchFollowTimelineForUser(c, c.Conn, c.CurrentUser)
followingItems, err = FetchFollowTimelineForUser(c, c.Conn, c.CurrentUser, lineageBuilder)
if err != nil {
c.Logger.Warn().Err(err).Msg("failed to fetch following feed")
}
@ -65,7 +65,7 @@ func Index(c *RequestContext) ResponseData {
for _, p := range featuredProjects {
featuredProjectIDs = append(featuredProjectIDs, p.Project.ID)
}
featuredItems, err = FetchTimeline(c, c.Conn, c.CurrentUser, TimelineQuery{
featuredItems, err = FetchTimeline(c, c.Conn, c.CurrentUser, lineageBuilder, hmndata.TimelineQuery{
ProjectIDs: featuredProjectIDs,
Limit: 100,
})
@ -73,7 +73,7 @@ func Index(c *RequestContext) ResponseData {
c.Logger.Warn().Err(err).Msg("failed to fetch featured feed")
}
recentItems, err = FetchTimeline(c, c.Conn, c.CurrentUser, TimelineQuery{
recentItems, err = FetchTimeline(c, c.Conn, c.CurrentUser, lineageBuilder, hmndata.TimelineQuery{
Limit: 100,
})
if err != nil {

7
src/website/projects.go
View File

@ -467,7 +467,12 @@ func ProjectHomepage(c *RequestContext) ResponseData {
}
}
templateData.RecentActivity, err = FetchTimeline(c, c.Conn, c.CurrentUser, TimelineQuery{
c.Perf.StartBlock("SQL", "Fetch subforum tree")
subforumTree := models.GetFullSubforumTree(c, c.Conn)
lineageBuilder := models.MakeSubforumLineageBuilder(subforumTree)
c.Perf.EndBlock()
templateData.RecentActivity, err = FetchTimeline(c, c.Conn, c.CurrentUser, lineageBuilder, hmndata.TimelineQuery{
ProjectIDs: []int{c.CurrentProject.ID},
})
if err != nil {

260
src/website/timeline_helper.go
View File

@ -18,10 +18,9 @@ import (
"git.handmade.network/hmn/hmn/src/oops"
"git.handmade.network/hmn/hmn/src/perf"
"git.handmade.network/hmn/hmn/src/templates"
"git.handmade.network/hmn/hmn/src/utils"
)
func FetchFollowTimelineForUser(ctx context.Context, conn db.ConnOrTx, user *models.User) ([]templates.TimelineItem, error) {
func FetchFollowTimelineForUser(ctx context.Context, conn db.ConnOrTx, user *models.User, lineageBuilder *models.SubforumLineageBuilder) ([]templates.TimelineItem, error) {
perf := perf.ExtractPerf(ctx)
perf.StartBlock("FOLLOW", "Assemble follow data")
@ -46,153 +45,30 @@ func FetchFollowTimelineForUser(ctx context.Context, conn db.ConnOrTx, user *mod
}
}
timelineItems, err := FetchTimeline(ctx, conn, user, TimelineQuery{
UserIDs: userIDs,
timelineItems := []templates.TimelineItem{}
if len(userIDs)+len(projectIDs) > 0 {
timelineItems, err = FetchTimeline(ctx, conn, user, lineageBuilder, hmndata.TimelineQuery{
OwnerIDs: userIDs,
ProjectIDs: projectIDs,
})
}
perf.EndBlock()
return timelineItems, err
}
type TimelineQuery struct {
UserIDs []int
ProjectIDs []int
Limit int
}
func FetchTimeline(ctx context.Context, conn db.ConnOrTx, currentUser *models.User, q TimelineQuery) ([]templates.TimelineItem, error) {
perf := perf.ExtractPerf(ctx)
// var users []*models.User
// var projects []hmndata.ProjectAndStuff
var snippets []hmndata.SnippetAndStuff
var posts []hmndata.PostAndStuff
// var streamers []hmndata.TwitchStreamer
// var streams []*models.TwitchStreamHistory
var err error
perf.StartBlock("TIMELINE", "Fetch timeline data")
{
snippets, err = hmndata.FetchSnippets(ctx, conn, currentUser, hmndata.SnippetQuery{
OwnerIDs: q.UserIDs,
ProjectIDs: q.ProjectIDs,
Limit: q.Limit,
})
func FetchTimeline(ctx context.Context, conn db.ConnOrTx, currentUser *models.User, lineageBuilder *models.SubforumLineageBuilder, q hmndata.TimelineQuery) ([]templates.TimelineItem, error) {
results, err := hmndata.FetchTimeline(ctx, conn, currentUser, q)
if err != nil {
return nil, oops.New(err, "failed to fetch timeline snippets")
logging.Error().Err(err).Msg("Fail")
}
posts, err = hmndata.FetchPosts(ctx, conn, currentUser, hmndata.PostsQuery{
UserIDs: q.UserIDs,
ProjectIDs: q.ProjectIDs,
SortDescending: true,
Limit: q.Limit,
})
if err != nil {
return nil, oops.New(err, "failed to fetch timeline posts")
return nil, err
}
// streamers, err = hmndata.FetchTwitchStreamers(ctx, conn, hmndata.TwitchStreamersQuery{
// UserIDs: validUserIDs,
// ProjectIDs: validProjectIDs,
// })
// if err != nil {
// return nil, oops.New(err, "failed to fetch streamers")
// }
// twitchLogins := make([]string, 0, len(streamers))
// for _, s := range streamers {
// twitchLogins = append(twitchLogins, s.TwitchLogin)
// }
// streams, err = db.Query[models.TwitchStreamHistory](ctx, conn,
// `
// SELECT $columns FROM twitch_stream_history WHERE twitch_login = ANY ($1)
// `,
// twitchLogins,
// )
// if err != nil {
// return nil, oops.New(err, "failed to fetch stream histories")
// }
}
perf.EndBlock()
perf.StartBlock("TIMELINE", "Construct timeline items")
timelineItems := make([]templates.TimelineItem, 0, len(snippets)+len(posts))
if len(posts) > 0 {
perf.StartBlock("SQL", "Fetch subforum tree")
subforumTree := models.GetFullSubforumTree(ctx, conn)
lineageBuilder := models.MakeSubforumLineageBuilder(subforumTree)
perf.EndBlock()
for _, post := range posts {
timelineItems = append(timelineItems, PostToTimelineItem(
hmndata.UrlContextForProject(&post.Project),
lineageBuilder,
&post.Post,
&post.Thread,
post.Author,
))
}
}
for _, s := range snippets {
item := SnippetToTimelineItem(
&s.Snippet,
s.Asset,
s.DiscordMessage,
s.Projects,
s.Owner,
false,
)
timelineItems = append(timelineItems, item)
}
// for _, s := range streams {
// ownerAvatarUrl := ""
// ownerName := ""
// ownerUrl := ""
// for _, streamer := range streamers {
// if streamer.TwitchLogin == s.TwitchLogin {
// if streamer.UserID != nil {
// for _, u := range users {
// if u.ID == *streamer.UserID {
// ownerAvatarUrl = templates.UserAvatarUrl(u)
// ownerName = u.BestName()
// ownerUrl = hmnurl.BuildUserProfile(u.Username)
// break
// }
// }
// } else if streamer.ProjectID != nil {
// for _, p := range projects {
// if p.Project.ID == *streamer.ProjectID {
// ownerAvatarUrl = templates.ProjectLogoUrl(&p.Project, p.LogoLightAsset, p.LogoDarkAsset)
// ownerName = p.Project.Name
// ownerUrl = hmndata.UrlContextForProject(&p.Project).BuildHomepage()
// }
// break
// }
// }
// break
// }
// }
// item := TwitchStreamToTimelineItem(s, ownerAvatarUrl, ownerName, ownerUrl)
// timelineItems = append(timelineItems, item)
// }
perf.StartBlock("TIMELINE", "Sort timeline")
sort.Slice(timelineItems, func(i, j int) bool {
return timelineItems[j].Date.Before(timelineItems[i].Date)
})
perf.EndBlock()
perf.EndBlock()
if q.Limit > 0 {
timelineItems = utils.ClampSlice(timelineItems, q.Limit)
timelineItems := make([]templates.TimelineItem, 0, len(results))
for _, r := range results {
timelineItems = append(timelineItems, TimelineItemToTemplate(r, lineageBuilder, false))
}
return timelineItems, nil
@ -503,3 +379,113 @@ func unknownMediaItem(asset *models.Asset) templates.TimelineItemMedia {
FileSize: asset.Size,
}
}
func TimelineItemToTemplate(item *hmndata.TimelineItemAndStuff, lineageBuilder *models.SubforumLineageBuilder, editable bool) templates.TimelineItem {
filterTitle := ""
typeTitle := ""
url := ""
var breadcrumbs []templates.Breadcrumb
switch item.Item.Type {
case models.TimelineItemTypeSnippet:
filterTitle = "Snippets"
typeTitle = "Snippet"
url = hmnurl.BuildSnippet(item.Item.ID)
case models.TimelineItemTypePost:
urlContext := hmndata.UrlContextForProject(&item.Projects[0].Project)
if item.Item.ThreadType == models.ThreadTypeProjectBlogPost {
filterTitle = "Blogs"
if item.Item.FirstPost {
typeTitle = "New blog post"
} else {
typeTitle = "Blog comment"
}
url = urlContext.BuildBlogThreadWithPostHash(item.Item.ThreadID, item.Item.Title, item.Item.ID)
breadcrumbs = []templates.Breadcrumb{
{
Name: urlContext.ProjectName,
Url: urlContext.BuildHomepage(),
},
{
Name: "Blog",
Url: urlContext.BuildBlog(1),
},
}
} else if item.Item.ThreadType == models.ThreadTypeForumPost {
filterTitle = "Forums"
if item.Item.FirstPost {
typeTitle = "New forum thread"
} else {
typeTitle = "Forum reply"
}
url = urlContext.BuildForumPost(lineageBuilder.GetSubforumLineageSlugs(item.Item.SubforumID), item.Item.ThreadID, item.Item.ID)
breadcrumbs = SubforumBreadcrumbs(urlContext, lineageBuilder, item.Item.SubforumID)
}
}
ownerTmpl := templates.UserToTemplate(item.Owner)
ti := templates.TimelineItem{
ID: strconv.Itoa(item.Item.ID),
Date: item.Item.Date,
Title: item.Item.Title,
TypeTitle: typeTitle,
FilterTitle: filterTitle,
Breadcrumbs: breadcrumbs,
Url: url,
DiscordMessageUrl: "",
OwnerAvatarUrl: ownerTmpl.AvatarUrl,
OwnerName: ownerTmpl.Name,
OwnerUrl: ownerTmpl.ProfileUrl,
Projects: nil,
Description: template.HTML(item.Item.ParsedDescription),
RawDescription: item.Item.RawDescription,
Media: nil,
ForumLayout: item.Item.Type == models.TimelineItemTypePost,
AllowTitleWrap: false,
TruncateDescription: false,
CanShowcase: item.Item.Type == models.TimelineItemTypeSnippet,
Editable: item.Item.Type == models.TimelineItemTypeSnippet && editable,
}
if item.Asset != nil {
if strings.HasPrefix(item.Asset.MimeType, "image/") {
ti.Media = append(ti.Media, imageMediaItem(item.Asset))
} else if strings.HasPrefix(item.Asset.MimeType, "video/") {
ti.Media = append(ti.Media, videoMediaItem(item.Asset))
} else if strings.HasPrefix(item.Asset.MimeType, "audio/") {
ti.Media = append(ti.Media, audioMediaItem(item.Asset))
} else {
ti.Media = append(ti.Media, unknownMediaItem(item.Asset))
}
}
if item.Item.ExternalUrl != nil {
if videoId := getYoutubeVideoID(*item.Item.ExternalUrl); videoId != "" {
ti.Media = append(ti.Media, youtubeMediaItem(videoId))
ti.CanShowcase = false
}
}
if len(ti.Media) == 0 ||
(len(ti.Media) > 0 && (ti.Media[0].Width == 0 || ti.Media[0].Height == 0)) {
ti.CanShowcase = false
}
if item.DiscordMessage != nil {
ti.DiscordMessageUrl = item.DiscordMessage.Url
}
sort.Slice(item.Projects, func(i, j int) bool {
return item.Projects[i].Project.Name < item.Projects[j].Project.Name
})
for _, proj := range item.Projects {
ti.Projects = append(ti.Projects, templates.ProjectAndStuffToTemplate(proj))
}
return ti
}

9
src/website/user.go
View File

@ -115,8 +115,13 @@ func UserProfile(c *RequestContext) ResponseData {
}
c.Perf.EndBlock()
timelineItems, err := FetchTimeline(c, c.Conn, c.CurrentUser, TimelineQuery{
UserIDs: []int{profileUser.ID},
c.Perf.StartBlock("SQL", "Fetch subforum tree")
subforumTree := models.GetFullSubforumTree(c, c.Conn)
lineageBuilder := models.MakeSubforumLineageBuilder(subforumTree)
c.Perf.EndBlock()
timelineItems, err := FetchTimeline(c, c.Conn, c.CurrentUser, lineageBuilder, hmndata.TimelineQuery{
OwnerIDs: []int{profileUser.ID},
})
if err != nil {
return c.ErrorResponse(http.StatusInternalServerError, err)