diff --git a/go.mod b/go.mod index 3bafdee..3352aad 100644 --- a/go.mod +++ b/go.mod @@ -62,6 +62,7 @@ require ( golang.org/x/text v0.3.6 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect + mvdan.cc/xurls/v2 v2.4.0 // indirect ) replace ( diff --git a/go.sum b/go.sum index dded924..ec19f54 100644 --- a/go.sum +++ b/go.sum @@ -272,6 +272,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -290,6 +291,7 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= @@ -526,4 +528,6 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc= +mvdan.cc/xurls/v2 v2.4.0/go.mod h1:+GEjq9uNjqs8LQfM9nVnM8rff0OQ5Iash5rzX+N1CSg= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= diff --git a/public/js/showcase.js b/public/js/showcase.js index aa2b5ef..2b85612 100644 --- a/public/js/showcase.js +++ b/public/js/showcase.js @@ -8,7 +8,7 @@ const TimelineMediaTypes = { const showcaseItemTemplate = makeTemplateCloner("showcase_item"); const modalTemplate = makeTemplateCloner("timeline_modal"); -const tagTemplate = makeTemplateCloner("timeline_item_tag"); +const projectLinkTemplate = makeTemplateCloner("project_link"); function showcaseTimestamp(rawDate) { const date = new Date(rawDate*1000); @@ -97,14 +97,16 @@ function makeShowcaseItem(timelineItem) { modalEl.date.textContent = timestamp; modalEl.date.setAttribute("href", timelineItem.snippet_url); - if (timelineItem.tags.length === 0) { - modalEl.tags.remove(); + if (timelineItem.projects.length === 0) { + modalEl.projects.remove(); } else { - for (const tag of timelineItem.tags) { - const tagItem = tagTemplate(); - tagItem.tag.innerText = tag.text; + for (const proj of timelineItem.projects) { + const projectLink = projectLinkTemplate(); + projectLink.root.href = proj.url; + projectLink.logo.src = proj.logo; + projectLink.name.textContent = proj.name; - modalEl.tags.appendChild(tagItem.root); + modalEl.projects.appendChild(projectLink.root); } } diff --git a/public/js/snippetedit.js b/public/js/snippetedit.js new file mode 100644 index 0000000..6cc81ed --- /dev/null +++ b/public/js/snippetedit.js @@ -0,0 +1,359 @@ +const snippetEditTemplate = makeTemplateCloner("snippet-edit"); +const snippetEditProjectTemplate = makeTemplateCloner("snippet-edit-project"); + +function readableByteSize(numBytes) { + const scales = [ + " bytes", + "kb", + "mb", + "gb" + ]; + let scale = 0; + while (numBytes > 1024 && scale < scales.length-1) { + numBytes /= 1024; + scale++; + } + return new Intl.NumberFormat([], { maximumFractionDigits: (scale > 0 ? 2 : 0) }).format(numBytes) + scales[scale]; +} + +function makeSnippetEdit(ownerName, ownerAvatar, ownerUrl, date, text, attachmentElement, projectIds, snippetId, originalSnippetEl) { + let snippetEdit = snippetEditTemplate(); + let projectSelector = null; + let originalAttachment = null; + let originalText = text; + let attachmentChanged = false; + let hasAttachment = false; + snippetEdit.redirect.value = location.href; + snippetEdit.avatarImg.src = ownerAvatar; + snippetEdit.avatarLink.href = ownerUrl; + snippetEdit.username.textContent = ownerName; + snippetEdit.username.href = ownerUrl; + snippetEdit.date.textContent = new Intl.DateTimeFormat([], {month: "2-digit", day: "2-digit", year: "numeric"}).format(date); + snippetEdit.text.value = text; + if (attachmentElement) { + originalAttachment = attachmentElement.cloneNode(true); + clearAttachment(true); + } + if (snippetId !== undefined && snippetId !== null) { + snippetEdit.snippetId.value = snippetId; + } else { + snippetEdit.deleteButton.remove(); + } + + for (let i = 0; i < projectIds.length; ++i) { + let proj = null; + for (let j = 0; j < availableProjects.length; ++j) { + if (projectIds[i] == availableProjects[j].id) { + proj = availableProjects[j]; + break; + } + } + + if (proj) { + addProject(proj); + } + } + updateProjectSelector(); + + function addProject(proj) { + let projEl = snippetEditProjectTemplate(); + projEl.projectId.value = proj.id; + projEl.projectLogo.src = proj.logo; + projEl.projectName.textContent = proj.name; + projEl.removeButton.addEventListener("click", function(ev) { + projEl.root.remove(); + updateProjectSelector(); + }); + snippetEdit.projectList.appendChild(projEl.root); + } + + function updateProjectSelector() { + if (projectSelector) { + projectSelector.remove(); + } + + let remainingProjects = []; + let projInputs = snippetEdit.projectList.querySelectorAll("input[name=project_id]"); + let assignedIds = []; + for (let i = 0; i < projInputs.length; ++i) { + let id = parseInt(projInputs[i].value, 10); + if (!isNaN(id)) { + assignedIds.push(id); + } + } + for (let i = 0; i < availableProjects.length; ++i) { + let found = false; + for (let j = 0; j < assignedIds.length; ++j) { + if (assignedIds[j] == availableProjects[i].id) { + found = true; + break; + } + } + if (!found) { + remainingProjects.push(availableProjects[i]); + } + } + + if (remainingProjects.length > 0) { + projectSelector = document.createElement("SELECT"); + let option = document.createElement("OPTION"); + option.textContent = "Add to project..."; + option.selected = true; + projectSelector.appendChild(option); + for (let i = 0; i < remainingProjects.length; ++i) { + option = document.createElement("OPTION"); + option.value = remainingProjects[i].id; + option.selected = false; + option.textContent = remainingProjects[i].name; + projectSelector.appendChild(option); + } + projectSelector.addEventListener("change", function(ev) { + if (projectSelector.selectedOptions.length > 0) { + let selected = projectSelector.selectedOptions[0]; + if (selected.value != "") { + let id = parseInt(selected.value, 10); + if (!isNaN(id)) { + for (let i = 0; i < availableProjects.length; ++i) { + if (availableProjects[i].id == id) { + addProject(availableProjects[i]); + break; + } + } + } + updateProjectSelector(); + } + } + }); + snippetEdit.projectList.appendChild(projectSelector); + } + + } + + function setFile(file) { + let dt = new DataTransfer(); + dt.items.add(file); + snippetEdit.file.files = dt.files; + + attachmentChanged = true; + snippetEdit.removeAttachment.value = "false"; + hasAttachment = true; + + let el = null; + if (file.type.startsWith("image/")) { + el = document.createElement("img"); + el.src = URL.createObjectURL(file); + } else if (file.type.startsWith("video/")) { + el = document.createElement("video"); + el.src = URL.createObjectURL(file); + el.controls = true; + } else if (file.type.startsWith("audio/")) { + el = document.createElement("audio"); + el.src = URL.createObjectURL(file); + } else { + el = document.createElement("div"); + el.classList.add("project-card", "br2", "pv1", "ph2"); + let anchor = document.createElement("a"); + anchor.href = URL.createObjectURL(file); + anchor.setAttribute("target", "_blank"); + anchor.textContent = file.name + " (" + readableByteSize(file.size) + ")"; + el.appendChild(anchor); + } + setPreview(el); + validate(); + } + + function clearAttachment(restoreOriginal) { + snippetEdit.file.value = ""; + let el = null; + attachmentChanged = false; + hasAttachment = false; + snippetEdit.removeAttachment.value = "false"; + if (originalAttachment) { + if (restoreOriginal) { + hasAttachment = true; + el = originalAttachment; + } else { + attachmentChanged = true; + snippetEdit.removeAttachment.value = "true"; + } + } + setPreview(el); + validate(); + } + + function setPreview(el) { + if (el) { + snippetEdit.uploadBox.style.display = "none"; + snippetEdit.previewBox.style.display = "block"; + snippetEdit.uploadResetLink.style.display = "none"; + snippetEdit.previewContent = emptyElement(snippetEdit.previewContent); + snippetEdit.previewContent.appendChild(el); + snippetEdit.resetLink.style.display = (!originalAttachment || el == originalAttachment) ? "none" : "inline-block"; + } else { + snippetEdit.uploadBox.style.display = "flex"; + snippetEdit.previewBox.style.display = "none"; + if (originalAttachment) { + snippetEdit.uploadResetLink.style.display = "block"; + } + } + } + + function validate() { + let sizeGood = true; + if (snippetEdit.file.files.length > 0 && snippetEdit.file.files[0].size > maxFilesize) { + // NOTE(asaf): Writing this out in bytes to make the limit exactly clear to the user. + let readableSize = new Intl.NumberFormat([], { useGrouping: "always" }).format(maxFilesize); + snippetEdit.errors.textContent = "File is too big! Max filesize is " + readableSize + " bytes"; + sizeGood = false; + } else { + snippetEdit.errors.textContent = ""; + } + + let hasText = snippetEdit.text.value.trim().length > 0; + + if ((hasText || hasAttachment) && sizeGood) { + snippetEdit.saveButton.disabled = false; + } else { + snippetEdit.saveButton.disabled = true; + } + } + + snippetEdit.uploadLink.addEventListener("click", function() { + snippetEdit.file.click(); + }); + + snippetEdit.removeLink.addEventListener("click", function() { + clearAttachment(false); + }); + + snippetEdit.replaceLink.addEventListener("click", function() { + snippetEdit.file.click(); + }); + + snippetEdit.resetLink.addEventListener("click", function() { + clearAttachment(true); + }); + + snippetEdit.uploadResetLink.addEventListener("click", function() { + clearAttachment(true); + }); + + snippetEdit.file.addEventListener("change", function() { + if (snippetEdit.file.files.length > 0) { + setFile(snippetEdit.file.files[0]); + } + }); + + snippetEdit.root.addEventListener("dragover", function(ev) { + let effect = "none"; + for (let i = 0; i < ev.dataTransfer.items.length; ++i) { + if (ev.dataTransfer.items[i].kind.toLowerCase() == "file") { + effect = "copy"; + break; + } + } + ev.dataTransfer.dropEffect = effect; + ev.preventDefault(); + }); + + let enterCounter = 0; + + snippetEdit.root.addEventListener("dragenter", function(ev) { + enterCounter++; + if (ev.dataTransfer && ev.dataTransfer.files && ev.dataTransfer.files.length > 0) { + snippetEdit.root.classList.add("drop"); + } + }); + + snippetEdit.root.addEventListener("dragleave", function(ev) { + enterCounter--; + if (enterCounter == 0) { + snippetEdit.root.classList.remove("drop"); + } + }); + + snippetEdit.root.addEventListener("drop", function(ev) { + enterCounter = 0; + snippetEdit.root.classList.remove("drop"); + + if (ev.dataTransfer && ev.dataTransfer.files && ev.dataTransfer.files.length > 0) { + setFile(ev.dataTransfer.files[0]); + } + + ev.preventDefault(); + }); + + snippetEdit.text.addEventListener("paste", function(ev) { + const files = ev.clipboardData?.files ?? []; + if (files.length > 0) { + setFile(files[0]); + } + }); + + snippetEdit.text.addEventListener("input", function(ev) { + validate(); + }); + + snippetEdit.saveButton.addEventListener("click", function(ev) { + let projectsChanged = false; + let projInputs = snippetEdit.projectList.querySelectorAll("input[name=project_id]"); + let assignedIds = []; + for (let i = 0; i < projInputs.length; ++i) { + let id = parseInt(projInputs[i].value, 10); + if (!isNaN(id)) { + assignedIds.push(id); + } + } + if (projectIds.length != assignedIds.length) { + projectsChanged = true; + } else { + for (let i = 0; i < projectIds.length; ++i) { + let found = false; + for (let j = 0; j < assignedIds.length; ++j) { + if (projectIds[i] == assignedIds[j]) { + found = true; + } + } + if (!found) { + projectsChanged = true; + break; + } + } + } + + if (originalSnippetEl && (!attachmentChanged && originalText == snippetEdit.text.value.trim() && !projectsChanged)) { + // NOTE(asaf): We're in edit mode and nothing changed, so no need to submit to the server. + ev.preventDefault(); + snippetEdit.root.parentElement.insertBefore(originalSnippetEl, snippetEdit.root); + snippetEdit.root.remove(); + } + }); + + snippetEdit.deleteButton.addEventListener("click", function(ev) { + snippetEdit.file.value = ""; + }); + + validate(); + + return snippetEdit; +} + +function editTimelineSnippet(timelineItemEl) { + let ownerName = timelineItemEl.querySelector(".user")?.textContent; + let ownerUrl = timelineItemEl.querySelector(".user")?.href; + let ownerAvatar = timelineItemEl.querySelector(".avatar-icon")?.src; + let creationDate = new Date(timelineItemEl.querySelector("time").dateTime); + let rawDesc = timelineItemEl.querySelector(".rawdesc").textContent; + let attachment = timelineItemEl.querySelector(".timeline-content-box")?.children?.[0]; + let projectIds = []; + let projectEls = timelineItemEl.querySelectorAll(".projects > a"); + for (let i = 0; i < projectEls.length; ++i) { + let projid = projectEls[i].getAttribute("data-projid"); + if (projid) { + projectIds.push(projid); + } + } + let snippetEdit = makeSnippetEdit(ownerName, ownerAvatar, ownerUrl, creationDate, rawDesc, attachment, projectIds, timelineItemEl.getAttribute("data-id"), timelineItemEl); + timelineItemEl.parentElement.insertBefore(snippetEdit.root, timelineItemEl); + timelineItemEl.remove(); +} diff --git a/public/style.css b/public/style.css index cb58f12..ca3b1a4 100644 --- a/public/style.css +++ b/public/style.css @@ -7461,6 +7461,30 @@ article code { .minh-6 { min-height: 32rem; } +.h1-5 { + height: 1.5rem; } + +.gap0 { + gap: 0; } + +.gap1 { + gap: 0.25rem; } + +.gap2 { + gap: 0.5rem; } + +.gap3 { + gap: 1rem; } + +.gap4 { + gap: 2rem; } + +.gap5 { + gap: 4rem; } + +.pre-line { + white-space: pre-line; } + .fira { font-family: "Fira Sans", sans-serif; } diff --git a/src/admintools/admintools.go b/src/admintools/admintools.go index 7a1f583..05354a6 100644 --- a/src/admintools/admintools.go +++ b/src/admintools/admintools.go @@ -466,5 +466,48 @@ func init() { moveThreadsToSubforumCommand.MarkFlagRequired("subforum_slug") adminCommand.AddCommand(moveThreadsToSubforumCommand) + fixupSnippetAssociation := &cobra.Command{ + Use: "fixupsnippets", + Short: "Associates tagged snippets with the right projects", + Run: func(cmd *cobra.Command, args []string) { + ctx := context.Background() + conn := db.NewConn() + defer conn.Close(ctx) + + type snippetProject struct { + SnippetID int `db:"snippet_tag.snippet_id"` + ProjectID int `db:"project.id"` + } + res, err := db.Query[snippetProject](ctx, conn, + ` + SELECT $columns + FROM snippet_tag + JOIN project ON project.tag = snippet_tag.tag_id + `, + ) + if err != nil { + panic(err) + } + for _, sp := range res { + _, err = conn.Exec(ctx, + ` + INSERT INTO snippet_project (snippet_id, project_id, kind) + VALUES ($1, $2, $3) + ON CONFLICT DO NOTHING; + `, + sp.SnippetID, + sp.ProjectID, + models.SnippetProjectKindDiscord, + ) + if err != nil { + panic(err) + } + } + + fmt.Printf("Done!\n") + }, + } + adminCommand.AddCommand(fixupSnippetAssociation) + addProjectCommands(adminCommand) } diff --git a/src/discord/message_handling.go b/src/discord/message_handling.go index 91332c8..cfd9df1 100644 --- a/src/discord/message_handling.go +++ b/src/discord/message_handling.go @@ -783,9 +783,6 @@ func HandleSnippetForInternedMessage(ctx context.Context, dbConn db.ConnOrTx, in // Match only tags for projects in which the current user is a collaborator. messageTags := getDiscordTags(existingSnippet.Description) - var desiredTags []int - var allTags []int - // Fetch projects so we know what tags the user can apply to their snippet. projects, err := hmndata.FetchProjects(ctx, tx, interned.HMNUser, hmndata.ProjectsQuery{ OwnerIDs: []int{interned.HMNUser.ID}, @@ -794,61 +791,40 @@ func HandleSnippetForInternedMessage(ctx context.Context, dbConn db.ConnOrTx, in return oops.New(err, "failed to look up user projects") } - projectIDs := make([]int, len(projects)) - for i, p := range projects { - projectIDs[i] = p.Project.ID - } - - userTags, err := db.Query[models.Tag](ctx, tx, - ` - SELECT $columns{tag} - FROM - tag - JOIN project ON project.tag = tag.id - WHERE - project.id = ANY ($1) - `, - projectIDs, - ) - if err != nil { - return oops.New(err, "failed to fetch tags for user projects") - } - - for _, tag := range userTags { - allTags = append(allTags, tag.ID) - for _, messageTag := range messageTags { - if strings.EqualFold(tag.Text, messageTag) { - desiredTags = append(desiredTags, tag.ID) - } - } - } - _, err = tx.Exec(ctx, ` - DELETE FROM snippet_tag + DELETE FROM snippet_project WHERE snippet_id = $1 - AND tag_id = ANY ($2) + AND kind = $2 `, existingSnippet.ID, - allTags, + models.SnippetProjectKindDiscord, ) if err != nil { - return oops.New(err, "failed to clear tags from snippet") + return oops.New(err, "failed to clear project association for snippet") } - for _, tagID := range desiredTags { - _, err = tx.Exec(ctx, - ` - INSERT INTO snippet_tag (snippet_id, tag_id) - VALUES ($1, $2) - ON CONFLICT DO NOTHING - `, - existingSnippet.ID, - tagID, - ) - if err != nil { - return oops.New(err, "failed to associate snippet with tag") + for _, p := range projects { + if p.Tag != nil { + for _, messageTag := range messageTags { + if strings.EqualFold(p.Tag.Text, messageTag) { + _, err = tx.Exec(ctx, + ` + INSERT INTO snippet_project (project_id, snippet_id, kind) + VALUES ($1, $2, $3) + ON CONFLICT DO NOTHING + `, + p.Project.ID, + existingSnippet.ID, + models.SnippetProjectKindDiscord, + ) + if err != nil { + return oops.New(err, "failed to associate snippet with project") + } + break + } + } } } } @@ -876,6 +852,7 @@ func getDiscordTags(content string) []string { // NOTE(ben): This is maybe redundant with the regexes we use for markdown. But // do we actually want to reuse those, or should we keep them separate? +// TODO(asaf): Centralize this var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`) func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msg *models.DiscordMessage) (*uuid.UUID, *string, error) { diff --git a/src/embed/embed.go b/src/embed/embed.go new file mode 100644 index 0000000..3912e90 --- /dev/null +++ b/src/embed/embed.go @@ -0,0 +1,199 @@ +package embed + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "net/url" + "regexp" + "strings" + "time" + + "git.handmade.network/hmn/hmn/src/logging" + "git.handmade.network/hmn/hmn/src/utils" +) + +var DownloadTooBigError = errors.New("download too big") +var NoEmbedFound = errors.New("no embed found") + +type Embeddable struct { + Url string + File *Embed +} + +type Embed struct { + Data []byte + ContentType string + Filename string +} + +var EmbeddableUrlRegex = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`) + +func IsUrlEmbeddable(u string) bool { + return EmbeddableUrlRegex.MatchString(u) +} + +func GetEmbeddableFromUrls(ctx context.Context, urls []string, maxSize int, httpTimeout time.Duration, httpMaxAttempts int) (*Embeddable, error) { + embedError := NoEmbedFound + for _, urlStr := range urls { + u, err := url.Parse(urlStr) + if err != nil { + continue + } + if u.Scheme == "" { + u.Scheme = "https" + urlStr = u.String() + } + + if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" { + continue + } + + if IsUrlEmbeddable(urlStr) { + result := Embeddable{ + Url: urlStr, + } + return &result, nil + } + + if httpMaxAttempts > 0 { + httpMaxAttempts -= 1 + embed, err := FetchEmbed(ctx, urlStr, httpTimeout, maxSize) + if err != nil { + embedError = err + continue + } + result := Embeddable{ + File: embed, + } + return &result, nil + } + } + return nil, embedError +} + +// If the url points to a file, downloads and returns the file. +// If the url points to an html page, parses opengraph and tries to fetch an image/video/audio file according to that. +// maxSize only limits the actual filesize. In the case of html we always fetch up to 100kb even if maxSize is smaller. +func FetchEmbed(ctx context.Context, urlStr string, timeout time.Duration, maxSize int) (*Embed, error) { + logging.ExtractLogger(ctx).Debug().Msg("Fetching embed") + client := &http.Client{ + Timeout: timeout, + } + req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil) + if err != nil { + return nil, err + } + res, err := client.Do(req) + if err != nil { + return nil, err + } + if res.StatusCode < 200 || res.StatusCode > 299 { + return nil, NoEmbedFound + } + contentType := res.Header.Get("Content-Type") + logging.ExtractLogger(ctx).Debug().Str("type", contentType).Msg("Got first result") + if strings.HasPrefix(contentType, "text/html") || strings.HasPrefix(contentType, "application/html") { + var buffer bytes.Buffer + _, err := io.CopyN(&buffer, res.Body, 100*1024) // NOTE(asaf): If the opengraph stuff isn't in the first 100kb, we don't care. + res.Body.Close() + if err != nil && !errors.Is(err, io.EOF) { + return nil, err + } + partialHtml := buffer.Bytes() + urlStr = ExtractEmbedFromOpenGraph(partialHtml) + logging.ExtractLogger(ctx).Debug().Str("url", urlStr).Msg("Got ograph") + if urlStr == "" { + return nil, NoEmbedFound + } + + req, err = http.NewRequestWithContext(ctx, "GET", urlStr, nil) + if err != nil { + return nil, err + } + res, err = client.Do(req) + if err != nil { + return nil, err + } + if res.StatusCode < 200 || res.StatusCode > 299 { + return nil, NoEmbedFound + } + contentType = res.Header.Get("Content-Type") + } + + var buffer bytes.Buffer + n, err := io.CopyN(&buffer, res.Body, int64(maxSize+1)) + res.Body.Close() + if err != nil && !errors.Is(err, io.EOF) { + return nil, err + } + + filename := "" + u, err := url.Parse(urlStr) + if err == nil { + lastSlash := utils.IntMax(strings.LastIndex(u.Path, "/"), 0) + filename = u.Path[lastSlash:] + } + result := Embed{ + Data: buffer.Bytes(), + ContentType: contentType, + Filename: filename, + } + if n == int64(maxSize+1) { + err = DownloadTooBigError + } else { + err = nil + } + return &result, err +} + +var metaRegex = regexp.MustCompile(`]+)/?>`) +var metaAttrRegex = regexp.MustCompile(`(?P\w+)="(?P[^"]+)"`) + +var OGKeys = []string{ + "og:audio", + "og:video", + "og:image", + "og:audio:url", + "og:image:url", + "og:video:url", + "og:audio:secure_url", + "og:image:secure_url", + "og:video:secure_url", + "twitter:image", +} + +// Tries to find an opengraph image/video/audio url in the provided html +// Since we only need to look at meta tags in the head, we don't need the full html document. +func ExtractEmbedFromOpenGraph(partialHtml []byte) string { + keyIdx := metaAttrRegex.SubexpIndex("key") + valueIdx := metaAttrRegex.SubexpIndex("value") + html := string(partialHtml) + matches := metaRegex.FindAllStringSubmatch(html, -1) + for _, m := range matches { + if len(m) > 1 { + content := "" + prop := "" + attrs := metaAttrRegex.FindAllStringSubmatch(m[1], -1) + for _, attr := range attrs { + key := attr[keyIdx] + value := attr[valueIdx] + if key == "name" || key == "property" { + for _, ogKey := range OGKeys { + if value == ogKey { + prop = value + } + } + } else if key == "content" { + content = value + } + } + if content != "" && prop != "" { + return content + } + } + } + return "" +} diff --git a/src/hmndata/snippet_helper.go b/src/hmndata/snippet_helper.go index a15e996..e942bec 100644 --- a/src/hmndata/snippet_helper.go +++ b/src/hmndata/snippet_helper.go @@ -12,6 +12,7 @@ import ( type SnippetQuery struct { IDs []int OwnerIDs []int + ProjectIDs []int Tags []int DiscordMessageIDs []string @@ -24,6 +25,7 @@ type SnippetAndStuff struct { Asset *models.Asset `db:"asset"` DiscordMessage *models.DiscordMessage `db:"discord_message"` Tags []*models.Tag + Projects []*ProjectAndStuff } func FetchSnippets( @@ -42,6 +44,7 @@ func FetchSnippets( } defer tx.Rollback(ctx) + var tagSnippetIDs []int if len(q.Tags) > 0 { // Get snippet IDs with this tag, then use that in the main query snippetIDs, err := db.QueryScalar[int](ctx, tx, @@ -64,7 +67,32 @@ func FetchSnippets( return nil, nil } - q.IDs = snippetIDs + tagSnippetIDs = snippetIDs + } + + var projectSnippetIDs []int + if len(q.ProjectIDs) > 0 { + // Get snippet IDs for these projects, then use that in the main query + snippetIDs, err := db.QueryScalar[int](ctx, tx, + ` + SELECT DISTINCT snippet_id + FROM + snippet_project + WHERE + project_id = ANY ($1) + `, + q.ProjectIDs, + ) + if err != nil { + return nil, oops.New(err, "failed to get snippet IDs for tag") + } + + // special early-out: no snippets found for these projects at all + if len(snippetIDs) == 0 { + return nil, nil + } + + projectSnippetIDs = snippetIDs } var qb db.QueryBuilder @@ -74,7 +102,7 @@ func FetchSnippets( FROM snippet LEFT JOIN hmn_user AS owner ON snippet.owner_id = owner.id - LEFT JOIN asset AS owner_avatar ON owner_avatar.id = owner.avatar_asset_id + LEFT JOIN asset AS avatar ON avatar.id = owner.avatar_asset_id LEFT JOIN asset ON snippet.asset_id = asset.id LEFT JOIN discord_message ON snippet.discord_message_id = discord_message.id WHERE @@ -84,6 +112,12 @@ func FetchSnippets( if len(q.IDs) > 0 { qb.Add(`AND snippet.id = ANY ($?)`, q.IDs) } + if len(tagSnippetIDs) > 0 { + qb.Add(`AND snippet.id = ANY ($?)`, tagSnippetIDs) + } + if len(projectSnippetIDs) > 0 { + qb.Add(`AND snippet.id = ANY ($?)`, projectSnippetIDs) + } if len(q.OwnerIDs) > 0 { qb.Add(`AND snippet.owner_id = ANY ($?)`, q.OwnerIDs) } @@ -115,6 +149,7 @@ func FetchSnippets( type resultRow struct { Snippet models.Snippet `db:"snippet"` Owner *models.User `db:"owner"` + AvatarAsset *models.Asset `db:"avatar"` Asset *models.Asset `db:"asset"` DiscordMessage *models.DiscordMessage `db:"discord_message"` } @@ -127,6 +162,9 @@ func FetchSnippets( result := make([]SnippetAndStuff, len(results)) // allocate extra space because why not snippetIDs := make([]int, len(results)) for i, row := range results { + if results[i].Owner != nil { + results[i].Owner.AvatarAsset = results[i].AvatarAsset + } result[i] = SnippetAndStuff{ Snippet: row.Snippet, Owner: row.Owner, @@ -167,6 +205,42 @@ func FetchSnippets( item.Tags = append(item.Tags, snippetTag.Tag) } + // Fetch projects + type snippetProjectRow struct { + SnippetID int `db:"snippet_id"` + ProjectID int `db:"project_id"` + } + snippetProjects, err := db.Query[snippetProjectRow](ctx, tx, + ` + SELECT $columns + FROM snippet_project + WHERE snippet_id = ANY($1) + `, + snippetIDs, + ) + if err != nil { + return nil, oops.New(err, "failed to fetch project ids for snippets") + } + var projectIds []int + for _, sp := range snippetProjects { + projectIds = append(projectIds, sp.ProjectID) + } + projects, err := FetchProjects(ctx, tx, currentUser, ProjectsQuery{ProjectIDs: projectIds}) + if err != nil { + return nil, oops.New(err, "failed to fetch projects for snippets") + } + projectMap := make(map[int]*ProjectAndStuff) + for i := range projects { + projectMap[projects[i].Project.ID] = &projects[i] + } + for _, sp := range snippetProjects { + snip, hasResult := resultBySnippetId[sp.SnippetID] + proj, hasProj := projectMap[sp.ProjectID] + if hasResult && hasProj { + snip.Projects = append(snip.Projects, proj) + } + } + err = tx.Commit(ctx) if err != nil { return nil, oops.New(err, "failed to commit transaction") diff --git a/src/hmnurl/hmnurl_test.go b/src/hmnurl/hmnurl_test.go index 38a35ba..a295743 100644 --- a/src/hmnurl/hmnurl_test.go +++ b/src/hmnurl/hmnurl_test.go @@ -120,6 +120,10 @@ func TestSnippet(t *testing.T) { AssertRegexMatch(t, BuildSnippet(15), RegexSnippet, map[string]string{"snippetid": "15"}) } +func TestSnippetSubmit(t *testing.T) { + AssertRegexMatch(t, BuildSnippetSubmit(), RegexSnippetSubmit, nil) +} + func TestFeed(t *testing.T) { AssertRegexMatch(t, BuildFeed(), RegexFeed, nil) assert.Equal(t, BuildFeed(), BuildFeedWithPage(1)) diff --git a/src/hmnurl/urls.go b/src/hmnurl/urls.go index 5432b7b..c2d6138 100644 --- a/src/hmnurl/urls.go +++ b/src/hmnurl/urls.go @@ -262,6 +262,13 @@ func BuildSnippet(snippetId int) string { return Url("/snippet/"+strconv.Itoa(snippetId), nil) } +var RegexSnippetSubmit = regexp.MustCompile(`^/snippet$`) + +func BuildSnippetSubmit() string { + defer CatchPanic() + return Url("/snippet", nil) +} + /* * Feed */ diff --git a/src/migration/migrations/2022-06-26T115703Z_SnippetProjectAssociation.go b/src/migration/migrations/2022-06-26T115703Z_SnippetProjectAssociation.go new file mode 100644 index 0000000..b1072e6 --- /dev/null +++ b/src/migration/migrations/2022-06-26T115703Z_SnippetProjectAssociation.go @@ -0,0 +1,50 @@ +package migrations + +import ( + "context" + "time" + + "git.handmade.network/hmn/hmn/src/migration/types" + "github.com/jackc/pgx/v4" +) + +func init() { + registerMigration(SnippetProjectAssociation{}) +} + +type SnippetProjectAssociation struct{} + +func (m SnippetProjectAssociation) Version() types.MigrationVersion { + return types.MigrationVersion(time.Date(2022, 6, 26, 11, 57, 3, 0, time.UTC)) +} + +func (m SnippetProjectAssociation) Name() string { + return "SnippetProjectAssociation" +} + +func (m SnippetProjectAssociation) Description() string { + return "Table for associating a snippet with projects" +} + +func (m SnippetProjectAssociation) Up(ctx context.Context, tx pgx.Tx) error { + _, err := tx.Exec(ctx, + ` + CREATE TABLE snippet_project ( + snippet_id INTEGER NOT NULL REFERENCES snippet (id) ON DELETE CASCADE, + project_id INTEGER NOT NULL REFERENCES project (id) ON DELETE CASCADE, + kind INTEGER NOT NULL, + UNIQUE (snippet_id, project_id) + ); + `, + ) + return err +} + +func (m SnippetProjectAssociation) Down(ctx context.Context, tx pgx.Tx) error { + _, err := tx.Exec(ctx, + ` + DROP TABLE snippet_project; + `, + ) + return err +} diff --git a/src/models/snippet.go b/src/models/snippet.go index 9310e6e..fdd8ed4 100644 --- a/src/models/snippet.go +++ b/src/models/snippet.go @@ -6,6 +6,13 @@ import ( "github.com/google/uuid" ) +type SnippetProjectAssociationKind int + +const ( + SnippetProjectKindDiscord SnippetProjectAssociationKind = iota + 1 + SnippetProjectKindWebsite +) + type Snippet struct { ID int `db:"id"` OwnerID int `db:"owner_id"` diff --git a/src/rawdata/scss/_core.scss b/src/rawdata/scss/_core.scss index e5e9a79..2392009 100644 --- a/src/rawdata/scss/_core.scss +++ b/src/rawdata/scss/_core.scss @@ -333,6 +333,38 @@ article code { min-height: $height-6; } +.h1-5 { + height: 1.5rem; +} + +.gap0 { + gap: $spacing-none; +} + +.gap1 { + gap: $spacing-extra-small; +} + +.gap2 { + gap: $spacing-small; +} + +.gap3 { + gap: $spacing-medium; +} + +.gap4 { + gap: $spacing-large; +} + +.gap5 { + gap: $spacing-extra-large; +} + +.pre-line { + white-space: pre-line; +} + .fira { font-family: "Fira Sans", sans-serif; } diff --git a/src/templates/mapping.go b/src/templates/mapping.go index dee349b..837927f 100644 --- a/src/templates/mapping.go +++ b/src/templates/mapping.go @@ -78,6 +78,7 @@ func ProjectToTemplate( url string, ) Project { return Project{ + ID: p.ID, Name: p.Name, Subdomain: p.Subdomain(), Color1: p.Color1, @@ -297,8 +298,6 @@ func LinkToTemplate(link *models.Link) Link { return tlink } -var controlCharRegex = regexp.MustCompile(`\p{Cc}`) - func TimelineItemsToJSON(items []TimelineItem) string { // NOTE(asaf): As of 2021-06-22: This only serializes the data necessary for snippet showcase. builder := strings.Builder{} @@ -378,22 +377,26 @@ func TimelineItemsToJSON(items []TimelineItem) string { builder.WriteString(item.DiscordMessageUrl) builder.WriteString(`",`) - builder.WriteString(`"tags":[`) - for i, tag := range item.Tags { - builder.WriteString(`{`) + builder.WriteString(`"projects":[`) + for j, proj := range item.Projects { + if j > 0 { + builder.WriteRune(',') + } + builder.WriteRune('{') - builder.WriteString(`"text":"`) - builder.WriteString(tag.Text) + builder.WriteString(`"name":"`) + builder.WriteString(proj.Name) + builder.WriteString(`",`) + + builder.WriteString(`"logo":"`) + builder.WriteString(proj.Logo) builder.WriteString(`",`) builder.WriteString(`"url":"`) - builder.WriteString(tag.Url) + builder.WriteString(proj.Url) builder.WriteString(`"`) - builder.WriteString(`}`) - if i < len(item.Tags)-1 { - builder.WriteString(`,`) - } + builder.WriteRune('}') } builder.WriteString(`]`) @@ -403,6 +406,33 @@ func TimelineItemsToJSON(items []TimelineItem) string { return builder.String() } +func SnippetEditProjectsToJSON(projects []Project) string { + builder := strings.Builder{} + builder.WriteRune('[') + for i, proj := range projects { + if i > 0 { + builder.WriteRune(',') + } + builder.WriteRune('{') + + builder.WriteString(`"id":`) + builder.WriteString(strconv.FormatInt(int64(proj.ID), 10)) + builder.WriteRune(',') + + builder.WriteString(`"name":"`) + builder.WriteString(proj.Name) + builder.WriteString(`",`) + + builder.WriteString(`"logo":"`) + builder.WriteString(proj.Logo) + builder.WriteRune('"') + + builder.WriteRune('}') + } + builder.WriteRune(']') + return builder.String() +} + func PodcastToTemplate(podcast *models.Podcast, imageFilename string) Podcast { imageUrl := "" if imageFilename != "" { diff --git a/src/templates/src/atom.xml b/src/templates/src/atom.xml index 68317ef..3f17f0e 100644 --- a/src/templates/src/atom.xml +++ b/src/templates/src/atom.xml @@ -55,7 +55,7 @@
- {{ .Description }} + {{ cleancontrolchars .Description }}
{{ range .EmbedMedia }}
diff --git a/src/templates/src/include/showcase_templates.html b/src/templates/src/include/showcase_templates.html index afaa96c..6af8ec2 100644 --- a/src/templates/src/include/showcase_templates.html +++ b/src/templates/src/include/showcase_templates.html @@ -27,10 +27,10 @@
-
+
Unknown description
-
+
View original message on Discord
@@ -40,6 +40,9 @@
-