Snippet creation and editing

snippet_edit
Asaf Gartner 2 months ago
parent 41c2b6e111
commit efc7d76cb7
  1. 1
      go.mod
  2. 4
      go.sum
  3. 16
      public/js/showcase.js
  4. 359
      public/js/snippetedit.js
  5. 24
      public/style.css
  6. 43
      src/admintools/admintools.go
  7. 73
      src/discord/message_handling.go
  8. 199
      src/embed/embed.go
  9. 78
      src/hmndata/snippet_helper.go
  10. 4
      src/hmnurl/hmnurl_test.go
  11. 7
      src/hmnurl/urls.go
  12. 50
      src/migration/migrations/2022-06-26T115703Z_SnippetProjectAssociation.go
  13. 7
      src/models/snippet.go
  14. 32
      src/rawdata/scss/_core.scss
  15. 54
      src/templates/mapping.go
  16. 2
      src/templates/src/atom.xml
  17. 11
      src/templates/src/include/showcase_templates.html
  18. 53
      src/templates/src/include/snippet_edit.html
  19. 21
      src/templates/src/include/timeline_item.html
  20. 42
      src/templates/src/project_homepage.html
  21. 57
      src/templates/src/snippet.html
  22. 51
      src/templates/src/user_profile.html
  23. 6
      src/templates/templates.go
  24. 14
      src/templates/types.go
  25. 2
      src/website/assets.go
  26. 2
      src/website/feed.go
  27. 2
      src/website/jam.go
  28. 2
      src/website/landing.go
  29. 30
      src/website/projects.go
  30. 3
      src/website/routes.go
  31. 2
      src/website/showcase.go
  32. 301
      src/website/snippet.go
  33. 18
      src/website/timeline_helper.go
  34. 16
      src/website/user.go

@ -62,6 +62,7 @@ require (
golang.org/x/text v0.3.6 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c // indirect
mvdan.cc/xurls/v2 v2.4.0 // indirect
)
replace (

@ -272,6 +272,7 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@ -290,6 +291,7 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7z
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
@ -526,4 +528,6 @@ honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWh
honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
mvdan.cc/xurls/v2 v2.4.0 h1:tzxjVAj+wSBmDcF6zBB7/myTy3gX9xvi8Tyr28AuQgc=
mvdan.cc/xurls/v2 v2.4.0/go.mod h1:+GEjq9uNjqs8LQfM9nVnM8rff0OQ5Iash5rzX+N1CSg=
rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=

@ -8,7 +8,7 @@ const TimelineMediaTypes = {
const showcaseItemTemplate = makeTemplateCloner("showcase_item");
const modalTemplate = makeTemplateCloner("timeline_modal");
const tagTemplate = makeTemplateCloner("timeline_item_tag");
const projectLinkTemplate = makeTemplateCloner("project_link");
function showcaseTimestamp(rawDate) {
const date = new Date(rawDate*1000);
@ -97,14 +97,16 @@ function makeShowcaseItem(timelineItem) {
modalEl.date.textContent = timestamp;
modalEl.date.setAttribute("href", timelineItem.snippet_url);
if (timelineItem.tags.length === 0) {
modalEl.tags.remove();
if (timelineItem.projects.length === 0) {
modalEl.projects.remove();
} else {
for (const tag of timelineItem.tags) {
const tagItem = tagTemplate();
tagItem.tag.innerText = tag.text;
for (const proj of timelineItem.projects) {
const projectLink = projectLinkTemplate();
projectLink.root.href = proj.url;
projectLink.logo.src = proj.logo;
projectLink.name.textContent = proj.name;
modalEl.tags.appendChild(tagItem.root);
modalEl.projects.appendChild(projectLink.root);
}
}

@ -0,0 +1,359 @@
const snippetEditTemplate = makeTemplateCloner("snippet-edit");
const snippetEditProjectTemplate = makeTemplateCloner("snippet-edit-project");
function readableByteSize(numBytes) {
const scales = [
" bytes",
"kb",
"mb",
"gb"
];
let scale = 0;
while (numBytes > 1024 && scale < scales.length-1) {
numBytes /= 1024;
scale++;
}
return new Intl.NumberFormat([], { maximumFractionDigits: (scale > 0 ? 2 : 0) }).format(numBytes) + scales[scale];
}
function makeSnippetEdit(ownerName, ownerAvatar, ownerUrl, date, text, attachmentElement, projectIds, snippetId, originalSnippetEl) {
let snippetEdit = snippetEditTemplate();
let projectSelector = null;
let originalAttachment = null;
let originalText = text;
let attachmentChanged = false;
let hasAttachment = false;
snippetEdit.redirect.value = location.href;
snippetEdit.avatarImg.src = ownerAvatar;
snippetEdit.avatarLink.href = ownerUrl;
snippetEdit.username.textContent = ownerName;
snippetEdit.username.href = ownerUrl;
snippetEdit.date.textContent = new Intl.DateTimeFormat([], {month: "2-digit", day: "2-digit", year: "numeric"}).format(date);
snippetEdit.text.value = text;
if (attachmentElement) {
originalAttachment = attachmentElement.cloneNode(true);
clearAttachment(true);
}
if (snippetId !== undefined && snippetId !== null) {
snippetEdit.snippetId.value = snippetId;
} else {
snippetEdit.deleteButton.remove();
}
for (let i = 0; i < projectIds.length; ++i) {
let proj = null;
for (let j = 0; j < availableProjects.length; ++j) {
if (projectIds[i] == availableProjects[j].id) {
proj = availableProjects[j];
break;
}
}
if (proj) {
addProject(proj);
}
}
updateProjectSelector();
function addProject(proj) {
let projEl = snippetEditProjectTemplate();
projEl.projectId.value = proj.id;
projEl.projectLogo.src = proj.logo;
projEl.projectName.textContent = proj.name;
projEl.removeButton.addEventListener("click", function(ev) {
projEl.root.remove();
updateProjectSelector();
});
snippetEdit.projectList.appendChild(projEl.root);
}
function updateProjectSelector() {
if (projectSelector) {
projectSelector.remove();
}
let remainingProjects = [];
let projInputs = snippetEdit.projectList.querySelectorAll("input[name=project_id]");
let assignedIds = [];
for (let i = 0; i < projInputs.length; ++i) {
let id = parseInt(projInputs[i].value, 10);
if (!isNaN(id)) {
assignedIds.push(id);
}
}
for (let i = 0; i < availableProjects.length; ++i) {
let found = false;
for (let j = 0; j < assignedIds.length; ++j) {
if (assignedIds[j] == availableProjects[i].id) {
found = true;
break;
}
}
if (!found) {
remainingProjects.push(availableProjects[i]);
}
}
if (remainingProjects.length > 0) {
projectSelector = document.createElement("SELECT");
let option = document.createElement("OPTION");
option.textContent = "Add to project...";
option.selected = true;
projectSelector.appendChild(option);
for (let i = 0; i < remainingProjects.length; ++i) {
option = document.createElement("OPTION");
option.value = remainingProjects[i].id;
option.selected = false;
option.textContent = remainingProjects[i].name;
projectSelector.appendChild(option);
}
projectSelector.addEventListener("change", function(ev) {
if (projectSelector.selectedOptions.length > 0) {
let selected = projectSelector.selectedOptions[0];
if (selected.value != "") {
let id = parseInt(selected.value, 10);
if (!isNaN(id)) {
for (let i = 0; i < availableProjects.length; ++i) {
if (availableProjects[i].id == id) {
addProject(availableProjects[i]);
break;
}
}
}
updateProjectSelector();
}
}
});
snippetEdit.projectList.appendChild(projectSelector);
}
}
function setFile(file) {
let dt = new DataTransfer();
dt.items.add(file);
snippetEdit.file.files = dt.files;
attachmentChanged = true;
snippetEdit.removeAttachment.value = "false";
hasAttachment = true;
let el = null;
if (file.type.startsWith("image/")) {
el = document.createElement("img");
el.src = URL.createObjectURL(file);
} else if (file.type.startsWith("video/")) {
el = document.createElement("video");
el.src = URL.createObjectURL(file);
el.controls = true;
} else if (file.type.startsWith("audio/")) {
el = document.createElement("audio");
el.src = URL.createObjectURL(file);
} else {
el = document.createElement("div");
el.classList.add("project-card", "br2", "pv1", "ph2");
let anchor = document.createElement("a");
anchor.href = URL.createObjectURL(file);
anchor.setAttribute("target", "_blank");
anchor.textContent = file.name + " (" + readableByteSize(file.size) + ")";
el.appendChild(anchor);
}
setPreview(el);
validate();
}
function clearAttachment(restoreOriginal) {
snippetEdit.file.value = "";
let el = null;
attachmentChanged = false;
hasAttachment = false;
snippetEdit.removeAttachment.value = "false";
if (originalAttachment) {
if (restoreOriginal) {
hasAttachment = true;
el = originalAttachment;
} else {
attachmentChanged = true;
snippetEdit.removeAttachment.value = "true";
}
}
setPreview(el);
validate();
}
function setPreview(el) {
if (el) {
snippetEdit.uploadBox.style.display = "none";
snippetEdit.previewBox.style.display = "block";
snippetEdit.uploadResetLink.style.display = "none";
snippetEdit.previewContent = emptyElement(snippetEdit.previewContent);
snippetEdit.previewContent.appendChild(el);
snippetEdit.resetLink.style.display = (!originalAttachment || el == originalAttachment) ? "none" : "inline-block";
} else {
snippetEdit.uploadBox.style.display = "flex";
snippetEdit.previewBox.style.display = "none";
if (originalAttachment) {
snippetEdit.uploadResetLink.style.display = "block";
}
}
}
function validate() {
let sizeGood = true;
if (snippetEdit.file.files.length > 0 && snippetEdit.file.files[0].size > maxFilesize) {
// NOTE(asaf): Writing this out in bytes to make the limit exactly clear to the user.
let readableSize = new Intl.NumberFormat([], { useGrouping: "always" }).format(maxFilesize);
snippetEdit.errors.textContent = "File is too big! Max filesize is " + readableSize + " bytes";
sizeGood = false;
} else {
snippetEdit.errors.textContent = "";
}
let hasText = snippetEdit.text.value.trim().length > 0;
if ((hasText || hasAttachment) && sizeGood) {
snippetEdit.saveButton.disabled = false;
} else {
snippetEdit.saveButton.disabled = true;
}
}
snippetEdit.uploadLink.addEventListener("click", function() {
snippetEdit.file.click();
});
snippetEdit.removeLink.addEventListener("click", function() {
clearAttachment(false);
});
snippetEdit.replaceLink.addEventListener("click", function() {
snippetEdit.file.click();
});
snippetEdit.resetLink.addEventListener("click", function() {
clearAttachment(true);
});
snippetEdit.uploadResetLink.addEventListener("click", function() {
clearAttachment(true);
});
snippetEdit.file.addEventListener("change", function() {
if (snippetEdit.file.files.length > 0) {
setFile(snippetEdit.file.files[0]);
}
});
snippetEdit.root.addEventListener("dragover", function(ev) {
let effect = "none";
for (let i = 0; i < ev.dataTransfer.items.length; ++i) {
if (ev.dataTransfer.items[i].kind.toLowerCase() == "file") {
effect = "copy";
break;
}
}
ev.dataTransfer.dropEffect = effect;
ev.preventDefault();
});
let enterCounter = 0;
snippetEdit.root.addEventListener("dragenter", function(ev) {
enterCounter++;
if (ev.dataTransfer && ev.dataTransfer.files && ev.dataTransfer.files.length > 0) {
snippetEdit.root.classList.add("drop");
}
});
snippetEdit.root.addEventListener("dragleave", function(ev) {
enterCounter--;
if (enterCounter == 0) {
snippetEdit.root.classList.remove("drop");
}
});
snippetEdit.root.addEventListener("drop", function(ev) {
enterCounter = 0;
snippetEdit.root.classList.remove("drop");
if (ev.dataTransfer && ev.dataTransfer.files && ev.dataTransfer.files.length > 0) {
setFile(ev.dataTransfer.files[0]);
}
ev.preventDefault();
});
snippetEdit.text.addEventListener("paste", function(ev) {
const files = ev.clipboardData?.files ?? [];
if (files.length > 0) {
setFile(files[0]);
}
});
snippetEdit.text.addEventListener("input", function(ev) {
validate();
});
snippetEdit.saveButton.addEventListener("click", function(ev) {
let projectsChanged = false;
let projInputs = snippetEdit.projectList.querySelectorAll("input[name=project_id]");
let assignedIds = [];
for (let i = 0; i < projInputs.length; ++i) {
let id = parseInt(projInputs[i].value, 10);
if (!isNaN(id)) {
assignedIds.push(id);
}
}
if (projectIds.length != assignedIds.length) {
projectsChanged = true;
} else {
for (let i = 0; i < projectIds.length; ++i) {
let found = false;
for (let j = 0; j < assignedIds.length; ++j) {
if (projectIds[i] == assignedIds[j]) {
found = true;
}
}
if (!found) {
projectsChanged = true;
break;
}
}
}
if (originalSnippetEl && (!attachmentChanged && originalText == snippetEdit.text.value.trim() && !projectsChanged)) {
// NOTE(asaf): We're in edit mode and nothing changed, so no need to submit to the server.
ev.preventDefault();
snippetEdit.root.parentElement.insertBefore(originalSnippetEl, snippetEdit.root);
snippetEdit.root.remove();
}
});
snippetEdit.deleteButton.addEventListener("click", function(ev) {
snippetEdit.file.value = "";
});
validate();
return snippetEdit;
}
function editTimelineSnippet(timelineItemEl) {
let ownerName = timelineItemEl.querySelector(".user")?.textContent;
let ownerUrl = timelineItemEl.querySelector(".user")?.href;
let ownerAvatar = timelineItemEl.querySelector(".avatar-icon")?.src;
let creationDate = new Date(timelineItemEl.querySelector("time").dateTime);
let rawDesc = timelineItemEl.querySelector(".rawdesc").textContent;
let attachment = timelineItemEl.querySelector(".timeline-content-box")?.children?.[0];
let projectIds = [];
let projectEls = timelineItemEl.querySelectorAll(".projects > a");
for (let i = 0; i < projectEls.length; ++i) {
let projid = projectEls[i].getAttribute("data-projid");
if (projid) {
projectIds.push(projid);
}
}
let snippetEdit = makeSnippetEdit(ownerName, ownerAvatar, ownerUrl, creationDate, rawDesc, attachment, projectIds, timelineItemEl.getAttribute("data-id"), timelineItemEl);
timelineItemEl.parentElement.insertBefore(snippetEdit.root, timelineItemEl);
timelineItemEl.remove();
}

@ -7461,6 +7461,30 @@ article code {
.minh-6 {
min-height: 32rem; }
.h1-5 {
height: 1.5rem; }
.gap0 {
gap: 0; }
.gap1 {
gap: 0.25rem; }
.gap2 {
gap: 0.5rem; }
.gap3 {
gap: 1rem; }
.gap4 {
gap: 2rem; }
.gap5 {
gap: 4rem; }
.pre-line {
white-space: pre-line; }
.fira {
font-family: "Fira Sans", sans-serif; }

@ -466,5 +466,48 @@ func init() {
moveThreadsToSubforumCommand.MarkFlagRequired("subforum_slug")
adminCommand.AddCommand(moveThreadsToSubforumCommand)
fixupSnippetAssociation := &cobra.Command{
Use: "fixupsnippets",
Short: "Associates tagged snippets with the right projects",
Run: func(cmd *cobra.Command, args []string) {
ctx := context.Background()
conn := db.NewConn()
defer conn.Close(ctx)
type snippetProject struct {
SnippetID int `db:"snippet_tag.snippet_id"`
ProjectID int `db:"project.id"`
}
res, err := db.Query[snippetProject](ctx, conn,
`
SELECT $columns
FROM snippet_tag
JOIN project ON project.tag = snippet_tag.tag_id
`,
)
if err != nil {
panic(err)
}
for _, sp := range res {
_, err = conn.Exec(ctx,
`
INSERT INTO snippet_project (snippet_id, project_id, kind)
VALUES ($1, $2, $3)
ON CONFLICT DO NOTHING;
`,
sp.SnippetID,
sp.ProjectID,
models.SnippetProjectKindDiscord,
)
if err != nil {
panic(err)
}
}
fmt.Printf("Done!\n")
},
}
adminCommand.AddCommand(fixupSnippetAssociation)
addProjectCommands(adminCommand)
}

@ -783,9 +783,6 @@ func HandleSnippetForInternedMessage(ctx context.Context, dbConn db.ConnOrTx, in
// Match only tags for projects in which the current user is a collaborator.
messageTags := getDiscordTags(existingSnippet.Description)
var desiredTags []int
var allTags []int
// Fetch projects so we know what tags the user can apply to their snippet.
projects, err := hmndata.FetchProjects(ctx, tx, interned.HMNUser, hmndata.ProjectsQuery{
OwnerIDs: []int{interned.HMNUser.ID},
@ -794,61 +791,40 @@ func HandleSnippetForInternedMessage(ctx context.Context, dbConn db.ConnOrTx, in
return oops.New(err, "failed to look up user projects")
}
projectIDs := make([]int, len(projects))
for i, p := range projects {
projectIDs[i] = p.Project.ID
}
userTags, err := db.Query[models.Tag](ctx, tx,
`
SELECT $columns{tag}
FROM
tag
JOIN project ON project.tag = tag.id
WHERE
project.id = ANY ($1)
`,
projectIDs,
)
if err != nil {
return oops.New(err, "failed to fetch tags for user projects")
}
for _, tag := range userTags {
allTags = append(allTags, tag.ID)
for _, messageTag := range messageTags {
if strings.EqualFold(tag.Text, messageTag) {
desiredTags = append(desiredTags, tag.ID)
}
}
}
_, err = tx.Exec(ctx,
`
DELETE FROM snippet_tag
DELETE FROM snippet_project
WHERE
snippet_id = $1
AND tag_id = ANY ($2)
AND kind = $2
`,
existingSnippet.ID,
allTags,
models.SnippetProjectKindDiscord,
)
if err != nil {
return oops.New(err, "failed to clear tags from snippet")
return oops.New(err, "failed to clear project association for snippet")
}
for _, tagID := range desiredTags {
_, err = tx.Exec(ctx,
`
INSERT INTO snippet_tag (snippet_id, tag_id)
VALUES ($1, $2)
ON CONFLICT DO NOTHING
`,
existingSnippet.ID,
tagID,
)
if err != nil {
return oops.New(err, "failed to associate snippet with tag")
for _, p := range projects {
if p.Tag != nil {
for _, messageTag := range messageTags {
if strings.EqualFold(p.Tag.Text, messageTag) {
_, err = tx.Exec(ctx,
`
INSERT INTO snippet_project (project_id, snippet_id, kind)
VALUES ($1, $2, $3)
ON CONFLICT DO NOTHING
`,
p.Project.ID,
existingSnippet.ID,
models.SnippetProjectKindDiscord,
)
if err != nil {
return oops.New(err, "failed to associate snippet with project")
}
break
}
}
}
}
}
@ -876,6 +852,7 @@ func getDiscordTags(content string) []string {
// NOTE(ben): This is maybe redundant with the regexes we use for markdown. But
// do we actually want to reuse those, or should we keep them separate?
// TODO(asaf): Centralize this
var RESnippetableUrl = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`)
func getSnippetAssetOrUrl(ctx context.Context, tx db.ConnOrTx, msg *models.DiscordMessage) (*uuid.UUID, *string, error) {

@ -0,0 +1,199 @@
package embed
import (
"bytes"
"context"
"errors"
"io"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"git.handmade.network/hmn/hmn/src/logging"
"git.handmade.network/hmn/hmn/src/utils"
)
var DownloadTooBigError = errors.New("download too big")
var NoEmbedFound = errors.New("no embed found")
type Embeddable struct {
Url string
File *Embed
}
type Embed struct {
Data []byte
ContentType string
Filename string
}
var EmbeddableUrlRegex = regexp.MustCompile(`^https?://(youtu\.be|(www\.)?youtube\.com/watch)`)
func IsUrlEmbeddable(u string) bool {
return EmbeddableUrlRegex.MatchString(u)
}
func GetEmbeddableFromUrls(ctx context.Context, urls []string, maxSize int, httpTimeout time.Duration, httpMaxAttempts int) (*Embeddable, error) {
embedError := NoEmbedFound
for _, urlStr := range urls {
u, err := url.Parse(urlStr)
if err != nil {
continue
}
if u.Scheme == "" {
u.Scheme = "https"
urlStr = u.String()
}
if (u.Scheme != "http" && u.Scheme != "https") || u.Host == "" {
continue
}
if IsUrlEmbeddable(urlStr) {
result := Embeddable{
Url: urlStr,
}
return &result, nil
}
if httpMaxAttempts > 0 {
httpMaxAttempts -= 1
embed, err := FetchEmbed(ctx, urlStr, httpTimeout, maxSize)
if err != nil {
embedError = err
continue
}
result := Embeddable{
File: embed,
}
return &result, nil
}
}
return nil, embedError
}
// If the url points to a file, downloads and returns the file.
// If the url points to an html page, parses opengraph and tries to fetch an image/video/audio file according to that.
// maxSize only limits the actual filesize. In the case of html we always fetch up to 100kb even if maxSize is smaller.
func FetchEmbed(ctx context.Context, urlStr string, timeout time.Duration, maxSize int) (*Embed, error) {
logging.ExtractLogger(ctx).Debug().Msg("Fetching embed")
client := &http.Client{
Timeout: timeout,
}
req, err := http.NewRequestWithContext(ctx, "GET", urlStr, nil)
if err != nil {
return nil, err
}
res, err := client.Do(req)
if err != nil {
return nil, err
}
if res.StatusCode < 200 || res.StatusCode > 299 {
return nil, NoEmbedFound
}
contentType := res.Header.Get("Content-Type")
logging.ExtractLogger(ctx).Debug().Str("type", contentType).Msg("Got first result")
if strings.HasPrefix(contentType, "text/html") || strings.HasPrefix(contentType, "application/html") {
var buffer bytes.Buffer
_, err := io.CopyN(&buffer, res.Body, 100*1024) // NOTE(asaf): If the opengraph stuff isn't in the first 100kb, we don't care.
res.Body.Close()
if err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
partialHtml := buffer.Bytes()
urlStr = ExtractEmbedFromOpenGraph(partialHtml)
logging.ExtractLogger(ctx).Debug().Str("url", urlStr).Msg("Got ograph")
if urlStr == "" {
return nil, NoEmbedFound
}
req, err = http.NewRequestWithContext(ctx, "GET", urlStr, nil)
if err != nil {
return nil, err
}
res, err = client.Do(req)
if err != nil {
return nil, err
}
if res.StatusCode < 200 || res.StatusCode > 299 {
return nil, NoEmbedFound
}
contentType = res.Header.Get("Content-Type")
}
var buffer bytes.Buffer
n, err := io.CopyN(&buffer, res.Body, int64(maxSize+1))
res.Body.Close()
if err != nil && !errors.Is(err, io.EOF) {
return nil, err
}
filename := ""
u, err := url.Parse(urlStr)
if err == nil {
lastSlash := utils.IntMax(strings.LastIndex(u.Path, "/"), 0)
filename = u.Path[lastSlash:]
}
result := Embed{
Data: buffer.Bytes(),
ContentType: contentType,
Filename: filename,
}
if n == int64(maxSize+1) {
err = DownloadTooBigError
} else {
err = nil
}
return &result, err
}
var metaRegex = regexp.MustCompile(`<meta\s+([^>]+)/?>`)
var metaAttrRegex = regexp.MustCompile(`(?P<key>\w+)="(?P<value>[^"]+)"`)
var OGKeys = []string{
"og:audio",
"og:video",
"og:image",
"og:audio:url",
"og:image:url",
"og:video:url",
"og:audio:secure_url",
"og:image:secure_url",
"og:video:secure_url",
"twitter:image",
}
// Tries to find an opengraph image/video/audio url in the provided html
// Since we only need to look at meta tags in the head, we don't need the full html document.
func ExtractEmbedFromOpenGraph(partialHtml []byte) string {
keyIdx := metaAttrRegex.SubexpIndex("key")
valueIdx := metaAttrRegex.SubexpIndex("value")
html := string(partialHtml)
matches := metaRegex.FindAllStringSubmatch(html, -1)
for _, m := range matches {
if len(m) > 1 {
content := ""
prop := ""
attrs := metaAttrRegex.FindAllStringSubmatch(m[1], -1)
for _, attr := range attrs {
key := attr[keyIdx]
value := attr[valueIdx]
if key == "name" || key == "property" {
for _, ogKey := range OGKeys {
if value == ogKey {
prop = value
}
}
} else if key == "content" {
content = value
}
}
if content != "" && prop != "" {
return content
}
}
}
return ""
}

@ -12,6 +12,7 @@ import (
type SnippetQuery struct {
IDs []int
OwnerIDs []int
ProjectIDs []int
Tags []int
DiscordMessageIDs []string
@ -24,6 +25,7 @@ type SnippetAndStuff struct {
Asset *models.Asset `db:"asset"`
DiscordMessage *models.DiscordMessage `db:"discord_message"`
Tags []*models.Tag
Projects []*ProjectAndStuff
}
func FetchSnippets(
@ -42,6 +44,7 @@ func FetchSnippets(
}
defer tx.Rollback(ctx)
var tagSnippetIDs []int
if len(q.Tags) > 0 {
// Get snippet IDs with this tag, then use that in the main query
snippetIDs, err := db.QueryScalar[int](ctx, tx,
@ -64,7 +67,32 @@ func FetchSnippets(
return nil, nil
}
q.IDs = snippetIDs
tagSnippetIDs = snippetIDs
}
var projectSnippetIDs []int
if len(q.ProjectIDs) > 0 {
// Get snippet IDs for these projects, then use that in the main query
snippetIDs, err := db.QueryScalar[int](ctx, tx,
`
SELECT DISTINCT snippet_id
FROM
snippet_project
WHERE
project_id = ANY ($1)
`,
q.ProjectIDs,
)
if err != nil {
return nil, oops.New(err, "failed to get snippet IDs for tag")
}
// special early-out: no snippets found for these projects at all
if len(snippetIDs) == 0 {
return nil, nil
}
projectSnippetIDs = snippetIDs
}
var qb db.QueryBuilder
@ -74,7 +102,7 @@ func FetchSnippets(
FROM
snippet
LEFT JOIN hmn_user AS owner ON snippet.owner_id = owner.id
LEFT JOIN asset AS owner_avatar ON owner_avatar.id = owner.avatar_asset_id
LEFT JOIN asset AS avatar ON avatar.id = owner.avatar_asset_id
LEFT JOIN asset ON snippet.asset_id = asset.id
LEFT JOIN discord_message ON snippet.discord_message_id = discord_message.id
WHERE
@ -84,6 +112,12 @@ func FetchSnippets(
if len(q.IDs) > 0 {
qb.Add(`AND snippet.id = ANY ($?)`, q.IDs)
}
if len(tagSnippetIDs) > 0 {
qb.Add(`AND snippet.id = ANY ($?)`, tagSnippetIDs)
}
if len(projectSnippetIDs) > 0 {
qb.Add(`AND snippet.id = ANY ($?)`, projectSnippetIDs)
}
if len(q.OwnerIDs) > 0 {
qb.Add(`AND snippet.owner_id = ANY ($?)`, q.OwnerIDs)
}
@ -115,6 +149,7 @@ func FetchSnippets(
type resultRow struct {
Snippet models.Snippet `db:"snippet"`
Owner *models.User `db:"owner"`
AvatarAsset *models.Asset `db:"avatar"`
Asset *models.Asset `db:"asset"`
DiscordMessage *models.DiscordMessage `db:"discord_message"`
}
@ -127,6 +162,9 @@ func FetchSnippets(
result := make([]SnippetAndStuff, len(results)) // allocate extra space because why not
snippetIDs := make([]int, len(results))
for i, row := range results {
if results[i].Owner != nil {
results[i].Owner.AvatarAsset = results[i].AvatarAsset
}
result[i] = SnippetAndStuff{
Snippet: row.Snippet,
Owner: row.Owner,
@ -167,6 +205,42 @@ func FetchSnippets(
item.Tags = append(item.Tags, snippetTag.Tag)
}
// Fetch projects
type snippetProjectRow struct {
SnippetID int `db:"snippet_id"`
ProjectID int `db:"project_id"`
}
snippetProjects, err := db.Query[snippetProjectRow](ctx, tx,
`
SELECT $columns
FROM snippet_project
WHERE snippet_id = ANY($1)
`,
snippetIDs,
)
if err != nil {
return nil, oops.New(err, "failed to fetch project ids for snippets")
}
var projectIds []int
for _, sp := range snippetProjects {
projectIds = append(projectIds, sp.ProjectID)
}
projects, err := FetchProjects(ctx, tx, currentUser, ProjectsQuery{ProjectIDs: projectIds})
if err != nil {
return nil, oops.New(err, "failed to fetch projects for snippets")
}
projectMap := make(map[int]*ProjectAndStuff)
for i := range projects {
projectMap[projects[i].Project.ID] = &projects[i]
}
for _, sp := range snippetProjects {
snip, hasResult := resultBySnippetId[sp.SnippetID]
proj, hasProj := projectMap[sp.ProjectID]
if hasResult && hasProj {
snip.Projects = append(snip.Projects, proj)
}
}
err = tx.Commit(ctx)
if err != nil {
return nil, oops.New(err, "failed to commit transaction")

@ -120,6 +120,10 @@ func TestSnippet(t *testing.T) {
AssertRegexMatch(t, BuildSnippet(15), RegexSnippet, map[string]string{"snippetid": "15"})
}
func TestSnippetSubmit(t *testing.T) {
AssertRegexMatch(t, BuildSnippetSubmit(), RegexSnippetSubmit, nil)
}
func TestFeed(t *testing.T) {
AssertRegexMatch(t, BuildFeed(), RegexFeed, nil)
assert.Equal(t, BuildFeed(), BuildFeedWithPage(1))

@ -262,6 +262,13 @@ func BuildSnippet(snippetId int) string {
return Url("/snippet/"+strconv.Itoa(snippetId), nil)
}
var RegexSnippetSubmit = regexp.MustCompile(`^/snippet$`)
func BuildSnippetSubmit() string {
defer CatchPanic()
return Url("/snippet", nil)
}
/*
* Feed
*/

@ -0,0 +1,50 @@
package migrations
import (
"context"
"time"
"git.handmade.network/hmn/hmn/src/migration/types"
"github.com/jackc/pgx/v4"
)
func init() {
registerMigration(SnippetProjectAssociation{})
}
type SnippetProjectAssociation struct{}
func (m SnippetProjectAssociation) Version() types.MigrationVersion {
return types.MigrationVersion(time.Date(2022, 6, 26, 11, 57, 3, 0, time.UTC))
}
func (m SnippetProjectAssociation) Name() string {
return "SnippetProjectAssociation"
}
func (m SnippetProjectAssociation) Description() string {
return "Table for associating a snippet with projects"
}
func (m SnippetProjectAssociation) Up(ctx context.Context, tx pgx.Tx) error {
_, err := tx.Exec(ctx,
`
CREATE TABLE snippet_project (
snippet_id INTEGER NOT NULL REFERENCES snippet (id) ON DELETE CASCADE,
project_id INTEGER NOT NULL REFERENCES project (id) ON DELETE CASCADE,
kind INTEGER NOT NULL,
UNIQUE (snippet_id, project_id)
);
`,
)
return err
}
func (m SnippetProjectAssociation) Down(ctx context.Context, tx pgx.Tx) error {
_, err := tx.Exec(ctx,
`
DROP TABLE snippet_project;
`,
)
return err
}

@ -6,6 +6,13 @@ import (
"github.com/google/uuid"
)
type SnippetProjectAssociationKind int
const (
SnippetProjectKindDiscord SnippetProjectAssociationKind = iota + 1
SnippetProjectKindWebsite
)
type Snippet struct {
ID int `db:"id"`
OwnerID int `db:"owner_id"`

@ -333,6 +333,38 @@ article code {
min-height: $height-6;
}
.h1-5 {
height: 1.5rem;
}
.gap0 {
gap: $spacing-none;
}
.gap1 {
gap: $spacing-extra-small;
}
.gap2 {
gap: $spacing-small;
}
.gap3 {
gap: $spacing-medium;
}
.gap4 {
gap: $spacing-large;
}
.gap5 {
gap: $spacing-extra-large;
}
.pre-line {
white-space: pre-line;
}
.fira {
font-family: "Fira Sans", sans-serif;
}

@ -78,6 +78,7 @@ func ProjectToTemplate(
url string,
) Project {
return Project{
ID: p.ID,
Name: p.Name,
Subdomain: p.Subdomain(),
Color1: p.Color1,
@ -297,8 +298,6 @@ func LinkToTemplate(link *models.Link) Link {
return tlink
}
var controlCharRegex = regexp.MustCompile(`\p{Cc}`)
func TimelineItemsToJSON(items []TimelineItem) string {
// NOTE(asaf): As of 2021-06-22: This only serializes the data necessary for snippet showcase.
builder := strings.Builder{}
@ -378,22 +377,26 @@ func TimelineItemsToJSON(items []TimelineItem) string {
builder.WriteString(item.DiscordMessageUrl)
builder.WriteString(`",`)
builder.WriteString(`"tags":[`)
for i, tag := range item.Tags {
builder.WriteString(`{`)
builder.WriteString(`"projects":[`)
for j, proj := range item.Projects {
if j > 0 {
builder.WriteRune(',')
}
builder.WriteRune('{')
builder.WriteString(`"name":"`)
builder.WriteString(proj.Name)
builder.WriteString(`",`)
builder.WriteString(`"text":"`)
builder.WriteString(tag.Text)
builder.WriteString(`"logo":"`)
builder.WriteString(proj.Logo)
builder.WriteString(`",`)
builder.WriteString(`"url":"`)
builder.WriteString(tag.Url)
builder.WriteString(proj.Url)
builder.WriteString(`"`)
builder.WriteString(`}`)
if i < len(item.Tags)-1 {
builder.WriteString(`,`)
}
builder.WriteRune('}')
}
builder.WriteString(`]`)
@ -403,6 +406,33 @@ func TimelineItemsToJSON(items []TimelineItem) string {
return builder.String()
}
func SnippetEditProjectsToJSON(projects []Project) string {
builder := strings.Builder{}
builder.WriteRune('[')
for i, proj := range projects {
if i > 0 {
builder.WriteRune(',')
}
builder.WriteRune('{')
builder.WriteString(`"id":`)
builder.WriteString(strconv.FormatInt(int64(proj.ID), 10))
builder.WriteRune(',')
builder.WriteString(`"name":"`)
builder.WriteString(proj.Name)
builder.WriteString(`",`)
builder.WriteString(`"logo":"`)
builder.WriteString(proj.Logo)
builder.WriteRune('"')
builder.WriteRune('}')
}
builder.WriteRune(']')
return builder.String()
}
func PodcastToTemplate(podcast *models.Podcast, imageFilename string) Podcast {
imageUrl := ""
if imageFilename != "" {

@ -55,7 +55,7 @@
<content type="xhtml">
<div xmlns="http://www.w3.org/1999/xhtml">
<div>
{{ .Description }}
{{ cleancontrolchars .Description }}
</div>
{{ range .EmbedMedia }}
<div>

@ -27,10 +27,10 @@
<a class="user" data-tmpl="userLink"></a>
<a data-tmpl="date" class="datetime tr" style="flex: 1 1 auto;"></a>
</div>
<div class="pre overflow-auto" data-tmpl="description">
<div class="pre-line overflow-auto" data-tmpl="description">
Unknown description
</div>
<div data-tmpl="tags" class="pt2 flex"></div>
<div data-tmpl="projects" class="pt2 flex gap2"></div>
<div class="i f7 pt2">
<a data-tmpl="discord_link" target="_blank">View original message on Discord</a>
</div>
@ -40,6 +40,9 @@
</div>
</template>
<template id="timeline_item_tag">
<div data-tmpl="tag" class="bg-theme-dimmer ph2 pv1 br2"></div>
<template id="project_link">
<a data-tmpl="root" class="flex flex-row items-center bg-theme-dimmer ph2 pv1 br2">
<img data-tmpl="logo" class="db mr1 br1 h1-5" />
<div data-tmpl="name"></div>
</a>
</template>

@ -0,0 +1,53 @@
<template id="snippet-edit">
<form data-tmpl="root" class="timeline-item pa3 mb2 br3" method="POST" action="{{ .SnippetEdit.SubmitUrl }}" enctype="multipart/form-data">
{{ csrftoken .Session }}
<input data-tmpl="redirect" type="hidden" name="redirect" />
<input data-tmpl="snippetId" type="hidden" name="snippet_id" />
<input data-tmpl="removeAttachment" type="hidden" name="remove_attachment" value="false" />
<input data-tmpl="file" type="file" name="file" class="dn" />
<div class="flex items-center">
<a data-tmpl="avatarLink" class="flex-shrink-0"><img data-tmpl="avatarImg" class="avatar-icon lite mr2" /></a>
<a data-tmpl="username" class="flex-shrink-0"></a>
<div class="spacer flex-grow-1"></div>
<span data-tmpl="date" class="flex-shrink-0">Date</span>
</div>
<textarea data-tmpl="text" placeholder="Description and/or links" class="w-100 h4 mt3" name="text"></textarea>
<div class="mv3">
<div data-tmpl="uploadBox" class="placeholder flex flex-column items-center">
<img src="" class="db w4 h4"/>
<a data-tmpl="uploadResetLink" class="mt3 dn" href="javascript:;">Restore</a>
<a data-tmpl="uploadLink" class="mt3" href="javascript:;">Upload image, video, or other file</a>
</div>
<div data-tmpl="previewBox" class="preview dn">
<div class="actions">
<a data-tmpl="removeLink" href="javascript:;">Remove</a>
<a data-tmpl="resetLink" href="javascript:;">Restore</a>
<a data-tmpl="replaceLink" href="javascript:;">Replace</a>
</div>
<div data-tmpl="previewContent">
</div>
</div>
</div>
<div data-tmpl="projectList" class="flex flex-wrap gap2"></div>
<div class="flex items-center">
<div data-tmpl="errors"></div>
<div class="flex-grow-1"></div>
<input data-tmpl="deleteButton" class="flex-grow-0 flex-shrink-0 mr3" type="submit" name="action" value="Delete" />
<input data-tmpl="saveButton" class="flex-grow-0 flex-shrink-0" type="submit" name="action" value="Save" />
</div>
</form>
</template>
<template id="snippet-edit-project">
<div data-tmpl="root" class="flex flex-row items-center bg-theme-dimmer ph2 pv1 br2">
<input data-tmpl="projectId" type="hidden" name="project_id" />
<img data-tmpl="projectLogo" class="db mr1 br1 h1-5"/>
<div data-tmpl="projectName"></div>
<a data-tmpl="removeButton" class="ml1" href="javascript:;">&#10006;</a>
</div>
</template>
<script>
const maxFilesize = {{ .SnippetEdit.AssetMaxSize }};
const availableProjects = JSON.parse("{{ .SnippetEdit.AvailableProjectsJSON }}");
</script>
<script src="{{ static "js/snippetedit.js" }}"></script>

@ -1,4 +1,4 @@
<div class="timeline-item flex flex-column pa3 mb2 br3" {{ with .FilterTitle }}data-filter-title="{{ . }}"{{ end }}>
<div class="timeline-item flex flex-column pa3 mb2 br3" data-id="{{ .ID }}" {{ with .FilterTitle }}data-filter-title="{{ . }}"{{ end }}>
{{/* top bar - avatar, info, date */}}
<div class="flex items-center">
{{ if .OwnerAvatarUrl }}
@ -28,12 +28,16 @@
{{ if .SmallInfo }}
<a href="{{ .Url }}">{{ timehtml (relativedate .Date) .Date }}</a>
{{ end }}
{{ if .Editable }}
<a href="javascript:;" class="edit ml2">&#9998;</a>
<div class="dn rawdesc">{{ .RawDescription }}</div>
{{ end }}
</div>