Added basic usage tracking for posts.

public/js/base64.js

@@ -0,0 +1,173 @@
+// NOTE(asaf): The built-in base64 capabilities can't handle non-ascii characters.
+// This code is copied from MDN at https://developer.mozilla.org/en-US/docs/Glossary/Base64
+//
+// Usage:
+// var sMyInput = "Base 64 \u2014 Mozilla Developer Network";
+// var aMyUTF8Input = strToUTF8Arr(sMyInput);
+// var sMyBase64 = base64EncArr(aMyUTF8Input);
+// alert(sMyBase64);
+// 
+// var aMyUTF8Output = base64DecToArr(sMyBase64);
+// var sMyOutput = UTF8ArrToStr(aMyUTF8Output);
+// alert(sMyOutput);
+
+function b64ToUint6 (nChr) {
+
+  return nChr > 64 && nChr < 91 ?
+      nChr - 65
+    : nChr > 96 && nChr < 123 ?
+      nChr - 71
+    : nChr > 47 && nChr < 58 ?
+      nChr + 4
+    : nChr === 43 ?
+      62
+    : nChr === 47 ?
+      63
+    :
+      0;
+
+}
+
+function base64DecToArr (sBase64, nBlocksSize) {
+
+  var
+    sB64Enc = sBase64.replace(/[^A-Za-z0-9\+\/]/g, ""), nInLen = sB64Enc.length,
+    nOutLen = nBlocksSize ? Math.ceil((nInLen * 3 + 1 >> 2) / nBlocksSize) * nBlocksSize : nInLen * 3 + 1 >> 2, taBytes = new Uint8Array(nOutLen);
+
+  for (var nMod3, nMod4, nUint24 = 0, nOutIdx = 0, nInIdx = 0; nInIdx < nInLen; nInIdx++) {
+    nMod4 = nInIdx & 3;
+    nUint24 |= b64ToUint6(sB64Enc.charCodeAt(nInIdx)) << 6 * (3 - nMod4);
+    if (nMod4 === 3 || nInLen - nInIdx === 1) {
+      for (nMod3 = 0; nMod3 < 3 && nOutIdx < nOutLen; nMod3++, nOutIdx++) {
+        taBytes[nOutIdx] = nUint24 >>> (16 >>> nMod3 & 24) & 255;
+      }
+      nUint24 = 0;
+
+    }
+  }
+
+  return taBytes;
+}
+
+/* Base64 string to array encoding */
+
+function uint6ToB64 (nUint6) {
+
+  return nUint6 < 26 ?
+      nUint6 + 65
+    : nUint6 < 52 ?
+      nUint6 + 71
+    : nUint6 < 62 ?
+      nUint6 - 4
+    : nUint6 === 62 ?
+      43
+    : nUint6 === 63 ?
+      47
+    :
+      65;
+
+}
+
+function base64EncArr (aBytes) {
+
+  var nMod3 = 2, sB64Enc = "";
+
+  for (var nLen = aBytes.length, nUint24 = 0, nIdx = 0; nIdx < nLen; nIdx++) {
+    nMod3 = nIdx % 3;
+    if (nIdx > 0 && (nIdx * 4 / 3) % 76 === 0) { sB64Enc += "\r\n"; }
+    nUint24 |= aBytes[nIdx] << (16 >>> nMod3 & 24);
+    if (nMod3 === 2 || aBytes.length - nIdx === 1) {
+      sB64Enc += String.fromCharCode(uint6ToB64(nUint24 >>> 18 & 63), uint6ToB64(nUint24 >>> 12 & 63), uint6ToB64(nUint24 >>> 6 & 63), uint6ToB64(nUint24 & 63));
+      nUint24 = 0;
+    }
+  }
+
+  return sB64Enc.substr(0, sB64Enc.length - 2 + nMod3) + (nMod3 === 2 ? '' : nMod3 === 1 ? '=' : '==');
+
+}
+
+/* UTF-8 array to DOMString and vice versa */
+
+function UTF8ArrToStr (aBytes) {
+
+  var sView = "";
+
+  for (var nPart, nLen = aBytes.length, nIdx = 0; nIdx < nLen; nIdx++) {
+    nPart = aBytes[nIdx];
+    sView += String.fromCharCode(
+      nPart > 251 && nPart < 254 && nIdx + 5 < nLen ? /* six bytes */
+        /* (nPart - 252 << 30) may be not so safe in ECMAScript! So...: */
+        (nPart - 252) * 1073741824 + (aBytes[++nIdx] - 128 << 24) + (aBytes[++nIdx] - 128 << 18) + (aBytes[++nIdx] - 128 << 12) + (aBytes[++nIdx] - 128 << 6) + aBytes[++nIdx] - 128
+      : nPart > 247 && nPart < 252 && nIdx + 4 < nLen ? /* five bytes */
+        (nPart - 248 << 24) + (aBytes[++nIdx] - 128 << 18) + (aBytes[++nIdx] - 128 << 12) + (aBytes[++nIdx] - 128 << 6) + aBytes[++nIdx] - 128
+      : nPart > 239 && nPart < 248 && nIdx + 3 < nLen ? /* four bytes */
+        (nPart - 240 << 18) + (aBytes[++nIdx] - 128 << 12) + (aBytes[++nIdx] - 128 << 6) + aBytes[++nIdx] - 128
+      : nPart > 223 && nPart < 240 && nIdx + 2 < nLen ? /* three bytes */
+        (nPart - 224 << 12) + (aBytes[++nIdx] - 128 << 6) + aBytes[++nIdx] - 128
+      : nPart > 191 && nPart < 224 && nIdx + 1 < nLen ? /* two bytes */
+        (nPart - 192 << 6) + aBytes[++nIdx] - 128
+      : /* nPart < 127 ? */ /* one byte */
+        nPart
+    );
+  }
+
+  return sView;
+
+}
+
+function strToUTF8Arr (sDOMStr) {
+
+  var aBytes, nChr, nStrLen = sDOMStr.length, nArrLen = 0;
+
+  /* mapping... */
+
+  for (var nMapIdx = 0; nMapIdx < nStrLen; nMapIdx++) {
+    nChr = sDOMStr.charCodeAt(nMapIdx);
+    nArrLen += nChr < 0x80 ? 1 : nChr < 0x800 ? 2 : nChr < 0x10000 ? 3 : nChr < 0x200000 ? 4 : nChr < 0x4000000 ? 5 : 6;
+  }
+
+  aBytes = new Uint8Array(nArrLen);
+
+  /* transcription... */
+
+  for (var nIdx = 0, nChrIdx = 0; nIdx < nArrLen; nChrIdx++) {
+    nChr = sDOMStr.charCodeAt(nChrIdx);
+    if (nChr < 128) {
+      /* one byte */
+      aBytes[nIdx++] = nChr;
+    } else if (nChr < 0x800) {
+      /* two bytes */
+      aBytes[nIdx++] = 192 + (nChr >>> 6);
+      aBytes[nIdx++] = 128 + (nChr & 63);
+    } else if (nChr < 0x10000) {
+      /* three bytes */
+      aBytes[nIdx++] = 224 + (nChr >>> 12);
+      aBytes[nIdx++] = 128 + (nChr >>> 6 & 63);
+      aBytes[nIdx++] = 128 + (nChr & 63);
+    } else if (nChr < 0x200000) {
+      /* four bytes */
+      aBytes[nIdx++] = 240 + (nChr >>> 18);
+      aBytes[nIdx++] = 128 + (nChr >>> 12 & 63);
+      aBytes[nIdx++] = 128 + (nChr >>> 6 & 63);
+      aBytes[nIdx++] = 128 + (nChr & 63);
+    } else if (nChr < 0x4000000) {
+      /* five bytes */
+      aBytes[nIdx++] = 248 + (nChr >>> 24);
+      aBytes[nIdx++] = 128 + (nChr >>> 18 & 63);
+      aBytes[nIdx++] = 128 + (nChr >>> 12 & 63);
+      aBytes[nIdx++] = 128 + (nChr >>> 6 & 63);
+      aBytes[nIdx++] = 128 + (nChr & 63);
+    } else /* if (nChr <= 0x7fffffff) */ {
+      /* six bytes */
+      aBytes[nIdx++] = 252 + (nChr >>> 30);
+      aBytes[nIdx++] = 128 + (nChr >>> 24 & 63);
+      aBytes[nIdx++] = 128 + (nChr >>> 18 & 63);
+      aBytes[nIdx++] = 128 + (nChr >>> 12 & 63);
+      aBytes[nIdx++] = 128 + (nChr >>> 6 & 63);
+      aBytes[nIdx++] = 128 + (nChr & 63);
+    }
+  }
+
+  return aBytes;
+
+}

src/assets/assets.go

@@ -63,7 +63,7 @@ func SanitizeFilename(filename string) string {
 	if filename == "" {
 		return "unnamed"
 	}
-	return REIllegalFilenameChars.ReplaceAllString(filename, "")
+	return REIllegalFilenameChars.ReplaceAllString(filename, "_")
 }
 
 func AssetKey(id, filename string) string {

src/hmnurl/hmnurl.go

@@ -3,6 +3,7 @@ package hmnurl
 import (
 	"fmt"
 	"net/url"
+	"regexp"
 	"time"
 
 	"git.handmade.network/hmn/hmn/src/config"
@@ -54,6 +55,7 @@ func SetCacheBust(newCacheBust string) {
 
 func SetS3BaseUrl(base string) {
 	S3BaseUrl = base
+	RegexS3Asset = regexp.MustCompile(fmt.Sprintf("%s(?P<key>[\\w\\-./]+)", regexp.QuoteMeta(S3BaseUrl)))
 }
 
 func GetBaseHost() string {

src/hmnurl/hmnurl_test.go

@@ -313,6 +313,10 @@ func TestForumMarkRead(t *testing.T) {
 	AssertSubdomain(t, BuildForumMarkRead("hero", 5), "hero")
 }
 
+func TestS3Asset(t *testing.T) {
+	AssertRegexMatchFull(t, BuildS3Asset("hello"), RegexS3Asset, map[string]string{"key": "hello"})
+}
+
 func AssertSubdomain(t *testing.T, fullUrl string, expectedSubdomain string) {
 	t.Helper()
 
@@ -343,8 +347,15 @@ func AssertRegexMatch(t *testing.T, fullUrl string, regex *regexp.Regexp, params
 	if len(requestPath) == 0 {
 		requestPath = "/"
 	}
-	match := regex.FindStringSubmatch(requestPath)
+
-	assert.NotNilf(t, match, "Url did not match regex: [%s] vs [%s]", requestPath, regex.String())
+	AssertRegexMatchFull(t, requestPath, regex, paramsToVerify)
+}
+
+func AssertRegexMatchFull(t *testing.T, fullUrl string, regex *regexp.Regexp, paramsToVerify map[string]string) {
+	t.Helper()
+
+	match := regex.FindStringSubmatch(fullUrl)
+	assert.NotNilf(t, match, "Url did not match regex: [%s] vs [%s]", fullUrl, regex.String())
 
 	if paramsToVerify != nil {
 		subexpNames := regex.SubexpNames()
@@ -386,5 +397,4 @@ func AssertRegexNoMatch(t *testing.T, fullUrl string, regex *regexp.Regexp) {
 func TestThingsThatDontNeedCoverage(t *testing.T) {
 	// look the other way ಠ_ಠ
 	BuildPodcastEpisodeFile("foo", "bar")
-	BuildS3Asset("ha ha")
 }

src/hmnurl/urls.go

@@ -638,10 +638,12 @@ func BuildEditorPreviewsJS() string {
 	return Url("/assets/editorpreviews.js", nil)
 }
 
-// NOTE(asaf): No Regex or tests for remote assets, since we don't parse it ourselves
+var RegexS3Asset *regexp.Regexp
+
 func BuildS3Asset(s3key string) string {
 	defer CatchPanic()
-	return fmt.Sprintf("%s%s", S3BaseUrl, s3key)
+	res := fmt.Sprintf("%s%s", S3BaseUrl, s3key)
+	return res
 }
 
 var RegexPublic = regexp.MustCompile("^/public/.+$")

src/migration/migrations/2021-09-22T182718Z_AddHandmadePostAssetUsage.go

@@ -0,0 +1,64 @@
+package migrations
+
+import (
+	"context"
+	"time"
+
+	"git.handmade.network/hmn/hmn/src/migration/types"
+	"git.handmade.network/hmn/hmn/src/oops"
+	"github.com/jackc/pgx/v4"
+)
+
+func init() {
+	registerMigration(AddHandmadePostAssetUsage{})
+}
+
+type AddHandmadePostAssetUsage struct{}
+
+func (m AddHandmadePostAssetUsage) Version() types.MigrationVersion {
+	return types.MigrationVersion(time.Date(2021, 9, 22, 18, 27, 18, 0, time.UTC))
+}
+
+func (m AddHandmadePostAssetUsage) Name() string {
+	return "AddHandmadePostAssetUsage"
+}
+
+func (m AddHandmadePostAssetUsage) Description() string {
+	return "Add table for tracking asset usage in posts, and a unique index on handmade_asset.s3_key"
+}
+
+func (m AddHandmadePostAssetUsage) Up(ctx context.Context, tx pgx.Tx) error {
+	_, err := tx.Exec(ctx,
+		`
+		CREATE TABLE handmade_post_asset_usage (
+			post_id INTEGER NOT NULL REFERENCES handmade_post(id) ON DELETE CASCADE,
+			asset_id UUID NOT NULL REFERENCES handmade_asset(id) ON DELETE CASCADE,
+			CONSTRAINT handmade_post_asset_usage_unique UNIQUE(post_id, asset_id)
+		);
+
+		CREATE INDEX handmade_post_asset_usage_post_id ON handmade_post_asset_usage(post_id);
+		CREATE INDEX handmade_post_asset_usage_asset_id ON handmade_post_asset_usage(asset_id);
+
+		ALTER TABLE handmade_asset
+			ADD CONSTRAINT handmade_asset_s3_key UNIQUE(s3_key);
+		`,
+	)
+	if err != nil {
+		return oops.New(err, "failed to add table and indexes")
+	}
+	return nil
+}
+
+func (m AddHandmadePostAssetUsage) Down(ctx context.Context, tx pgx.Tx) error {
+	_, err := tx.Exec(ctx,
+		`
+		DROP INDEX handmade_post_asset_usage_post_id;
+		DROP INDEX handmade_post_asset_usage_asset_id;
+		DROP TABLE handmade_post_asset_usage;
+		`,
+	)
+	if err != nil {
+		return oops.New(err, "failed to drop table and indexes")
+	}
+	return nil
+}

src/templates/src/editor.html

@@ -2,6 +2,7 @@
 
 {{ define "extrahead" }}
     <script src="{{ static "go_wasm_exec.js" }}"></script>
+    <script src="{{ static "js/base64.js" }}"></script>
     <script>
         const previewWorker = new Worker('/assets/editorpreviews.js');
     </script>
@@ -298,7 +299,7 @@
 			if (items[i].file) {
 				fileCounter++;
 				toInsert += makeUploadString(fileCounter, items[i].file.name) + "\n";
-				startUpload(fileCounter, items[i].file);
+				queueUpload(fileCounter, items[i].file);
 			} else {
 				toInsert += `${items[i].error}\n`;
 			}
@@ -306,6 +307,7 @@
 
 		textField.value = textField.value.substring(0, cursorStart) + toInsert + textField.value.substring(cursorEnd, textField.value.length);
 		doMarkdown();
+		uploadNext();
 	}
 
 	function replaceUploadString(upload, newString) {
@@ -327,7 +329,11 @@
 		doMarkdown();
 	}
 
-	function startUpload(uploadNumber, file) {
+	function replaceUploadStringError(upload) {
+		replaceUploadString(upload, `There was a problem uploading your file \`${upload.file.name}\`.`);
+	}
+
+	function queueUpload(uploadNumber, file) {
 		uploadQueue.push({
 			uploadNumber: uploadNumber,
 			file: file
@@ -335,7 +341,6 @@
 
 		currentBatchSize++;
 		uploadProgressText.textContent = `Uploading files ${currentBatchDone+1}/${currentBatchSize}`;
-		uploadNext();
 	}
 
 	function uploadDone(ev) {
@@ -352,14 +357,14 @@
 				} else if (currentXhr.response.error) {
 					replaceUploadString(currentUpload, `Upload failed for \`${currentUpload.file.name}\`: ${currentXhr.response.error}.`);
 				} else {
-					replaceUploadString(currentUpload, `There was a problem uploading your file \`${currentUpload.file.name}\`.`);
+					replaceUploadStringError(currentUpload);
 				}
 			} else {
-				replaceUploadString(currentUpload, `There was a problem uploading your file \`${currentUpload.file.name}\`.`);
+				replaceUploadStringError(currentUpload);
 			}
 		} catch (err) {
-			replaceUploadString(currentUpload, `There was a problem uploading your file \`${currentUpload.file.name}\`.`);
 			console.error(err);
+			replaceUploadStringError(currentUpload);
 		}
 		currentUpload = null;
 		currentXhr = null;
@@ -384,15 +389,23 @@
 				submitButton.disabled = true;
 				submitButton.value = "Uploading files...";
 
+				try {
+					let utf8Filename = strToUTF8Arr(next.file.name);
+					let base64Filename = base64EncArr(utf8Filename);
 					// NOTE(asaf): We use XHR because fetch can't do upload progress reports. Womp womp. https://youtu.be/Pubd-spHN-0?t=2
 					currentXhr = new XMLHttpRequest();
 					currentXhr.upload.addEventListener("progress", updateUploadProgress);
 					currentXhr.open("POST", uploadUrl, true);
-				currentXhr.setRequestHeader("Hmn-Upload-Filename", next.file.name);
+					currentXhr.setRequestHeader("Hmn-Upload-Filename", base64Filename);
 					currentXhr.responseType = "json";
 					currentXhr.addEventListener("loadend", uploadDone);
 					currentXhr.send(next.file);
 					currentUpload = next;
+				} catch (err) {
+					replaceUploadStringError(next);
+					console.error(err);
+					uploadNext();
+				}
 			} else {
 				submitButton.disabled = false;
 				submitButton.value = submitText;

src/website/assets.go

@@ -2,6 +2,7 @@ package website
 
 import (
 	"bytes"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"image"
@@ -51,9 +52,12 @@ func AssetUpload(c *RequestContext) ResponseData {
 	}
 
 	filenameHeader, hasFilename := c.Req.Header["Hmn-Upload-Filename"]
-	originalFilename := ""
+	originalFilename := "upload"
 	if hasFilename {
-		originalFilename = strings.ReplaceAll(filenameHeader[0], " ", "_")
+		decodedFilename, err := base64.StdEncoding.DecodeString(filenameHeader[0])
+		if err == nil {
+			originalFilename = string(decodedFilename)
+		}
 	}
 
 	bodyReader := http.MaxBytesReader(c.Res, c.Req.Body, int64(maxFilesize))

src/website/threads_and_posts_helper.go

@@ -10,10 +10,12 @@ import (
 	"time"
 
 	"git.handmade.network/hmn/hmn/src/db"
+	"git.handmade.network/hmn/hmn/src/hmnurl"
 	"git.handmade.network/hmn/hmn/src/logging"
 	"git.handmade.network/hmn/hmn/src/models"
 	"git.handmade.network/hmn/hmn/src/oops"
 	"git.handmade.network/hmn/hmn/src/parsing"
+	"github.com/google/uuid"
 	"github.com/jackc/pgx/v4"
 )
 
@@ -429,6 +431,51 @@ func CreatePostVersion(ctx context.Context, tx pgx.Tx, postId int, unparsedConte
 		panic(oops.New(err, "failed to set current post version and preview"))
 	}
 
+	// Update asset usage
+
+	_, err = tx.Exec(ctx,
+		`
+		DELETE FROM handmade_post_asset_usage
+		WHERE post_id = $1
+		`,
+		postId,
+	)
+
+	matches := hmnurl.RegexS3Asset.FindAllStringSubmatch(unparsedContent, -1)
+	keyIdx := hmnurl.RegexS3Asset.SubexpIndex("key")
+
+	var keys []string
+	for _, match := range matches {
+		key := match[keyIdx]
+		keys = append(keys, key)
+	}
+
+	type assetId struct {
+		AssetID uuid.UUID `db:"id"`
+	}
+	assetResult, err := db.Query(ctx, tx, assetId{},
+		`
+		SELECT $columns
+		FROM handmade_asset
+		WHERE s3_key = ANY($1)
+		`,
+		keys,
+	)
+	if err != nil {
+		panic(oops.New(err, "failed to get assets matching keys"))
+	}
+
+	var values [][]interface{}
+
+	for _, asset := range assetResult.ToSlice() {
+		values = append(values, []interface{}{postId, asset.(*assetId).AssetID})
+	}
+
+	_, err = tx.CopyFrom(ctx, pgx.Identifier{"handmade_post_asset_usage"}, []string{"post_id", "asset_id"}, pgx.CopyFromRows(values))
+	if err != nil {
+		panic(oops.New(err, "failed to insert post asset usage"))
+	}
+
 	return
 }