From 7063fd560441e5fed0c57e63e7b86ce103fb847c Mon Sep 17 00:00:00 2001 From: Asaf Gartner Date: Wed, 22 Sep 2021 22:18:39 +0300 Subject: [PATCH] Added basic usage tracking for posts. --- public/js/base64.js | 173 ++++++++++++++++++ src/assets/assets.go | 4 +- src/hmnurl/hmnurl.go | 2 + src/hmnurl/hmnurl_test.go | 16 +- src/hmnurl/urls.go | 6 +- ...09-22T182718Z_AddHandmadePostAssetUsage.go | 64 +++++++ src/templates/src/editor.html | 43 +++-- src/website/assets.go | 8 +- src/website/threads_and_posts_helper.go | 47 +++++ 9 files changed, 339 insertions(+), 24 deletions(-) create mode 100644 public/js/base64.js create mode 100644 src/migration/migrations/2021-09-22T182718Z_AddHandmadePostAssetUsage.go diff --git a/public/js/base64.js b/public/js/base64.js new file mode 100644 index 0000000..e1f3626 --- /dev/null +++ b/public/js/base64.js @@ -0,0 +1,173 @@ +// NOTE(asaf): The built-in base64 capabilities can't handle non-ascii characters. +// This code is copied from MDN at https://developer.mozilla.org/en-US/docs/Glossary/Base64 +// +// Usage: +// var sMyInput = "Base 64 \u2014 Mozilla Developer Network"; +// var aMyUTF8Input = strToUTF8Arr(sMyInput); +// var sMyBase64 = base64EncArr(aMyUTF8Input); +// alert(sMyBase64); +// +// var aMyUTF8Output = base64DecToArr(sMyBase64); +// var sMyOutput = UTF8ArrToStr(aMyUTF8Output); +// alert(sMyOutput); + +function b64ToUint6 (nChr) { + + return nChr > 64 && nChr < 91 ? + nChr - 65 + : nChr > 96 && nChr < 123 ? + nChr - 71 + : nChr > 47 && nChr < 58 ? + nChr + 4 + : nChr === 43 ? + 62 + : nChr === 47 ? + 63 + : + 0; + +} + +function base64DecToArr (sBase64, nBlocksSize) { + + var + sB64Enc = sBase64.replace(/[^A-Za-z0-9\+\/]/g, ""), nInLen = sB64Enc.length, + nOutLen = nBlocksSize ? Math.ceil((nInLen * 3 + 1 >> 2) / nBlocksSize) * nBlocksSize : nInLen * 3 + 1 >> 2, taBytes = new Uint8Array(nOutLen); + + for (var nMod3, nMod4, nUint24 = 0, nOutIdx = 0, nInIdx = 0; nInIdx < nInLen; nInIdx++) { + nMod4 = nInIdx & 3; + nUint24 |= b64ToUint6(sB64Enc.charCodeAt(nInIdx)) << 6 * (3 - nMod4); + if (nMod4 === 3 || nInLen - nInIdx === 1) { + for (nMod3 = 0; nMod3 < 3 && nOutIdx < nOutLen; nMod3++, nOutIdx++) { + taBytes[nOutIdx] = nUint24 >>> (16 >>> nMod3 & 24) & 255; + } + nUint24 = 0; + + } + } + + return taBytes; +} + +/* Base64 string to array encoding */ + +function uint6ToB64 (nUint6) { + + return nUint6 < 26 ? + nUint6 + 65 + : nUint6 < 52 ? + nUint6 + 71 + : nUint6 < 62 ? + nUint6 - 4 + : nUint6 === 62 ? + 43 + : nUint6 === 63 ? + 47 + : + 65; + +} + +function base64EncArr (aBytes) { + + var nMod3 = 2, sB64Enc = ""; + + for (var nLen = aBytes.length, nUint24 = 0, nIdx = 0; nIdx < nLen; nIdx++) { + nMod3 = nIdx % 3; + if (nIdx > 0 && (nIdx * 4 / 3) % 76 === 0) { sB64Enc += "\r\n"; } + nUint24 |= aBytes[nIdx] << (16 >>> nMod3 & 24); + if (nMod3 === 2 || aBytes.length - nIdx === 1) { + sB64Enc += String.fromCharCode(uint6ToB64(nUint24 >>> 18 & 63), uint6ToB64(nUint24 >>> 12 & 63), uint6ToB64(nUint24 >>> 6 & 63), uint6ToB64(nUint24 & 63)); + nUint24 = 0; + } + } + + return sB64Enc.substr(0, sB64Enc.length - 2 + nMod3) + (nMod3 === 2 ? '' : nMod3 === 1 ? '=' : '=='); + +} + +/* UTF-8 array to DOMString and vice versa */ + +function UTF8ArrToStr (aBytes) { + + var sView = ""; + + for (var nPart, nLen = aBytes.length, nIdx = 0; nIdx < nLen; nIdx++) { + nPart = aBytes[nIdx]; + sView += String.fromCharCode( + nPart > 251 && nPart < 254 && nIdx + 5 < nLen ? /* six bytes */ + /* (nPart - 252 << 30) may be not so safe in ECMAScript! So...: */ + (nPart - 252) * 1073741824 + (aBytes[++nIdx] - 128 << 24) + (aBytes[++nIdx] - 128 << 18) + (aBytes[++nIdx] - 128 << 12) + (aBytes[++nIdx] - 128 << 6) + aBytes[++nIdx] - 128 + : nPart > 247 && nPart < 252 && nIdx + 4 < nLen ? /* five bytes */ + (nPart - 248 << 24) + (aBytes[++nIdx] - 128 << 18) + (aBytes[++nIdx] - 128 << 12) + (aBytes[++nIdx] - 128 << 6) + aBytes[++nIdx] - 128 + : nPart > 239 && nPart < 248 && nIdx + 3 < nLen ? /* four bytes */ + (nPart - 240 << 18) + (aBytes[++nIdx] - 128 << 12) + (aBytes[++nIdx] - 128 << 6) + aBytes[++nIdx] - 128 + : nPart > 223 && nPart < 240 && nIdx + 2 < nLen ? /* three bytes */ + (nPart - 224 << 12) + (aBytes[++nIdx] - 128 << 6) + aBytes[++nIdx] - 128 + : nPart > 191 && nPart < 224 && nIdx + 1 < nLen ? /* two bytes */ + (nPart - 192 << 6) + aBytes[++nIdx] - 128 + : /* nPart < 127 ? */ /* one byte */ + nPart + ); + } + + return sView; + +} + +function strToUTF8Arr (sDOMStr) { + + var aBytes, nChr, nStrLen = sDOMStr.length, nArrLen = 0; + + /* mapping... */ + + for (var nMapIdx = 0; nMapIdx < nStrLen; nMapIdx++) { + nChr = sDOMStr.charCodeAt(nMapIdx); + nArrLen += nChr < 0x80 ? 1 : nChr < 0x800 ? 2 : nChr < 0x10000 ? 3 : nChr < 0x200000 ? 4 : nChr < 0x4000000 ? 5 : 6; + } + + aBytes = new Uint8Array(nArrLen); + + /* transcription... */ + + for (var nIdx = 0, nChrIdx = 0; nIdx < nArrLen; nChrIdx++) { + nChr = sDOMStr.charCodeAt(nChrIdx); + if (nChr < 128) { + /* one byte */ + aBytes[nIdx++] = nChr; + } else if (nChr < 0x800) { + /* two bytes */ + aBytes[nIdx++] = 192 + (nChr >>> 6); + aBytes[nIdx++] = 128 + (nChr & 63); + } else if (nChr < 0x10000) { + /* three bytes */ + aBytes[nIdx++] = 224 + (nChr >>> 12); + aBytes[nIdx++] = 128 + (nChr >>> 6 & 63); + aBytes[nIdx++] = 128 + (nChr & 63); + } else if (nChr < 0x200000) { + /* four bytes */ + aBytes[nIdx++] = 240 + (nChr >>> 18); + aBytes[nIdx++] = 128 + (nChr >>> 12 & 63); + aBytes[nIdx++] = 128 + (nChr >>> 6 & 63); + aBytes[nIdx++] = 128 + (nChr & 63); + } else if (nChr < 0x4000000) { + /* five bytes */ + aBytes[nIdx++] = 248 + (nChr >>> 24); + aBytes[nIdx++] = 128 + (nChr >>> 18 & 63); + aBytes[nIdx++] = 128 + (nChr >>> 12 & 63); + aBytes[nIdx++] = 128 + (nChr >>> 6 & 63); + aBytes[nIdx++] = 128 + (nChr & 63); + } else /* if (nChr <= 0x7fffffff) */ { + /* six bytes */ + aBytes[nIdx++] = 252 + (nChr >>> 30); + aBytes[nIdx++] = 128 + (nChr >>> 24 & 63); + aBytes[nIdx++] = 128 + (nChr >>> 18 & 63); + aBytes[nIdx++] = 128 + (nChr >>> 12 & 63); + aBytes[nIdx++] = 128 + (nChr >>> 6 & 63); + aBytes[nIdx++] = 128 + (nChr & 63); + } + } + + return aBytes; + +} diff --git a/src/assets/assets.go b/src/assets/assets.go index bf02a50..ea1c639 100644 --- a/src/assets/assets.go +++ b/src/assets/assets.go @@ -57,13 +57,13 @@ type CreateInput struct { Width, Height int } -var REIllegalFilenameChars = regexp.MustCompile(`[^\w \-.]`) +var REIllegalFilenameChars = regexp.MustCompile(`[^\w\-.]`) func SanitizeFilename(filename string) string { if filename == "" { return "unnamed" } - return REIllegalFilenameChars.ReplaceAllString(filename, "") + return REIllegalFilenameChars.ReplaceAllString(filename, "_") } func AssetKey(id, filename string) string { diff --git a/src/hmnurl/hmnurl.go b/src/hmnurl/hmnurl.go index 134aebe..354f4d1 100644 --- a/src/hmnurl/hmnurl.go +++ b/src/hmnurl/hmnurl.go @@ -3,6 +3,7 @@ package hmnurl import ( "fmt" "net/url" + "regexp" "time" "git.handmade.network/hmn/hmn/src/config" @@ -54,6 +55,7 @@ func SetCacheBust(newCacheBust string) { func SetS3BaseUrl(base string) { S3BaseUrl = base + RegexS3Asset = regexp.MustCompile(fmt.Sprintf("%s(?P[\\w\\-./]+)", regexp.QuoteMeta(S3BaseUrl))) } func GetBaseHost() string { diff --git a/src/hmnurl/hmnurl_test.go b/src/hmnurl/hmnurl_test.go index 9c45b90..3194a2f 100644 --- a/src/hmnurl/hmnurl_test.go +++ b/src/hmnurl/hmnurl_test.go @@ -313,6 +313,10 @@ func TestForumMarkRead(t *testing.T) { AssertSubdomain(t, BuildForumMarkRead("hero", 5), "hero") } +func TestS3Asset(t *testing.T) { + AssertRegexMatchFull(t, BuildS3Asset("hello"), RegexS3Asset, map[string]string{"key": "hello"}) +} + func AssertSubdomain(t *testing.T, fullUrl string, expectedSubdomain string) { t.Helper() @@ -343,8 +347,15 @@ func AssertRegexMatch(t *testing.T, fullUrl string, regex *regexp.Regexp, params if len(requestPath) == 0 { requestPath = "/" } - match := regex.FindStringSubmatch(requestPath) - assert.NotNilf(t, match, "Url did not match regex: [%s] vs [%s]", requestPath, regex.String()) + + AssertRegexMatchFull(t, requestPath, regex, paramsToVerify) +} + +func AssertRegexMatchFull(t *testing.T, fullUrl string, regex *regexp.Regexp, paramsToVerify map[string]string) { + t.Helper() + + match := regex.FindStringSubmatch(fullUrl) + assert.NotNilf(t, match, "Url did not match regex: [%s] vs [%s]", fullUrl, regex.String()) if paramsToVerify != nil { subexpNames := regex.SubexpNames() @@ -386,5 +397,4 @@ func AssertRegexNoMatch(t *testing.T, fullUrl string, regex *regexp.Regexp) { func TestThingsThatDontNeedCoverage(t *testing.T) { // look the other way ಠ_ಠ BuildPodcastEpisodeFile("foo", "bar") - BuildS3Asset("ha ha") } diff --git a/src/hmnurl/urls.go b/src/hmnurl/urls.go index a33b83d..4122218 100644 --- a/src/hmnurl/urls.go +++ b/src/hmnurl/urls.go @@ -638,10 +638,12 @@ func BuildEditorPreviewsJS() string { return Url("/assets/editorpreviews.js", nil) } -// NOTE(asaf): No Regex or tests for remote assets, since we don't parse it ourselves +var RegexS3Asset *regexp.Regexp + func BuildS3Asset(s3key string) string { defer CatchPanic() - return fmt.Sprintf("%s%s", S3BaseUrl, s3key) + res := fmt.Sprintf("%s%s", S3BaseUrl, s3key) + return res } var RegexPublic = regexp.MustCompile("^/public/.+$") diff --git a/src/migration/migrations/2021-09-22T182718Z_AddHandmadePostAssetUsage.go b/src/migration/migrations/2021-09-22T182718Z_AddHandmadePostAssetUsage.go new file mode 100644 index 0000000..1db02b9 --- /dev/null +++ b/src/migration/migrations/2021-09-22T182718Z_AddHandmadePostAssetUsage.go @@ -0,0 +1,64 @@ +package migrations + +import ( + "context" + "time" + + "git.handmade.network/hmn/hmn/src/migration/types" + "git.handmade.network/hmn/hmn/src/oops" + "github.com/jackc/pgx/v4" +) + +func init() { + registerMigration(AddHandmadePostAssetUsage{}) +} + +type AddHandmadePostAssetUsage struct{} + +func (m AddHandmadePostAssetUsage) Version() types.MigrationVersion { + return types.MigrationVersion(time.Date(2021, 9, 22, 18, 27, 18, 0, time.UTC)) +} + +func (m AddHandmadePostAssetUsage) Name() string { + return "AddHandmadePostAssetUsage" +} + +func (m AddHandmadePostAssetUsage) Description() string { + return "Add table for tracking asset usage in posts, and a unique index on handmade_asset.s3_key" +} + +func (m AddHandmadePostAssetUsage) Up(ctx context.Context, tx pgx.Tx) error { + _, err := tx.Exec(ctx, + ` + CREATE TABLE handmade_post_asset_usage ( + post_id INTEGER NOT NULL REFERENCES handmade_post(id) ON DELETE CASCADE, + asset_id UUID NOT NULL REFERENCES handmade_asset(id) ON DELETE CASCADE, + CONSTRAINT handmade_post_asset_usage_unique UNIQUE(post_id, asset_id) + ); + + CREATE INDEX handmade_post_asset_usage_post_id ON handmade_post_asset_usage(post_id); + CREATE INDEX handmade_post_asset_usage_asset_id ON handmade_post_asset_usage(asset_id); + + ALTER TABLE handmade_asset + ADD CONSTRAINT handmade_asset_s3_key UNIQUE(s3_key); + `, + ) + if err != nil { + return oops.New(err, "failed to add table and indexes") + } + return nil +} + +func (m AddHandmadePostAssetUsage) Down(ctx context.Context, tx pgx.Tx) error { + _, err := tx.Exec(ctx, + ` + DROP INDEX handmade_post_asset_usage_post_id; + DROP INDEX handmade_post_asset_usage_asset_id; + DROP TABLE handmade_post_asset_usage; + `, + ) + if err != nil { + return oops.New(err, "failed to drop table and indexes") + } + return nil +} diff --git a/src/templates/src/editor.html b/src/templates/src/editor.html index a569bfa..6c1fe7e 100644 --- a/src/templates/src/editor.html +++ b/src/templates/src/editor.html @@ -2,6 +2,7 @@ {{ define "extrahead" }} + @@ -298,7 +299,7 @@ if (items[i].file) { fileCounter++; toInsert += makeUploadString(fileCounter, items[i].file.name) + "\n"; - startUpload(fileCounter, items[i].file); + queueUpload(fileCounter, items[i].file); } else { toInsert += `${items[i].error}\n`; } @@ -306,6 +307,7 @@ textField.value = textField.value.substring(0, cursorStart) + toInsert + textField.value.substring(cursorEnd, textField.value.length); doMarkdown(); + uploadNext(); } function replaceUploadString(upload, newString) { @@ -327,7 +329,11 @@ doMarkdown(); } - function startUpload(uploadNumber, file) { + function replaceUploadStringError(upload) { + replaceUploadString(upload, `There was a problem uploading your file \`${upload.file.name}\`.`); + } + + function queueUpload(uploadNumber, file) { uploadQueue.push({ uploadNumber: uploadNumber, file: file @@ -335,7 +341,6 @@ currentBatchSize++; uploadProgressText.textContent = `Uploading files ${currentBatchDone+1}/${currentBatchSize}`; - uploadNext(); } function uploadDone(ev) { @@ -352,14 +357,14 @@ } else if (currentXhr.response.error) { replaceUploadString(currentUpload, `Upload failed for \`${currentUpload.file.name}\`: ${currentXhr.response.error}.`); } else { - replaceUploadString(currentUpload, `There was a problem uploading your file \`${currentUpload.file.name}\`.`); + replaceUploadStringError(currentUpload); } } else { - replaceUploadString(currentUpload, `There was a problem uploading your file \`${currentUpload.file.name}\`.`); + replaceUploadStringError(currentUpload); } } catch (err) { - replaceUploadString(currentUpload, `There was a problem uploading your file \`${currentUpload.file.name}\`.`); console.error(err); + replaceUploadStringError(currentUpload); } currentUpload = null; currentXhr = null; @@ -384,15 +389,23 @@ submitButton.disabled = true; submitButton.value = "Uploading files..."; - // NOTE(asaf): We use XHR because fetch can't do upload progress reports. Womp womp. https://youtu.be/Pubd-spHN-0?t=2 - currentXhr = new XMLHttpRequest(); - currentXhr.upload.addEventListener("progress", updateUploadProgress); - currentXhr.open("POST", uploadUrl, true); - currentXhr.setRequestHeader("Hmn-Upload-Filename", next.file.name); - currentXhr.responseType = "json"; - currentXhr.addEventListener("loadend", uploadDone); - currentXhr.send(next.file); - currentUpload = next; + try { + let utf8Filename = strToUTF8Arr(next.file.name); + let base64Filename = base64EncArr(utf8Filename); + // NOTE(asaf): We use XHR because fetch can't do upload progress reports. Womp womp. https://youtu.be/Pubd-spHN-0?t=2 + currentXhr = new XMLHttpRequest(); + currentXhr.upload.addEventListener("progress", updateUploadProgress); + currentXhr.open("POST", uploadUrl, true); + currentXhr.setRequestHeader("Hmn-Upload-Filename", base64Filename); + currentXhr.responseType = "json"; + currentXhr.addEventListener("loadend", uploadDone); + currentXhr.send(next.file); + currentUpload = next; + } catch (err) { + replaceUploadStringError(next); + console.error(err); + uploadNext(); + } } else { submitButton.disabled = false; submitButton.value = submitText; diff --git a/src/website/assets.go b/src/website/assets.go index 6e4fe19..541cd60 100644 --- a/src/website/assets.go +++ b/src/website/assets.go @@ -2,6 +2,7 @@ package website import ( "bytes" + "encoding/base64" "encoding/json" "fmt" "image" @@ -51,9 +52,12 @@ func AssetUpload(c *RequestContext) ResponseData { } filenameHeader, hasFilename := c.Req.Header["Hmn-Upload-Filename"] - originalFilename := "" + originalFilename := "upload" if hasFilename { - originalFilename = strings.ReplaceAll(filenameHeader[0], " ", "_") + decodedFilename, err := base64.StdEncoding.DecodeString(filenameHeader[0]) + if err == nil { + originalFilename = string(decodedFilename) + } } bodyReader := http.MaxBytesReader(c.Res, c.Req.Body, int64(maxFilesize)) diff --git a/src/website/threads_and_posts_helper.go b/src/website/threads_and_posts_helper.go index 6d39085..87c6008 100644 --- a/src/website/threads_and_posts_helper.go +++ b/src/website/threads_and_posts_helper.go @@ -10,10 +10,12 @@ import ( "time" "git.handmade.network/hmn/hmn/src/db" + "git.handmade.network/hmn/hmn/src/hmnurl" "git.handmade.network/hmn/hmn/src/logging" "git.handmade.network/hmn/hmn/src/models" "git.handmade.network/hmn/hmn/src/oops" "git.handmade.network/hmn/hmn/src/parsing" + "github.com/google/uuid" "github.com/jackc/pgx/v4" ) @@ -429,6 +431,51 @@ func CreatePostVersion(ctx context.Context, tx pgx.Tx, postId int, unparsedConte panic(oops.New(err, "failed to set current post version and preview")) } + // Update asset usage + + _, err = tx.Exec(ctx, + ` + DELETE FROM handmade_post_asset_usage + WHERE post_id = $1 + `, + postId, + ) + + matches := hmnurl.RegexS3Asset.FindAllStringSubmatch(unparsedContent, -1) + keyIdx := hmnurl.RegexS3Asset.SubexpIndex("key") + + var keys []string + for _, match := range matches { + key := match[keyIdx] + keys = append(keys, key) + } + + type assetId struct { + AssetID uuid.UUID `db:"id"` + } + assetResult, err := db.Query(ctx, tx, assetId{}, + ` + SELECT $columns + FROM handmade_asset + WHERE s3_key = ANY($1) + `, + keys, + ) + if err != nil { + panic(oops.New(err, "failed to get assets matching keys")) + } + + var values [][]interface{} + + for _, asset := range assetResult.ToSlice() { + values = append(values, []interface{}{postId, asset.(*assetId).AssetID}) + } + + _, err = tx.CopyFrom(ctx, pgx.Identifier{"handmade_post_asset_usage"}, []string{"post_id", "asset_id"}, pgx.CopyFromRows(values)) + if err != nil { + panic(oops.New(err, "failed to insert post asset usage")) + } + return }