hmn
/
hmn
4
Fork 9
Code Issues 59 Pull Requests Releases Activity

Add education-specific extensions 

not all of them yet
This commit is contained in:
Ben Visness 2022-09-01 20:16:09 -05:00
parent d5bdb9a7af
commit 98b9ef6589
12 changed files with 237 additions and 189 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

130
public/go_wasm_exec.js
View File

@ -2,47 +2,18 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
"use strict";
(() => {
// Map multiple JavaScript environments to a single common API,
// preferring web standards over Node.js API.
//
// Environments considered:
// - Browsers
// - Node.js
// - Electron
// - Parcel
// - Webpack
if (typeof global !== "undefined") {
// global already exists
} else if (typeof window !== "undefined") {
window.global = window;
} else if (typeof self !== "undefined") {
self.global = self;
} else {
throw new Error("cannot export Go (neither global, window nor self is defined)");
}
if (!global.require && typeof require !== "undefined") {
global.require = require;
}
if (!global.fs && global.require) {
const fs = require("fs");
if (typeof fs === "object" && fs !== null && Object.keys(fs).length !== 0) {
global.fs = fs;
}
}
const enosys = () => {
const err = new Error("not implemented");
err.code = "ENOSYS";
return err;
};
if (!global.fs) {
if (!globalThis.fs) {
let outputBuf = "";
global.fs = {
globalThis.fs = {
constants: { O_WRONLY: -1, O_RDWR: -1, O_CREAT: -1, O_TRUNC: -1, O_APPEND: -1, O_EXCL: -1 }, // unused
writeSync(fd, buf) {
outputBuf += decoder.decode(buf);
@ -87,8 +58,8 @@
};
}
if (!global.process) {
global.process = {
if (!globalThis.process) {
globalThis.process = {
getuid() { return -1; },
getgid() { return -1; },
geteuid() { return -1; },
@ -102,47 +73,26 @@
}
}
if (!global.crypto && global.require) {
const nodeCrypto = require("crypto");
global.crypto = {
getRandomValues(b) {
nodeCrypto.randomFillSync(b);
},
};
}
if (!global.crypto) {
throw new Error("global.crypto is not available, polyfill required (getRandomValues only)");
if (!globalThis.crypto) {
throw new Error("globalThis.crypto is not available, polyfill required (crypto.getRandomValues only)");
}
if (!global.performance) {
global.performance = {
now() {
const [sec, nsec] = process.hrtime();
return sec * 1000 + nsec / 1000000;
},
};
if (!globalThis.performance) {
throw new Error("globalThis.performance is not available, polyfill required (performance.now only)");
}
if (!global.TextEncoder && global.require) {
global.TextEncoder = require("util").TextEncoder;
}
if (!global.TextEncoder) {
throw new Error("global.TextEncoder is not available, polyfill required");
if (!globalThis.TextEncoder) {
throw new Error("globalThis.TextEncoder is not available, polyfill required");
}
if (!global.TextDecoder && global.require) {
global.TextDecoder = require("util").TextDecoder;
if (!globalThis.TextDecoder) {
throw new Error("globalThis.TextDecoder is not available, polyfill required");
}
if (!global.TextDecoder) {
throw new Error("global.TextDecoder is not available, polyfill required");
}
// End of polyfills for common API.
const encoder = new TextEncoder("utf-8");
const decoder = new TextDecoder("utf-8");
global.Go = class {
globalThis.Go = class {
constructor() {
this.argv = ["js"];
this.env = {};
@ -296,8 +246,8 @@
setInt64(sp + 8, (timeOrigin + performance.now()) * 1000000);
},
// func walltime1() (sec int64, nsec int32)
"runtime.walltime1": (sp) => {
// func walltime() (sec int64, nsec int32)
"runtime.walltime": (sp) => {
sp >>>= 0;
const msec = (new Date).getTime();
setInt64(sp + 8, msec / 1000);
@ -401,6 +351,7 @@
storeValue(sp + 56, result);
this.mem.setUint8(sp + 64, 1);
} catch (err) {
sp = this._inst.exports.getsp() >>> 0; // see comment above
storeValue(sp + 56, err);
this.mem.setUint8(sp + 64, 0);
}
@ -417,6 +368,7 @@
storeValue(sp + 40, result);
this.mem.setUint8(sp + 48, 1);
} catch (err) {
sp = this._inst.exports.getsp() >>> 0; // see comment above
storeValue(sp + 40, err);
this.mem.setUint8(sp + 48, 0);
}
@ -433,6 +385,7 @@
storeValue(sp + 40, result);
this.mem.setUint8(sp + 48, 1);
} catch (err) {
sp = this._inst.exports.getsp() >>> 0; // see comment above
storeValue(sp + 40, err);
this.mem.setUint8(sp + 48, 0);
}
@ -514,7 +467,7 @@
null,
true,
false,
global,
globalThis,
this,
];
this._goRefCounts = new Array(this._values.length).fill(Infinity); // number of references that Go has to a JS value, indexed by reference id
@ -523,7 +476,7 @@
[null, 2],
[true, 3],
[false, 4],
[global, 5],
[globalThis, 5],
[this, 6],
]);
this._idPool = []; // unused ids that have been garbage collected
@ -564,6 +517,13 @@
offset += 8;
});
// The linker guarantees global data starts from at least wasmMinDataAddr.
// Keep in sync with cmd/link/internal/ld/data.go:wasmMinDataAddr.
const wasmMinDataAddr = 4096 + 8192;
if (offset >= wasmMinDataAddr) {
throw new Error("total length of command line and environment variables exceeds limit");
}
this._inst.exports.run(argc, argv);
if (this.exited) {
this._resolveExitPromise();
@ -591,36 +551,4 @@
};
}
}
if (
typeof module !== "undefined" &&
global.require &&
global.require.main === module &&
global.process &&
global.process.versions &&
!global.process.versions.electron
) {
if (process.argv.length < 3) {
console.error("usage: go_js_wasm_exec [wasm binary] [arguments]");
process.exit(1);
}
const go = new Go();
go.argv = process.argv.slice(2);
go.env = Object.assign({ TMPDIR: require("os").tmpdir() }, process.env);
go.exit = process.exit;
WebAssembly.instantiate(fs.readFileSync(process.argv[2]), go.importObject).then((result) => {
process.on("exit", (code) => { // Node.js exits if no event handler is pending
if (code === 0 && !go.exited) {
// deadlock, make Go print error and stack traces
go._pendingEvent = { id: 0 };
go._resume();
}
});
return go.run(result.instance);
}).catch((err) => {
console.error(err);
process.exit(1);
});
}
})();

BIN
public/parsing.wasm
View File

Binary file not shown.

171
src/parsing/bbcode.go
View File

@ -28,87 +28,45 @@ var reTag = regexp.MustCompile(`\[\s*(?P<opentagname>[a-zA-Z0-9]+)|\[\s*\/\s*(?P
var previewBBCodeCompiler = bbcode.NewCompiler(false, false)
var realBBCodeCompiler = bbcode.NewCompiler(false, false)
var eduPreviewBBCodeCompiler = bbcode.NewCompiler(false, false)
var eduRealBBCodeCompiler = bbcode.NewCompiler(false, false)
var REYoutubeVidOnly = regexp.MustCompile(`^[a-zA-Z0-9_-]{11}$`)
func init() {
type attr struct {
Name, Value string
}
all := []bbcode.Compiler{previewBBCodeCompiler, realBBCodeCompiler, eduPreviewBBCodeCompiler, eduRealBBCodeCompiler}
real := []bbcode.Compiler{realBBCodeCompiler, eduRealBBCodeCompiler}
preview := []bbcode.Compiler{previewBBCodeCompiler, eduPreviewBBCodeCompiler}
education := []bbcode.Compiler{eduPreviewBBCodeCompiler, eduRealBBCodeCompiler}
addSimpleTag := func(name, tag string, notext bool, attrs ...attr) {
var tagFunc bbcode.TagCompilerFunc = func(bn *bbcode.BBCodeNode) (*bbcode.HTMLTag, bool) {
if notext {
var newChildren []*bbcode.BBCodeNode
for _, child := range bn.Children {
if child.ID != bbcode.TEXT {
newChildren = append(newChildren, child)
}
}
bn.Children = newChildren
}
addSimpleTag(all, "h1", "h1", false, nil)
addSimpleTag(all, "h2", "h3", false, nil)
addSimpleTag(all, "h3", "h3", false, nil)
addSimpleTag(all, "m", "span", false, attrs{"class": "monospace"})
addSimpleTag(all, "ol", "ol", true, nil)
addSimpleTag(all, "ul", "ul", true, nil)
addSimpleTag(all, "li", "li", false, nil)
addSimpleTag(all, "spoiler", "span", false, attrs{"class": "spoiler"})
addSimpleTag(all, "table", "table", true, nil)
addSimpleTag(all, "tr", "tr", true, nil)
addSimpleTag(all, "th", "th", false, nil)
addSimpleTag(all, "td", "td", false, nil)
out := bbcode.NewHTMLTag("")
out.Name = tag
for _, a := range attrs {
out.Attrs[a.Name] = a.Value
}
return out, true
}
previewBBCodeCompiler.SetTag(name, tagFunc)
realBBCodeCompiler.SetTag(name, tagFunc)
}
addTag := func(name string, f bbcode.TagCompilerFunc) {
previewBBCodeCompiler.SetTag(name, f)
realBBCodeCompiler.SetTag(name, f)
}
previewBBCodeCompiler.SetTag("youtube", makeYoutubeBBCodeFunc(true))
realBBCodeCompiler.SetTag("youtube", makeYoutubeBBCodeFunc(false))
addSimpleTag("h1", "h1", false)
addSimpleTag("h2", "h3", false)
addSimpleTag("h3", "h3", false)
addSimpleTag("m", "span", false, attr{"class", "monospace"})
addSimpleTag("ol", "ol", true)
addSimpleTag("ul", "ul", true)
addSimpleTag("li", "li", false)
addSimpleTag("spoiler", "span", false, attr{"class", "spoiler"})
addSimpleTag("table", "table", true)
addSimpleTag("tr", "tr", true)
addSimpleTag("th", "th", false)
addSimpleTag("td", "td", false)
addTag("quote", func(bn *bbcode.BBCodeNode) (*bbcode.HTMLTag, bool) {
addTag(all, "quote", func(bn *bbcode.BBCodeNode) (*bbcode.HTMLTag, bool) {
cite := bn.GetOpeningTag().Value
if cite == "" {
out := bbcode.NewHTMLTag("")
out.Name = "blockquote"
return out, true
return htm("blockquote", nil), true
} else {
out := bbcode.NewHTMLTag("")
out.Name = "blockquote"
out.Attrs["cite"] = cite
a := bbcode.NewHTMLTag("")
a.Name = "a"
a.Attrs = map[string]string{
"href": hmnurl.BuildUserProfile(cite),
"class": "quotewho",
}
a.AppendChild(bbcode.NewHTMLTag(cite))
br := bbcode.NewHTMLTag("")
br.Name = "br"
out.AppendChild(a)
out.AppendChild(br)
return out, true
return htm("blockquote", attrs{"cite": cite},
htm("a", attrs{"href": hmnurl.BuildUserProfile(cite), "class": "quotewho"},
bbcode.NewHTMLTag(cite),
),
htm("br", nil),
), true
}
})
addTag("code", func(bn *bbcode.BBCodeNode) (*bbcode.HTMLTag, bool) {
addTag(all, "code", func(bn *bbcode.BBCodeNode) (*bbcode.HTMLTag, bool) {
lang := ""
if tagvalue := bn.GetOpeningTag().Value; tagvalue != "" {
lang = tagvalue
@ -150,6 +108,63 @@ func init() {
return out, false
})
addTag(preview, "youtube", makeYoutubeBBCodeFunc(true))
addTag(real, "youtube", makeYoutubeBBCodeFunc(false))
addTag(education, "glossary", func(bn *bbcode.BBCodeNode) (*bbcode.HTMLTag, bool) {
term := bn.GetOpeningTag().Value
return htm("a", attrs{"href": hmnurl.BuildEducationGlossary(term), "class": "glossary-term", "data-term": term}), true
})
addSimpleTag(education, "note", "div", false, attrs{"class": "education-note"})
}
type attrs map[string]string
// Generates a "simple" tag, i.e. one that is more or less just HTML.
//
// name is the bbcode tag name.
// tag is the HTML tag to generate.
// notext prevents the tag from having any direct text children (useful for e.g. [ol] and [ul])
// attrs are any HTML attributes to add to the tag.
func addSimpleTag(compilers []bbcode.Compiler, name, tag string, notext bool, attributes attrs) {
var tagFunc bbcode.TagCompilerFunc = func(bn *bbcode.BBCodeNode) (*bbcode.HTMLTag, bool) {
if notext {
var newChildren []*bbcode.BBCodeNode
for _, child := range bn.Children {
if child.ID != bbcode.TEXT {
newChildren = append(newChildren, child)
}
}
bn.Children = newChildren
}
out := bbcode.NewHTMLTag("")
out.Name = tag
out.Attrs = attributes
return out, true
}
for _, compiler := range compilers {
compiler.SetTag(name, tagFunc)
}
}
func addTag(compilers []bbcode.Compiler, name string, f bbcode.TagCompilerFunc) {
for _, compiler := range compilers {
compiler.SetTag(name, f)
}
}
// html was taken
func htm(tag string, attributes attrs, children ...*bbcode.HTMLTag) *bbcode.HTMLTag {
res := bbcode.NewHTMLTag("")
res.Name = tag
res.Attrs = attributes
for _, child := range children {
res.AppendChild(child)
}
return res
}
func makeYoutubeBBCodeFunc(preview bool) bbcode.TagCompilerFunc {
@ -233,7 +248,8 @@ func makeYoutubeBBCodeFunc(preview bool) bbcode.TagCompilerFunc {
// ----------------------
type bbcodeParser struct {
Preview bool
Preview bool
Education bool
}
var _ parser.InlineParser = &bbcodeParser{}
@ -291,9 +307,15 @@ func (s bbcodeParser) Parse(parent gast.Node, block text.Reader, pc parser.Conte
unparsedBBCode := restOfSource[:endIndex]
block.Advance(len(unparsedBBCode))
compiler := realBBCodeCompiler
if s.Preview {
var compiler bbcode.Compiler
if s.Preview && s.Education {
compiler = eduPreviewBBCodeCompiler
} else if s.Preview && !s.Education {
compiler = previewBBCodeCompiler
} else if !s.Preview && s.Education {
compiler = eduRealBBCodeCompiler
} else {
compiler = realBBCodeCompiler
}
return NewBBCode(compiler.Compile(string(unparsedBBCode)))
@ -368,12 +390,13 @@ func (r *BBCodeHTMLRenderer) renderBBCode(w util.BufWriter, source []byte, n gas
// ----------------------
type BBCodeExtension struct {
Preview bool
Preview bool
Education bool
}
func (e BBCodeExtension) Extend(m goldmark.Markdown) {
m.Parser().AddOptions(parser.WithInlineParsers(
util.Prioritized(bbcodeParser{Preview: e.Preview}, BBCodePriority),
util.Prioritized(bbcodeParser{Preview: e.Preview, Education: e.Education}, BBCodePriority),
))
m.Renderer().AddOptions(renderer.WithNodeRenderers(
util.Prioritized(NewBBCodeHTMLRenderer(), BBCodePriority),

26
src/parsing/parsing.go
View File

@ -46,6 +46,24 @@ var DiscordMarkdown = makeGoldmark(
goldmark.WithRendererOptions(html.WithHardWraps()),
)
// Used for rendering real-time previews of post content.
var EducationPreviewMarkdown = makeGoldmark(
goldmark.WithExtensions(makeGoldmarkExtensions(MarkdownOptions{
Previews: true,
Embeds: true,
Education: true,
})...),
)
// Used for generating the final HTML for a post.
var EducationRealMarkdown = makeGoldmark(
goldmark.WithExtensions(makeGoldmarkExtensions(MarkdownOptions{
Previews: false,
Embeds: true,
Education: true,
})...),
)
func ParseMarkdown(source string, md goldmark.Markdown) string {
var buf bytes.Buffer
if err := md.Convert([]byte(source), &buf); err != nil {
@ -56,8 +74,9 @@ func ParseMarkdown(source string, md goldmark.Markdown) string {
}
type MarkdownOptions struct {
Previews bool
Embeds bool
Previews bool
Embeds bool
Education bool
}
func makeGoldmark(opts ...goldmark.Option) goldmark.Markdown {
@ -114,7 +133,8 @@ func makeGoldmarkExtensions(opts MarkdownOptions) []goldmark.Extender {
extenders = append(extenders,
MathjaxExtension{},
BBCodeExtension{
Preview: opts.Previews,
Preview: opts.Previews,
Education: opts.Education,
},
)

16
src/parsing/parsing_test.go
View File

@ -61,6 +61,22 @@ func main() {
assert.Contains(t, html, "Hello, world!")
})
})
t.Run("education", func(t *testing.T) {
t.Run("[glossary]", func(t *testing.T) {
html := ParseMarkdown("[glossary=foo]Foo Protocol[/glossary]", EducationRealMarkdown)
t.Log(html)
assert.Equal(t, 1, strings.Count(html, "<a"))
assert.Contains(t, html, `class="glossary-term"`)
assert.Contains(t, html, `data-term="foo"`)
})
t.Run("[note]", func(t *testing.T) {
html := ParseMarkdown("[note]This should only appear to editors![/note]", EducationRealMarkdown)
t.Log(html)
assert.Equal(t, 1, strings.Count(html, "<div"))
assert.Contains(t, html, `class="education-note"`)
})
})
}
func TestSharlock(t *testing.T) {

57
src/parsing/wasm/build.go Normal file
View File

@ -0,0 +1,57 @@
//go:build !js
package main
import (
"fmt"
"go/build"
"io"
"os"
"os/exec"
"path/filepath"
"git.handmade.network/hmn/hmn/src/utils"
)
func main() {
const publicDir = "../../../public"
compile := exec.Command("go", "build", "-o", filepath.Join(publicDir, "parsing.wasm"))
compile.Env = append(os.Environ(), "GOOS=js", "GOARCH=wasm")
run(compile)
utils.Must(copy(
fmt.Sprintf("%s/misc/wasm/wasm_exec.js", build.Default.GOROOT),
filepath.Join(publicDir, "go_wasm_exec.js"),
))
}
func run(cmd *exec.Cmd) {
output, err := cmd.CombinedOutput()
fmt.Print(string(output))
if err != nil {
fmt.Println(err)
if exit, ok := err.(*exec.ExitError); ok {
os.Exit(exit.ExitCode())
}
}
}
func copy(src string, dst string) error {
s, err := os.Open(src)
if err != nil {
return err
}
defer s.Close()
d, err := os.Create(dst)
if err != nil {
return err
}
if _, err := io.Copy(d, s); err != nil {
d.Close()
return err
}
return d.Close()
}

6
src/parsing/wasm/build.sh
View File

@ -1,6 +0,0 @@
#!/bin/bash
PUBLIC_DIR=../../../public
GOOS=js GOARCH=wasm go build -o $PUBLIC_DIR/parsing.wasm
cp "$(go env GOROOT)/misc/wasm/wasm_exec.js" $PUBLIC_DIR/go_wasm_exec.js

5
src/parsing/wasm/parsingmain.go
View File

@ -1,4 +1,4 @@
// +build js
//go:build js
package main
@ -12,6 +12,9 @@ func main() {
js.Global().Set("parseMarkdown", js.FuncOf(func(this js.Value, args []js.Value) interface{} {
return parsing.ParseMarkdown(args[0].String(), parsing.ForumPreviewMarkdown)
}))
js.Global().Set("parseMarkdownEdu", js.FuncOf(func(this js.Value, args []js.Value) interface{} {
return parsing.ParseMarkdown(args[0].String(), parsing.EducationPreviewMarkdown)
}))
var done chan bool
<-done // block forever

1
src/templates/src/include/markdown_previews.html
View File

@ -97,6 +97,7 @@
previewWorker.postMessage({
elementID: inputEl.id,
markdown: inputEl.value,
parserName: '{{ or .ParserName "parseMarkdown" }}',
});
}

11
src/templates/src/markdown_worker.js
View File

@ -1,5 +1,8 @@
importScripts('/public/go_wasm_exec.js');
// wowee good javascript yeah
const global = Function('return this')();
/*
NOTE(ben): The structure here is a little funny but allows for some debouncing. Any postMessages
that got queued up can run all at once, then it can process the latest one.
@ -9,8 +12,8 @@ let wasmLoaded = false;
let jobs = {};
onmessage = ({ data }) => {
const { elementID, markdown } = data;
jobs[elementID] = markdown;
const { elementID, markdown, parserName } = data;
jobs[elementID] = { markdown, parserName };
setTimeout(doPreview, 0);
}
@ -27,8 +30,8 @@ const doPreview = () => {
return;
}
for (const [elementID, markdown] of Object.entries(jobs)) {
const html = parseMarkdown(markdown);
for (const [elementID, { markdown, parserName }] of Object.entries(jobs)) {
const html = global[parserName](markdown);
postMessage({
elementID: elementID,
html: html,

2
src/website/education.go
View File

@ -311,6 +311,8 @@ func getEditorDataForEduArticle(
MaxFileSize: AssetMaxSize(currentUser),
UploadUrl: urlContext.BuildAssetUpload(),
ShowEduOptions: true,
ParserName: "parseMarkdownEdu",
}
if article != nil {

1
src/website/forums.go
View File

@ -50,6 +50,7 @@ type editorData struct {
PostReplyingTo *templates.Post
ShowEduOptions bool
ParserName string
MaxFileSize int
UploadUrl string
}