Annotation-Pushers
/
Annotation-System
1
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

hmmlib2: hmmldump -x option for extra output, afl fuzz testing

This commit is contained in:
Alex Baines 2021-04-04 20:39:35 +01:00
parent 1993e9f1dd
commit 678be54e51
6 changed files with 222 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hmmlib2/.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
/utils/hmmldump
/utils/fuzz/fuzz.gcno
/utils/fuzz/fuzz.gcda
/utils/fuzz/output
/utils/fuzz/fuzz

285
hmmlib2/utils/dump.c
View File

@ -1,6 +1,8 @@
#define HMMLIB_IMPLEMENTATION #define HMMLIB_IMPLEMENTATION
#include "hmmlib.h" #include "hmmlib.h"
#include "stb_sb.h" #include "stb_sb.h"
#include <stdbool.h>
#include <getopt.h>
typedef struct { typedef struct {
char* text; char* text;
@ -9,29 +11,50 @@ typedef struct {
static Index* index_find(Index* base, const char* text) static Index* index_find(Index* base, const char* text)
{ {
for(size_t i = 0; i < sb_count(base); ++i){ for(size_t i = 0; i < sb_count(base); ++i){
if(strcmp(base[i].text, text) == 0){ if(strcmp(base[i].text, text) == 0){
return base + i; return base + i;
} }
} }
return NULL; return NULL;
} }
void hmml_dump(HMML_Output* hmml) void hmml_dump(HMML_Output* hmml, bool extra)
{ {
if(!hmml){ if(!hmml){
puts("(null)"); puts("(null)");
return; return;
} }
if(!hmml->well_formed){ if(!hmml->well_formed){
printf("Error:%d:%d %s\n", hmml->error.line, hmml->error.col, hmml->error.message); printf("Error:%d:%d %s\n", hmml->error.line, hmml->error.col, hmml->error.message);
return; return;
} }
puts("Annotations:"); if(extra) {
for(size_t i = 0; i < hmml->annotation_count; ++i){ puts("Metadata:");
HMML_Annotation* a = hmml->annotations + i; static const char* meta[] = { "member", "stream_platform", "stream_username", "project", "title", "vod_platform", "id", "output", "template", "medium" };
for(size_t i = 0; i < countof(meta); ++i) {
const char* value = ((char**)&hmml->metadata)[i];
printf(" %s = %s\n", meta[i], value);
}
puts(" Credits:");
for(size_t i = 0; i < hmml->metadata.credit_count; ++i) {
HMML_Credit* c = hmml->metadata.credits + i;
printf(" %s [%s]\n", c->name, c->role);
}
puts(" Custom:");
for(size_t i = 0; i < hmml->metadata.custom_count; ++i) {
HMML_VideoCustomMetaData* m = hmml->metadata.custom + i;
printf(" %s = %s\n", m->key, m->value);
}
}
puts("Annotations:");
for(size_t i = 0; i < hmml->annotation_count; ++i){
HMML_Annotation* a = hmml->annotations + i;
char time_buf[256]; char time_buf[256];
char* tp = time_buf; char* tp = time_buf;
@ -43,129 +66,152 @@ void hmml_dump(HMML_Output* hmml)
sprintf(tp, " %2d:%02d", a->m, a->s); sprintf(tp, " %2d:%02d", a->m, a->s);
} }
printf("\t%3d [%s] [%s]\n", a->line, time_buf, a->text); printf("\t%3d [%s] [%s]\n", a->line, time_buf, a->text);
} }
Index* authors = NULL; Index* authors = NULL;
Index* markers[HMML_MARKER_COUNT] = {}; Index* markers[HMML_MARKER_COUNT] = {};
int max_text_len = 0; int max_text_len = 0;
for(size_t i = 0; i < hmml->annotation_count; ++i){ for(size_t i = 0; i < hmml->annotation_count; ++i){
HMML_Annotation* a = hmml->annotations + i; HMML_Annotation* a = hmml->annotations + i;
if(a->author){ if(a->author){
int len = strlen(a->author); int len = strlen(a->author);
if(len > max_text_len){ if(len > max_text_len){
max_text_len = len; max_text_len = len;
} }
Index* idx; Index* idx;
if(!(idx = index_find(authors, a->author))){ if(!(idx = index_find(authors, a->author))){
Index x = { .text = a->author }; Index x = { .text = a->author };
sb_push(authors, x); sb_push(authors, x);
idx = &sb_last(authors); idx = &sb_last(authors);
} }
sb_push(idx->lines, a->line); sb_push(idx->lines, a->line);
} }
} }
for(size_t i = 0; i < hmml->annotation_count; ++i){ for(size_t i = 0; i < hmml->annotation_count; ++i){
HMML_Annotation* a = hmml->annotations + i; HMML_Annotation* a = hmml->annotations + i;
for(size_t j = 0; j < a->marker_count; ++j){ for(size_t j = 0; j < a->marker_count; ++j){
int type = a->markers[j].type; int type = a->markers[j].type;
char* text = a->markers[j].marker; char* text = a->markers[j].marker;
int len = strlen(text); int len = strlen(text);
if(len > max_text_len){ if(len > max_text_len){
max_text_len = len; max_text_len = len;
} }
Index* idx; Index* idx;
if(!(idx = index_find(markers[type], text))){ if(!(idx = index_find(markers[type], text))){
Index x = { .text = text }; Index x = { .text = text };
sb_push(markers[type], x); sb_push(markers[type], x);
idx = &sb_last(markers[type]); idx = &sb_last(markers[type]);
} }
sb_push(idx->lines, a->line); sb_push(idx->lines, a->line);
} }
} }
puts("Authors:"); puts("Authors:");
for(size_t i = 0; i < sb_count(authors); ++i){ for(size_t i = 0; i < sb_count(authors); ++i){
printf("\t %*s: ", max_text_len, authors[i].text); printf("\t %*s: ", max_text_len, authors[i].text);
for(size_t j = 0; j < sb_count(authors[i].lines); ++j){ for(size_t j = 0; j < sb_count(authors[i].lines); ++j){
printf("%3d ", authors[i].lines[j]); printf("%3d ", authors[i].lines[j]);
} }
puts(""); puts("");
} }
static const char* m_tags[HMML_MARKER_COUNT] = { "Categories", "Members", "Projects" }; static const char* m_tags[HMML_MARKER_COUNT] = { "Categories", "Members", "Projects" };
for(size_t i = 0; i < HMML_MARKER_COUNT; ++i){ for(size_t i = 0; i < HMML_MARKER_COUNT; ++i){
printf("%s:\n", m_tags[i]); printf("%s:\n", m_tags[i]);
for(size_t j = 0; j < sb_count(markers[i]); ++j){ for(size_t j = 0; j < sb_count(markers[i]); ++j){
printf("\t %*s: ", max_text_len, markers[i][j].text); printf("\t %*s: ", max_text_len, markers[i][j].text);
for(size_t k = 0; k < sb_count(markers[i][j].lines); ++k){ for(size_t k = 0; k < sb_count(markers[i][j].lines); ++k){
printf("%3d ", markers[i][j].lines[k]); printf("%3d ", markers[i][j].lines[k]);
} }
puts(""); puts("");
} }
} }
static const char* r_tags[] = { "Site", "Page", "URL", "Title", "Article", "Author", "Editor", "Publisher", "ISBN" }; static const char* r_tags[] = { "Site", "Page", "URL", "Title", "Article", "Author", "Editor", "Publisher", "ISBN" };
puts("References:"); puts("References:");
for(size_t i = 0; i < hmml->annotation_count; ++i){ for(size_t i = 0; i < hmml->annotation_count; ++i){
HMML_Annotation* a = hmml->annotations + i; HMML_Annotation* a = hmml->annotations + i;
for(size_t j = 0; j < a->reference_count; ++j){ for(size_t j = 0; j < a->reference_count; ++j){
printf("\t%3d ", a->line); printf("\t%3d ", a->line);
HMML_Reference* r = a->references + j; HMML_Reference* r = a->references + j;
for(size_t k = 0; k < countof(r_tags); ++k){ for(size_t k = 0; k < countof(r_tags); ++k){
char* item = ((char**)r)[k]; char* item = ((char**)r)[k];
if(item){ if(item){
printf("[%s = %s] ", r_tags[k], item); printf("[%s = %s] ", r_tags[k], item);
} }
} }
puts(""); puts("");
} }
} }
puts("Quotes:"); puts("Quotes:");
for(size_t i = 0; i < hmml->annotation_count; ++i){ for(size_t i = 0; i < hmml->annotation_count; ++i){
HMML_Annotation* a = hmml->annotations + i; HMML_Annotation* a = hmml->annotations + i;
if(a->quote.present){ if(a->quote.present){
if(a->quote.author){ if(a->quote.author){
printf("\t%3d [Quote #%d, by %s]", a->line, a->quote.id, a->quote.author); printf("\t%3d [Quote #%d, by %s]", a->line, a->quote.id, a->quote.author);
} else { } else {
printf("\t%3d [Quote #%d]", a->line, a->quote.id); printf("\t%3d [Quote #%d]", a->line, a->quote.id);
} }
puts(""); puts("");
} }
} }
for(size_t i = 0; i < sb_count(authors); ++i){ for(size_t i = 0; i < sb_count(authors); ++i){
sb_free(authors[i].lines); sb_free(authors[i].lines);
} }
sb_free(authors); sb_free(authors);
for(size_t i = 0; i < HMML_MARKER_COUNT; ++i){ for(size_t i = 0; i < HMML_MARKER_COUNT; ++i){
for(size_t j = 0; j < sb_count(markers[i]); ++j){ for(size_t j = 0; j < sb_count(markers[i]); ++j){
sb_free(markers[i][j].lines); sb_free(markers[i][j].lines);
} }
sb_free(markers[i]); sb_free(markers[i]);
} }
}
void usage(char* argv0, FILE* out)
{
fprintf(out, "Usage: %s [-bx] [file]\n", argv0);
} }
int main(int argc, char** argv) int main(int argc, char** argv)
{ {
if(argc < 2) { bool dump_extra = false;
fprintf(stderr, "Usage: %s [file]\n", argv[0]); bool breakpoint = false;
int opt;
while((opt = getopt(argc, argv, "bx")) != -1) {
if(opt == 'x') {
dump_extra = true;
} else if(opt == 'b') {
breakpoint = true;
} else {
usage(argv[0], stderr);
return 1;
}
}
if(optind >= argc){
usage(argv[0], stderr);
return 1; return 1;
} }
argc -= (optind-1);
argv += (optind-1);
FILE* f = fopen(argv[1], "r"); FILE* f = fopen(argv[1], "r");
if(!f) { if(!f) {
perror(argv[1]); perror(argv[1]);
@ -185,8 +231,11 @@ int main(int argc, char** argv)
HMML_Output out = hmml_parse(mem); HMML_Output out = hmml_parse(mem);
free(mem); free(mem);
// asm("int3"); if(breakpoint) {
hmml_dump(&out); asm("int3");
}
hmml_dump(&out, dump_extra);
hmml_free(&out); hmml_free(&out);
} }

16
hmmlib2/utils/fuzz/Makefile Normal file
View File

@ -0,0 +1,16 @@
fuzz: fuzz.c ../../hmmlib.h
afl-gcc -I../.. -g -D_GNU_SOURCE -fprofile-arcs -ftest-coverage $< -o $@
run: fuzz | output
afl-fuzz -i tests -o output ./$<
cov: fuzz | output
afl-cov -d output --coverage-cmd 'cat AFL_FILE | ./fuzz' --code-dir . --enable-branch-coverage --overwrite
output:
mkdir -p $@
clean:
rm -rf fuzz fuzz.gcno fuzz.gcda output
.PHONY: run cov clean

20
hmmlib2/utils/fuzz/fuzz.c Normal file
View File

@ -0,0 +1,20 @@
#define HMMLIB_IMPLEMENTATION
#include "hmmlib.h"
int main(int argc, char** argv)
{
char* mem = NULL;
size_t size = 0;
char buf[BUFSIZ];
while(fgets(buf, BUFSIZ, stdin)) {
size_t n = strlen(buf) + 1;
mem = realloc(mem, size + n);
memcpy(mem + size, buf, n);
size += n - 1;
}
HMML_Output out = hmml_parse(mem);
free(mem);
hmml_free(&out);
}

11
hmmlib2/utils/fuzz/tests/t1 Normal file
View File

@ -0,0 +1,11 @@
[video member=nothings credit="inso:programmer" annotator=Miblo]
[2:01][Recap and update the TODO list]
[38:58][Continue implementing parse_tag()][:parsing :test]
[41:40][@miblo][Ohhh, right. Yeah, the \[video\] and \[/video\] tags are the only :ones that [:have a s d f] that open-close format. All the [~other] [@tags abc] are "single" tags]
[45:17][Enable parse_tag() to tokenise the \[video\] node][:parsing][quote bob 1]
[3:42:11][@miblo][@nothings2: That's fine, yeah! They are also in: [ref
site="GitLab: Annotation-Pushers / Annotation-Game"
page="projects/nothings/obbg"
url=http://git.handmadedev.org/Annotation-Pushers/Annotation-Game/tree/master/projects/nothings/obbg]]
[3:50:44][Take a break][quote 5]
[/video]

3
hmmlib2/utils/fuzz/tests/t2 Normal file
View File

@ -0,0 +1,3 @@
[video credit=inso:programmer]
[2:01][Recap and :update the ~TODO @list]
[/video]