Annotation-Pushers
/
Annotation-System
1
Fork 0
Code Issues 1 Pull Requests Projects Releases Wiki Activity

hmmlib2: hmmldump -x option for extra output, afl fuzz testing

This commit is contained in:
Alex Baines 2021-04-04 20:39:35 +01:00
parent 1993e9f1dd
commit 678be54e51
6 changed files with 222 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
hmmlib2/.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
/utils/hmmldump
/utils/fuzz/fuzz.gcno
/utils/fuzz/fuzz.gcda
/utils/fuzz/output
/utils/fuzz/fuzz

285
hmmlib2/utils/dump.c
View File

@ -1,6 +1,8 @@
#define HMMLIB_IMPLEMENTATION
#include "hmmlib.h"
#include "stb_sb.h"
#include <stdbool.h>
#include <getopt.h>
typedef struct {
char* text;
@ -9,29 +11,50 @@ typedef struct {
static Index* index_find(Index* base, const char* text)
{
for(size_t i = 0; i < sb_count(base); ++i){
if(strcmp(base[i].text, text) == 0){
return base + i;
}
}
return NULL;
for(size_t i = 0; i < sb_count(base); ++i){
if(strcmp(base[i].text, text) == 0){
return base + i;
}
}
return NULL;
}
void hmml_dump(HMML_Output* hmml)
void hmml_dump(HMML_Output* hmml, bool extra)
{
if(!hmml){
puts("(null)");
return;
}
if(!hmml){
puts("(null)");
return;
}
if(!hmml->well_formed){
printf("Error:%d:%d %s\n", hmml->error.line, hmml->error.col, hmml->error.message);
return;
}
if(!hmml->well_formed){
printf("Error:%d:%d %s\n", hmml->error.line, hmml->error.col, hmml->error.message);
return;
}
puts("Annotations:");
for(size_t i = 0; i < hmml->annotation_count; ++i){
HMML_Annotation* a = hmml->annotations + i;
if(extra) {
puts("Metadata:");
static const char* meta[] = { "member", "stream_platform", "stream_username", "project", "title", "vod_platform", "id", "output", "template", "medium" };
for(size_t i = 0; i < countof(meta); ++i) {
const char* value = ((char**)&hmml->metadata)[i];
printf(" %s = %s\n", meta[i], value);
}
puts(" Credits:");
for(size_t i = 0; i < hmml->metadata.credit_count; ++i) {
HMML_Credit* c = hmml->metadata.credits + i;
printf(" %s [%s]\n", c->name, c->role);
}
puts(" Custom:");
for(size_t i = 0; i < hmml->metadata.custom_count; ++i) {
HMML_VideoCustomMetaData* m = hmml->metadata.custom + i;
printf(" %s = %s\n", m->key, m->value);
}
}
puts("Annotations:");
for(size_t i = 0; i < hmml->annotation_count; ++i){
HMML_Annotation* a = hmml->annotations + i;
char time_buf[256];
char* tp = time_buf;
@ -43,129 +66,152 @@ void hmml_dump(HMML_Output* hmml)
sprintf(tp, " %2d:%02d", a->m, a->s);
}
printf("\t%3d [%s] [%s]\n", a->line, time_buf, a->text);
}
printf("\t%3d [%s] [%s]\n", a->line, time_buf, a->text);
}
Index* authors = NULL;
Index* markers[HMML_MARKER_COUNT] = {};
int max_text_len = 0;
Index* authors = NULL;
Index* markers[HMML_MARKER_COUNT] = {};
int max_text_len = 0;
for(size_t i = 0; i < hmml->annotation_count; ++i){
HMML_Annotation* a = hmml->annotations + i;
for(size_t i = 0; i < hmml->annotation_count; ++i){
HMML_Annotation* a = hmml->annotations + i;
if(a->author){
int len = strlen(a->author);
if(len > max_text_len){
max_text_len = len;
}
if(a->author){
int len = strlen(a->author);
if(len > max_text_len){
max_text_len = len;
}
Index* idx;
if(!(idx = index_find(authors, a->author))){
Index x = { .text = a->author };
sb_push(authors, x);
idx = &sb_last(authors);
}
Index* idx;
if(!(idx = index_find(authors, a->author))){
Index x = { .text = a->author };
sb_push(authors, x);
idx = &sb_last(authors);
}
sb_push(idx->lines, a->line);
}
}
sb_push(idx->lines, a->line);
}
}
for(size_t i = 0; i < hmml->annotation_count; ++i){
HMML_Annotation* a = hmml->annotations + i;
for(size_t i = 0; i < hmml->annotation_count; ++i){
HMML_Annotation* a = hmml->annotations + i;
for(size_t j = 0; j < a->marker_count; ++j){
int type = a->markers[j].type;
char* text = a->markers[j].marker;
for(size_t j = 0; j < a->marker_count; ++j){
int type = a->markers[j].type;
char* text = a->markers[j].marker;
int len = strlen(text);
if(len > max_text_len){
max_text_len = len;
}
int len = strlen(text);
if(len > max_text_len){
max_text_len = len;
}
Index* idx;
if(!(idx = index_find(markers[type], text))){
Index x = { .text = text };
sb_push(markers[type], x);
idx = &sb_last(markers[type]);
}
Index* idx;
if(!(idx = index_find(markers[type], text))){
Index x = { .text = text };
sb_push(markers[type], x);
idx = &sb_last(markers[type]);
}
sb_push(idx->lines, a->line);
}
}
sb_push(idx->lines, a->line);
}
}
puts("Authors:");
for(size_t i = 0; i < sb_count(authors); ++i){
printf("\t %*s: ", max_text_len, authors[i].text);
for(size_t j = 0; j < sb_count(authors[i].lines); ++j){
printf("%3d ", authors[i].lines[j]);
}
puts("");
}
puts("Authors:");
for(size_t i = 0; i < sb_count(authors); ++i){
printf("\t %*s: ", max_text_len, authors[i].text);
for(size_t j = 0; j < sb_count(authors[i].lines); ++j){
printf("%3d ", authors[i].lines[j]);
}
puts("");
}
static const char* m_tags[HMML_MARKER_COUNT] = { "Categories", "Members", "Projects" };
static const char* m_tags[HMML_MARKER_COUNT] = { "Categories", "Members", "Projects" };
for(size_t i = 0; i < HMML_MARKER_COUNT; ++i){
printf("%s:\n", m_tags[i]);
for(size_t j = 0; j < sb_count(markers[i]); ++j){
printf("\t %*s: ", max_text_len, markers[i][j].text);
for(size_t k = 0; k < sb_count(markers[i][j].lines); ++k){
printf("%3d ", markers[i][j].lines[k]);
}
puts("");
}
}
for(size_t i = 0; i < HMML_MARKER_COUNT; ++i){
printf("%s:\n", m_tags[i]);
for(size_t j = 0; j < sb_count(markers[i]); ++j){
printf("\t %*s: ", max_text_len, markers[i][j].text);
for(size_t k = 0; k < sb_count(markers[i][j].lines); ++k){
printf("%3d ", markers[i][j].lines[k]);
}
puts("");
}
}
static const char* r_tags[] = { "Site", "Page", "URL", "Title", "Article", "Author", "Editor", "Publisher", "ISBN" };
puts("References:");
for(size_t i = 0; i < hmml->annotation_count; ++i){
HMML_Annotation* a = hmml->annotations + i;
for(size_t j = 0; j < a->reference_count; ++j){
printf("\t%3d ", a->line);
HMML_Reference* r = a->references + j;
for(size_t k = 0; k < countof(r_tags); ++k){
char* item = ((char**)r)[k];
if(item){
printf("[%s = %s] ", r_tags[k], item);
}
}
puts("");
}
}
static const char* r_tags[] = { "Site", "Page", "URL", "Title", "Article", "Author", "Editor", "Publisher", "ISBN" };
puts("References:");
for(size_t i = 0; i < hmml->annotation_count; ++i){
HMML_Annotation* a = hmml->annotations + i;
for(size_t j = 0; j < a->reference_count; ++j){
printf("\t%3d ", a->line);
HMML_Reference* r = a->references + j;
for(size_t k = 0; k < countof(r_tags); ++k){
char* item = ((char**)r)[k];
if(item){
printf("[%s = %s] ", r_tags[k], item);
}
}
puts("");
}
}
puts("Quotes:");
for(size_t i = 0; i < hmml->annotation_count; ++i){
HMML_Annotation* a = hmml->annotations + i;
if(a->quote.present){
if(a->quote.author){
printf("\t%3d [Quote #%d, by %s]", a->line, a->quote.id, a->quote.author);
} else {
printf("\t%3d [Quote #%d]", a->line, a->quote.id);
}
puts("");
}
}
puts("Quotes:");
for(size_t i = 0; i < hmml->annotation_count; ++i){
HMML_Annotation* a = hmml->annotations + i;
if(a->quote.present){
if(a->quote.author){
printf("\t%3d [Quote #%d, by %s]", a->line, a->quote.id, a->quote.author);
} else {
printf("\t%3d [Quote #%d]", a->line, a->quote.id);
}
puts("");
}
}
for(size_t i = 0; i < sb_count(authors); ++i){
sb_free(authors[i].lines);
}
sb_free(authors);
for(size_t i = 0; i < sb_count(authors); ++i){
sb_free(authors[i].lines);
}
sb_free(authors);
for(size_t i = 0; i < HMML_MARKER_COUNT; ++i){
for(size_t j = 0; j < sb_count(markers[i]); ++j){
sb_free(markers[i][j].lines);
}
sb_free(markers[i]);
}
for(size_t i = 0; i < HMML_MARKER_COUNT; ++i){
for(size_t j = 0; j < sb_count(markers[i]); ++j){
sb_free(markers[i][j].lines);
}
sb_free(markers[i]);
}
}
void usage(char* argv0, FILE* out)
{
fprintf(out, "Usage: %s [-bx] [file]\n", argv0);
}
int main(int argc, char** argv)
{
if(argc < 2) {
fprintf(stderr, "Usage: %s [file]\n", argv[0]);
bool dump_extra = false;
bool breakpoint = false;
int opt;
while((opt = getopt(argc, argv, "bx")) != -1) {
if(opt == 'x') {
dump_extra = true;
} else if(opt == 'b') {
breakpoint = true;
} else {
usage(argv[0], stderr);
return 1;
}
}
if(optind >= argc){
usage(argv[0], stderr);
return 1;
}
argc -= (optind-1);
argv += (optind-1);
FILE* f = fopen(argv[1], "r");
if(!f) {
perror(argv[1]);
@ -185,8 +231,11 @@ int main(int argc, char** argv)
HMML_Output out = hmml_parse(mem);
free(mem);
// asm("int3");
hmml_dump(&out);
if(breakpoint) {
asm("int3");
}
hmml_dump(&out, dump_extra);
hmml_free(&out);
}

16
hmmlib2/utils/fuzz/Makefile Normal file
View File

@ -0,0 +1,16 @@
fuzz: fuzz.c ../../hmmlib.h
afl-gcc -I../.. -g -D_GNU_SOURCE -fprofile-arcs -ftest-coverage $< -o $@
run: fuzz | output
afl-fuzz -i tests -o output ./$<
cov: fuzz | output
afl-cov -d output --coverage-cmd 'cat AFL_FILE | ./fuzz' --code-dir . --enable-branch-coverage --overwrite
output:
mkdir -p $@
clean:
rm -rf fuzz fuzz.gcno fuzz.gcda output
.PHONY: run cov clean

20
hmmlib2/utils/fuzz/fuzz.c Normal file
View File

@ -0,0 +1,20 @@
#define HMMLIB_IMPLEMENTATION
#include "hmmlib.h"
int main(int argc, char** argv)
{
char* mem = NULL;
size_t size = 0;
char buf[BUFSIZ];
while(fgets(buf, BUFSIZ, stdin)) {
size_t n = strlen(buf) + 1;
mem = realloc(mem, size + n);
memcpy(mem + size, buf, n);
size += n - 1;
}
HMML_Output out = hmml_parse(mem);
free(mem);
hmml_free(&out);
}

11
hmmlib2/utils/fuzz/tests/t1 Normal file
View File

@ -0,0 +1,11 @@
[video member=nothings credit="inso:programmer" annotator=Miblo]
[2:01][Recap and update the TODO list]
[38:58][Continue implementing parse_tag()][:parsing :test]
[41:40][@miblo][Ohhh, right. Yeah, the \[video\] and \[/video\] tags are the only :ones that [:have a s d f] that open-close format. All the [~other] [@tags abc] are "single" tags]
[45:17][Enable parse_tag() to tokenise the \[video\] node][:parsing][quote bob 1]
[3:42:11][@miblo][@nothings2: That's fine, yeah! They are also in: [ref
site="GitLab: Annotation-Pushers / Annotation-Game"
page="projects/nothings/obbg"
url=http://git.handmadedev.org/Annotation-Pushers/Annotation-Game/tree/master/projects/nothings/obbg]]
[3:50:44][Take a break][quote 5]
[/video]

3
hmmlib2/utils/fuzz/tests/t2 Normal file
View File

@ -0,0 +1,3 @@
[video credit=inso:programmer]
[2:01][Recap and :update the ~TODO @list]
[/video]