From ce2feef7f63e17157ce92ac833e4bce5efcd7e4f Mon Sep 17 00:00:00 2001
From: Alex Baines <alex@abaines.me.uk>
Date: Mon, 6 Mar 2017 13:06:22 +0000
Subject: [PATCH] fix marker after ref bug + handle dos newlines

---
 hmmlib/hmmlib.l | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/hmmlib/hmmlib.l b/hmmlib/hmmlib.l
index fae3858..0ab5344 100644
--- a/hmmlib/hmmlib.l
+++ b/hmmlib/hmmlib.l
@@ -53,13 +53,14 @@
 %option extra-type="HMML_ParseState*"
 %option noyywrap
 
-S               [\t ]*
+S               [\t \r]*
 ATTR_SIMPLE     [^\" \]\t\r\n][^ \]\t\r\n]*
 ATTR_ALNUM      [0-9a-zA-Z][0-9a-zA-Z_]*
 ATTR_QUOTED     \"([^\n\"\\]|\\.)*\"
 TAG_VIDEO_OPEN  \[video
 TIMECODE        \[[0-9]{1,2}(:[0-5][0-9]){1,2}\]
 BAD_TIMECODE    \[[0-9]{1,2}(:[6-9][0-9]){1,2}\]
+TEXT_BREAK      [^\\:@~\[\]\r\n\t ]
 LB              \[
 RB              \]
 
@@ -80,7 +81,7 @@ RB              \]
 %%
 
 <<EOF>> { HMML_ERR("Unexpected EOF, video close tag not found."); }
-\n      { yyextra->line++; }
+\r\n|\n { yyextra->line++; }
 
 <INITIAL>{TAG_VIDEO_OPEN} { BEGIN(VIDEO); }
 <INITIAL>.                { HMML_ERR("Missing video tag."); }
@@ -113,19 +114,19 @@ RB              \]
 <TEXT_START>{LB}  { yyless(0); BEGIN(TEXT); }
 <TEXT_START>.     { HMML_ERR("Unknown character '%c' after timecode.", *yytext); }
 
-<TEXT>[^\\:@~\]\n\[ ]+ { memcpy(sb_add(yyextra->an.text, yyleng), yytext, yyleng); }
-<TEXT>\\.              { CHECKESCAPE(yytext[1]); memcpy(sb_add(yyextra->an.text, yyleng-1), yytext+1, yyleng-1); }
-<TEXT>[ ]:             { sb_push(yyextra->an.text, ' '); M_(CATEGORY, TEXT); BEGIN(MARKER); }
-<TEXT>[ ]@             { sb_push(yyextra->an.text, ' '); M_(MEMBER  , TEXT); BEGIN(MARKER); }
-<TEXT>[ ]~             { sb_push(yyextra->an.text, ' '); M_(PROJECT , TEXT); BEGIN(MARKER); }
-<TEXT>{LB}:            { M_(CATEGORY, MARKER_XTRA); BEGIN(MARKER); }
-<TEXT>{LB}@            { M_(MEMBER  , MARKER_XTRA); BEGIN(MARKER); }
-<TEXT>{LB}~            { M_(PROJECT , MARKER_XTRA); BEGIN(MARKER); }
-<TEXT>\]               { BEGIN(AFTERTEXT); }
-<TEXT>{LB}ref          { yyextra->ref.offset = sb_count(yyextra->an.text); BEGIN(REF); }
+<TEXT>{TEXT_BREAK}+ { memcpy(sb_add(yyextra->an.text, yyleng), yytext, yyleng); }
+<TEXT>\\.           { CHECKESCAPE(yytext[1]); memcpy(sb_add(yyextra->an.text, yyleng-1), yytext+1, yyleng-1); }
+<TEXT>[ \r\t]+:     { sb_push(yyextra->an.text, ' '); M_(CATEGORY, TEXT); BEGIN(MARKER); }
+<TEXT>[ \r\t]+@     { sb_push(yyextra->an.text, ' '); M_(MEMBER  , TEXT); BEGIN(MARKER); }
+<TEXT>[ \r\t]+~     { sb_push(yyextra->an.text, ' '); M_(PROJECT , TEXT); BEGIN(MARKER); }
+<TEXT>{LB}:         { M_(CATEGORY, MARKER_XTRA); BEGIN(MARKER); }
+<TEXT>{LB}@         { M_(MEMBER  , MARKER_XTRA); BEGIN(MARKER); }
+<TEXT>{LB}~         { M_(PROJECT , MARKER_XTRA); BEGIN(MARKER); }
+<TEXT>\]            { BEGIN(AFTERTEXT); }
+<TEXT>{LB}ref       { yyextra->ref.offset = sb_count(yyextra->an.text); BEGIN(REF); }
 <TEXT>{LB}
-<TEXT>{S}{S}           { sb_push(yyextra->an.text, ' '); }
-<TEXT>.                { sb_push(yyextra->an.text, *yytext); }
+<TEXT>[ \r\t]{1,2}  { sb_push(yyextra->an.text, ' '); }
+<TEXT>.             { sb_push(yyextra->an.text, *yytext); }
 
 <MARKER>{ATTR_ALNUM}  { M_ADD(yytext  , yyleng  ); BEGIN(yyextra->mnext); };
 <MARKER>{ATTR_QUOTED} { M_ADD(yytext+1, yyleng-2); BEGIN(yyextra->mnext); };
@@ -135,7 +136,7 @@ RB              \]
 <MARKER_XTRA>\] { BEGIN(TEXT); }
 <MARKER_XTRA>. 
 
-<REF>[\t ]
+<REF>{S}
 <REF>site{S}=      { yyextra->attr = R_(site);      BEGIN(R_ATTR); }
 <REF>page{S}=      { yyextra->attr = R_(page);      BEGIN(R_ATTR); }
 <REF>url{S}=       { yyextra->attr = R_(url);       BEGIN(R_ATTR); }