From bbca5bb3bbacd2bb8c85cfe54293b8c4da29dbf9 Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Wed, 14 Sep 2016 15:47:51 +0200
Subject: Allow tabs after setext header line.

See jgm/commonmark.js#109
---
 src/scanners.c      | 72 +++++++++++++++++++++++++++++++++--------------------
 src/scanners.re     |  4 +--
 test/regression.txt | 10 ++++++++
 3 files changed, 57 insertions(+), 29 deletions(-)

diff --git a/src/scanners.c b/src/scanners.c
index 8a07d48..21a433d 100644
--- a/src/scanners.c
+++ b/src/scanners.c
@@ -18338,18 +18338,18 @@ bufsize_t _scan_setext_heading_line(const unsigned char *p) {
   {
     unsigned char yych;
     static const unsigned char yybm[] = {
-        0, 0,  0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
-        0, 0,  0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
-        0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0,
-        0, 0,  0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
-        0, 0,  0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
-        0, 0,  0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
-        0, 0,  0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
-        0, 0,  0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
-        0, 0,  0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
-        0, 0,  0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
-        0, 0,  0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
-        0, 0,  0, 0, 0, 0, 0, 0, 0, 0, 0,  0, 0, 0,
+        0, 0,  0, 0, 0, 0, 0, 0, 0, 32, 0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
+        0, 0,  0, 0, 0, 0, 0, 0, 0, 0,  32, 0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
+        0, 64, 0, 0, 0, 0, 0, 0, 0, 0,  0,  0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0,
+        0, 0,  0, 0, 0, 0, 0, 0, 0, 0,  0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
+        0, 0,  0, 0, 0, 0, 0, 0, 0, 0,  0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
+        0, 0,  0, 0, 0, 0, 0, 0, 0, 0,  0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
+        0, 0,  0, 0, 0, 0, 0, 0, 0, 0,  0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
+        0, 0,  0, 0, 0, 0, 0, 0, 0, 0,  0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
+        0, 0,  0, 0, 0, 0, 0, 0, 0, 0,  0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
+        0, 0,  0, 0, 0, 0, 0, 0, 0, 0,  0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
+        0, 0,  0, 0, 0, 0, 0, 0, 0, 0,  0,  0, 0, 0, 0, 0, 0, 0,   0, 0, 0, 0,
+        0, 0,  0, 0, 0, 0, 0, 0, 0, 0,  0,  0, 0, 0,
     };
     yych = *(marker = p);
     if (yych <= 0xC1) {
@@ -18394,30 +18394,34 @@ bufsize_t _scan_setext_heading_line(const unsigned char *p) {
       goto yy1201;
     }
     if (yych <= '\f') {
-      if (yych == '\n')
-        goto yy1199;
+      if (yych <= 0x08)
+        goto yy1179;
+      if (yych <= '\n')
+        goto yy1198;
       goto yy1179;
     } else {
       if (yych <= '\r')
-        goto yy1199;
+        goto yy1198;
       if (yych == ' ')
-        goto yy1197;
+        goto yy1198;
       goto yy1179;
     }
   yy1181:
     yych = *(marker = ++p);
-    if (yybm[0 + yych] & 32) {
-      goto yy1191;
+    if (yybm[0 + yych] & 64) {
+      goto yy1195;
     }
     if (yych <= '\f') {
-      if (yych == '\n')
-        goto yy1193;
+      if (yych <= 0x08)
+        goto yy1179;
+      if (yych <= '\n')
+        goto yy1192;
       goto yy1179;
     } else {
       if (yych <= '\r')
-        goto yy1193;
-      if (yych == '-')
-        goto yy1195;
+        goto yy1192;
+      if (yych == ' ')
+        goto yy1192;
       goto yy1179;
     }
   yy1182:
@@ -18477,10 +18481,13 @@ bufsize_t _scan_setext_heading_line(const unsigned char *p) {
   yy1191:
     ++p;
     yych = *p;
+  yy1192:
     if (yybm[0 + yych] & 32) {
       goto yy1191;
     }
-    if (yych == '\n')
+    if (yych <= 0x08)
+      goto yy1184;
+    if (yych <= '\n')
       goto yy1193;
     if (yych != '\r')
       goto yy1184;
@@ -18494,7 +18501,9 @@ bufsize_t _scan_setext_heading_line(const unsigned char *p) {
       goto yy1191;
     }
     if (yych <= '\f') {
-      if (yych == '\n')
+      if (yych <= 0x08)
+        goto yy1184;
+      if (yych <= '\n')
         goto yy1193;
       goto yy1184;
     } else {
@@ -18507,8 +18516,13 @@ bufsize_t _scan_setext_heading_line(const unsigned char *p) {
   yy1197:
     ++p;
     yych = *p;
+  yy1198:
     if (yych <= '\f') {
-      if (yych != '\n')
+      if (yych <= 0x08)
+        goto yy1184;
+      if (yych <= '\t')
+        goto yy1197;
+      if (yych >= '\v')
         goto yy1184;
     } else {
       if (yych <= '\r')
@@ -18527,7 +18541,11 @@ bufsize_t _scan_setext_heading_line(const unsigned char *p) {
       goto yy1201;
     }
     if (yych <= '\f') {
-      if (yych == '\n')
+      if (yych <= 0x08)
+        goto yy1184;
+      if (yych <= '\t')
+        goto yy1197;
+      if (yych <= '\n')
         goto yy1199;
       goto yy1184;
     } else {
diff --git a/src/scanners.re b/src/scanners.re
index 7ecf358..77d30cd 100644
--- a/src/scanners.re
+++ b/src/scanners.re
@@ -263,8 +263,8 @@ bufsize_t _scan_setext_heading_line(const unsigned char *p)
 {
   const unsigned char *marker = NULL;
 /*!re2c
-  [=]+ [ ]* [\r\n] { return 1; }
-  [-]+ [ ]* [\r\n] { return 2; }
+  [=]+ [ \t]* [\r\n] { return 1; }
+  [-]+ [ \t]* [\r\n] { return 2; }
   .? { return 0; }
 */
 }
diff --git a/test/regression.txt b/test/regression.txt
index 54872da..d5a1e5d 100644
--- a/test/regression.txt
+++ b/test/regression.txt
@@ -56,3 +56,13 @@ Issue jgm/CommonMark#430:  h2..h6 not recognized as block tags.
 <h6>lorem</h6>
 ````````````````````````````````
 
+Issue jgm/commonmark.js#109 - tabs after setext header line
+
+
+```````````````````````````````` example
+hi
+--→
+.
+<h2>hi</h2>
+````````````````````````````````
+
-- 
cgit v1.2.3