Re-added tools/{makespec.py, template.html, template.tex}.

These were inadvertently dropped in the commit that described
them as being moved to tools/.

4 files changed, 504 insertions, 0 deletions

diff --git a/tools/makespec.py b/tools/makespec.py
new file mode 100755
index 0000000..84663f2
--- /dev/null
+++ b/tools/makespec.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+import re
+import sys
+from subprocess import *
+from string import Template
+
+if len(sys.argv) == 2:
+    specformat = sys.argv[1]
+    if not (specformat in ["html", "markdown"]):
+        sys.stderr.write("Format must be html or markdown\n")
+        exit(1)
+else:
+    sys.stderr.write("Usage:  makespec.py [html|markdown]\n")
+    exit(1)
+
+def toIdentifier(s):
+   return re.sub(r'\s+', '-', re.sub(r'\W+', ' ', s.strip().lower()))
+
+def parseYaml(yaml):
+    metadata = {}
+    def parseField(match):
+        key = match.group(1)
+        val = match.group(2).strip()
+        if re.match(r'^\'', val):
+            val = val[1:len(val) - 1]
+        metadata[key] = val
+    fieldre = re.compile('^(\w+):(.*)$', re.MULTILINE)
+    re.sub(fieldre, parseField, yaml)
+    return metadata
+
+def pipe_through_prog(prog, text):
+    p1 = Popen(prog.split(), stdout=PIPE, stdin=PIPE, stderr=PIPE)
+    [result, err] = p1.communicate(input=text.encode('utf-8'))
+    return [p1.returncode, result.decode('utf-8'), err]
+
+def replaceAnchor(match):
+    refs.append("[{0}]: #{1}".format(match.group(1), match.group(2)))
+    if specformat == "html":
+        return '<a id="{1}" href="#{1}" class="definition">{0}</a>'.format(match.group(1), match.group(2))
+    else:
+        return match.group(0)
+
+stage = 0
+example = 0
+section = ""
+sections = []
+mdlines = []
+refs = []
+lastnum = []
+finishedMeta = False
+yamllines = []
+
+with open('spec.txt', 'r', encoding='utf-8') as spec:
+    for ln in spec:
+        if not finishedMeta:
+            yamllines.append(ln)
+            if re.match(r'^\.\.\.$', ln):
+                finishedMeta = True
+        elif re.match(r'^\.$', ln):
+            if stage == 0:
+                example += 1
+                mdlines.append("\n<div class=\"example\" id=\"example-{0}\" data-section=\"{1}\">\n".format(example, section))
+                mdlines.append("<div class=\"examplenum\"><a href=\"#example-{0}\">Example {0}</a>&nbsp;&nbsp;<a class=\"dingus\" title=\"open in interactive dingus\">(interact)</a></div>\n\n".format(example))
+                mdlines.append("````````````````````````````````````````````````````````` markdown\n")
+                stage = 1
+            elif stage == 1:
+                mdlines.append("`````````````````````````````````````````````````````````\n\n")
+                mdlines.append("````````````````````````````````````````````````````````` html\n")
+                stage = 2
+            elif stage == 2:
+                mdlines.append("`````````````````````````````````````````````````````````\n\n")
+                mdlines.append("</div>\n")
+                stage = 0
+            else:
+                sys.stderr.out("Encountered unknown stage {0}\n".format(stage))
+                sys.exit(1)
+        else:
+            if stage == 0:
+                match = re.match(r'^(#{1,6}) *(.*)', ln)
+                if match:
+                    section = match.group(2)
+                    lastlevel = len(lastnum)
+                    level = len(match.group(1))
+                    if re.search(r'{-}$', section):
+                        section = re.sub(r' *{-} *$', '', section)
+                        if specformat == 'html':
+                            ln = re.sub(r' *{-} *$', '', ln)
+                        number = ''
+                    else:
+                        if lastlevel == level:
+                            lastnum[level - 1] = lastnum[level - 1] + 1
+                        elif lastlevel < level:
+                            while len(lastnum) < level:
+                                lastnum.append(1)
+                        else: # lastlevel > level
+                            lastnum = lastnum[0:level]
+                            lastnum[level - 1] = lastnum[level - 1] + 1
+                        number = '.'.join([str(x) for x in lastnum])
+                    ident = toIdentifier(section)
+                    ln = re.sub(r' ', ' ' + number + ' ', ln, count=1)
+                    sections.append(dict(level=level,
+                                         contents=section,
+                                         ident=ident,
+                                         number=number))
+                    refs.append("[{0}]: #{1}".format(section, ident))
+                    ln = re.sub(r'# +', '# <a id="{0}"></a> '.format(ident),
+                                ln, count=1)
+                else:
+                    ln = re.sub(r'\[([^]]*)\]\(@([^)]*)\)', replaceAnchor, ln)
+            else:
+                ln = re.sub(r' ', '␣', ln)
+            mdlines.append(ln)
+
+mdtext = ''.join(mdlines) + '\n\n' + '\n'.join(refs) + '\n'
+yaml = ''.join(yamllines)
+metadata = parseYaml(yaml)
+
+if specformat == "markdown":
+    sys.stdout.write(yaml + '\n\n' + mdtext)
+elif specformat == "html":
+    with open("template.html", "r", encoding="utf-8") as templatefile:
+        template = Template(templatefile.read())
+    toclines = []
+    for section in sections:
+        indent = '    ' * (section['level'] - 1)
+        toclines.append(indent + '* [' + section['number'] + ' ' +
+                        section['contents'] + '](#' + section['ident'] + ')')
+    toc = '<div id="TOC">\n\n' + '\n'.join(toclines) + '\n\n</div>\n\n'
+    prog = "build/src/cmark"
+    [retcode, result, err] = pipe_through_prog(prog, toc + mdtext)
+    if retcode == 0:
+        result = re.sub(r'␣', '<span class="space"> </span>', result)
+        result = re.sub(r'<h([1-6])><a id="([^\"]*)"><\/a> ',
+                        "<h\\1 id=\"\\2\">", result)
+        # put plural s inside links for better visuals:
+        result = re.sub(r'<\/a>s', "s</a>", result)
+        sys.stdout.write(template.substitute(metadata, body=result))
+
+        # check for errors:
+        idents = []
+        for ident in re.findall(r'id="([^"]*)"', result):
+            if ident in idents:
+                sys.stderr.write("WARNING: duplicate identifier '" + ident +
+                                 "'\n")
+            else:
+                idents.append(ident)
+        for href in re.findall(r'href="#([^"]*)"', result):
+            if not (href in idents):
+                sys.stderr.write("WARNING: internal link with no anchor '" +
+                                 href + "'\n")
+        reftexts = []
+        for ref in refs:
+            ref = re.sub('].*',']',ref).upper()
+            if ref in reftexts:
+                sys.stderr.write("WARNING: duplicate reference link '" +
+                                 ref + "'\n")
+            else:
+                reftexts.append(ref)
+
+    else:
+        sys.stderr.write("Error converting markdown version of spec:\n")
+        sys.stderr.write(err)
+        exit(1)
+
+exit(0)
diff --git a/tools/mkcasefold.pl b/tools/mkcasefold.pl
index 740ce77..740ce77 100644..100755
--- a/tools/mkcasefold.pl
+++ b/tools/mkcasefold.pl
diff --git a/tools/template.html b/tools/template.html
new file mode 100644
index 0000000..a5cc081
--- /dev/null
+++ b/tools/template.html
@@ -0,0 +1,110 @@
+<!DOCTYPE html>
+<html>
+<head>
+<meta charset="UTF-8">
+<title>${title}</title>
+<style type="text/css">
+  body { font-family: Helvetica, arial, freesans, clean, sans-serif;
+    line-height: 1.4;
+    max-width: 48em;
+    margin: auto;
+    color: #333333;
+    background-color: #fff;
+    font-size: 13pt;
+  }
+div#TOC ul { list-style: none; }
+h1 { font-size: 140%; font-weight: bold; border-top: 1px solid gray; padding-top: 0.5em; }
+h2 { font-size: 120%; font-weight: bold; }
+h3 { font-size: 110%; font-weight: bold; }
+h4 { font-size: 100%; font-weight: bold; }
+a.definition { font-weight: bold; }
+span.space { position: relative; }
+span.space:after {
+  content: "·";
+  position: absolute;
+  /* create a mark that indicates a space (trick from D. Greenspan) */
+  top: 0px; bottom: 7px; left: 1px; right: 1px;
+  color: #AAA;
+}
+div.example { overflow: hidden; }
+p { text-align: justify; }
+pre { padding: 0.5em; margin-left: 0; margin-right: 0; margin-top: 0.2em;
+  margin-bottom: 0.5em; font-size: 88%; }
+pre {
+ white-space: pre-wrap;       /* css-3 */
+ white-space: -moz-pre-wrap;  /* Mozilla, since 1999 */
+ white-space: -pre-wrap;      /* Opera 4-6 */
+ white-space: -o-pre-wrap;    /* Opera 7 */
+ word-wrap: break-word;       /* Internet Explorer 5.5+ */
+}
+code { font-family: monospace; background-color: #D3E1E4; }
+pre > code { background-color: transparent; }
+div.example > pre { float:left; width: 48%; }
+div.example > pre:nth-child(2) { clear:left; background-color: #D3E1E4; }
+div.example > pre:nth-child(3) { clear:right; background-color: #C9CaCE; }
+#watermark {
+ position:fixed;
+ bottom:0px;
+ left:0px;
+ padding: 1em;
+ width: 100%;
+ font-size: 120%;
+ opacity:0.7;
+ z-index:99;
+ color: white;
+}
+#watermark a { color: white; }
+div.examplenum { font-size: 82%; text-align: left; }
+a.dingus { color: red; cursor: pointer; }
+a.footnoteRef > sup:before {
+  content: "[";
+}
+a.footnoteRef > sup:after {
+  content: "]";
+}
+a.footnoteRef > sup {
+  vertical-align: baseline;
+  font-size: 100%;
+}
+</style>
+<script src="//code.jquery.com/jquery-1.11.0.min.js"></script>
+<script type="text/javascript">
+$$(document).ready(function() {
+  $$("div.example").each(function(e) {
+    var t = $$(this).find('code.markdown').text();
+    $$(this).find('a.dingus').click(function(f) {
+      window.open('/dingus.html?text=' +
+        encodeURIComponent(t.replace(/→/g,"\t")));
+    });
+  });
+  $$("code.markdown").dblclick(function(e) { window.open('/dingus.html?text=' +
+      encodeURIComponent($$(this).find('code').text()));
+  });
+});
+</script>
+</head>
+<body>
+<h1 class="title">${title}</h1>
+<div class="version">Version ${version} (${date})</div>
+<div class="authors">
+    <span class="author">${author}</span>
+</div>
+<div class="license">
+<a rel="license"
+   href="http://creativecommons.org/licenses/by-sa/4.0/"><img alt="Creative
+   Commons BY-SA" style="border-width:0"
+   src="https://i.creativecommons.org/l/by-sa/4.0/80x15.png"
+   /></a><br/><span style="display:none"><span xmlns:dct="http://purl.org/dc/terms/"
+   href="http://purl.org/dc/dcmitype/Text" property="dct:title"
+   rel="dct:type">CommonMark Spec</span> by
+   <a xmlns:cc="http://creativecommons.org/ns#"
+   href="http://spec.commonmark.org" property="cc:attributionName"
+   rel="cc:attributionURL">John MacFarlane</a> is licensed under a
+   <a rel="license"
+   href="http://creativecommons.org/licenses/by-sa/4.0/">Creative
+   Commons Attribution-ShareAlike 4.0 International License</a>.</span>
+</div>
+<div id="watermark"></div>
+${body}
+</body>
+</html>
diff --git a/tools/template.tex b/tools/template.tex
new file mode 100644
index 0000000..d083b72
--- /dev/null
+++ b/tools/template.tex
@@ -0,0 +1,229 @@
+\documentclass[$if(fontsize)$$fontsize$,$endif$$if(lang)$$lang$,$endif$$if(papersize)$$papersize$,$endif$$for(classoption)$$classoption$$sep$,$endfor$]{$documentclass$}
+$if(fontfamily)$
+\usepackage{$fontfamily$}
+$else$
+\usepackage{lmodern}
+$endif$
+$if(linestretch)$
+\usepackage{setspace}
+\setstretch{$linestretch$}
+$endif$
+\usepackage{amssymb,amsmath}
+\usepackage{ifxetex,ifluatex}
+\usepackage{fixltx2e} % provides \textsubscript
+\ifnum 0\ifxetex 1\fi\ifluatex 1\fi=0 % if pdftex
+  \usepackage[T1]{fontenc}
+  \usepackage[utf8]{inputenc}
+$if(euro)$
+  \usepackage{eurosym}
+$endif$
+\else % if luatex or xelatex
+  \ifxetex
+    \usepackage{mathspec}
+    \usepackage{xltxtra,xunicode}
+  \else
+    \usepackage{fontspec}
+  \fi
+  \defaultfontfeatures{Mapping=tex-text,Scale=MatchLowercase}
+  \newcommand{\euro}{€}
+$if(mainfont)$
+    \setmainfont{$mainfont$}
+$endif$
+$if(sansfont)$
+    \setsansfont{$sansfont$}
+$endif$
+$if(monofont)$
+    \setmonofont[Mapping=tex-ansi]{$monofont$}
+$endif$
+$if(mathfont)$
+    \setmathfont(Digits,Latin,Greek){$mathfont$}
+$endif$
+\fi
+% use upquote if available, for straight quotes in verbatim environments
+\IfFileExists{upquote.sty}{\usepackage{upquote}}{}
+% use microtype if available
+\IfFileExists{microtype.sty}{\usepackage{microtype}}{}
+\usepackage[margin=1in]{geometry}
+$if(natbib)$
+\usepackage{natbib}
+\bibliographystyle{$if(biblio-style)$$biblio-style$$else$plainnat$endif$}
+$endif$
+$if(biblatex)$
+\usepackage{biblatex}
+$if(biblio-files)$
+\bibliography{$biblio-files$}
+$endif$
+$endif$
+$if(listings)$
+\usepackage{listings}
+$endif$
+$if(lhs)$
+\lstnewenvironment{code}{\lstset{language=Haskell,basicstyle=\small\ttfamily}}{}
+$endif$
+\usepackage{fancyvrb}
+\usepackage{color,framed}
+\newcommand{\VerbBar}{|}
+\newcommand{\VERB}{\Verb[commandchars=\\\{\}]}
+\DefineVerbatimEnvironment{Highlighting}{Verbatim}{commandchars=\\\{\},fontsize=\small}
+% Add ',fontsize=\small' for more characters per line
+\definecolor{shadecolor}{gray}{1}
+\newenvironment{Shaded}{\begin{snugshade}}{\end{snugshade}}
+\newcommand{\NormalTok}[1]{{#1}}
+\let\KeywordTok\NormalTok
+\let\DataTypeTok\NormalTok
+\let\DecValTok\NormalTok
+\let\BaseNTok\NormalTok
+\let\FloatTok\NormalTok
+\let\CharTok\NormalTok
+\let\StringTok\NormalTok
+\let\CommentTok\NormalTok
+\let\OtherTok\NormalTok
+\let\AlertTok\NormalTok
+\let\FunctionTok\NormalTok
+\let\RegionMarkerTok\NormalTok
+\let\ErrorTok\NormalTok
+%\newcommand{\KeywordTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{\textbf{{#1}}}}
+%\newcommand{\DataTypeTok}[1]{\textcolor[rgb]{0.56,0.13,0.00}{{#1}}}
+%\newcommand{\DecValTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
+%\newcommand{\BaseNTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
+%\newcommand{\FloatTok}[1]{\textcolor[rgb]{0.25,0.63,0.44}{{#1}}}
+%\newcommand{\CharTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
+%\newcommand{\StringTok}[1]{\textcolor[rgb]{0.25,0.44,0.63}{{#1}}}
+%\newcommand{\CommentTok}[1]{\textcolor[rgb]{0.38,0.63,0.69}{\textit{{#1}}}}
+%\newcommand{\OtherTok}[1]{\textcolor[rgb]{0.00,0.44,0.13}{{#1}}}
+%\newcommand{\AlertTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
+%\newcommand{\FunctionTok}[1]{\textcolor[rgb]{0.02,0.16,0.49}{{#1}}}
+%\newcommand{\RegionMarkerTok}[1]{{#1}}
+%\newcommand{\ErrorTok}[1]{\textcolor[rgb]{1.00,0.00,0.00}{\textbf{{#1}}}}
+$if(verbatim-in-note)$
+\usepackage{fancyvrb}
+$endif$
+$if(tables)$
+\usepackage{longtable,booktabs}
+$endif$
+$if(graphics)$
+\usepackage{graphicx}
+\makeatletter
+\def\maxwidth{\ifdim\Gin@nat@width>\linewidth\linewidth\else\Gin@nat@width\fi}
+\def\maxheight{\ifdim\Gin@nat@height>\textheight\textheight\else\Gin@nat@height\fi}
+\makeatother
+% Scale images if necessary, so that they will not overflow the page
+% margins by default, and it is still possible to overwrite the defaults
+% using explicit options in \includegraphics[width, height, ...]{}
+\setkeys{Gin}{width=\maxwidth,height=\maxheight,keepaspectratio}
+$endif$
+\ifxetex
+  \usepackage[setpagesize=false, % page size defined by xetex
+              unicode=false, % unicode breaks when used with xetex
+              xetex]{hyperref}
+\else
+  \usepackage[unicode=true]{hyperref}
+\fi
+\hypersetup{breaklinks=true,
+            bookmarks=true,
+            pdfauthor={$author-meta$},
+            pdftitle={$title-meta$},
+            colorlinks=true,
+            citecolor=$if(citecolor)$$citecolor$$else$blue$endif$,
+            urlcolor=$if(urlcolor)$$urlcolor$$else$blue$endif$,
+            linkcolor=$if(linkcolor)$$linkcolor$$else$magenta$endif$,
+            pdfborder={0 0 0}}
+\urlstyle{same}  % don't use monospace font for urls
+$if(links-as-notes)$
+% Make links footnotes instead of hotlinks:
+\renewcommand{\href}[2]{#2\footnote{\url{#1}}}
+$endif$
+$if(strikeout)$
+\usepackage[normalem]{ulem}
+% avoid problems with \sout in headers with hyperref:
+\pdfstringdefDisableCommands{\renewcommand{\sout}{}}
+$endif$
+\setlength{\parindent}{0pt}
+\setlength{\parskip}{6pt plus 2pt minus 1pt}
+\setlength{\emergencystretch}{3em}  % prevent overfull lines
+$if(numbersections)$
+\setcounter{secnumdepth}{5}
+$else$
+\setcounter{secnumdepth}{0}
+$endif$
+$if(verbatim-in-note)$
+\VerbatimFootnotes % allows verbatim text in footnotes
+$endif$
+$if(lang)$
+\ifxetex
+  \usepackage{polyglossia}
+  \setmainlanguage{$mainlang$}
+\else
+  \usepackage[$lang$]{babel}
+\fi
+$endif$
+
+\usepackage{titlesec}
+\titleformat{\chapter}[hang]{\Huge\bfseries}{\thechapter\ }{0pt}{\Huge\bfseries}
+
+\usepackage{fancyhdr}
+\pagestyle{fancy}
+\pagenumbering{arabic}
+\lhead{\itshape $title$}
+\chead{}
+\rhead{\itshape{\nouppercase{\rightmark}}}
+\lfoot{v$version$ ($date$)}
+\cfoot{}
+\rfoot{\thepage}
+
+$if(title)$
+\title{$title$$if(subtitle)$\\\vspace{0.5em}{\large $subtitle$}$endif$}
+$endif$
+$if(author)$
+\author{$for(author)$$author$$sep$ \and $endfor$}
+$endif$
+\date{$date$}
+$for(header-includes)$
+$header-includes$
+$endfor$
+
+\begin{document}
+$if(title)$
+\maketitle
+$endif$
+$if(abstract)$
+\begin{abstract}
+$abstract$
+\end{abstract}
+$endif$
+
+$for(include-before)$
+$include-before$
+
+$endfor$
+$if(toc)$
+{
+\hypersetup{linkcolor=black}
+\setcounter{tocdepth}{$toc-depth$}
+\tableofcontents
+}
+$endif$
+$body$
+
+$if(natbib)$
+$if(biblio-files)$
+$if(biblio-title)$
+$if(book-class)$
+\renewcommand\bibname{$biblio-title$}
+$else$
+\renewcommand\refname{$biblio-title$}
+$endif$
+$endif$
+\bibliography{$biblio-files$}
+
+$endif$
+$endif$
+$if(biblatex)$
+\printbibliography$if(biblio-title)$[title=$biblio-title$]$endif$
+
+$endif$
+$for(include-after)$
+$include-after$
+
+$endfor$
+\end{document}