From 1998db887e385e786c0eac4b1973bcddc6406bdc Mon Sep 17 00:00:00 2001
From: John MacFarlane <jgm@berkeley.edu>
Date: Sun, 5 Nov 2017 10:53:23 -0800
Subject: Add allowed failures to pathological_tests.py.

This allows us to include tests that we don't yet know
how to pass.
---
 test/pathological_tests.py | 39 +++++++++++++++++++++++++++++----------
 1 file changed, 29 insertions(+), 10 deletions(-)

(limited to 'test')

diff --git a/test/pathological_tests.py b/test/pathological_tests.py
index 40552de..b86f4c0 100644
--- a/test/pathological_tests.py
+++ b/test/pathological_tests.py
@@ -17,6 +17,8 @@ if __name__ == "__main__":
             default=None, help='directory containing dynamic library')
     args = parser.parse_args(sys.argv[1:])
 
+allowed_failures = {"many references": True}
+
 cmark = CMark(prog=args.program, library_dir=args.library_dir)
 
 # list of pairs consisting of input and a regex that must match the output.
@@ -74,40 +76,57 @@ pathological = {
 
 whitespace_re = re.compile('/s+/')
 
-results = {'passed': 0, 'errored': 0, 'failed': 0}
+results = {'passed': [], 'errored': [], 'failed': [], 'ignored': []}
 
 def run_pathological_test(description, results):
     (inp, regex) = pathological[description]
     [rc, actual, err] = cmark.to_html(inp)
+    extra = ""
     if rc != 0:
         print(description, '[ERRORED (return code %d)]' %rc)
         print(err)
-        results['errored'] += 1
+        if allowed_failures[description]:
+            results['ignored'].append(description)
+        else:
+            results['errored'].append(description)
     elif regex.search(actual):
         print(description, '[PASSED]')
-        results['passed'] += 1
+        results['passed'].append(description)
     else:
         print(description, '[FAILED]')
         print(repr(actual))
-        results['failed'] += 1
-
+        if allowed_failures[description]:
+            results['ignored'].append(description)
+        else:
+            results['failed'].append(description)
 
 print("Testing pathological cases:")
 for description in pathological:
     p = multiprocessing.Process(target=run_pathological_test,
               args=(description, results,))
     p.start()
-    # wait 8 seconds or until it finishes
-    p.join(8)
+    # wait 4 seconds or until it finishes
+    p.join(4)
     # kill it if still active
     if p.is_alive():
         print(description, '[TIMEOUT]')
+        if allowed_failures[description]:
+            results['ignored'].append(description)
+        else:
+            results['errored'].append(description)
         p.terminate()
-        results['errored'] += 1
         p.join()
 
-print("%d passed, %d failed, %d errored" %
-          (results['passed'], results['failed'], results['errored']))
+passed  = len(results['passed'])
+failed  = len(results['failed'])
+errored = len(results['errored'])
+ignored = len(results['ignored'])
+
+print("%d passed, %d failed, %d errored" % (passed, failed, errored))
+if ignored > 0:
+    print("Ignoring these allowed failures:")
+    for x in results['ignored']:
+        print(x)
 if (results['failed'] == 0 and results['errored'] == 0):
     exit(0)
 else:
-- 
cgit v1.2.3