summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJohn MacFarlane <jgm@berkeley.edu>2017-11-05 10:53:23 -0800
committerJohn MacFarlane <jgm@berkeley.edu>2017-11-05 10:53:23 -0800
commit1998db887e385e786c0eac4b1973bcddc6406bdc (patch)
treec2dd5d9d5326663d8b6ee72ce3584334ff870245
parent4db1f75834602aa1215b58cbad737593c99ccdc2 (diff)
Add allowed failures to pathological_tests.py.
This allows us to include tests that we don't yet know how to pass.
-rw-r--r--test/pathological_tests.py39
1 files changed, 29 insertions, 10 deletions
diff --git a/test/pathological_tests.py b/test/pathological_tests.py
index 40552de..b86f4c0 100644
--- a/test/pathological_tests.py
+++ b/test/pathological_tests.py
@@ -17,6 +17,8 @@ if __name__ == "__main__":
default=None, help='directory containing dynamic library')
args = parser.parse_args(sys.argv[1:])
+allowed_failures = {"many references": True}
+
cmark = CMark(prog=args.program, library_dir=args.library_dir)
# list of pairs consisting of input and a regex that must match the output.
@@ -74,40 +76,57 @@ pathological = {
whitespace_re = re.compile('/s+/')
-results = {'passed': 0, 'errored': 0, 'failed': 0}
+results = {'passed': [], 'errored': [], 'failed': [], 'ignored': []}
def run_pathological_test(description, results):
(inp, regex) = pathological[description]
[rc, actual, err] = cmark.to_html(inp)
+ extra = ""
if rc != 0:
print(description, '[ERRORED (return code %d)]' %rc)
print(err)
- results['errored'] += 1
+ if allowed_failures[description]:
+ results['ignored'].append(description)
+ else:
+ results['errored'].append(description)
elif regex.search(actual):
print(description, '[PASSED]')
- results['passed'] += 1
+ results['passed'].append(description)
else:
print(description, '[FAILED]')
print(repr(actual))
- results['failed'] += 1
-
+ if allowed_failures[description]:
+ results['ignored'].append(description)
+ else:
+ results['failed'].append(description)
print("Testing pathological cases:")
for description in pathological:
p = multiprocessing.Process(target=run_pathological_test,
args=(description, results,))
p.start()
- # wait 8 seconds or until it finishes
- p.join(8)
+ # wait 4 seconds or until it finishes
+ p.join(4)
# kill it if still active
if p.is_alive():
print(description, '[TIMEOUT]')
+ if allowed_failures[description]:
+ results['ignored'].append(description)
+ else:
+ results['errored'].append(description)
p.terminate()
- results['errored'] += 1
p.join()
-print("%d passed, %d failed, %d errored" %
- (results['passed'], results['failed'], results['errored']))
+passed = len(results['passed'])
+failed = len(results['failed'])
+errored = len(results['errored'])
+ignored = len(results['ignored'])
+
+print("%d passed, %d failed, %d errored" % (passed, failed, errored))
+if ignored > 0:
+ print("Ignoring these allowed failures:")
+ for x in results['ignored']:
+ print(x)
if (results['failed'] == 0 and results['errored'] == 0):
exit(0)
else: