From 5aa17dc1368bdb2adda676c13681dbcd87950e5e Mon Sep 17 00:00:00 2001 From: cjihrig Date: Wed, 2 Mar 2016 12:12:47 -0500 Subject: [PATCH] tools: support testing known issues This commit adds a known_issues directory to the test directory for scripts that reproduce known bugs. Since these scripts are expected to fail, it also adds a --expect-fail flag to test.py which reports tests as successful when they fail. Refs: https://github.com/nodejs/testing/issues/18 PR-URL: https://github.com/nodejs/node/pull/5528 Reviewed-By: James M Snell Reviewed-By: Wyatt Preul Reviewed-By: Rich Trott --- Makefile | 3 +++ test/known_issues/testcfg.py | 6 ++++++ tools/test.py | 10 +++++++--- vcbuild.bat | 1 + 4 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 test/known_issues/testcfg.py diff --git a/Makefile b/Makefile index 221dd74e9d..28d17e9235 100644 --- a/Makefile +++ b/Makefile @@ -192,6 +192,9 @@ test-internet: all test-debugger: all $(PYTHON) tools/test.py debugger +test-known-issues: all + $(PYTHON) tools/test.py known_issues --expect-fail + test-npm: $(NODE_EXE) NODE=$(NODE) tools/test-npm.sh diff --git a/test/known_issues/testcfg.py b/test/known_issues/testcfg.py new file mode 100644 index 0000000000..af4f56574b --- /dev/null +++ b/test/known_issues/testcfg.py @@ -0,0 +1,6 @@ +import sys, os +sys.path.append(os.path.join(os.path.dirname(__file__), '..')) +import testpy + +def GetConfiguration(context, root): + return testpy.SimpleTestConfiguration(context, root, 'known_issues') diff --git a/tools/test.py b/tools/test.py index 9d227b10ae..1c10c624f7 100755 --- a/tools/test.py +++ b/tools/test.py @@ -430,7 +430,7 @@ class TestCase(object): self.thread_id = 0 def IsNegative(self): - return False + return self.context.expect_fail def CompareTime(self, other): return cmp(other.duration, self.duration) @@ -778,13 +778,14 @@ TIMEOUT_SCALEFACTOR = { class Context(object): - def __init__(self, workspace, buildspace, verbose, vm, args, timeout, - processor, suppress_dialogs, store_unexpected_output): + def __init__(self, workspace, buildspace, verbose, vm, args, expect_fail, + timeout, processor, suppress_dialogs, store_unexpected_output): self.workspace = workspace self.buildspace = buildspace self.verbose = verbose self.vm_root = vm self.node_args = args + self.expect_fail = expect_fail self.timeout = timeout self.processor = processor self.suppress_dialogs = suppress_dialogs @@ -1289,6 +1290,8 @@ def BuildOptions(): result.add_option("--special-command", default=None) result.add_option("--node-args", dest="node_args", help="Args to pass through to Node", default=[], action="append") + result.add_option("--expect-fail", dest="expect_fail", + help="Expect test cases to fail", default=False, action="store_true") result.add_option("--valgrind", help="Run tests through valgrind", default=False, action="store_true") result.add_option("--cat", help="Print the source of the tests", @@ -1480,6 +1483,7 @@ def Main(): VERBOSE, shell, options.node_args, + options.expect_fail, options.timeout, processor, options.suppress_dialogs, diff --git a/vcbuild.bat b/vcbuild.bat index 8a9f1c3069..39f7ac6e64 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -62,6 +62,7 @@ if /i "%1"=="test-gc" set test_args=%test_args% gc&set buildnodeweak=1&got if /i "%1"=="test-internet" set test_args=%test_args% internet&goto arg-ok if /i "%1"=="test-pummel" set test_args=%test_args% pummel&goto arg-ok if /i "%1"=="test-all" set test_args=%test_args% sequential parallel message gc internet pummel&set buildnodeweak=1&set jslint=1&goto arg-ok +if /i "%1"=="test-known-issues" set test_args=%test_args% known_issues --expect-fail&goto arg-ok if /i "%1"=="jslint" set jslint=1&goto arg-ok if /i "%1"=="msi" set msi=1&set licensertf=1&set download_arg="--download=all"&set i18n_arg=small-icu&goto arg-ok if /i "%1"=="build-release" set build_release=1&goto arg-ok