From be4a8fc3e384317b893fef354bd209afe61f5c2a Mon Sep 17 00:00:00 2001 From: Alexis Campailla Date: Tue, 9 Jun 2015 16:11:28 +0200 Subject: [PATCH 1/6] test: support writing test output to file This is a minimal effort to support test output written both to stdout and file in order to get our buildbots understanding test output. Cherry picked from https://github.com/jbergstroem/io.js/commit/31940738e2d256e91c1c694c5219b8f840c629ff Original commit message follows: PR-URL: https://github.com/iojs/io.js/pull/934 Reviewed-By: Chris Dickinson Reviewed-By: Ben Noordhuis Conflicts: tools/test.py --- tools/test.py | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tools/test.py b/tools/test.py index 579d444f6c5..b85da6ae6c6 100755 --- a/tools/test.py +++ b/tools/test.py @@ -29,6 +29,7 @@ import imp +import logging import optparse import os import platform @@ -45,6 +46,8 @@ from datetime import datetime from Queue import Queue, Empty +logger = logging.getLogger('testrunner') + VERBOSE = False @@ -225,7 +228,7 @@ def HasRun(self, output): class TapProgressIndicator(SimpleProgressIndicator): def Starting(self): - print '1..%i' % len(self.cases) + logger.info('1..%i' % len(self.cases)) self._done = 0 def AboutToRun(self, case): @@ -238,16 +241,16 @@ def HasRun(self, output): status_line = 'not ok %i - %s' % (self._done, command) if FLAKY in output.test.outcomes and self.flaky_tests_mode == "dontcare": status_line = status_line + " # TODO : Fix flaky test" - print status_line + logger.info(status_line) for l in output.output.stderr.splitlines(): - print '#' + l + logger.info('#' + l) for l in output.output.stdout.splitlines(): - print '#' + l + logger.info('#' + l) else: status_line = 'ok %i - %s' % (self._done, command) if FLAKY in output.test.outcomes: status_line = status_line + " # TODO : Fix flaky test" - print status_line + logger.info(status_line) duration = output.test.duration @@ -255,9 +258,9 @@ def HasRun(self, output): total_seconds = (duration.microseconds + (duration.seconds + duration.days * 24 * 3600) * 10**6) / 10**6 - print ' ---' - print ' duration_ms: %d.%d' % (total_seconds, duration.microseconds / 1000) - print ' ...' + logger.info(' ---') + logger.info(' duration_ms: %d.%d' % (total_seconds, duration.microseconds / 1000)) + logger.info(' ...') def Done(self): pass @@ -1192,6 +1195,8 @@ def BuildOptions(): default='release') result.add_option("-v", "--verbose", help="Verbose output", default=False, action="store_true") + result.add_option('--logfile', dest='logfile', + help='write test output to file. NOTE: this only applies the tap progress indicator') result.add_option("-S", dest="scons_flags", help="Flag to pass through to scons", default=[], action="append") result.add_option("-p", "--progress", @@ -1368,6 +1373,13 @@ def Main(): parser.print_help() return 1 + ch = logging.StreamHandler(sys.stdout) + logger.addHandler(ch) + logger.setLevel('INFO') + if options.logfile: + fh = logging.FileHandler(options.logfile) + logger.addHandler(fh) + workspace = abspath(join(dirname(sys.argv[0]), '..')) suites = GetSuites(join(workspace, 'test')) repositories = [TestRepository(join(workspace, 'test', name)) for name in suites] From 162fca4cd3b12fb26cd3d5f6308a3add490ae804 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johan=20Bergstr=C3=B6m?= Date: Sat, 30 May 2015 13:46:12 +1000 Subject: [PATCH 2/6] tools: pass constant to logger instead of string On a few of our installations (namely CentOS), passing 'INFO' resulted in a silent loglevel. Use a logging constant instead. Cherry-picked from https://github.com/nodejs/io.js/commit/8606793999979829df002c2ab69235a05605fc4d Original commit metadata follows: Fixes: https://github.com/nodejs/build/issues/104 PR-URL: https://github.com/nodejs/io.js/pull/1842 Reviewed-By: Rod Vagg --- tools/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/test.py b/tools/test.py index b85da6ae6c6..e14ade37128 100755 --- a/tools/test.py +++ b/tools/test.py @@ -1375,7 +1375,7 @@ def Main(): ch = logging.StreamHandler(sys.stdout) logger.addHandler(ch) - logger.setLevel('INFO') + logger.setLevel(logging.INFO) if options.logfile: fh = logging.FileHandler(options.logfile) logger.addHandler(fh) From 20046544978ef1135c10f2098eb4c927ae6a2438 Mon Sep 17 00:00:00 2001 From: Alexis Campailla Date: Wed, 8 Jul 2015 16:39:37 +0200 Subject: [PATCH 3/6] build: support Jenkins via test-ci --- Makefile | 3 +++ vcbuild.bat | 13 ++++++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index f9c83225dd1..be37a36a940 100644 --- a/Makefile +++ b/Makefile @@ -127,6 +127,9 @@ test-all-http1: test-build test-all-valgrind: test-build $(PYTHON) tools/test.py --mode=debug,release --valgrind +test-ci: + $(PYTHON) tools/test.py -p tap --logfile test.tap --mode=release --arch=$(DESTCPU) simple message internet + test-release: test-build $(PYTHON) tools/test.py --mode=release diff --git a/vcbuild.bat b/vcbuild.bat index 96ffeed02f7..0e46a27d204 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -61,7 +61,8 @@ if /i "%1"=="test-simple" set test=test-simple&goto arg-ok if /i "%1"=="test-message" set test=test-message&goto arg-ok if /i "%1"=="test-gc" set test=test-gc&set buildnodeweak=1&goto arg-ok if /i "%1"=="test-all" set test=test-all&set buildnodeweak=1&goto arg-ok -if /i "%1"=="test" set test=test&goto arg-ok +if /i "%1"=="test-ci" set test=test-ci&set nosnapshot=1&goto arg-ok +if /i "%1"=="test" set test=test&set jslint=1&goto arg-ok @rem Include small-icu support with MSI installer if /i "%1"=="msi" set msi=1&set licensertf=1&set download_arg="--download=all"&set i18n_arg=small-icu&goto arg-ok if /i "%1"=="upload" set upload=1&goto arg-ok @@ -81,7 +82,6 @@ goto next-arg :args-done if defined upload goto upload -if defined jslint goto jslint if defined build_release ( set nosnapshot=1 @@ -197,12 +197,15 @@ if errorlevel 1 echo Failed to sign msi&goto exit :run @rem Run tests if requested. -if "%test%"=="" goto exit +if "%test%"=="" goto jslint if "%config%"=="Debug" set test_args=--mode=debug if "%config%"=="Release" set test_args=--mode=release +set test_args=%test_args% --arch=%target_arch% + if "%test%"=="test" set test_args=%test_args% simple message +if "%test%"=="test-ci" set test_args=%test_args% -p tap --logfile test.tap simple message internet if "%test%"=="test-internet" set test_args=%test_args% internet if "%test%"=="test-pummel" set test_args=%test_args% pummel if "%test%"=="test-simple" set test_args=%test_args% simple @@ -224,8 +227,7 @@ goto exit :run-tests echo running 'python tools/test.py %test_args%' python tools/test.py %test_args% -if "%test%"=="test" goto jslint -goto exit +goto jslint :create-msvs-files-failed echo Failed to create vc project files. @@ -243,6 +245,7 @@ scp Release\node.pdb node@nodejs.org:~/web/nodejs.org/dist/v%NODE_VERSION%/node. goto exit :jslint +if not defined jslint goto exit echo running jslint set PYTHONPATH=tools/closure_linter/ python tools/closure_linter/closure_linter/gjslint.py --unix_mode --strict --nojsdoc -r lib/ -r src/ --exclude_files lib/punycode.js From 9d66f02aa0d30ca278f98850205c5be3f7261e47 Mon Sep 17 00:00:00 2001 From: Alexis Campailla Date: Wed, 8 Jul 2015 16:44:06 +0200 Subject: [PATCH 4/6] build: support flaky tests in test-ci Adding support for specifying flaky test mode to the test runner: - via an environment variable FLAKY_TESTS for Makefile - via an argument ignore-flaky for vcbuild.bat Conflicts: Makefile --- Makefile | 3 ++- vcbuild.bat | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index be37a36a940..246663648e1 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,7 @@ NINJA ?= ninja DESTDIR ?= SIGN ?= PREFIX ?= /usr/local +FLAKY_TESTS ?= run NODE ?= ./node @@ -128,7 +129,7 @@ test-all-valgrind: test-build $(PYTHON) tools/test.py --mode=debug,release --valgrind test-ci: - $(PYTHON) tools/test.py -p tap --logfile test.tap --mode=release --arch=$(DESTCPU) simple message internet + $(PYTHON) tools/test.py -p tap --logfile test.tap --mode=release --arch=$(DESTCPU) --flaky-tests=$(FLAKY_TESTS) simple message internet test-release: test-build $(PYTHON) tools/test.py --mode=release diff --git a/vcbuild.bat b/vcbuild.bat index 0e46a27d204..ab4dd003f5f 100644 --- a/vcbuild.bat +++ b/vcbuild.bat @@ -38,6 +38,7 @@ set noperfctr_msi_arg= set i18n_arg= set download_arg= set build_release= +set flaky_tests_arg= :next-arg if "%1"=="" goto args-done @@ -72,6 +73,7 @@ if /i "%1"=="full-icu" set i18n_arg=%1&goto arg-ok if /i "%1"=="intl-none" set i18n_arg=%1&goto arg-ok if /i "%1"=="download-all" set download_arg="--download=all"&goto arg-ok if /i "%1"=="build-release" set build_release=1&goto arg-ok +if /i "%1"=="ignore-flaky" set flaky_tests_arg=--flaky-tests=dontcare&goto arg-ok echo Warning: ignoring invalid command line option `%1`. @@ -205,7 +207,7 @@ if "%config%"=="Release" set test_args=--mode=release set test_args=%test_args% --arch=%target_arch% if "%test%"=="test" set test_args=%test_args% simple message -if "%test%"=="test-ci" set test_args=%test_args% -p tap --logfile test.tap simple message internet +if "%test%"=="test-ci" set test_args=%test_args% -p tap --logfile test.tap %flaky_tests_arg% simple message internet if "%test%"=="test-internet" set test_args=%test_args% internet if "%test%"=="test-pummel" set test_args=%test_args% pummel if "%test%"=="test-simple" set test_args=%test_args% simple From 9729d6244a3121e7af67b0a741b635d29d71fab9 Mon Sep 17 00:00:00 2001 From: Alexis Campailla Date: Tue, 7 Jul 2015 22:48:26 +0200 Subject: [PATCH 5/6] test: runner should return 0 on flaky tests Make the test runner return a 0 exit code when only flaky tests fail and --flaky-tests=dontcare is specified. --- tools/test.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tools/test.py b/tools/test.py index e14ade37128..aca101e0637 100755 --- a/tools/test.py +++ b/tools/test.py @@ -68,7 +68,9 @@ def __init__(self, cases, flaky_tests_mode): self.remaining = len(cases) self.total = len(cases) self.failed = [ ] + self.flaky_failed = [ ] self.crashed = 0 + self.flaky_crashed = 0 self.terminate = False self.lock = threading.Lock() @@ -129,9 +131,14 @@ def RunSingle(self): return self.lock.acquire() if output.UnexpectedOutput(): - self.failed.append(output) - if output.HasCrashed(): - self.crashed += 1 + if FLAKY in output.test.outcomes and self.flaky_tests_mode == "dontcare": + self.flaky_failed.append(output) + if output.HasCrashed(): + self.flaky_crashed += 1 + else: + self.failed.append(output) + if output.HasCrashed(): + self.crashed += 1 else: self.succeeded += 1 self.remaining -= 1 From c9ef4b3ae2a3d0a670a10621cdc79173bf54a4f3 Mon Sep 17 00:00:00 2001 From: Alexis Campailla Date: Wed, 8 Jul 2015 20:06:44 +0200 Subject: [PATCH 6/6] build: run-ci makefile rule Adding a single rule to be called from Jenkins. --- Makefile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 246663648e1..b3685e1479d 100644 --- a/Makefile +++ b/Makefile @@ -244,6 +244,11 @@ docopen: out/doc/api/all.html docclean: -rm -rf out/doc +run-ci: + $(PYTHON) ./configure --without-snapshot $(CONFIG_FLAGS) + $(MAKE) + $(MAKE) test-ci + RAWVER=$(shell $(PYTHON) tools/getnodeversion.py) VERSION=v$(RAWVER) NODE_DOC_VERSION=$(VERSION) @@ -443,4 +448,9 @@ cpplint: lint: jslint cpplint -.PHONY: lint cpplint jslint bench clean docopen docclean doc dist distclean check uninstall install install-includes install-bin all staticlib dynamiclib test test-all test-addons build-addons website-upload pkg blog blogclean tar binary release-only bench-http-simple bench-idle bench-all bench bench-misc bench-array bench-buffer bench-net bench-http bench-fs bench-tls +.PHONY: lint cpplint jslint bench clean docopen docclean doc dist distclean \ + check uninstall install install-includes install-bin all staticlib \ + dynamiclib test test-all test-addons build-addons website-upload pkg \ + blog blogclean tar binary release-only bench-http-simple bench-idle \ + bench-all bench bench-misc bench-array bench-buffer bench-net \ + bench-http bench-fs bench-tls run-ci