split the huge file into its natural modules (#460)

* split the huge file into its natural modules

* remove unused imports

Author: dothebart

Diff for: jenkins/helper/arangosh.py

@@ -0,0 +1,59 @@
+#!/bin/env python3
+""" launch a testing.js instance with given testsuite and arguments """
+
+from async_client import (
+    ArangoCLIprogressiveTimeoutExecutor,
+
+    make_logfile_params,
+    logfile_line_result,
+    delete_logfile_params,
+)
+
+class ArangoshExecutor(ArangoCLIprogressiveTimeoutExecutor):
+    """configuration"""
+
+    def __init__(self, site_config, slot_lock):
+        self.slot_lock = slot_lock
+        self.read_only = False
+        super().__init__(site_config, None)
+
+    def run_testing(self,
+                    testcase,
+                    testing_args,
+                    timeout,
+                    directory,
+                    logfile,
+                    identifier,
+                    verbose
+                    ):
+       # pylint: disable=R0913 disable=R0902
+        """ testing.js wrapper """
+        print('------')
+        print(testing_args)
+        args = [
+            '-c', str(self.cfg.cfgdir / 'arangosh.conf'),
+            "--log.foreground-tty", "true",
+            "--log.force-direct", "true",
+            '--log.level', 'warning',
+            "--log.level", "v8=debug",
+            '--server.endpoint', 'none',
+            '--javascript.allow-external-process-control', 'true',
+            '--javascript.execute', self.cfg.base_path / 'UnitTests' / 'unittest.js',
+            ]
+        run_cmd = args +[
+            '--',
+            testcase,
+            '--testOutput', directory ] + testing_args
+        params = make_logfile_params(verbose, logfile, self.cfg.trace)
+        ret = self.run_monitored(
+            self.cfg.bin_dir / "arangosh",
+            run_cmd,
+            params=params,
+            progressive_timeout=timeout,
+            deadline=self.cfg.deadline,
+            result_line_handler=logfile_line_result,
+            identifier=identifier
+        )
+        delete_logfile_params(params)
+        ret['error'] = params['error']
+        return ret

Diff for: jenkins/helper/dmesg.py

@@ -0,0 +1,58 @@
+#!/bin/env python3
+"""
+on supported systems launch a thread that will follow system messages
+to detect serious incidents like oom
+"""
+
+import psutil
+
+from async_client import (
+    ArangoCLIprogressiveTimeoutExecutor,
+
+    make_tail_params,
+    tail_line_result,
+    delete_tail_params
+)
+
+def dmesg_runner(dmesg):
+    """ thread to run dmesg in """
+    dmesg.launch()
+
+class DmesgWatcher(ArangoCLIprogressiveTimeoutExecutor):
+    """configuration"""
+
+    def __init__(self, site_config):
+        self.params = None
+        super().__init__(site_config, None)
+
+    def launch(self):
+       # pylint: disable=R0913 disable=R0902
+        """ dmesg wrapper """
+        print('------')
+        args = ['-wT']
+        verbose = False
+        self.params = make_tail_params(verbose,
+                                       'dmesg ',
+                                       self.cfg.test_report_dir / 'dmesg_log.txt')
+        ret = self.run_monitored(
+            "dmesg",
+            args,
+            params=self.params,
+            progressive_timeout=9999999,
+            deadline=self.cfg.deadline,
+            result_line_handler=tail_line_result,
+            identifier='0_dmesg'
+        )
+        #delete_logfile_params(params)
+        ret = {}
+        ret['error'] = self.params['error']
+        delete_tail_params(self.params)
+        return ret
+
+    def end_run(self):
+        """ terminate dmesg again """
+        print(f'killing dmesg {self.pid}')
+        try:
+            psutil.Process(self.pid).kill()
+        except psutil.NoSuchProcess:
+            print('dmesg already gone?')

Diff for: jenkins/helper/dump_handler.py

@@ -0,0 +1,16 @@
+#!/bin/env python3
+""" test the syntax of the test-definition.txt and output the results """
+def generate_dump_output(_, tests):
+    """ interactive version output to inspect comprehension """
+    def output(line):
+        """ output one line """
+        print(line)
+
+    for test in tests:
+        params = " ".join(f"{key}={value}" for key, value in test['params'].items())
+        output(f"{test['name']}")
+        output(f"\tpriority: {test['priority']}")
+        output(f"\tparallelity: {test['parallelity']}")
+        output(f"\tflags: {' '.join(test['flags'])}")
+        output(f"\tparams: {params}")
+        output(f"\targs: {' '.join(test['args'])}")

Diff for: jenkins/helper/launch_handler.py

@@ -0,0 +1,58 @@
+#!/bin/env python3
+""" launch tests from the config, write reports, etc. Control logic """
+from pathlib import Path
+import sys
+import time
+from threading  import Thread
+from traceback import print_exc
+
+from dmesg import DmesgWatcher, dmesg_runner
+from site_config import SiteConfig, IS_LINUX
+from testing_runner import TestingRunner
+
+# pylint: disable=broad-except
+def launch(args, tests):
+    """ Manage test execution on our own """
+    runner = None
+    try:
+        runner = TestingRunner(SiteConfig(Path(args.definitions).resolve()))
+    except Exception as exc:
+        print(exc)
+        raise exc
+    dmesg = DmesgWatcher(runner.cfg)
+    if IS_LINUX:
+        dmesg_thread = Thread(target=dmesg_runner, args=[dmesg])
+        dmesg_thread.start()
+        time.sleep(3)
+    for test in tests:
+        runner.register_test_func(args.cluster, test)
+    runner.sort_by_priority()
+    print(runner.scenarios)
+    create_report = True
+    if args.no_report:
+        print("won't generate report as you demanded!")
+        create_report = False
+    try:
+        runner.testing_runner()
+        runner.overload_report_fh.close()
+        runner.generate_report_txt()
+        if create_report:
+            runner.generate_test_report()
+            if not runner.cfg.is_asan:
+                runner.generate_crash_report()
+    except Exception as exc:
+        runner.success = False
+        sys.stderr.flush()
+        sys.stdout.flush()
+        print(exc, file=sys.stderr)
+        print_exc()
+    finally:
+        sys.stderr.flush()
+        sys.stdout.flush()
+        runner.create_log_file()
+        runner.create_testruns_file()
+        if IS_LINUX:
+            dmesg.end_run()
+            print('joining dmesg threads')
+            dmesg_thread.join()
+        runner.print_and_exit_closing_stance()

Diff for: jenkins/helper/monkeypatch_psutil.py

@@ -7,7 +7,7 @@
 import signal
 import subprocess
 import time
-import sys
+# import sys
 
 winver = platform.win32_ver()
 if winver[0]:
@@ -89,7 +89,7 @@ def terminate(self):
                 #         sys.exit(1)
                 #     # pylint: disable=unnecessary-pass
                 #     pass
-                # 
+                #
                 # original_sigint_handler = signal.getsignal(signal.SIGINT)
                 # signal.signal(signal.SIGINT, sigint_boomerang_handler)
                 # # only here on the wintendo: