Fix coverage runs (#479)

* only run coverage if tests are successfull

* we should allways have the reports

* Update helper.fish

* silence

* multi threadded unlimited core tar

* configure individual temp-subdirectories per run

* configure individual temp-subdirectories per run

* revert

* don't delete files that are still needed

* debug

* try to clean temp

* reduce sysetm capabilities in coverage tests

* no extensions...

* lint

* lint

* use proper environment variable already here

* lint

* lint

* blank

* launch cluster and single from one python run.

* lint

* suffix the runs

* fix prefix handling for both case

* remove debug

* strive to include missing directories

* attempt to set exit code

* improve message composition

* improve messaging for coredump zipping

* lint

* fix oskar python report switching integration

* re-enable code

* fix oskar python report switching integration

* fix generation

* work on exit code handling

* work on exit code handling

* Update jenkins/helper/test_launch_controller.py

Co-authored-by: Tobias Gödderz <[email protected]>

* Update jenkins/runCoverageRocksDB.fish

Co-authored-by: Tobias Gödderz <[email protected]>

* disable running for now

* fix path

* reenable

* preserve inbetween file

* link last missing files

* individualise slots, give gcov a little more

* switch wording

* switch wording

* set the suite to run early on.

* fix filtering, add missing enterprise filter

* cluster args is used 2-fold: to indicate cluster only tests, and later on to control that we are in the custer case

* fix syntax

* add negative way so its better switcheable

* ps

* quotes maybe?

* add negative way so its better switcheable

* add echo

* fix cluster parameter adding to the testcasse

---------

Co-authored-by: Vadim <[email protected]>
Co-authored-by: Tobias Gödderz <[email protected]>

Author: 3 people

Diff for: helper.fish

@@ -94,6 +94,7 @@ end
 
 function single ; set -gx TESTSUITE single ; end
 function cluster ; set -gx TESTSUITE cluster ; end
+function single_cluster ; set -gx TESTSUITE single_cluster ; end
 function resilience ; set -gx TESTSUITE resilience ; end
 function catchtest ; set -gx TESTSUITE gtest ; end
 function gtest ; set -gx TESTSUITE gtest ; end
@@ -1519,7 +1520,7 @@ function showConfig
   printf $fmt3 'SkipGrey'       $SKIPGREY      '(skipGrey/includeGrey)'
   printf $fmt3 'OnlyGrey'       $ONLYGREY      '(onlyGreyOn/onlyGreyOff)'
   printf $fmt3 'Storage engine' $STORAGEENGINE '(mmfiles/rocksdb)'
-  printf $fmt3 'Test suite'     $TESTSUITE     '(single/cluster/resilience/gtest)'
+  printf $fmt3 'Test suite'     $TESTSUITE     '(single/cluster/single_cluster/resilience/gtest)'
   printf $fmt2 'Log Levels'     (echo $LOG_LEVELS)
   echo
   echo 'Package Configuration'

Diff for: jenkins/helper/asciiprint.py

@@ -58,7 +58,7 @@ def ascii_convert(the_bytes: bytes):
 
 def ascii_convert_str(the_str: str):
     """convert string to only be ascii without control sequences"""
-    return ANSI_ESCAPE.sub(rb"", the_str)
+    return ANSI_ESCAPE.sub("", the_str)
 
 
 def ascii_print(string):

Diff for: jenkins/helper/launch_handler.py

@@ -25,7 +25,7 @@ def launch(args, tests):
         dmesg_thread.start()
         time.sleep(3)
     for test in tests:
-        runner.register_test_func(args.cluster, test)
+        runner.register_test_func(test)
     runner.sort_by_priority()
     print(runner.scenarios)
     create_report = True

Diff for: jenkins/helper/site_config.py

@@ -4,6 +4,7 @@
 import os
 from pathlib import Path
 import platform
+import re
 import shutil
 import signal
 import sys
@@ -55,8 +56,9 @@ def get_workspace():
     #        return workdir
     return Path.cwd() / 'work'
 
+ansi_escape = re.compile(r'\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])')
 for env in os.environ:
-    print(f'{env}={os.environ[env]}')
+    print(f"{env}={ansi_escape.sub('', os.environ[env])}")
 TEMP = Path("/tmp/")
 if 'TMP' in os.environ:
     TEMP = Path(os.environ['TMP'])
@@ -135,15 +137,23 @@ def __init__(self, definition_file):
         self.slots_to_parallelity_factor = self.max_load / self.available_slots
         self.rapid_fire = round(self.available_slots / 10)
         self.is_asan = 'SAN' in os.environ and os.environ['SAN'] == 'On'
-        if self.is_asan:
-            print('SAN enabled, reducing possible system capacity')
+        self.is_aulsan = self.is_asan and os.environ['SAN_MODE'] == 'AULSan'
+        self.is_gcov = 'COVERAGE' in os.environ and os.environ['COVERAGE'] == 'On'
+        san_gcov_msg = ""
+        if self.is_asan or self.is_gcov:
+            san_gcov_msg = ' - SAN '
+            slot_divisor = 4
+            if self.is_aulsan:
+                san_gcov_msg = ' - AUL-SAN '
+            elif self.is_gcov:
+                san_gcov_msg = ' - GCOV'
+                slot_divisor = 3
+            san_gcov_msg += ' enabled, reducing possible system capacity\n'
             self.rapid_fire = 1
-            self.available_slots /= 4
+            self.available_slots /= slot_divisor
             #self.timeout *= 1.5
             self.loop_sleep *= 2
             self.max_load /= 2
-            if os.environ['SAN_MODE'] == 'AULSan':
-                print('Aulsan must reduce even more!')
         self.deadline = datetime.now() + timedelta(seconds=self.timeout)
         self.hard_deadline = datetime.now() + timedelta(seconds=self.timeout + 660)
         if definition_file.is_file():
@@ -175,7 +185,7 @@ def __init__(self, definition_file):
  - Starting {str(datetime.now())} soft deadline will be: {str(self.deadline)} hard deadline will be: {str(self.hard_deadline)}
  - {self.core_dozend} / {self.loop_sleep} machine size / loop frequency
  - {socket_count} number of currently active tcp sockets
- """)
+{san_gcov_msg}""")
         self.cfgdir = base_source_dir / 'etc' / 'relative'
         self.bin_dir = bin_dir
         self.base_path = base_source_dir

Diff for: jenkins/helper/test_launch_controller.py

@@ -1,6 +1,7 @@
 #!/bin/env python3
 """ read test definition, and generate the output for the specified target """
 import argparse
+import copy
 import sys
 from traceback import print_exc
 
@@ -18,39 +19,64 @@
 
 def filter_tests(args, tests):
     """ filter testcase by operations target Single/Cluster/full """
+    for one in tests:
+        one['prefix'] = ""
     if args.all:
         return tests
 
-    filters = []
-    if args.cluster:
-        filters.append(lambda test: "single" not in test["flags"])
-    else:
-        filters.append(lambda test: "cluster" not in test["flags"])
-
-    if args.full:
-        filters.append(lambda test: "!full" not in test["flags"])
-    else:
-        filters.append(lambda test: "full" not in test["flags"])
-
-    if args.gtest:
-        filters.append(lambda test: "gtest" ==  test["name"])
-
-    if IS_WINDOWS:
-        filters.append(lambda test: "!windows" not in test["flags"])
-
-    if IS_MAC:
-        filters.append(lambda test: "!mac" not in test["flags"])
-
-    if IS_ARM:
-        filters.append(lambda test: "!arm" not in test["flags"])
-
-    if args.no_report:
-        print("Disabling report generation")
-        args.create_report = False
+    def list_generator(cluster):
+        # pylint: disable=too-many-branches
+        filters = []
+        if cluster:
+            filters.append(lambda test: "single" not in test["flags"])
+        else:
+            filters.append(lambda test: "cluster" not in test["flags"])
 
-    for one_filter in filters:
-        tests = filter(one_filter, tests)
-    return list(tests)
+        if args.full:
+            filters.append(lambda test: "!full" not in test["flags"])
+        else:
+            filters.append(lambda test: "full" not in test["flags"])
+
+        if args.gtest:
+            filters.append(lambda test: "gtest" ==  test["name"])
+
+        if not args.enterprise:
+            filters.append(lambda test: "enterprise" not in test["flags"])
+
+        if IS_WINDOWS:
+            filters.append(lambda test: "!windows" not in test["flags"])
+
+        if IS_MAC:
+            filters.append(lambda test: "!mac" not in test["flags"])
+
+        if IS_ARM:
+            filters.append(lambda test: "!arm" not in test["flags"])
+
+        if args.no_report:
+            print("Disabling report generation")
+            args.create_report = False
+
+        filtered = copy.deepcopy(tests)
+        for one_filter in filters:
+            filtered = filter(one_filter, filtered)
+        remaining_tests = list(filtered)
+        if cluster:
+            # after we filtered for cluster only tests, we now need to make sure
+            # that tests are actually launched as cluster tests:
+            for one in remaining_tests:
+                if not 'cluster' in one['flags']:
+                    one['flags'].append('cluster')
+        return remaining_tests
+
+    if args.single_cluster:
+        res_sg = list_generator(False)
+        for one in res_sg:
+            one['prefix'] = "sg_"
+        res_cl = list_generator(True)
+        for one in res_cl:
+            one['prefix'] = "cl_"
+        return res_sg + res_cl
+    return list_generator(args.cluster)
 
 formats = {
     "dump": generate_dump_output,
@@ -73,6 +99,7 @@ def filter_tests(args, tests):
 }
 
 known_parameter = {
+    "prefix": 'internal',
     "buckets": "number of buckets to use for this test",
     "suffix": "suffix that is appended to the tests folder name",
     "priority": "priority that controls execution order. Testsuites with lower priority are executed later",
@@ -104,6 +131,9 @@ def parse_arguments():
     parser.add_argument("--validate-only", help="validates the test definition file", action="store_true")
     parser.add_argument("--help-flags", help="prints information about available flags and exits", action="store_true")
     parser.add_argument("--cluster", help="output only cluster tests instead of single server", action="store_true")
+    parser.add_argument("--single_cluster", help="process cluster cluster and single tests", action="store_true")
+    parser.add_argument("--enterprise", help="add enterprise tests", action="store_true")
+    parser.add_argument("--no-enterprise", help="add enterprise tests", action="store_true")
     parser.add_argument("--full", help="output full test set", action="store_true")
     parser.add_argument("--gtest", help="only run gtest", action="store_true")
     parser.add_argument("--all", help="output all test, ignore other filters", action="store_true")

Diff for: jenkins/helper/testing_runner.py

@@ -10,6 +10,8 @@
 import sys
 import time
 from threading  import Thread, Lock
+from multiprocessing import Process
+import zipfile
 
 import psutil
 
@@ -41,6 +43,17 @@
 except ModuleNotFoundError:
     pass
 
+def zipp_this(filenames, target_dir):
+    """ worker function to zip one file a time in a subprocess """
+    # pylint: disable=consider-using-with
+    for corefile in filenames:
+        try:
+            print(f'zipping {corefile}')
+            zipfile.ZipFile(str(target_dir / (corefile.name + '.xz')),
+                            mode='w', compression=zipfile.ZIP_LZMA).write(str(corefile))
+        except Exception as exc:
+            print(f'skipping {corefile} since {exc}')
+
 def testing_runner(testing_instance, this, arangosh):
     """ operate one makedata instance """
     try:
@@ -343,16 +356,28 @@ def cleanup_unneeded_binary_files(self):
         """ delete all files not needed for the crashreport binaries """
         shutil.rmtree(str(self.cfg.bin_dir / 'tzdata'))
         needed = [
-            'arangod',
+            'tzdata',
+            'icudtl',
+            'fuertetest',
+            'arangovpack',
+            'arangobackup',
             'arangosh',
-            'arangodump',
-            'arangorestore',
+            'arangoexport',
+            'arangoinspect',
             'arangoimport',
-            'arangobackup',
-            'arangodbtests']
+            'arangoimp',
+            'arango-secure-installation',
+            'foxx-manager',
+            'arangorestore',
+            'arangobench',
+            'snowball',
+            'arangodbtests',
+            'arangod',
+            'arango-init-database',
+            'arangodump']
         for one_file in self.cfg.bin_dir.iterdir():
             if (one_file.suffix == '.lib' or
-                (one_file.stem not in needed) ):
+                (one_file.stem not in needed)):
                 print(f'Deleting {str(one_file)}')
                 one_file.unlink(missing_ok=True)
 
@@ -389,56 +414,45 @@ def generate_crash_report(self):
             print(f'Coredumps are not collected: {str(len(core_files_list))} coredumps found; coredumps max limit to collect is {str(core_max_count)}!')
             return
 
-        for one_file in core_files_list:
-            if one_file.is_file():
-                size = (one_file.stat().st_size / (1024 * 1024))
-                if 0 < MAX_COREFILE_SIZE_MB and MAX_COREFILE_SIZE_MB < size:
-                    print(f'deleting coredump {str(one_file)} its too big: {str(size)}')
-                    one_file.unlink(missing_ok=True)
-                    core_files_list.remove(one_file)
-            else:
-                print(f"removing file from list {str(one_file)}")
-                core_files_list.remove(one_file)
-
-        if len(core_files_list) > core_max_count and core_max_count > 0:
-            count = 0
-            for one_crash_file in core_files_list:
-                count += 1
-                if count > core_max_count:
-                    print(f'{core_max_count} reached. will not archive {one_crash_file}')
-                    one_crash_file.unlink(missing_ok=True)
-                    core_files_list.remove(one_crash_file)
-
-        if len(core_files_list) == 0:
-            return
-        self.crashed = True
-
         core_zip_dir = get_workspace() / 'coredumps'
         core_zip_dir.mkdir(parents=True, exist_ok=True)
+        zip_slots = psutil.cpu_count(logical=False)
+        count = 0
+        zip_slot_array = []
+        for _ in range(zip_slots):
+            zip_slot_array.append([])
         for one_file in core_files_list:
             if one_file.exists():
-                try:
-                    shutil.move(str(one_file.resolve()), str(core_zip_dir.resolve()))
-                except shutil.Error as ex:
-                    msg = f"generate_crash_report: failed to move file while while gathering coredumps: {ex}"
-                    self.append_report_txt('\n' + msg + '\n')
-                    print(msg)
-                except PermissionError as ex:
-                    print(f"won't move {str(one_file)} - not an owner! {str(ex)}")
-                    self.append_report_txt(f"won't move {str(one_file)} - not an owner! {str(ex)}")
+                zip_slot_array[count % zip_slots].append(one_file)
+                count += 1
+        zippers = []
+        print(f"coredump launching zipper sub processes {zip_slot_array}")
+        for zip_slot in zip_slot_array:
+            if len(zip_slot) > 0:
+                proc = Process(target=zipp_this, args=(zip_slot, core_zip_dir))
+                proc.start()
+                zippers.append(proc)
+        for zipper in zippers:
+            zipper.join()
+        print("compressing files done")
+
+        for one_file in core_files_list:
+            if one_file.is_file():
+                one_file.unlink(missing_ok=True)
 
         crash_report_file = get_workspace() / datetime.now(tz=None).strftime(f"crashreport-{self.cfg.datetime_format}")
         print(f"creating crashreport: {str(crash_report_file)} with {str(core_files_list)}")
         sys.stdout.flush()
         try:
             shutil.make_archive(str(crash_report_file),
-                                ZIPFORMAT,
+                                'tar',
                                 (core_zip_dir / '..').resolve(),
                                 core_zip_dir.name,
                                 True)
         except Exception as ex:
             print("Failed to create binaries zip: " + str(ex))
             self.append_report_txt("Failed to create binaries zip: " + str(ex))
+
         self.cleanup_unneeded_binary_files()
         binary_report_file = get_workspace() / datetime.now(tz=None).strftime(f"binaries-{self.cfg.datetime_format}")
         print("creating crashreport binary support zip: " + str(binary_report_file))
@@ -536,29 +550,23 @@ def create_testruns_file(self):
 </table>
 ''')
 
-    def register_test_func(self, cluster, test):
+    def register_test_func(self, test):
         """ print one test function """
         args = test["args"]
         params = test["params"]
         suffix = params.get("suffix", "")
-        name = test["name"]
+        name = test['prefix'] + test["name"]
         if suffix:
             name += f"_{suffix}"
 
         if test["parallelity"] :
             parallelity = test["parallelity"]
-        if 'single' in test['flags'] and cluster:
-            return
-        if 'cluster' in test['flags'] and not cluster:
-            return
-        if cluster:
+        if 'cluster' in test['flags']:
             self.cluster = True
             if parallelity == 1:
                 parallelity = 4
             args += ['--cluster', 'true',
                      '--dumpAgencyOnError', 'true']
-        if "enterprise" in test["flags"]:
-            return
         if "ldap" in test["flags"] and not 'LDAPHOST' in os.environ:
             return