QA-585: harvest resource logs (#614)

* switch location

* have commandline to identify scripts

* log ppid

* write a second jsonl file decyphering logfile -> testing.js PID

* write a second jsonl file decyphering logfile -> testing.js PID

* write a second jsonl file decyphering logfile -> testing.js PID

* write system information as well

* write system information as well

* write a second jsonl file decyphering logfile -> testing.js PID

* fix file mode

* resort, move into function, disable cleanup of lower layers

* disable delta measuring, we do that while evaluating the results

Author: dothebart

Diff for: jenkins/helper/aggregate_coverage.py

@@ -183,7 +183,7 @@ def launch(self):
                 Path(self.job[1]).rename(Path(self.params['output']))
             else:
                 print("none of our files found in the error message!")
-        else:
+        elif self.job[2]:
             for one_file in [self.job[0], self.job[1]]:
                 print('cleaning up')
                 cleanup_file = Path(one_file)
@@ -196,6 +196,9 @@ def launch(self):
                     print('file gone')
                 else:
                     print(f'file {str(cleanup_file)} already gone?')
+                print(f"skipping {self.job[0]}")
+            else:
+                print(f"skipping this layer with {self.job[0]} {self.job[1]}")
         print(f"launch(): returning {ret}")
         return ret
 
@@ -279,14 +282,16 @@ def combine_coverage_dirs_multi(cfg,
     if len(sub_jobs) == 1:
         print(sub_jobs)
         return sub_jobs[0]
+    layer = 0
     while len(sub_jobs) > 1:
         next_jobs = []
         jobs.append([])
         while len(sub_jobs) > 1:
             last_output = combined_dir / f'{jobcount}'
             this_subjob = [str(sub_jobs.pop()),
                            str(sub_jobs.pop()),
-                           str(last_output)]
+                           str(last_output),
+                           count < 5]
             jobs[count].append(this_subjob)
             next_jobs.append(this_subjob[2])
             jobcount += 1
@@ -346,39 +351,8 @@ def convert_lcov_to_cobertura(cfg, lcov_file, source_dir, binary, cobertura_xml,
     cov = LcovCobertura(cfg)
     cov.launch(lcov_file, source_dir, binary, cobertura_xml, excludes)
 
-def main():
-    """ go """
-    # pylint disable=too-many-locals disable=too-many-statements
-    base_dir = Path(sys.argv[1])
-    os.chdir(base_dir)
-    coverage_dir = base_dir / 'coverage'
-    if coverage_dir.exists():
-        shutil.rmtree(str(coverage_dir))
-    coverage_dir.mkdir()
-    gcov_dir = base_dir / sys.argv[2]
-    #try:
-    #    shutil.make_archive("/work/testresults2124",
-    #                        'tar.gz',
-    #                        "/work/gcov",
-    #                        "/work/gcov",
-    #                        True)
-    #except Exception as ex:
-    #    print(f"Failed to create zip: {str(ex)}")
-    cfg = SiteConfig(gcov_dir.resolve())
-    #import glob
-    #for filename in glob.iglob('/work/gcov**/**', recursive=True):
-    #    print(filename)
-    result_dir = combine_coverage_dirs_multi(
-        cfg,
-        gcov_dir,
-        psutil.cpu_count(logical=False))
-
-    sourcedir = base_dir / 'ArangoDB'
-    binary = sourcedir / 'build' / 'bin' / 'arangod'
-    lcov_file = gcov_dir / 'coverage.lcov'
-    print('converting to lcov file')
-    convert_to_lcov_file(cfg, result_dir, lcov_file)
-    # copy the source files from the sourcecode directory
+def copy_source_directory(sourcedir, coverage_dir):
+    """ copy the source files from the sourcecode directory """
     for copy_dir in [
             Path('lib'),
             Path('arangosh'),
@@ -401,7 +375,36 @@ def main():
                 for filename in files:
                     source = os.path.join(root, filename)
                     shutil.copy2(source, path / filename)
+    print('create a symlink into the jemalloc source:')
+    jmdir = sourcedir / '3rdParty' / 'jemalloc' / 'jemalloc' / 'include'
+    if not jmdir.exists():
+        jmdir = list((sourcedir / '3rdParty' / 'jemalloc').glob('v*'))[0] / 'include'
+    (sourcedir / 'include').symlink_to(jmdir)
 
+def main():
+    """ go """
+    # pylint disable=too-many-locals disable=too-many-statements
+    base_dir = Path(sys.argv[1])
+    os.chdir(base_dir)
+    coverage_dir = base_dir / 'coverage'
+    if coverage_dir.exists():
+        shutil.rmtree(str(coverage_dir))
+    coverage_dir.mkdir()
+    gcov_dir = base_dir / sys.argv[2]
+    cfg = SiteConfig(gcov_dir.resolve())
+    result_dir = combine_coverage_dirs_multi(
+        cfg,
+        gcov_dir,
+        psutil.cpu_count(logical=False))
+
+    sourcedir = base_dir / 'ArangoDB'
+    binary = sourcedir / 'build' / 'bin' / 'arangod'
+    lcov_file = gcov_dir / 'coverage.lcov'
+
+    copy_source_directory(sourcedir, coverage_dir)
+
+    print('converting to lcov file')
+    convert_to_lcov_file(cfg, result_dir, lcov_file)
     print('copy the gcno files from the build directory')
     buildir = sourcedir / 'build'
     baselen = len(str(buildir))
@@ -413,12 +416,6 @@ def main():
             source = os.path.join(root, filename)
             shutil.copy2(source, path / filename)
 
-    print('create a symlink into the jemalloc source:')
-    jmdir = sourcedir / '3rdParty' / 'jemalloc' / 'jemalloc' / 'include'
-    if not jmdir.exists():
-        jmdir = list((sourcedir / '3rdParty' / 'jemalloc').glob('v*'))[0] / 'include'
-    (sourcedir / 'include').symlink_to(jmdir)
-
     cobertura_xml = coverage_dir / 'coverage.xml'
     print('converting to cobertura report')
     convert_lcov_to_cobertura(cfg, lcov_file,

Diff for: jenkins/helper/arangosh.py

@@ -80,4 +80,5 @@ def run_testing(self,
         )
         delete_logfile_params(params)
         ret['error'] = params['error']
+        ret['pid'] = params['pid']
         return ret

Diff for: jenkins/helper/overload_thread.py

@@ -17,7 +17,7 @@ def overload_thread(sitecfg, _):
     """watcher thread to track system load"""
     continue_running = True
     print("starting load monitoring thread")
-    fn =sitecfg.basedir / "overloads.jsonl"
+    fn =sitecfg.run_root / "overloads.jsonl"
     print(f"report file: {str(fn)}")
     with open(fn, "w+", encoding="utf-8")  as jsonl_file:
         while continue_running:
@@ -30,7 +30,7 @@ def overload_thread(sitecfg, _):
             load = psutil.getloadavg()
             if (load[0] > sitecfg.max_load) or (load[1] > sitecfg.max_load1) or (load[0] > sitecfg.overload):
                 #print(f"{str(load)} <= {sitecfg.overload} Load to high - Disk I/O: " + str(psutil.swap_memory()))
-                jsonl_file.write(f'["{datetime.now ()}", {get_all_processes_stats_json()}]\n')
+                jsonl_file.write(f'["{datetime.now ()}", {get_all_processes_stats_json(load)}]\n')
             time.sleep(1)
             with END_THREAD_LOCK:
                 continue_running = not END_THREAD

Diff for: jenkins/helper/testing_runner.py

@@ -4,6 +4,7 @@
 import os
 from pathlib import Path
 import hashlib
+import json
 import pprint
 import re
 import shutil
@@ -168,6 +169,8 @@ def testing_runner(testing_instance, this, arangosh):
             print(traceback.format_exc())
             raise ex
         with arangosh.slot_lock:
+            with open((this.cfg.run_root / "job_to_pids.jsonl"), "a+", encoding="utf-8")  as jsonl_file:
+                jsonl_file.write(f'{json.dumps({"pid": ret["pid"], "logfile": str(this.log_file)})}\n')
             testing_instance.running_suites.remove(this.name_enum)
         testing_instance.done_job(this.parallelity)
 

Diff for: jenkins/helper/tools/killall.py

@@ -10,6 +10,8 @@ def list_all_processes():
     pseaf = "PID  Process"
     # pylint: disable=catching-non-exception
     for process in psutil.process_iter(["pid", "ppid", "name"]):
+        if  process.pid in [1, 2] or process.ppid() == 2:
+            continue
         cmdline = process.name
         try:
             cmdline = str(process.cmdline())
@@ -27,7 +29,6 @@ def list_all_processes():
 
 def kill_all_arango_processes():
     """list all processes for later reference"""
-    pseaf = "PID  Process"
     # pylint: disable=catching-non-exception
     for process in psutil.process_iter(["pid", "name"]):
         if (process.name().lower().find('arango') >= 0 or
@@ -50,6 +51,8 @@ def gather_process_thread_statistics(p):
     ret['process'] = [{
         'time': time.ctime(),
         'pid': p.pid,
+        'ppid': p.ppid(),
+        'cmdline': p.cmdline(),
         'name': p.name(),
         'percent': p.cpu_percent(),
         'iocounters': p.io_counters(),
@@ -72,25 +75,27 @@ def add_delta(p1, p2):
             p1[tid]['d_sys'] = p2[tid]['sys'] - p1[tid]['sys']
     p1['process'].append(p2['process'][0])
 
-def get_all_processes_stats_json():
+def get_all_processes_stats_json(load):
     """ aggregate a structure of all processes and their threads plus delta """
+    # pylint: disable=broad-exception-caught
     process_full_list = {}
-    for n in [True, False]:
-        processes = psutil.process_iter()
-        for process in processes:
-            name = ""
-            try:
-                name = process.name()
-                if  process.ppid() != 2 and process.pid not in [1, 2]:
-                    procstat = gather_process_thread_statistics(process)
-                    if n:
-                        process_full_list[f"p{process.pid}"] = procstat
-                    else:
-                        add_delta(process_full_list[f"p{process.pid}"], procstat)
-            except psutil.AccessDenied:
-                pass
-            except Exception as ex:
-                print(f"while inspecting {name}: {ex} ")
-        if n:
-            time.sleep(1)
+    process_full_list['sys'] = {
+        'load': load,
+        'vmem': psutil.virtual_memory(),
+        'mem': psutil.swap_memory(),
+        'diskio': psutil.disk_io_counters(perdisk=True, nowrap=True),
+        'netio': psutil.net_io_counters(pernic=True, nowrap=True),
+    }
+    processes = psutil.process_iter()
+    for process in processes:
+        name = ""
+        try:
+            name = process.name()
+            if  process.pid not in [1, 2] and process.ppid() != 2:
+                procstat = gather_process_thread_statistics(process)
+                process_full_list[f"p{process.pid}"] = procstat
+        except psutil.AccessDenied:
+            pass
+        except Exception as ex:
+            print(f"while inspecting {name}: {ex} ")
     return json.dumps(process_full_list)