QA-333: the clear name seems to sum up to too long filenames for several testes (#466)

* the clear name seems to sum up to too long filenames for several testcases

* build the ID earlier

* lint

* fix tempdir handling

* streamline parameter handling

* dmesg needs to be adjusted to PID via params as wel.

Author: dothebart

Diff for: jenkins/helper/arangosh.py

@@ -1,7 +1,6 @@
 #!/bin/env python3
 """ launch a testing.js instance with given testsuite and arguments """
 import os
-
 from async_client import (
     ArangoCLIprogressiveTimeoutExecutor,
 
@@ -16,15 +15,14 @@ class ArangoshExecutor(ArangoCLIprogressiveTimeoutExecutor):
     def __init__(self, site_config, slot_lock):
         self.slot_lock = slot_lock
         self.read_only = False
-        self.temp_dir = ""
         super().__init__(site_config, None)
 
-    def get_environment(self):
+    def get_environment(self, params):
         """ hook to implemnet custom environment variable setters """
         my_env = os.environ.copy()
-        my_env['TMPDIR'] = str(self.temp_dir)
-        my_env['TEMP'] = str(self.temp_dir)
-        my_env['TMP'] = str(self.temp_dir)
+        my_env['TMPDIR'] = str(params['temp_dir'])
+        my_env['TEMP'] = str(params['temp_dir'])
+        my_env['TMP'] = str(params['temp_dir'])
         return my_env
 
     def run_testing(self,
@@ -34,14 +32,13 @@ def run_testing(self,
                     directory,
                     logfile,
                     identifier,
-                    tempdir,
+                    temp_dir,
                     verbose
                     ):
        # pylint: disable=R0913 disable=R0902
         """ testing.js wrapper """
         print('------')
         print(testing_args)
-        self.temp_dir = tempdir
         args = [
             '-c', str(self.cfg.cfgdir / 'arangosh.conf'),
             "--log.foreground-tty", "true",
@@ -56,7 +53,7 @@ def run_testing(self,
             '--',
             testcase,
             '--testOutput', directory ] + testing_args
-        params = make_logfile_params(verbose, logfile, self.cfg.trace)
+        params = make_logfile_params(verbose, logfile, self.cfg.trace, temp_dir)
         ret = self.run_monitored(
             self.cfg.bin_dir / "arangosh",
             run_cmd,

Diff for: jenkins/helper/async_client.py

@@ -88,7 +88,7 @@ def delete_tail_params(params):
     params['output'].close()
     print(f"{params['identifier']} {params['lfn']} closed")
 
-def make_logfile_params(verbose, logfile, trace):
+def make_logfile_params(verbose, logfile, trace, temp_dir):
     """ create the structure to work with logfiles """
     return {
         "trace_io": True,
@@ -97,8 +97,10 @@ def make_logfile_params(verbose, logfile, trace):
         "verbose": verbose,
         "output": logfile.open('wb'),
         "identifier": "",
-        "lfn": str(logfile)
+        "lfn": str(logfile),
+        "temp_dir": temp_dir
     }
+
 def logfile_line_result(wait, line, params):
     """ Write the line to a logfile, print progress. """
     # pylint: disable=pointless-statement
@@ -269,19 +271,19 @@ def __init__(self, config, connect_instance, deadline_signal=-1):
             else:
                 self.deadline_signal = signal.SIGINT
 
-    def dig_for_children(self):
+    def dig_for_children(self, params):
         """ manual search for children that may be there without the self.pid still being there """
         children = []
         for process in psutil.process_iter(["pid", "ppid", "name"]):
-            if process.ppid() == self.pid:
+            if process.ppid() == params['pid']:
                 children.append(process)
             elif (process.ppid() == 1 and
                   (process.name().lower().find('arango') >= 0 or
                    process.name().lower().find('tshark') >= 0)):
                 children.append(process)
         return children
 
-    def get_environment(self):
+    def get_environment(self, params):
         """ hook to implemnet custom environment variable setters """
         return os.environ.copy()
 
@@ -357,10 +359,7 @@ def run_monitored(self,
         children = []
         if identifier == "":
             # pylint: disable=global-statement
-            global ID_COUNTER
-            my_no = ID_COUNTER
-            ID_COUNTER += 1
-            identifier = f"IO_{str(my_no)}"
+            identifier = f"IO_{str(params.my_id)}"
         print(params)
         params['identifier'] = identifier
         if not isinstance(deadline,datetime):
@@ -376,10 +375,10 @@ def run_monitored(self,
             stderr=PIPE,
             close_fds=ON_POSIX,
             cwd=self.cfg.test_data_dir.resolve(),
-            env=self.get_environment()
+            env=self.get_environment(params)
         ) as process:
             # pylint: disable=consider-using-f-string
-            self.pid = process.pid
+            params['pid'] = process.pid
             queue = Queue()
             thread1 = Thread(
                 name=f"readIO {identifier}",
@@ -452,15 +451,15 @@ def run_monitored(self,
                         try:
                             children = children + process.children(recursive=True)
                             rc_exit = process.wait(timeout=1)
-                            children = children + self.dig_for_children()
+                            children = children + self.dig_for_children(params)
                             add_message_to_report(
                                 params,
                                 f"{identifier} exited unexpectedly: {str(rc_exit)}",
                                 True, True)
                             kill_children(identifier, params, children)
                             break
                         except psutil.NoSuchProcess:
-                            children = children + self.dig_for_children()
+                            children = children + self.dig_for_children(params)
                             add_message_to_report(
                                 params,
                                 f"{identifier} exited unexpectedly: {str(rc_exit)}",
@@ -502,7 +501,7 @@ def run_monitored(self,
                     try:
                         process.send_signal(self.deadline_signal)
                     except psutil.NoSuchProcess:
-                        children = children + self.dig_for_children()
+                        children = children + self.dig_for_children(params)
                         print_log(f"{identifier} process already dead!", params)
                 elif have_deadline > 1 and datetime.now() > final_deadline:
                     try:

Diff for: jenkins/helper/dmesg.py

@@ -51,8 +51,8 @@ def launch(self):
 
     def end_run(self):
         """ terminate dmesg again """
-        print(f'killing dmesg {self.pid}')
+        print(f"killing dmesg {self.params['pid']}")
         try:
-            psutil.Process(self.pid).kill()
+            psutil.Process(self.params['pid']).kill()
         except psutil.NoSuchProcess:
             print('dmesg already gone?')

Diff for: jenkins/helper/testing_runner.py

@@ -71,6 +71,7 @@ def testing_runner(testing_instance, this, arangosh):
             this.summary += this.summary_file.read_text()
         else:
             print(f'{this.name_enum} no testreport!')
+        final_name = TEMP / this.name
         if this.crashed or not this.success:
             print(str(this.log_file.name))
             print(this.log_file.parent / ("FAIL_" + str(this.log_file.name))
@@ -85,9 +86,11 @@ def testing_runner(testing_instance, this, arangosh):
                 if this.crashed:
                     testing_instance.crashed = True
                 testing_instance.success = False
-            temp_dir = TEMP / ("FAIL_" + this.name)
-            this.temp_dir.rename(temp_dir)
-            this.temp_dir = temp_dir
+            final_name = TEMP / ("FAIL_" + this.name)
+        try:
+            this.temp_dir.rename(final_name)
+        except FileExistsError as ex:
+            print(f"can't expand the temp directory {ex} to {final_name}")
     finally:
         with arangosh.slot_lock:
             testing_instance.running_suites.remove(this.name_enum)
@@ -162,6 +165,8 @@ def launch_next(self, offset, counter, do_loadcheck):
             parallelity = self.scenarios[offset].parallelity
         self.used_slots += parallelity
         this = self.scenarios[offset]
+        this.counter = counter
+        this.temp_dir = TEMP / str(counter)
         this.name_enum = f"{this.name} {str(counter)}"
         print(f"launching {this.name_enum}")
         pp.pprint(this)