integrate_ui_automation_to_rta (#124)

Add UI tests

Author: MDAlFattah

README.md

@@ -20,10 +20,10 @@
 ## Linux
 
 - **debian** / **ubuntu**:
-  `apt-get install python3-yaml python3-requests python3-click python3-distro python3-psutil python3-pexpect python3-pyftpdlib python3-statsd python3-selenium python3-markdown python3-pip gdb`
+  `apt-get install python3-yaml python3-requests python3-click python3-distro python3-psutil python3-pexpect python3-pyftpdlib python3-statsd python3-selenium python3-pip gdb`
 
   the `python3-semver` on debian is to old - need to use the pip version instead:
-  `pip3 install semver beautifultable allure_python_commons certifi`
+  `pip3 install semver beautifultable allure_python_commons certifi tabulate`
 
   Ubuntu 16.40 pip3 system package is broken. Fix like this: 
   `dpkg -r python3-pip python3-pexpect` 
@@ -41,7 +41,7 @@
 ## Mac OS
 :
     `brew install gnu-tar`
-    `pip3 install click psutil requests pyyaml semver pexpect selenium beautifultable markdown allure_python_commons certifi`
+    `pip3 install click psutil requests pyyaml semver pexpect selenium beautifultable tabulate allure_python_commons certifi`
     `brew install gdb`
 if `python --version` is below 3.9 you also have to download ftplib:
     `pip3 install click ftplib`

attic_snippets/download_selenium_browser_drivers.py

@@ -23,7 +23,7 @@
 """
 
 
-class BaseSelenium:
+class BasePage:
     browser_name = None
     driver: WebDriver
 
@@ -78,9 +78,9 @@ def tear_down(cls):
         cls.driver.quit()
 
 # if __name__ == "__main__":
-#     selenium = BaseSelenium()
+#     selenium = BasePage()
 #     selenium.set_up_class()
 #     selenium.tear_down()
 
-BaseSelenium.set_up_class()
-BaseSelenium.tear_down()
+BasePage.set_up_class()
+BasePage.tear_down()

containers/docker_deb/Dockerfile

@@ -53,7 +53,7 @@ RUN if [ -z "$CHROME_DRIVER_VERSION" ]; \
   && sudo ln -fs /opt/selenium/chromedriver-$CHROME_DRIVER_VERSION /usr/bin/chromedriver
 
 
-RUN pip3 install semver selenium beautifultable allure_python_commons mss
+RUN pip3 install semver selenium beautifultable allure_python_commons mss tabulate
 
 run mkdir -p /home/release-test-automation \
  /home/package_cache \

containers/docker_rpm/Dockerfile

@@ -17,8 +17,8 @@ run mkdir -p /home/release-test-automation \
  /home/entrypoint
 RUN mkdir -p /home/entrypoint
 
-RUN yum -y update; yum install -y python3 python3-pyyaml python36-PyYAML python3-requests python3-click gcc platform-python-devel python3-distro python3-devel python36-distro python36-click python36-pexpect python3-pexpect python3-pyftpdlib python3-markdown initscripts file gdb chromedriver chromium;
-RUN pip3 install selenium psutil semver click requests pyyaml distro pexpect beautifultable allure_python_commons markdown certifi mss
+RUN yum -y update; yum install -y python3 python3-pyyaml python36-PyYAML python3-requests python3-click gcc platform-python-devel python3-distro python3-devel python36-distro python36-click python36-pexpect python3-pexpect python3-pyftpdlib initscripts file gdb chromedriver chromium python3-markdown;
+RUN pip3 install selenium psutil semver click requests pyyaml distro pexpect beautifultable allure_python_commons tabulate certifi mss
 
 RUN (cd /lib/systemd/system/sysinit.target.wants/; for i in ; do [ $i == systemd-tmpfiles-setup.service ] || rm -f $i; done); \
   rm -rf /lib/systemd/system/multi-user.target.wants/;\

containers/docker_tar/Dockerfile

@@ -5,7 +5,7 @@ ENV TZ=Europe/Berlin
 ARG CHROME_VERSION="google-chrome-stable"
 RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
 RUN apt-get update ; \
-    apt-get install -y wget unzip python3-yaml python3-requests python3-click python3-distro python3-psutil python3-pexpect python3-pyftpdlib python3-statsd python3-pip python3-markdown python-certifi gdb git python3-selenium fuse snapd snap-confine squashfuse sudo; \
+    apt-get install -y wget unzip python3-yaml python3-requests python3-click python3-distro python3-psutil python3-pexpect python3-pyftpdlib python3-statsd python3-pip python-certifi gdb git python3-selenium fuse snapd snap-confine squashfuse sudo python3-markdown; \
     wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
   && echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list \
   && apt-get update -qqy \
@@ -48,7 +48,7 @@ RUN if [ -z "$CHROME_DRIVER_VERSION" ]; \
 #VOLUME ["/sys/fs/cgroup"]
 # STOPSIGNAL SIGRTMIN+3
 
-RUN pip3 install semver beautifultable allure_python_commons mss
+RUN pip3 install semver beautifultable allure_python_commons mss tabulate
 RUN mkdir -p /home/entrypoint /home/release-test-automation /home/package_cache /home/versions /home/test_dir
 # ADD tarball_nightly_test.py /home/entrypoint/tarball_nightly_test.py
 # ENTRYPOINT ["/home/entrypoint/tarball_nightly_test.py"]

containers/this_version.txt

@@ -1 +1 @@
-1.1
+1.2

deployment/requirements.txt

@@ -3,5 +3,6 @@ semver
 distro
 pexpect
 allure_python_commons
-markdown
-certifi
+tabulate
+certifi
+markdown

release_tester/arangodb/async_client.py

@@ -50,6 +50,7 @@ def convert_result(result_array):
 
 class CliExecutionException(Exception):
     """transport CLI error texts"""
+
     def __init__(self, message, execution_result):
         self.execution_result = execution_result
         self.message = message

release_tester/arangodb/imp.py

@@ -7,11 +7,11 @@
 
 def get_type_args(filename):
     """guess the format by the filename"""
-    if filename.endswith("jsonl"):
+    if str(filename).endswith("jsonl"):
         return ["--type=jsonl"]
-    if filename.endswith("json"):
+    if str(filename).endswith("json"):
         return ["--type=json"]
-    if filename.endswith("csv"):
+    if str(filename).endswith("csv"):
         return ["--type=csv"]
     raise NotImplementedError("no filename type encoding implemented for " + filename)
 
@@ -57,7 +57,7 @@ def import_collection(self, collection_name, filename, more_args=[]):
     def import_smart_edge_collection(self, collection_name, filename, edge_relations, more_args=[]):
         """import into smart edge collection"""
         if len(edge_relations) == 1:
-            edge_relations[1] = edge_relations[0]
+            edge_relations.append(edge_relations[0])
         # fmt: off
         args = [
             '--from-collection-prefix', edge_relations[0],

release_tester/arangodb/installers/base.py

@@ -264,7 +264,7 @@ def calc_config_file_name(self):
             if "WORKSPACE_TMP" in os.environ:
                 wdtmp = Path(os.environ["WORKSPACE_TMP"])
                 wdtmp.mkdir(parents=True, exist_ok=True)
-                cfg_file =  wdtmp / "config.yml"
+                cfg_file = wdtmp / "config.yml"
             else:
                 cfg_file = Path("c:/") / "tmp/" / "config.yml"
         else:

release_tester/arangodb/instance.py

@@ -395,7 +395,7 @@ def get_local_url(self, login):
             port=self.port,
         )
 
-    def get_public_url(self, login):
+    def get_public_url(self, login=""):
         """our public url"""
         return "{protocol}://{login}{host}:{port}".format(
             protocol=self.get_http_protocol(),

release_tester/arangodb/starter/deployments/__init__.py

@@ -63,6 +63,11 @@ def make_runner(
     for one_installer_set in installer_set:
         assert len(one_installer_set) == 2, "no complete object config?"
 
+    # Configure Chrome to accept self-signed SSL certs and certs signed by unknown CA.
+    # FIXME: Add custom CA to Chrome to properly validate server cert.
+    if ssl:
+        selenium_driver_args += ("ignore-certificate-errors",)
+
     logging.debug("Factory for Runner of type: {0}".format(str(runner_type)))
     args = (
         runner_type,

release_tester/arangodb/starter/deployments/activefailover.py

@@ -157,6 +157,8 @@ def starter_run_impl(self):
     def finish_setup_impl(self):
         logging.info("instances are ready, detecting leader")
         self._detect_leader()
+        if self.selenium:
+            self.set_selenium_instances()
 
         # add data to leader
         self.makedata_instances.append(self.leader)
@@ -206,6 +208,9 @@ def test_setup_impl(self):
         )
         if not ret[0]:
             raise Exception("check data failed " + ret[1])
+        if self.selenium:
+            self.set_selenium_instances()
+            self.selenium.test_setup()
 
     def wait_for_restore_impl(self, backup_starter):
         backup_starter.wait_for_restore()
@@ -244,8 +249,7 @@ def upgrade_arangod_version_impl(self):
             node.detect_instance_pids()
         self.print_all_instances_table()
         if self.selenium:
-            self.selenium.web.refresh()  # version doesn't upgrade if we don't do this...
-            self.selenium.check_old(self.new_cfg, expect_follower_count=2, retry_count=10)
+            self.selenium.test_wait_for_upgrade()
 
     def upgrade_arangod_version_manual_impl(self):
         """manual upgrade this installation"""
@@ -264,12 +268,7 @@ def upgrade_arangod_version_manual_impl(self):
                 [
                     InstanceType.RESILIENT_SINGLE,
                 ],
-                [
-                    "--database.auto-upgrade",
-                    "true",
-                    "--javascript.copy-installation",
-                    "true",
-                ],
+                ["--database.auto-upgrade", "true", "--javascript.copy-installation", "true"],
             )
         self.progress(True, "step 3 - launch instances again")
         for node in self.starter_instances:
@@ -282,8 +281,7 @@ def upgrade_arangod_version_manual_impl(self):
         self.leader.maintainance(False, InstanceType.RESILIENT_SINGLE)
         self.print_all_instances_table()
         if self.selenium:
-            self.selenium.web.refresh()  # version doesn't upgrade if we don't do this...
-            self.selenium.check_old(self.new_cfg, expect_follower_count=2, retry_count=10)
+            self.selenium.test_wait_for_upgrade()
 
     def jam_attempt_impl(self):
         # pylint: disable=R0915
@@ -332,13 +330,9 @@ def jam_attempt_impl(self):
         self.set_frontend_instances()
 
         if self.selenium:
-            self.selenium.connect_server(
-                self.leader.get_frontends(),
-                "_system",
-                self.new_cfg if self.new_cfg else self.cfg,
-            )
-            cfg = self.new_cfg if self.new_cfg else self.cfg
-            self.selenium.check_old(cfg=cfg, expect_follower_count=1, retry_count=10)
+            # cfg = self.new_cfg if self.new_cfg else self.cfg
+            self.set_selenium_instances()
+            self.selenium.test_jam_attempt()
 
         prompt_user(
             self.basecfg,
@@ -370,8 +364,9 @@ def jam_attempt_impl(self):
 
         logging.info("state of this test is: %s", "Success" if self.success else "Failed")
         if self.selenium:
-            cfg = self.new_cfg if self.new_cfg else self.cfg
-            self.selenium.check_old(cfg=cfg, expect_follower_count=2, retry_count=10)
+            # cfg = self.new_cfg if self.new_cfg else self.cfg
+            self.set_selenium_instances()
+            self.selenium.test_wait_for_upgrade()
 
     def shutdown_impl(self):
         for node in self.starter_instances:
@@ -387,3 +382,13 @@ def before_backup_impl(self):
 
     def after_backup_impl(self):
         pass
+
+    def set_selenium_instances(self):
+        """set instances in selenium runner"""
+        self.selenium.set_instances(
+            self.cfg,
+            self.leader.arango_importer,
+            self.leader.arango_restore,
+            [x for x in self.leader.all_instances if x.instance_type == InstanceType.RESILIENT_SINGLE][0],
+            self.new_cfg,
+        )

release_tester/arangodb/starter/deployments/cluster.py

@@ -17,6 +17,8 @@
 arangoversions = {
     "370": semver.VersionInfo.parse("3.7.0"),
 }
+
+
 class Cluster(Runner):
     """this launches a cluster setup"""
 
@@ -158,7 +160,8 @@ def finish_setup_impl(self):
         self.set_frontend_instances()
 
     def test_setup_impl(self):
-        pass
+        if self.selenium:
+            self.selenium.test_setup()
 
     def wait_for_restore_impl(self, backup_starter):
         for starter in self.starter_instances:
@@ -180,7 +183,7 @@ def upgrade_arangod_version_impl(self):
 
         self.starter_instances[1].command_upgrade()
         if self.selenium:
-            self.selenium.upgrade_deployment(self.cfg, self.new_cfg, timeout=30)  # * 5s
+            self.selenium.test_wait_for_upgrade()  # * 5s
         self.starter_instances[1].wait_for_upgrade(300)
         if self.cfg.stress_upgrade:
             bench_instances[0].wait()
@@ -245,7 +248,7 @@ def upgrade_arangod_version_manual_impl(self):
         self.starter_instances[0].maintainance(False, InstanceType.COORDINATOR)
 
         if self.selenium:
-            self.selenium.upgrade_deployment(self.cfg, self.new_cfg, timeout=30)  # * 5s
+            self.selenium.test_wait_for_upgrade()  # * 5s
 
     @step
     def jam_attempt_impl(self):
@@ -266,7 +269,7 @@ def jam_attempt_impl(self):
 
         prompt_user(self.basecfg, "instance stopped")
         if self.selenium:
-            self.selenium.jam_step_1(self.new_cfg if self.new_cfg else self.cfg)
+            self.selenium.jam_step_1()
 
         ret = self.starter_instances[0].arangosh.check_test_data(
             "Cluster one node missing", True, ["--disabledDbserverUUID", uuid]
@@ -323,7 +326,7 @@ def jam_attempt_impl(self):
 
         prompt_user(self.basecfg, "cluster should be up")
         if self.selenium:
-            self.selenium.jam_step_2(self.new_cfg if self.new_cfg else self.cfg)
+            self.selenium.jam_step_2()
 
     def shutdown_impl(self):
         for node in self.starter_instances:
@@ -335,3 +338,13 @@ def before_backup_impl(self):
 
     def after_backup_impl(self):
         pass
+
+    def set_selenium_instances(self):
+        """set instances in selenium runner"""
+        self.selenium.set_instances(
+            self.cfg,
+            self.starter_instances[0].arango_importer,
+            self.starter_instances[0].arango_restore,
+            [x for x in self.starter_instances[0].all_instances if x.instance_type == InstanceType.COORDINATOR][0],
+            self.new_cfg,
+        )

release_tester/arangodb/starter/deployments/dc2dc.py

@@ -391,10 +391,7 @@ def test_setup_impl(self):
             raise Exception("check data on source cluster failed " + ret[1])
         self._get_in_sync(20)
 
-        self.cluster2["instance"].send_request(InstanceType.COORDINATOR,
-                                               requests.post,
-                                               "/_admin/routing/reload",
-                                               "")
+        self.cluster2["instance"].send_request(InstanceType.COORDINATOR, requests.post, "/_admin/routing/reload", "")
         ret = self.cluster2["instance"].arangosh.check_test_data(
             "dc2dc (post setup - dc2)", True, ["--readOnly", "true"]
         )