Refactor/docker compose anchors (#449)

* Eliminate `log_slave_updates` system variable, closes #410

- `log_slave_updates` modifies to which file binlog events is written.
- The variable is not applicable to python-mysql-replication, because no file is written

* Refactor docker-compose.yml to use extension and YAML anchors for improved readability and expandability

Changes:
- Updated version to 3.4 at which extension for docker compose was first introduced
- Use extension to hold common fields and anchors to reference the extensions

* fix: use folded block scalar

* fix: add volumes to test MariadbStartEncryptionEvent

* fix: docker-compose-test mysql server check query

* Eliminate log_slave_updates system variable in docker-compose-test.yml

* fix: docker-compose settings for mariadb

* fix: MYSQL_5_7_CTL_PORT casting

* add copy .mariadb

---------

Co-authored-by: Soul Lee <[email protected]>
Co-authored-by: lre12 <[email protected]>

Author: 3 people

docker-compose-test.yml

@@ -1,26 +1,56 @@
-version: '3.2'
+version: '3.4'
+
+x-mysql: &mysql
+  environment:
+    MYSQL_ALLOW_EMPTY_PASSWORD: true
+  command: >
+    mysqld
+    --log-bin=mysql-bin.log
+    --server-id 1
+    --binlog-format=row
+    --gtid_mode=on
+    --enforce-gtid-consistency=on
+
+x-mariadb: &mariadb
+  environment:
+    MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1
+  command: >
+    --server-id=1
+    --default-authentication-plugin=mysql_native_password
+    --log-bin=master-bin
+    --binlog-format=row
+
 services:
-  percona-5.7:
-    platform: linux/amd64
+  percona-5.7-ctl:
+    <<: *mysql
     image: percona:5.7
-    environment:
-      MYSQL_ALLOW_EMPTY_PASSWORD: true
-      MYSQL_DATABASE: pymysqlreplication_test
     ports:
-      - 3306:3306
-    command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates
-    restart: always
+      - "3307:3306"
     networks:
       - default
 
-  percona-5.7-ctl:
+  percona-5.7:
+    <<: *mysql
     image: percona:5.7
-    environment:
-      MYSQL_ALLOW_EMPTY_PASSWORD: true
-      MYSQL_DATABASE: pymysqlreplication_test
     ports:
-      - 3307:3307
-    command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates -P 3307
+      - "3306:3306"
+    networks:
+      - default
+
+  mariadb-10.6:
+    <<: *mariadb
+    image: mariadb:10.6
+    ports:
+      - "3308:3306"
+    volumes:
+      - type: bind
+        source: ./.mariadb
+        target: /opt/key_file
+      - type: bind
+        source: ./.mariadb/my.cnf
+        target: /etc/mysql/my.cnf
+    networks:
+      - default
 
   pymysqlreplication:
     build:
@@ -30,6 +60,9 @@ services:
         BASE_IMAGE: python:3.11-alpine
         MYSQL_5_7: percona-5.7
         MYSQL_5_7_CTL: percona-5.7-ctl
+        MYSQL_5_7_CTL_PORT: 3306
+        MARIADB_10_6: mariadb-10.6
+        MARIADB_10_6_PORT: 3306
 
     command:
       - /bin/sh
@@ -39,7 +72,7 @@ services:
         
         while :
         do
-          if mysql -h percona-5.7 --user=root --execute "USE pymysqlreplication_test;" 2>&1 >/dev/null && mysql -h percona-5.7-ctl --port=3307  --user=root --execute "USE pymysqlreplication_test;" 2>&1 >/dev/null; then
+          if mysql -h percona-5.7 --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-5.7-ctl --user=root --execute "SELECT version();" 2>&1 >/dev/null; then
             break
           fi
           sleep 1
@@ -56,4 +89,5 @@ services:
       - percona-5.7-ctl
 
 networks:
-  default: {}
+  default:
+    driver: bridge

docker-compose.yml

@@ -1,37 +1,47 @@
-version: '3.2'
+version: '3.4'
+
+x-mysql: &mysql
+  environment:
+    MYSQL_ALLOW_EMPTY_PASSWORD: true
+  command: >
+    mysqld
+    --log-bin=mysql-bin.log
+    --server-id 1
+    --binlog-format=row
+    --gtid_mode=on
+    --enforce-gtid-consistency=on
+
+x-mariadb: &mariadb
+  environment:
+    MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1
+  command: >
+    --log-bin=master-bin
+    --server-id=1
+    --default-authentication-plugin=mysql_native_password
+    --binlog-format=row
+
 services:
   percona-5.7:
+    <<: *mysql
     image: percona:5.7
-    environment:
-      MYSQL_ALLOW_EMPTY_PASSWORD: true
     ports:
-      - 3306:3306
-    command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates
+      - "3306:3306"
 
   percona-5.7-ctl:
+    <<: *mysql
     image: percona:5.7
-    environment:
-      MYSQL_ALLOW_EMPTY_PASSWORD: true
     ports:
-      - 3307:3307
-    command: mysqld --log-bin=mysql-bin.log --server-id 1 --binlog-format=row --gtid_mode=on --enforce-gtid-consistency=on --log_slave_updates -P 3307
+      - "3307:3306"
 
   mariadb-10.6:
+    <<: *mariadb
     image: mariadb:10.6
-    environment:
-      MARIADB_ALLOW_EMPTY_ROOT_PASSWORD: 1
     ports:
       - "3308:3306"
-    command: |
-      --server-id=1
-      --default-authentication-plugin=mysql_native_password
-      --log-bin=master-bin
-      --binlog-format=row
-      --log-slave-updates=on
     volumes:
       - type: bind
         source: ./.mariadb
         target: /opt/key_file
       - type: bind
         source: ./.mariadb/my.cnf
-        target: /etc/mysql/my.cnf  
+        target: /etc/mysql/my.cnf

pymysqlreplication/tests/base.py

@@ -125,10 +125,10 @@ class PyMySQLReplicationMariaDbTestCase(PyMySQLReplicationTestCase):
     def setUp(self):
         # default
         self.database = {
-            "host": "localhost",
+            "host": os.environ.get("MARIADB_10_6") or "localhost",
             "user": "root",
             "passwd": "",
-            "port": 3308,
+            "port": int(os.environ.get("MARIADB_10_6_PORT") or 3308),
             "use_unicode": True,
             "charset": "utf8",
             "db": "pymysqlreplication_test"

pymysqlreplication/tests/test_basic.py

@@ -771,9 +771,8 @@ def setUp(self):
         self.stream.close()
         ctl_db = copy.copy(self.database)
         ctl_db["db"] = None
-        ctl_db["port"] = 3307
-        if os.environ.get("MYSQL_5_7_CTL") is not None:
-            ctl_db["host"] = os.environ.get("MYSQL_5_7_CTL")
+        ctl_db["port"] = int(os.environ.get("MYSQL_5_7_CTL_PORT") or 3307)
+        ctl_db["host"] = os.environ.get("MYSQL_5_7_CTL") or "localhost"
         self.ctl_conn_control = pymysql.connect(**ctl_db)
         self.ctl_conn_control.cursor().execute("DROP DATABASE IF EXISTS pymysqlreplication_test")
         self.ctl_conn_control.cursor().execute("CREATE DATABASE pymysqlreplication_test")

test.Dockerfile

@@ -1,6 +1,7 @@
 ARG BASE_IMAGE=${BASE_IMAGE:-python:3.11-alpine}
 FROM ${BASE_IMAGE}
 
+COPY .mariadb .mariadb
 COPY pymysqlreplication pymysqlreplication
 COPY README.md README.md
 COPY setup.py setup.py
@@ -13,4 +14,13 @@ ARG MYSQL_5_7
 ENV MYSQL_5_7 ${MYSQL_5_7}
 
 ARG MYSQL_5_7_CTL
-ENV MYSQL_5_7_CTL ${MYSQL_5_7_CTL}
+ENV MYSQL_5_7_CTL ${MYSQL_5_7_CTL}
+
+ARG MYSQL_5_7_CTL_PORT
+ENV MYSQL_5_7_CTL_PORT ${MYSQL_5_7_CTL_PORT}
+
+ARG MARIADB_10_6
+ENV MARIADB_10_6 ${MARIADB_10_6}
+
+ARG MARIADB_10_6_PORT
+ENV MARIADB_10_6_PORT ${MARIADB_10_6_PORT}