Skip to content

Commit 9cbb705

Browse files
etametam
authored
Improve tests (#628)
1 parent 894b13c commit 9cbb705

16 files changed

+252
-159
lines changed

‎.github/ISSUE_TEMPLATE.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ Please specify the versions you are using. Exact version numbers are preferred.
33
- Pymyrepl (e.g., 1.0.2):
44
- OS (e.g., Ubuntu 18.04):
55
- Database and version (Remove unnecessary options):
6-
- MySQL:
7-
- MariaDB:
6+
- MySQL:
7+
- MariaDB:
88
- Percona:
99

1010
### System Variables

‎.github/PULL_REQUEST_TEMPLATE.md

-1
Original file line numberDiff line numberDiff line change
@@ -18,4 +18,3 @@
1818

1919
### Additional Information
2020
<!--If there's any additional information or context you'd like to provide for this PR, such as screenshots, reference documents, or release notes, please include them here.-->
21-

‎.github/workflows/coverage.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
11
name: Python Coverage
2-
on:
2+
on:
33
push:
44
pull_request:
55
workflow_dispatch:
66
jobs:
77
build:
88
runs-on: ubuntu-latest
99
timeout-minutes: 3
10-
10+
1111
steps:
1212
- name: Check out code
1313
uses: actions/checkout@v4

‎.github/workflows/pytest.yml

+3-6
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@ jobs:
1616
- {name: 'Pypy 3.7', python: 'pypy-3.7'}
1717
- {name: 'Pypy 3.10', python: 'pypy-3.10'}
1818
name: ${{ matrix.name }}
19-
runs-on: ubuntu-latest
19+
# ubuntu-latest does not have python 3.7
20+
runs-on: ubuntu-22.04
2021
timeout-minutes: 3
2122

2223
steps:
@@ -53,14 +54,10 @@ jobs:
5354
working-directory: pymysqlreplication/tests
5455
run: pytest -k "$PYTEST_SKIP_OPTION" --db=mysql-5
5556

56-
- name: Run tests for mysql-5-ctl
57-
working-directory: pymysqlreplication/tests
58-
run: pytest -k "$PYTEST_SKIP_OPTION" --db=mysql-5-ctl
59-
6057
- name: Run tests for mysql-8
6158
working-directory: pymysqlreplication/tests
6259
run: pytest -k "$PYTEST_SKIP_OPTION" --db=mysql-8
6360

6461
- name: Run tests for mariadb-10
6562
working-directory: pymysqlreplication/tests
66-
run: pytest -k "$PYTEST_SKIP_OPTION" -m mariadb --db=mariadb-10
63+
run: pytest -k "$PYTEST_SKIP_OPTION" --db=mariadb-10

‎.pre-commit-config.yaml

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,14 @@
11
repos:
22
- repo: https://github.com/pre-commit/pre-commit-hooks
3-
rev: v4.4.0
3+
rev: v5.0.0
44
hooks:
55
- id: trailing-whitespace
66
- id: end-of-file-fixer
77
- id: check-yaml
88
- id: check-added-large-files
99

1010
- repo: https://github.com/astral-sh/ruff-pre-commit
11-
rev: v0.0.286
11+
rev: v0.9.10
1212
hooks:
1313
- id: ruff
1414
args: [ --fix, --exit-non-zero-on-fix ]

‎docker-compose-test.yml

+17-26
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,3 @@
1-
version: '3.4'
2-
31
x-mysql: &mysql
42
environment:
53
MYSQL_ALLOW_EMPTY_PASSWORD: true
@@ -21,36 +19,22 @@ x-mariadb: &mariadb
2119
--binlog-format=row
2220
2321
services:
24-
percona-5.7-ctl:
25-
<<: *mysql
26-
image: percona:5.7
27-
ports:
28-
- "3307:3306"
29-
networks:
30-
- default
31-
3222
percona-5.7:
3323
<<: *mysql
3424
image: percona:5.7
35-
ports:
36-
- "3306:3306"
3725
networks:
3826
- default
3927

4028
percona-8.0:
4129
<<: *mysql
4230
image: percona:8.0
4331
platform: linux/amd64
44-
ports:
45-
- "3309:3306"
4632
networks:
4733
- default
4834

4935
mariadb-10.6:
5036
<<: *mariadb
5137
image: mariadb:10.6
52-
ports:
53-
- "3308:3306"
5438
volumes:
5539
- type: bind
5640
source: ./.mariadb
@@ -66,10 +50,7 @@ services:
6650
context: .
6751
dockerfile: test.Dockerfile
6852
args:
69-
BASE_IMAGE: python:3.11-alpine
7053
MYSQL_5_7: percona-5.7
71-
MYSQL_5_7_CTL: percona-5.7-ctl
72-
MYSQL_5_7_CTL_PORT: 3306
7354
MYSQL_8_0: percona-8.0
7455
MYSQL_8_0_PORT: 3306
7556
MARIADB_10_6: mariadb-10.6
@@ -81,24 +62,34 @@ services:
8162
- |
8263
echo "wait mysql server"
8364
84-
while :
85-
do
86-
if mysql -h percona-5.7 --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-5.7-ctl --user=root --execute "SELECT version();" 2>&1 >/dev/null && mysql -h percona-8.0 --user=root --execute "SELECT version();" 2>&1 >/dev/null; then
87-
break
88-
fi
65+
check_servers_up() {
66+
for s in percona-5.7 percona-8.0 mariadb-10.6; do
67+
mariadb -h "$$s" --user=root --execute "SELECT version();" || {
68+
echo "$$h not ready"
69+
return 1
70+
}
71+
done
72+
return 0
73+
}
74+
75+
while ! check_servers_up; do
8976
sleep 1
9077
done
9178
9279
echo "run pytest"
93-
pytest -k "not test_no_trailing_rotate_event and not test_end_log_pos"
80+
for db in mysql-5 mariadb-10 mysql-8; do
81+
echo "testing with $$db"
82+
pytest -vv -k "not test_no_trailing_rotate_event and not test_end_log_pos" --db "$$db"
83+
echo "tested with $$db"
84+
done
9485
9586
working_dir: /pymysqlreplication
9687
networks:
9788
- default
9889
depends_on:
9990
- percona-5.7
100-
- percona-5.7-ctl
10191
- percona-8.0
92+
- mariadb-10.6
10293

10394
networks:
10495
default:

‎examples/mysql_to_rabbitmq.py

+10-10
Original file line numberDiff line numberDiff line change
@@ -16,13 +16,13 @@
1616
WriteRowsEvent,
1717
)
1818

19-
MYSQL_SETTINGS = {"host": "127.0.0.1", "port": 3306, "user": "root",
19+
MYSQL_SETTINGS = {"host": "127.0.0.1", "port": 3306, "user": "root",
2020
"passwd": "password"}
2121

2222
def main():
2323
stream = BinLogStreamReader(
2424
connection_settings=MYSQL_SETTINGS,
25-
server_id=3,
25+
server_id=3,
2626
only_events=[DeleteRowsEvent, WriteRowsEvent, UpdateRowsEvent],
2727
)
2828

@@ -31,14 +31,14 @@ def main():
3131
password='password'
3232
)
3333
params = pika.ConnectionParameters('rabbitmq_host', credentials=credentials)
34-
34+
3535
# RabbitMQ Connection Settings
3636
conn = pika.BlockingConnection(params)
3737
channel = conn.channel()
3838
channel.queue_declare(queue='order')
3939
channel.exchange_declare(durable=True, exchange_type='direct', exchange='direct')
4040
channel.queue_bind(queue='order', exchange='direct', routing_key='order')
41-
41+
4242
for binlogevent in stream:
4343
for row in binlogevent.rows:
4444
if isinstance(binlogevent, DeleteRowsEvent):
@@ -48,23 +48,23 @@ def main():
4848
elif isinstance(binlogevent, UpdateRowsEvent):
4949
routing_key = "order"
5050
message_body = row["after_values"].items()
51-
51+
5252
elif isinstance(binlogevent, WriteRowsEvent):
5353
routing_key = "order"
5454
message_body = row["values"].items()
55-
56-
properties = pika.BasicProperties(content_type='application/json',
55+
56+
properties = pika.BasicProperties(content_type='application/json',
5757
delivery_mode=DeliveryMode.Transient)
5858
channel.basic_publish(
5959
exchange='direct',
6060
routing_key=routing_key,
6161
body=json.dumps(message_body, default=lambda x: str(x)),
6262
properties=properties
6363
)
64-
64+
6565
stream.close()
6666
conn.close()
67-
67+
6868

6969
if __name__ == '__main__':
70-
main()
70+
main()

‎getting-started.md

-1
Original file line numberDiff line numberDiff line change
@@ -1 +0,0 @@
1-

‎pymysqlreplication/tests/base.py

+22-1
Original file line numberDiff line numberDiff line change
@@ -45,8 +45,19 @@ def setUpDatabase(self, get_db):
4545
def setUp(self, charset="utf8"):
4646
# default
4747
self.conn_control = None
48+
49+
for i in ['host', 'port']:
50+
env_override = self.database.pop(f'{i}_env_override', None)
51+
if env_override:
52+
override = os.environ.get(env_override)
53+
if override:
54+
if i == 'port':
55+
override = int(override)
56+
self.database[i] = override
57+
4858
db = copy.copy(self.database)
4959
db["db"] = None
60+
db["charset"] = charset
5061
self.connect_conn_control(db)
5162
self.execute("DROP DATABASE IF EXISTS pymysqlreplication_test")
5263
self.execute("CREATE DATABASE pymysqlreplication_test")
@@ -55,8 +66,8 @@ def setUp(self, charset="utf8"):
5566
self.connect_conn_control(db)
5667
self.stream = None
5768
self.resetBinLog()
58-
self.isMySQL56AndMore()
5969
self.__is_mariaDB = None
70+
self.isMySQL56AndMore()
6071

6172
def getMySQLVersion(self):
6273
"""Return the MySQL version of the server
@@ -65,27 +76,37 @@ def getMySQLVersion(self):
6576
return self.execute("SELECT VERSION()").fetchone()[0].split("-")[0]
6677

6778
def isMySQL56AndMore(self):
79+
if self.isMariaDB():
80+
return False
6881
version = float(self.getMySQLVersion().rsplit(".", 1)[0])
6982
if version >= 5.6:
7083
return True
7184
return False
7285

7386
def isMySQL57(self):
87+
if self.isMariaDB():
88+
return False
7489
version = float(self.getMySQLVersion().rsplit(".", 1)[0])
7590
return version == 5.7
7691

7792
def isMySQL80AndMore(self):
93+
if self.isMariaDB():
94+
return False
7895
version = float(self.getMySQLVersion().rsplit(".", 1)[0])
7996
return version >= 8.0
8097

8198
def isMySQL8014AndMore(self):
99+
if self.isMariaDB():
100+
return False
82101
version = float(self.getMySQLVersion().rsplit(".", 1)[0])
83102
version_detail = int(self.getMySQLVersion().rsplit(".", 1)[1])
84103
if version > 8.0:
85104
return True
86105
return version == 8.0 and version_detail >= 14
87106

88107
def isMySQL8016AndMore(self):
108+
if self.isMariaDB():
109+
return False
89110
version = float(self.getMySQLVersion().rsplit(".", 1)[0])
90111
version_detail = int(self.getMySQLVersion().rsplit(".", 1)[1])
91112
if version > 8.0:

‎pymysqlreplication/tests/config.json

+9-15
Original file line numberDiff line numberDiff line change
@@ -5,34 +5,28 @@
55
"passwd": "",
66
"port": 3306,
77
"use_unicode": true,
8-
"charset": "utf8",
9-
"db": "pymysqlreplication_test"
10-
},
11-
"mysql-5-ctl": {
12-
"host": "localhost",
13-
"user": "root",
14-
"passwd": "",
15-
"port": 3307,
16-
"use_unicode": true,
17-
"charset": "utf8",
18-
"db": "pymysqlreplication_test"
8+
"db": "pymysqlreplication_test",
9+
"host_env_override": "MYSQL_5_7",
10+
"port_env_override": "MYSQL_5_7_PORT"
1911
},
2012
"mariadb-10": {
2113
"host": "localhost",
2214
"user": "root",
2315
"passwd": "",
2416
"port": 3308,
2517
"use_unicode": true,
26-
"charset": "utf8",
27-
"db": "pymysqlreplication_test"
18+
"db": "pymysqlreplication_test",
19+
"host_env_override": "MARIADB_10_6",
20+
"port_env_override": "MARIADB_10_6_PORT"
2821
},
2922
"mysql-8": {
3023
"host": "localhost",
3124
"user": "root",
3225
"passwd": "",
3326
"port": 3309,
3427
"use_unicode": true,
35-
"charset": "utf8",
36-
"db": "pymysqlreplication_test"
28+
"db": "pymysqlreplication_test",
29+
"host_env_override": "MYSQL_8_0",
30+
"port_env_override": "MYSQL_8_0_PORT"
3731
}
3832
}

0 commit comments

Comments
 (0)