Skip to content

Commit 3a6250c

Browse files
authored
[BUGFIX] Resolve boto3 client threading issue (#66)
* Added thread locks and updated pytest adapter version * Placing impl.py in the correct location
1 parent 8adeea2 commit 3a6250c

File tree

2 files changed

+8
-3
lines changed

2 files changed

+8
-3
lines changed

Diff for: dbt/adapters/athena/impl.py

100644100755
+7-2
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import boto3
55
from botocore.exceptions import ClientError
66
from typing import Optional
7+
from threading import Lock
78

89
from dbt.adapters.base import available
910
from dbt.adapters.sql import SQLAdapter
@@ -12,6 +13,8 @@
1213
from dbt.events import AdapterLogger
1314
logger = AdapterLogger("Athena")
1415

16+
boto3_client_lock = Lock()
17+
1518
class AthenaAdapter(SQLAdapter):
1619
ConnectionManager = AthenaConnectionManager
1720
Relation = AthenaRelation
@@ -52,7 +55,8 @@ def clean_up_partitions(
5255
conn = self.connections.get_thread_connection()
5356
client = conn.handle
5457

55-
glue_client = boto3.client('glue', region_name=client.region_name)
58+
with boto3_client_lock:
59+
glue_client = boto3.client('glue', region_name=client.region_name)
5660
s3_resource = boto3.resource('s3', region_name=client.region_name)
5761
partitions = glue_client.get_partitions(
5862
# CatalogId='123456789012', # Need to make this configurable if it is different from default AWS Account ID
@@ -77,7 +81,8 @@ def clean_up_table(
7781
# Look up Glue partitions & clean up
7882
conn = self.connections.get_thread_connection()
7983
client = conn.handle
80-
glue_client = boto3.client('glue', region_name=client.region_name)
84+
with boto3_client_lock:
85+
glue_client = boto3.client('glue', region_name=client.region_name)
8186
try:
8287
table = glue_client.get_table(
8388
DatabaseName=database_name,

Diff for: dev_requirements.txt

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
11
flake8==3.8.4
22

33
# Test requirements
4-
pytest-dbt-adapter==0.5.1
4+
pytest-dbt-adapter==0.6.0

0 commit comments

Comments
 (0)