Skip to content

Commit e32ccc3

Browse files
activesoullactivesoull
authored
Merge pull request #3029 from activeloopai/labelbox-integration-refactor
Labelbox integration refactor
2 parents 8ab9a25 + 675cdf4 commit e32ccc3

File tree

9 files changed

+316
-109
lines changed

9 files changed

+316
-109
lines changed

‎deeplake/core/storage/azure.py

+12
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,8 @@ def __init__(self, root: str, creds: Dict = {}, token: Optional[str] = None):
3636
self.expiration: Optional[str] = None
3737
self.db_engine: bool = False
3838
self.repository: Optional[str] = None
39+
self.hub_path = ""
40+
self.tag = ""
3941

4042
def _set_attrs(self):
4143
self.account_name, self.container_name, self.root_folder = self._get_attrs(
@@ -111,6 +113,7 @@ def __setitem__(self, path, content):
111113
retry_wait = 0
112114
for i in range(1, tries + 1):
113115
always_warn(f"Encountered connection error, retry {i} out of {tries}")
116+
self._check_update_creds(force=True)
114117
retry_wait = self._retry_wait_and_extend(retry_wait, ex)
115118
try:
116119
self._set(path, content)
@@ -132,6 +135,7 @@ def __getitem__(self, path):
132135
retry_wait = 0
133136
for i in range(1, tries + 1):
134137
always_warn(f"Encountered connection error, retry {i} out of {tries}")
138+
self._check_update_creds(force=True)
135139
retry_wait = self._retry_wait_and_extend(retry_wait, ex)
136140
try:
137141
res = self.get_bytes(path)
@@ -163,6 +167,7 @@ def __delitem__(self, path):
163167
retry_wait = 0
164168
for i in range(1, tries + 1):
165169
always_warn(f"Encountered connection error, retry {i} out of {tries}")
170+
self._check_update_creds(force=True)
166171
retry_wait = self._retry_wait_and_extend(retry_wait, ex)
167172
try:
168173
self._del(path)
@@ -228,6 +233,7 @@ def clear(self, prefix=""):
228233
retry_wait = 0
229234
for i in range(1, tries + 1):
230235
always_warn(f"Encountered connection error, retry {i} out of {tries}")
236+
self._check_update_creds(force=True)
231237
retry_wait = self._retry_wait_and_extend(retry_wait, ex)
232238
try:
233239
self._clear(prefix)
@@ -290,6 +296,8 @@ def __getstate__(self):
290296
"db_engine": self.db_engine,
291297
"repository": self.repository,
292298
"expiration": self.expiration,
299+
"tag": getattr(self, "tag", ""),
300+
"hub_path": getattr(self, "hub_path", ""),
293301
}
294302

295303
def __setstate__(self, state):
@@ -377,6 +385,7 @@ def get_object_size(self, path: str) -> int:
377385
retry_wait = 0
378386
for i in range(1, tries + 1):
379387
always_warn(f"Encountered connection error, retry {i} out of {tries}")
388+
self._check_update_creds(force=True)
380389
retry_wait = self._retry_wait_and_extend(retry_wait, ex)
381390
try:
382391
res = self._get_object_size(path)
@@ -464,13 +473,16 @@ def _get_object_from_full_url(self, url: str) -> bytes:
464473
return blob_client.download_blob().readall()
465474

466475
def get_object_from_full_url(self, url: str) -> bytes:
476+
from azure.core.exceptions import ClientAuthenticationError # type: ignore
477+
467478
try:
468479
return self._get_object_from_full_url(url)
469480
except ClientAuthenticationError as ex:
470481
tries = 5
471482
retry_wait = 0
472483
for i in range(1, tries + 1):
473484
always_warn(f"Encountered connection error, retry {i} out of {tries}")
485+
self._check_update_creds(force=True)
474486
retry_wait = self._retry_wait_and_extend(retry_wait, ex)
475487
try:
476488
res = self._get_object_from_full_url(url)

‎deeplake/integrations/__init__.py

+1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
from .pytorch import dataset_to_pytorch
22
from .tf import dataset_to_tensorflow
33
from .wandb import *
4+
from .labelbox import *

‎deeplake/integrations/labelbox/v3_converters.py ‎deeplake/integrations/labelbox/converters.py

+30-35
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,13 @@
22
import urllib.request
33
import numpy as np
44
import copy
5+
from deeplake.integrations.labelbox.deeplake_utils import *
56

67

78
def bbox_converter_(obj, converter, tensor_name, context, generate_labels):
89
ds = context["ds"]
910
try:
10-
ds.create_tensor(
11-
tensor_name,
12-
htype="bbox",
13-
dtype="int32",
14-
coords={"type": "pixel", "mode": "LTWH"},
15-
)
11+
ds.create_tensor(tensor_name, **bbox_tensor_create_kwargs_())
1612
except:
1713
pass
1814

@@ -73,9 +69,10 @@ def radio_converter_(obj, converter, tensor_name, context, generate_labels):
7369
try:
7470
ds.create_tensor(
7571
tensor_name,
76-
htype="class_label",
77-
class_names=list(converter.label_mappings[tensor_name].keys()),
78-
chunk_compression="lz4",
72+
**class_label_tensor_create_kwargs_(),
73+
)
74+
ds[tensor_name].update_metadata(
75+
{"class_names": list(converter.label_mappings[tensor_name].keys())}
7976
)
8077
except:
8178
pass
@@ -115,9 +112,10 @@ def checkbox_converter_(obj, converter, tensor_name, context, generate_labels):
115112
try:
116113
ds.create_tensor(
117114
tensor_name,
118-
htype="class_label",
119-
class_names=list(converter.label_mappings[tensor_name].keys()),
120-
chunk_compression="lz4",
115+
**class_label_tensor_create_kwargs_(),
116+
)
117+
ds[tensor_name].update_metadata(
118+
{"class_names": list(converter.label_mappings[tensor_name].keys())}
121119
)
122120
except:
123121
pass
@@ -149,7 +147,7 @@ def checkbox_converter_nested(row, obj):
149147
def point_converter_(obj, converter, tensor_name, context, generate_labels):
150148
ds = context["ds"]
151149
try:
152-
ds.create_tensor(tensor_name, htype="point", dtype="int32")
150+
ds.create_tensor(tensor_name, **point_tensor_create_kwargs_())
153151
except:
154152
pass
155153

@@ -187,7 +185,7 @@ def interpolator(start, end, progress):
187185
def line_converter_(obj, converter, tensor_name, context, generate_labels):
188186
ds = context["ds"]
189187
try:
190-
ds.create_tensor(tensor_name, htype="polygon", dtype="int32")
188+
ds.create_tensor(tensor_name, **polygon_tensor_create_kwargs_())
191189
except:
192190
pass
193191

@@ -230,20 +228,14 @@ def raster_segmentation_converter_(
230228
):
231229
ds = context["ds"]
232230
try:
233-
ds.create_tensor(
234-
tensor_name, htype="binary_mask", dtype="bool", sample_compression="lz4"
235-
)
231+
ds.create_tensor(tensor_name, **binary_mask_tensor_create_kwargs_())
236232
except:
237233
pass
238234

239235
try:
240236
if generate_labels:
241237
ds.create_tensor(
242-
f"{tensor_name}_labels",
243-
htype="class_label",
244-
dtype="int32",
245-
class_names=[],
246-
chunk_compression="lz4",
238+
f"{tensor_name}_labels", **class_label_tensor_create_kwargs_()
247239
)
248240
converter.label_mappings[f"{tensor_name}_labels"] = dict()
249241
except:
@@ -270,30 +262,33 @@ def mask_converter(row, obj):
270262
converter.label_mappings[f"{tensor_name}_labels"][tool_name] = (
271263
len(converter.label_mappings[f"{tensor_name}_labels"])
272264
)
273-
ds[f"{tensor_name}_labels"].info.update(
274-
class_names=list(
275-
converter.label_mappings[f"{tensor_name}_labels"].keys()
276-
)
265+
ds[f"{tensor_name}_labels"].update_metadata(
266+
{
267+
"class_names": list(
268+
converter.label_mappings[
269+
f"{tensor_name}_labels"
270+
].keys()
271+
)
272+
}
277273
)
278274
val = []
279275
try:
280-
val = (
281-
ds[f"{tensor_name}_labels"][row].numpy(aslist=True).tolist()
282-
)
276+
val = ds[f"{tensor_name}_labels"].value(row, aslist=True)
283277
except (KeyError, IndexError):
284278
pass
285-
286279
val.append(
287280
converter.label_mappings[f"{tensor_name}_labels"][tool_name]
288281
)
289-
ds[f"{tensor_name}_labels"][row] = val
282+
ds[f"{tensor_name}_labels"].set_value(row, val)
290283

291284
mask = np.array(Image.open(response)).astype(np.bool_)
292285
mask = mask[..., np.newaxis]
293286
try:
294287
if generate_labels:
295-
val = ds[tensor_name][row].numpy()
288+
val = ds[tensor_name].value(row)
296289
labels = ds[f"{tensor_name}_labels"].info["class_names"]
290+
if val is None:
291+
raise IndexError()
297292
if len(labels) != val.shape[-1]:
298293
val = np.concatenate(
299294
[val, np.zeros_like(mask)],
@@ -302,11 +297,11 @@ def mask_converter(row, obj):
302297
idx = labels.index(tool_name)
303298
val[:, :, idx] = np.logical_or(val[:, :, idx], mask[:, :, 0])
304299
else:
305-
val = np.logical_or(ds[tensor_name][row].numpy(), mask)
300+
val = np.logical_or(ds[tensor_name].value(row), mask)
306301
except (KeyError, IndexError):
307302
val = mask
308303

309-
ds[tensor_name][row] = val
304+
ds[tensor_name].set_value(row, val)
310305
except Exception as e:
311306
print(f"Error downloading mask: {e}")
312307

@@ -316,7 +311,7 @@ def mask_converter(row, obj):
316311
def text_converter_(obj, converter, tensor_name, context, generate_labels):
317312
ds = context["ds"]
318313
try:
319-
ds.create_tensor(tensor_name, htype="text", dtype="str")
314+
ds.create_tensor(tensor_name, **text_tensor_create_kwargs_())
320315
except:
321316
pass
322317

0 commit comments

Comments
 (0)