Skip to content

Commit b987e66

Browse files
authored
fix(studio): wait for studio metrics publish to complete on end (#827)
1 parent 2672989 commit b987e66

File tree

2 files changed

+32
-0
lines changed

2 files changed

+32
-0
lines changed

Diff for: src/dvclive/live.py

+7
Original file line numberDiff line numberDiff line change
@@ -905,6 +905,11 @@ def worker():
905905

906906
self._studio_queue.put(self)
907907

908+
def _wait_for_studio_updates_posted(self):
909+
if self._studio_queue:
910+
logger.debug("Waiting for studio updates to be posted")
911+
self._studio_queue.join()
912+
908913
def end(self):
909914
"""
910915
Signals that the current experiment has ended.
@@ -946,6 +951,8 @@ def end(self):
946951

947952
self.save_dvc_exp()
948953

954+
self._wait_for_studio_updates_posted()
955+
949956
# Mark experiment as done
950957
post_to_studio(self, "done")
951958

Diff for: tests/test_post_to_studio.py

+25
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
from pathlib import Path
22

33
import pytest
4+
import time
45
from dvc.env import DVC_EXP_GIT_REMOTE
56
from dvc_studio_client import DEFAULT_STUDIO_URL
67
from dvc_studio_client.env import DVC_STUDIO_REPO_URL, DVC_STUDIO_TOKEN
@@ -186,6 +187,30 @@ def test_post_to_studio_done_only_once(tmp_dir, mocked_dvc_repo, mocked_studio_p
186187
assert expected_done_calls == actual_done_calls
187188

188189

190+
def test_studio_updates_posted_on_end(tmp_path, mocked_dvc_repo, mocked_studio_post):
191+
mocked_post, valid_response = mocked_studio_post
192+
metrics_file = tmp_path / "metrics.json"
193+
metrics_content = "metrics"
194+
195+
def long_post(*args, **kwargs):
196+
# in case of `data` `long_post` should be called from a separate thread,
197+
# meanwhile main thread go forward without slowing down, so if there is no
198+
# some kind of wait in the Live main thread, then it will complete before
199+
# we even can have a chance to write the file below
200+
if kwargs["json"]["type"] == "data":
201+
time.sleep(1)
202+
metrics_file.write_text(metrics_content)
203+
204+
return valid_response
205+
206+
mocked_post.side_effect = long_post
207+
208+
with Live() as live:
209+
live.log_metric("foo", 1)
210+
211+
assert metrics_file.read_text() == metrics_content
212+
213+
189214
@pytest.mark.studio()
190215
def test_post_to_studio_skip_start_and_done_on_env_var(
191216
tmp_dir, mocked_dvc_repo, mocked_studio_post, monkeypatch

0 commit comments

Comments
 (0)