Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix the number of training batches used in the training loop #653

Merged
merged 6 commits into from
Jan 5, 2020
+9 −10
Merged

Fix the number of training batches used in the training loop #653

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
63c937a
Fix the number of processed training batches
kuynzereb Dec 27, 2019
693ec81
Fix tests
kuynzereb Dec 27, 2019
3fb3961
fix tests
kuynzereb Dec 27, 2019
f49aee4
fix tests
kuynzereb Dec 27, 2019
95a46cd
One more attempt
kuynzereb Dec 27, 2019
d1fa76d
Fix another test
kuynzereb Dec 27, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 4 additions & 5 deletions pytorch_lightning/trainer/training_loop.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -367,6 +367,10 @@ def run_training_epoch(self):

# run epoch
for batch_idx, batch in enumerate(self.get_train_dataloader()):
# stop epoch if we limited the number of training batches
if batch_idx >= self.num_training_batches:
break

self.batch_idx = batch_idx

model = self.get_model()
Expand Down Expand Up @@ -413,11 +417,6 @@ def run_training_epoch(self):
if early_stop_epoch or self.fast_dev_run:
break

# stop epoch if we limited the number of training batches
met_batch_limit = batch_idx >= self.num_training_batches
if met_batch_limit:
break

# epoch end hook
if self.is_function_implemented('on_epoch_end'):
model = self.get_model()
Expand Down
2 changes: 1 addition & 1 deletion tests/test_logging.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,7 +309,7 @@ def version(self):

trainer_options = dict(
max_epochs=1,
train_percent_check=0.01,
train_percent_check=0.05,
logger=logger,
default_save_path=tmpdir
)
Expand Down
6 changes: 3 additions & 3 deletions tests/test_restore_models.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,12 +269,12 @@ def test_cpu_restore_training(tmpdir):
logger = tutils.get_test_tube_logger(tmpdir, False, version=test_logger_version)

trainer_options = dict(
max_epochs=2,
max_epochs=4,
val_check_interval=0.50,
val_percent_check=0.2,
train_percent_check=0.2,
logger=logger,
checkpoint_callback=ModelCheckpoint(tmpdir)
checkpoint_callback=ModelCheckpoint(tmpdir, save_top_k=-1)
)

# fit model
Expand All @@ -290,7 +290,7 @@ def test_cpu_restore_training(tmpdir):
# we want to see if the weights come back correctly
new_logger = tutils.get_test_tube_logger(tmpdir, False, version=test_logger_version)
trainer_options = dict(
max_epochs=2,
max_epochs=4,
val_check_interval=0.50,
val_percent_check=0.2,
train_percent_check=0.2,
Expand Down
2 changes: 1 addition & 1 deletion tests/test_trainer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -393,7 +393,7 @@ class CurrentTestModel(
default_save_path=tmpdir,
max_epochs=1,
val_percent_check=0.1,
train_percent_check=0.1,
train_percent_check=0.2,
)

# fit model
Expand Down