CarGDev/sheepOp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fixing memory

Browse Source
This commit is contained in:
Carlos Gutierrez
2025-11-16 16:52:26 -05:00
parent 9f17e1db24
commit fb1ca67be9
1 changed files with 76 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
training/__init__.py
View File

@@ -131,6 +131,7 @@ class Trainer:
disable=False, # Explicitly enable disable=False, # Explicitly enable
) )
try:
for batch_idx, batch in enumerate(progress_bar): for batch_idx, batch in enumerate(progress_bar):
input_ids = batch['input_ids'].to(self.device) input_ids = batch['input_ids'].to(self.device)
labels = batch['labels'].to(self.device) labels = batch['labels'].to(self.device)
@@ -191,7 +192,7 @@ class Trainer:
# Logging # Logging
if self.global_step % self.log_interval == 0: if self.global_step % self.log_interval == 0:
avg_loss = total_loss / num_batches avg_loss = total_loss / num_batches if num_batches > 0 else 0.0
lr = self.optimizer.param_groups[0]['lr'] lr = self.optimizer.param_groups[0]['lr']
progress_bar.set_postfix({ progress_bar.set_postfix({
'loss': f'{avg_loss:.4f}', 'loss': f'{avg_loss:.4f}',
@@ -207,15 +208,18 @@ class Trainer:
train_loss=avg_loss, train_loss=avg_loss,
lr=lr, lr=lr,
) )
except KeyboardInterrupt:
# Re-raise to be handled by outer try-except
raise
# Evaluation # Evaluation (only reached if no interruption)
if self.val_loader is not None and self.global_step % self.eval_interval == 0: if self.val_loader is not None and self.global_step % self.eval_interval == 0:
val_loss = self.evaluate() val_loss = self.evaluate()
if val_loss < self.best_val_loss: if val_loss < self.best_val_loss:
self.best_val_loss = val_loss self.best_val_loss = val_loss
self.save_checkpoint(is_best=True) self.save_checkpoint(is_best=True)
avg_loss = total_loss / num_batches avg_loss = total_loss / num_batches if num_batches > 0 else 0.0
# Log epoch metrics # Log epoch metrics
self.metrics.log( self.metrics.log(