Unverified Commit 50748503 authored by Adrià Arrufat's avatar Adrià Arrufat Committed by GitHub
Browse files

fix backtracking when losses stay at inf (fixes #2206) (#2209)

* fix backtracking when losses stay at inf

* always backtrack when there is an inf value
parent a1f15837
...@@ -1113,10 +1113,10 @@ namespace dlib ...@@ -1113,10 +1113,10 @@ namespace dlib
while (previous_loss_values_to_keep_until_disk_sync.size() > 2 * gradient_updates_since_last_sync) while (previous_loss_values_to_keep_until_disk_sync.size() > 2 * gradient_updates_since_last_sync)
previous_loss_values_to_keep_until_disk_sync.pop_front(); previous_loss_values_to_keep_until_disk_sync.pop_front();
// Always retry if there are any nan values // Always retry if there are any nan or inf values
for (auto x : previous_loss_values_to_keep_until_disk_sync) for (auto x : previous_loss_values_to_keep_until_disk_sync)
{ {
if (std::isnan(x)) if (std::isnan(x) || std::isinf(x))
return true; return true;
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment