"git@developer.sourcefind.cn:yangql/googletest.git" did not exist on "edbcd6294e399be1ec2400c33b9d3aa9f6dbbf85"
Unverified Commit 8032b8d2 authored by Michael Baumgartner's avatar Michael Baumgartner Committed by GitHub
Browse files

make errors more general

parent 2aa85703
......@@ -191,10 +191,11 @@ class MemoryEstimatorDetection(MemoryEstimator):
device = torch.device("cuda", self.gpu_id)
logger.info(f"Estimating on {device} with shape {shape} and "
f"batch size {self.batch_size} and num_instances {num_instances}")
loss = None
opt = None
inp = None
block_tensor = None
try:
loss = None
opt = None
inp = None
with cudnn_deterministic():
torch.cuda.reset_peak_memory_stats()
network = network.to(device)
......@@ -237,17 +238,16 @@ class MemoryEstimatorDetection(MemoryEstimator):
scaler.step(opt)
scaler.update()
dyn_mem = torch.cuda.memory_reserved()
except (RuntimeError,) as e:
except Exception as e:
logger.info(f"Caught error (If out of memory error do not worry): {e}")
empty_mem = 0
fixed_mem = float('Inf')
dyn_mem = float('Inf')
finally:
del loss
del opt
del inp
del block_tensor
del opt
del inp
del block_tensor
network.cpu()
torch.cuda.empty_cache()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment