Unverified Commit 27b757ec authored by Yifan Xiong's avatar Yifan Xiong Committed by GitHub
Browse files

Bug Fix - Fix race condition issue for multi ranks (#117)

Fix race condition issue when multi ranks rotating the same directory.
parent 43620c3f
......@@ -23,7 +23,7 @@ def rotate_dir(target_dir):
if target_dir.is_dir() and any(target_dir.iterdir()):
logger.warning('Directory %s is not empty.', str(target_dir))
for i in itertools.count(start=1):
backup_dir = target_dir.with_name(f'{target_dir.name}.{i}')
backup_dir = target_dir.with_name(f'{target_dir.name}.bak{i}')
if not backup_dir.is_dir():
target_dir.rename(backup_dir)
break
......
......@@ -131,9 +131,8 @@ def __get_benchmark_dir(self, benchmark_name):
benchmark_output_dir = self._output_path / 'benchmarks' / benchmark_name
for rank_env in ['PROC_RANK', 'LOCAL_RANK']:
if os.getenv(rank_env):
benchmark_output_dir /= 'rank{}'.format(os.getenv(rank_env))
break
return benchmark_output_dir
return benchmark_output_dir / 'rank{}'.format(os.getenv(rank_env))
return benchmark_output_dir / 'rank0'
def __create_benchmark_dir(self, benchmark_name):
"""Create output directory for benchmark.
......@@ -141,7 +140,7 @@ def __create_benchmark_dir(self, benchmark_name):
Args:
benchmark_name (str): Benchmark name.
"""
rotate_dir(self._output_path / 'benchmarks' / benchmark_name)
rotate_dir(self.__get_benchmark_dir(benchmark_name))
try:
self.__get_benchmark_dir(benchmark_name).mkdir(mode=0o755, parents=True, exist_ok=True)
except Exception:
......
......@@ -92,7 +92,7 @@ def test_get_arguments(self):
def test_create_benchmark_dir(self):
"""Test __create_benchmark_dir."""
foo_path = Path(self.sb_output_dir, 'benchmarks', 'foo')
foo_path = Path(self.sb_output_dir, 'benchmarks', 'foo', 'rank0')
self.executor._SuperBenchExecutor__create_benchmark_dir('foo')
self.assertTrue(foo_path.is_dir())
self.assertFalse(any(foo_path.iterdir()))
......@@ -102,20 +102,20 @@ def test_create_benchmark_dir(self):
self.assertTrue(foo_path.is_dir())
self.assertFalse(any(foo_path.iterdir()))
self.assertFalse((foo_path / 'bar.txt').is_file())
self.assertTrue(foo_path.with_name('foo.1').is_dir())
self.assertTrue((foo_path.with_name('foo.1') / 'bar.txt').is_file())
self.assertTrue(foo_path.with_name('rank0.bak1').is_dir())
self.assertTrue((foo_path.with_name('rank0.bak1') / 'bar.txt').is_file())
(foo_path / 'bar.json').touch()
self.executor._SuperBenchExecutor__create_benchmark_dir('foo')
self.assertTrue(foo_path.is_dir())
self.assertFalse(any(foo_path.iterdir()))
self.assertFalse((foo_path / 'bar.json').is_file())
self.assertTrue(foo_path.with_name('foo.2').is_dir())
self.assertTrue((foo_path.with_name('foo.2') / 'bar.json').is_file())
self.assertTrue(foo_path.with_name('rank0.bak2').is_dir())
self.assertTrue((foo_path.with_name('rank0.bak2') / 'bar.json').is_file())
def test_write_benchmark_results(self):
"""Test __write_benchmark_results."""
foobar_path = Path(self.sb_output_dir, 'benchmarks', 'foobar')
foobar_path = Path(self.sb_output_dir, 'benchmarks', 'foobar', 'rank0')
foobar_results_path = foobar_path / 'results.json'
self.executor._SuperBenchExecutor__create_benchmark_dir('foobar')
foobar_results = {
......@@ -144,5 +144,5 @@ def test_exec_default_benchmarks(self, mock_exec_benchmark):
self.assertTrue(Path(self.sb_output_dir, 'benchmarks').is_dir())
for benchmark_name in self.executor._sb_benchmarks:
self.assertTrue(Path(self.sb_output_dir, 'benchmarks', benchmark_name).is_dir())
self.assertTrue(Path(self.sb_output_dir, 'benchmarks', benchmark_name, 'results.json').is_file())
self.assertTrue(Path(self.sb_output_dir, 'benchmarks', benchmark_name, 'rank0').is_dir())
self.assertTrue(Path(self.sb_output_dir, 'benchmarks', benchmark_name, 'rank0', 'results.json').is_file())
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment