Bug Fix - Fix race condition issue for multi ranks (#117)

Fix race condition issue when multi ranks rotating the same directory.

superbench/common/utils/file_handler.py

@@ -23,7 +23,7 @@ def rotate_dir(target_dir):
         if target_dir.is_dir() and any(target_dir.iterdir()):
             logger.warning('Directory %s is not empty.', str(target_dir))
             for i in itertools.count(start=1):
-                backup_dir = target_dir.with_name(f'{target_dir.name}.{i}')
+                backup_dir = target_dir.with_name(f'{target_dir.name}.bak{i}')
                 if not backup_dir.is_dir():
                     target_dir.rename(backup_dir)
                     break

superbench/executor/executor.py

@@ -131,9 +131,8 @@ def __get_benchmark_dir(self, benchmark_name):
         benchmark_output_dir = self._output_path / 'benchmarks' / benchmark_name
         for rank_env in ['PROC_RANK', 'LOCAL_RANK']:
             if os.getenv(rank_env):
-                benchmark_output_dir /= 'rank{}'.format(os.getenv(rank_env))
-                break
-        return benchmark_output_dir
+                return benchmark_output_dir / 'rank{}'.format(os.getenv(rank_env))
+        return benchmark_output_dir / 'rank0'
 
     def __create_benchmark_dir(self, benchmark_name):
         """Create output directory for benchmark.
@@ -141,7 +140,7 @@ def __create_benchmark_dir(self, benchmark_name):
         Args:
             benchmark_name (str): Benchmark name.
         """
-        rotate_dir(self._output_path / 'benchmarks' / benchmark_name)
+        rotate_dir(self.__get_benchmark_dir(benchmark_name))
         try:
             self.__get_benchmark_dir(benchmark_name).mkdir(mode=0o755, parents=True, exist_ok=True)
         except Exception:

tests/executor/test_executor.py

@@ -92,7 +92,7 @@ def test_get_arguments(self):
 
     def test_create_benchmark_dir(self):
         """Test __create_benchmark_dir."""
-        foo_path = Path(self.sb_output_dir, 'benchmarks', 'foo')
+        foo_path = Path(self.sb_output_dir, 'benchmarks', 'foo', 'rank0')
         self.executor._SuperBenchExecutor__create_benchmark_dir('foo')
         self.assertTrue(foo_path.is_dir())
         self.assertFalse(any(foo_path.iterdir()))
@@ -102,20 +102,20 @@ def test_create_benchmark_dir(self):
         self.assertTrue(foo_path.is_dir())
         self.assertFalse(any(foo_path.iterdir()))
         self.assertFalse((foo_path / 'bar.txt').is_file())
-        self.assertTrue(foo_path.with_name('foo.1').is_dir())
-        self.assertTrue((foo_path.with_name('foo.1') / 'bar.txt').is_file())
+        self.assertTrue(foo_path.with_name('rank0.bak1').is_dir())
+        self.assertTrue((foo_path.with_name('rank0.bak1') / 'bar.txt').is_file())
 
         (foo_path / 'bar.json').touch()
         self.executor._SuperBenchExecutor__create_benchmark_dir('foo')
         self.assertTrue(foo_path.is_dir())
         self.assertFalse(any(foo_path.iterdir()))
         self.assertFalse((foo_path / 'bar.json').is_file())
-        self.assertTrue(foo_path.with_name('foo.2').is_dir())
-        self.assertTrue((foo_path.with_name('foo.2') / 'bar.json').is_file())
+        self.assertTrue(foo_path.with_name('rank0.bak2').is_dir())
+        self.assertTrue((foo_path.with_name('rank0.bak2') / 'bar.json').is_file())
 
     def test_write_benchmark_results(self):
         """Test __write_benchmark_results."""
-        foobar_path = Path(self.sb_output_dir, 'benchmarks', 'foobar')
+        foobar_path = Path(self.sb_output_dir, 'benchmarks', 'foobar', 'rank0')
         foobar_results_path = foobar_path / 'results.json'
         self.executor._SuperBenchExecutor__create_benchmark_dir('foobar')
         foobar_results = {
@@ -144,5 +144,5 @@ def test_exec_default_benchmarks(self, mock_exec_benchmark):
 
         self.assertTrue(Path(self.sb_output_dir, 'benchmarks').is_dir())
         for benchmark_name in self.executor._sb_benchmarks:
-            self.assertTrue(Path(self.sb_output_dir, 'benchmarks', benchmark_name).is_dir())
-            self.assertTrue(Path(self.sb_output_dir, 'benchmarks', benchmark_name, 'results.json').is_file())
+            self.assertTrue(Path(self.sb_output_dir, 'benchmarks', benchmark_name, 'rank0').is_dir())
+            self.assertTrue(Path(self.sb_output_dir, 'benchmarks', benchmark_name, 'rank0', 'results.json').is_file())