"...composable_kernel_rocm.git" did not exist on "e9d4e893e589ca2e2588365bee4ff3fe59408ce6"
Commit 17b131a2 authored by Jonas Kaufmann's avatar Jonas Kaufmann Committed by Antoine Kaufmann
Browse files

orchestration: serialize and deserialize via file interface

Instead of first converting to strings. Should reduce memory consumption when
dealing with large logs.
parent cc465599
...@@ -21,6 +21,7 @@ ...@@ -21,6 +21,7 @@
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import json import json
import pathlib
import time import time
from simbricks.orchestration.experiments import Experiment from simbricks.orchestration.experiments import Experiment
...@@ -60,9 +61,12 @@ class ExpOutput(object): ...@@ -60,9 +61,12 @@ class ExpOutput(object):
} }
self.sims[sim.full_name()] = obj self.sims[sim.full_name()] = obj
def dumps(self): def dump(self, outpath: str):
return json.dumps(self.__dict__) pathlib.Path(outpath).parent.mkdir(parents=True, exist_ok=True)
with open(outpath, 'w', encoding='utf-8') as file:
json.dump(self.__dict__, file)
def loads(self, json_s): def load(self, file: str):
for k, v in json.loads(json_s).items(): with open(file, 'r', encoding='utf-8') as fp:
self.__dict__[k] = v for k, v in json.load(fp).items():
self.__dict__[k] = v
...@@ -21,7 +21,6 @@ ...@@ -21,7 +21,6 @@
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import asyncio import asyncio
import pathlib
import typing as tp import typing as tp
from simbricks.orchestration import proxy from simbricks.orchestration import proxy
...@@ -76,9 +75,7 @@ class DistributedSimpleRuntime(Runtime): ...@@ -76,9 +75,7 @@ class DistributedSimpleRuntime(Runtime):
print( print(
f'Writing collected output of run {run.name()} to JSON file ...' f'Writing collected output of run {run.name()} to JSON file ...'
) )
pathlib.Path(run.outpath).parent.mkdir(parents=True, exist_ok=True) run.output.dump(run.outpath)
with open(run.outpath, 'w', encoding='utf-8') as f:
f.write(run.output.dumps())
async def start(self): async def start(self):
for run in self.runnable: for run in self.runnable:
......
...@@ -21,7 +21,6 @@ ...@@ -21,7 +21,6 @@
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import asyncio import asyncio
import pathlib
import typing as tp import typing as tp
from simbricks.orchestration import exectools from simbricks.orchestration import exectools
...@@ -68,9 +67,7 @@ class LocalSimpleRuntime(Runtime): ...@@ -68,9 +67,7 @@ class LocalSimpleRuntime(Runtime):
print( print(
f'Writing collected output of run {run.name()} to JSON file ...' f'Writing collected output of run {run.name()} to JSON file ...'
) )
pathlib.Path(run.outpath).parent.mkdir(parents=True, exist_ok=True) run.output.dump(run.outpath)
with open(run.outpath, 'w', encoding='utf-8') as f:
f.write(run.output.dumps())
async def start(self): async def start(self):
"""Execute the runs defined in `self.runnable`.""" """Execute the runs defined in `self.runnable`."""
...@@ -144,9 +141,7 @@ class LocalParallelRuntime(Runtime): ...@@ -144,9 +141,7 @@ class LocalParallelRuntime(Runtime):
print( print(
f'Writing collected output of run {run.name()} to JSON file ...' f'Writing collected output of run {run.name()} to JSON file ...'
) )
pathlib.Path(run.outpath).parent.mkdir(parents=True, exist_ok=True) run.output.dump(run.outpath)
with open(run.outpath, 'w', encoding='utf-8') as f:
f.write(run.output.dumps())
print('finished run ', run.name()) print('finished run ', run.name())
return run return run
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment