Skip to content

GitLab

Unverified Commit c1e66968 authored by Jakob Görgen's avatar Jakob Görgen
Browse files

minor fixes + cleanups

parent 11cb92b8
Hide whitespace changes
Inline Side-by-side
Showing with 181 additions and 171 deletions
experiments/simbricks/orchestration/system/base.py
View file @ c1e66968
......@@ -20,36 +20,36 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
from abc import (abstractmethod, ABC)
import typing as tp
import abc
class System():
""" Defines System configuration of the whole simulation """
class System:
"""Defines System configuration of the whole simulation"""
def __init__(self) -> None:
self.hosts: tp.List[Component] = []
self.hosts: list[Component] = []
def add_component(self, c: Component) -> None:
assert c.system == self
class Component(ABC):
class Component(abc.ABC):
def __init__(self, s: System) -> None:
s.system = s
s.add_component(self)
@abstractmethod
def interfaces(self) -> tp.List[Interface]:
return None
@abc.abstractmethod
def interfaces(self) -> list[Interface]:
return []
def channels(self) -> tp.List[Channel]:
def channels(self) -> list[Channel]:
return [i.channel for i in self.interfaces() if i.is_connected()]
class Interface(ABC):
class Interface(abc.ABC):
def __init__(self, c: Component) -> None:
self.component = c
self.channel: tp.Optional[Channel] = None
self.channel: Channel | None = None
def is_connected(self) -> bool:
return self.channel is not None
......@@ -62,17 +62,17 @@ class Interface(ABC):
self.channel = c
class Channel(ABC):
class Channel(abc.ABC):
def __init__(self, a: Interface, b: Interface) -> None:
self.latency = 500
self.a: Interface = a
self.b: Interface = b
def interfaces(self) -> tp.List[Interface]:
def interfaces(self) -> list[Interface]:
return [self.a, self.b]
def disconnect(self):
# Note AK: this is a bit ugly, this leaves the channel dangling. But
# it's not referenced anywhere, so that's fine I guess.
self.a.disconnect()
self.b.disconnect()
\ No newline at end of file
self.b.disconnect()
experiments/simbricks/orchestration/system/eth.py
View file @ c1e66968
......@@ -20,9 +20,7 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import typing as tp
import simbricks.orchestration.system.base as base
from simbricks.orchestration.system import base
class EthInterface(base.Interface):
......@@ -50,7 +48,7 @@ class EthSimpleNIC(base.Component):
class EthSwitch(base.Component):
def __init__(self, s: base.System) -> None:
super().__init__(s)
self.eth_ifs: tp.List[EthInterface] = []
self.eth_ifs: list[EthInterface] = []
def if_add(self, i: EthInterface) -> None:
self.eth_ifs.append(i)
\ No newline at end of file
experiments/simbricks/orchestration/system/host/__init__.py
View file @ c1e66968
......@@ -19,98 +19,3 @@
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import typing as tp
from abc import (ABC)
import os.path
import simbricks.orchestration.system.base as base
import simbricks.orchestration.system.eth as eth
import simbricks.orchestration.system.mem as mem
import simbricks.orchestration.system.pcie as pcie
from simbricks.orchestration.system.host.disk_images import *
from simbricks.orchestration.system.host.app import *
class Host(base.Component):
def __init__(self, s: base.System):
super().__init__(s)
self.ifs: tp.List[pcie.PCIeHostInterface] = []
self.applications: tp.List[Application]
def interfaces(self) -> tp.List[base.Interface]:
return self.pcie_ifs + self.eth_ifs + self.mem_ifs
def add_if(self, i: base.Interface) -> None:
self.ifs.append(i)
def add_app(self, a: Application) -> None:
self.applications.append(a)
class FullSystemHost(Host):
def __init__(self, s: base.System) -> None:
super().__init__(s)
self.memory = 512
self.cores = 1
self.cpu_freq = '3GHz'
self.disks: tp.List[DiskImage] = []
def add_disk(self, disk: DiskImage) -> None:
self.disks.append(disk)
class LinuxHost(FullSystemHost):
def __init__(self, s: base.System) -> None:
super().__init__(s)
self.applications: tp.List[LinuxApplication] = []
self.load_modules = []
self.kcmd_append = ''
def add_app(self, a: LinuxApplication) -> None:
self.applications.append(a)
def run_cmds(self, env: expenv.ExpEnv) -> tp.List[str]:
"""Commands to run on node."""
return self.app.run_cmds(self)
def cleanup_cmds(self, env: expenv.ExpEnv) -> tp.List[str]:
"""Commands to run to cleanup node."""
return []
def config_files(self, env: expenv.ExpEnv) -> tp.Dict[str, tp.IO]:
"""
Additional files to put inside the node, which are mounted under
`/tmp/guest/`.
Specified in the following format: `filename_inside_node`:
`IO_handle_of_file`
"""
cfg_files = {}
for app in self.applications:
cfg_files |= self.app.config_files(env)
return cfg_files
def prepare_pre_cp(self, env: expenv.ExpEnv) -> tp.List[str]:
"""Commands to run to prepare node before checkpointing."""
return [
'set -x',
'export HOME=/root',
'export LANG=en_US',
'export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:' + \
'/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"'
]
def prepare_post_cp(self, env: expenv.ExpEnv) -> tp.List[str]:
"""Commands to run to prepare node after checkpoint restore."""
return []
def strfile(self, s: str) -> io.BytesIO:
"""
Helper function to convert a string to an IO handle for usage in
`config_files()`.
Using this, you can create a file with the string as its content on the
simulated node.
"""
return io.BytesIO(bytes(s, encoding='UTF-8'))
\ No newline at end of file
experiments/simbricks/orchestration/system/host/app.py
View file @ c1e66968
......@@ -21,34 +21,32 @@
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import typing as tp
from abc import (ABC, abstractmethod)
import abc
import io
from simbricks.orchestration.system.host import base
from simbricks.orchestration.experiment import experiment_environment as expenv
if tp.TYPE_CHECKING: # prevent cyclic import
import simbricks.orchestration.system.host as host
import simbricks.orchestration.experiment.experiment_environment as expenv
class Application(ABC):
class Application(abc.ABC):
def __init__(self, h: host.Host) -> None:
self.host = h
# Note AK: Maybe we can factor most of the duplicate calls with the host out
# into a separate module.
class LinuxApplication(ABC):
class LinuxApplication(abc.ABC):
def __init__(self, h: host.LinuxHost) -> None:
self.host = h
def run_cmds(self, env: expenv.ExpEnv) -> tp.List[str]:
def run_cmds(self, env: expenv.ExpEnv) -> list[str]:
"""Commands to run on node."""
return self.app.run_cmds(self)
def cleanup_cmds(self, env: expenv.ExpEnv) -> tp.List[str]:
def cleanup_cmds(self, env: expenv.ExpEnv) -> list[str]:
"""Commands to run to cleanup node."""
return []
def config_files(self, env: expenv.ExpEnv) -> tp.Dict[str, tp.IO]:
def config_files(self, env: expenv.ExpEnv) -> dict[str, tp.IO]:
"""
Additional files to put inside the node, which are mounted under
`/tmp/guest/`.
......@@ -58,17 +56,17 @@ class LinuxApplication(ABC):
"""
return {}
def prepare_pre_cp(self, env: expenv.ExpEnv) -> tp.List[str]:
def prepare_pre_cp(self, env: expenv.ExpEnv) -> list[str]:
"""Commands to run to prepare node before checkpointing."""
return [
'set -x',
'export HOME=/root',
'export LANG=en_US',
'export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:' + \
'/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"'
"set -x",
"export HOME=/root",
"export LANG=en_US",
'export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:'
+ '/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"',
]
def prepare_post_cp(self, env: expenv.ExpEnv) -> tp.List[str]:
def prepare_post_cp(self, env: expenv.ExpEnv) -> list[str]:
"""Commands to run to prepare node after checkpoint restore."""
return []
......@@ -80,4 +78,4 @@ class LinuxApplication(ABC):
Using this, you can create a file with the string as its content on the
simulated node.
"""
return io.BytesIO(bytes(s, encoding='UTF-8'))
\ No newline at end of file
return io.BytesIO(bytes(s, encoding="UTF-8"))
experiments/simbricks/orchestration/system/host/base.py 0 → 100644
View file @ c1e66968
# Copyright 2024 Max Planck Institute for Software Systems, and
# National University of Singapore
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import typing as tp
import io
from os import path
from simbricks.orchestration.system import base
from simbricks.orchestration.system import eth
from simbricks.orchestration.system import mem
from simbricks.orchestration.system import pcie
from simbricks.orchestration.system.host import disk_images
from simbricks.orchestration.system.host import app
class Host(base.Component):
def __init__(self, s: base.System):
super().__init__(s)
self.ifs: list[pcie.PCIeHostInterface] = []
self.applications: list[Application]
def interfaces(self) -> list[base.Interface]:
return self.pcie_ifs + self.eth_ifs + self.mem_ifs
def add_if(self, i: base.Interface) -> None:
self.ifs.append(i)
def add_app(self, a: Application) -> None:
self.applications.append(a)
class FullSystemHost(Host):
def __init__(self, s: base.System) -> None:
super().__init__(s)
self.memory = 512
self.cores = 1
self.cpu_freq = '3GHz'
self.disks: list[disk_images.DiskImage] = []
def add_disk(self, disk: DiskImage) -> None:
self.disks.append(disk)
class LinuxHost(FullSystemHost):
def __init__(self, s: base.System) -> None:
super().__init__(s)
self.applications: list[LinuxApplication] = []
self.load_modules = []
self.kcmd_append = ''
def add_app(self, a: LinuxApplication) -> None:
self.applications.append(a)
def run_cmds(self, env: expenv.ExpEnv) -> list[str]:
"""Commands to run on node."""
return self.app.run_cmds(self)
def cleanup_cmds(self, env: expenv.ExpEnv) -> list[str]:
"""Commands to run to cleanup node."""
return []
def config_files(self, env: expenv.ExpEnv) -> dict[str, tp.IO]:
"""
Additional files to put inside the node, which are mounted under
`/tmp/guest/`.
Specified in the following format: `filename_inside_node`:
`IO_handle_of_file`
"""
cfg_files = {}
for app in self.applications:
cfg_files |= self.app.config_files(env)
return cfg_files
def prepare_pre_cp(self, env: expenv.ExpEnv) -> list[str]:
"""Commands to run to prepare node before checkpointing."""
return [
'set -x',
'export HOME=/root',
'export LANG=en_US',
'export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:' + \
'/usr/bin:/sbin:/bin:/usr/games:/usr/local/games"'
]
def prepare_post_cp(self, env: expenv.ExpEnv) -> list[str]:
"""Commands to run to prepare node after checkpoint restore."""
return []
def strfile(self, s: str) -> io.BytesIO:
"""
Helper function to convert a string to an IO handle for usage in
`config_files()`.
Using this, you can create a file with the string as its content on the
simulated node.
"""
return io.BytesIO(bytes(s, encoding='UTF-8'))
\ No newline at end of file
experiments/simbricks/orchestration/system/host/disk_images.py
View file @ c1e66968
......@@ -20,24 +20,21 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import typing as tp
from abc import (ABC, abstractmethod)
import abc
import os.path
from simbricks.orchestration.system.host import base
from simbricks.orchestration.experiment import experiment_environment as expenv
if tp.TYPE_CHECKING: # prevent cyclic import
import simbricks.orchestration.system.host as host
import simbricks.orchestration.experiment.experiment_environment as expenv
class DiskImage(ABC):
class DiskImage(abc.ABC):
def __init__(self, h: host.Host) -> None:
self.host = h
@abstractmethod
def available_formats(self) -> tp.List[str]:
@abc.abstractmethod
def available_formats(self) -> list[str]:
return []
@abstractmethod
@abc.abstractmethod
async def prepare_image_path(self, env: expenv.ExpEnv, format: str) -> str:
pass
......@@ -47,9 +44,9 @@ class ExternalDiskImage(DiskImage):
def __init__(self, h: host.FullSystemHost, path: str) -> None:
super().__init__(h)
self.path = path
self.formats = ['raw', 'qcow2']
self.formats = ["raw", "qcow2"]
def available_formats(self) -> tp.List[str]:
def available_formats(self) -> list[str]:
return self.formats
async def prepare_image_path(self, env: expenv.ExpEnv, format: str) -> str:
......@@ -62,19 +59,19 @@ class DistroDiskImage(DiskImage):
def __init__(self, h: host.FullSystemHost, name: str) -> None:
super().__init__(h)
self.name = name
self.formats = ['raw', 'qcow2']
self.formats = ["raw", "qcow2"]
def available_formats(self) -> tp.List[str]:
def available_formats(self) -> list[str]:
return self.formats
async def prepare_image_path(self, env: expenv.ExpEnv, format: str) -> str:
path = env.hd_path(self.name)
if format == 'raw':
path += '.raw'
elif format == 'qcow':
if format == "raw":
path += ".raw"
elif format == "qcow":
pass
else:
raise RuntimeError('Unsupported disk format')
raise RuntimeError("Unsupported disk format")
assert os.path.isfile(self.path)
return self.path
......@@ -84,8 +81,8 @@ class LinuxConfigDiskImage(DiskImage):
def __init__(self, h: host.LinuxHost) -> None:
super().__init__(h)
def available_formats(self) -> tp.List[str]:
return ['raw']
def available_formats(self) -> list[str]:
return ["raw"]
async def prepare_image_path(self, env: expenv.ExpEnv, format: str) -> str:
# TODO: build tar from host path parameters and then return path
......@@ -100,9 +97,9 @@ class PackerDiskImage(DiskImage):
super().__init__(h)
self.config_path = packer_config_path
def available_formats(self) -> tp.List[str]:
return ['raw', 'qcow']
def available_formats(self) -> list[str]:
return ["raw", "qcow"]
async def prepare_image_path(self, env: expenv.ExpEnv, format: str) -> str:
# TODO: invoke packer to build the image if necessary
pass
\ No newline at end of file
pass
experiments/simbricks/orchestration/system/mem.py
View file @ c1e66968
......@@ -20,9 +20,7 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import typing as tp
import simbricks.orchestration.system.base as base
from simbricks.orchestration.system import base
class MemHostInterface(base.Interface):
......@@ -59,5 +57,5 @@ class MemSimpleDevice(base.Component):
super().__init__(s)
self.mem_if = MemDeviceInterface()
def interfaces(self) -> tp.List[base.Interface]:
def interfaces(self) -> list[base.Interface]:
return [self.mem_if]
\ No newline at end of file
experiments/simbricks/orchestration/system/nic.py
View file @ c1e66968
......@@ -20,28 +20,29 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import typing as tp
import simbricks.orchestration.system.base as base
import simbricks.orchestration.system.pcie as pcie
import simbricks.orchestration.system.eth as eth
from simbricks.orchestration.system import base
from simbricks.orchestration.system import pcie
from simbricks.orchestration.system import eth
class SimplePCIeNIC(pcie.PCIeSimpleDevice, eth.EthSimpleNIC):
def __init__(self, s: base.System) -> None:
super().__init__(s)
def interfaces(self) -> tp.List[base.Interface]:
def interfaces(self) -> list[base.Interface]:
return [self.pci_if, self.eth_if]
class IntelI40eNIC(SimplePCIeNIC):
pass
def __init__(self, s: base.System) -> None:
super().__init__(s)
class IntelE1000NIC(SimplePCIeNIC):
pass
def __init__(self, s: base.System) -> None:
super().__init__(s)
class CorundumNIC(SimplePCIeNIC):
pass
\ No newline at end of file
def __init__(self, s: base.System) -> None:
super().__init__(s)
experiments/simbricks/orchestration/system/pcie.py
View file @ c1e66968
......@@ -20,9 +20,7 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import typing as tp
import simbricks.orchestration.system.base as base
from simbricks.orchestration.system import base
class PCIeHostInterface(base.Interface):
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment