add dtk-container-toolkit

internal/config/image/devices.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package image
+
+import "strings"
+
+// VisibleDevices represents the devices selected in a container image
+// through the DTK_VISIBLE_DEVICES or other environment variables
+type VisibleDevices interface {
+	List() []string
+	Has(string) bool
+}
+
+var _ VisibleDevices = (*all)(nil)
+var _ VisibleDevices = (*none)(nil)
+var _ VisibleDevices = (*void)(nil)
+var _ VisibleDevices = (*devices)(nil)
+
+// NewVisibleDevices creates a VisibleDevices based on the value of the specified envvar.
+func NewVisibleDevices(envvars ...string) VisibleDevices {
+	for _, envvar := range envvars {
+		if envvar == "all" {
+			return all{}
+		}
+		if envvar == "none" {
+			return none{}
+		}
+		if envvar == "" || envvar == "void" {
+			return void{}
+		}
+	}
+
+	return newDevices(envvars...)
+}
+
+type all struct{}
+
+// List returns ["all"] for all devices
+func (a all) List() []string {
+	return []string{"all"}
+}
+
+// Has for all devices is true for any id except the empty ID
+func (a all) Has(id string) bool {
+	return id != ""
+}
+
+type none struct{}
+
+// List returns [""] for the none devices
+func (n none) List() []string {
+	return []string{""}
+}
+
+// Has for none devices is false for any id
+func (n none) Has(string) bool {
+	return false
+}
+
+type void struct {
+	none
+}
+
+// List returns nil for the void devices
+func (v void) List() []string {
+	return nil
+}
+
+type devices struct {
+	len    int
+	lookup map[string]int
+}
+
+func newDevices(idOrCommaSeparated ...string) devices {
+	lookup := make(map[string]int)
+
+	i := 0
+	for _, commaSeparated := range idOrCommaSeparated {
+		for _, id := range strings.Split(commaSeparated, ",") {
+			lookup[id] = i
+			i++
+		}
+	}
+
+	d := devices{
+		len:    i,
+		lookup: lookup,
+	}
+	return d
+}
+
+// List returns the list of requested devices
+func (d devices) List() []string {
+	list := make([]string, d.len)
+
+	for id, i := range d.lookup {
+		list[i] = id
+	}
+	return list
+}
+
+// Has checks whether the specified ID is in the set of requested devices
+func (d devices) Has(id string) bool {
+	if id == "" {
+		return false
+	}
+
+	_, exist := d.lookup[id]
+	return exist
+}

internal/config/image/dtk_image.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package image
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"tags.cncf.io/container-device-interface/pkg/parser"
+)
+
+// DTK represents a DTK image that can be used for HCU computing. This wraps
+// a map of environment variable to values that can be used to perform lookups
+// such as requirements.
+type DTK struct {
+	env    map[string]string
+	mounts []specs.Mount
+}
+
+// NewDTKImageFromSpec creates a DTK image from the input OCI runtime spec.
+// The process environment is read (if present) to construct the DTK Image.
+func NewDTKImageFromSpec(spec *specs.Spec) (DTK, error) {
+	var env []string
+	if spec != nil && spec.Process != nil {
+		env = spec.Process.Env
+	}
+
+	return New(
+		WithEnv(env),
+		WithMounts(spec.Mounts),
+	)
+}
+
+// LoadHyhalMethod
+func (i DTK) LoadHyhalMethod(envVar string) bool {
+	if devs, ok := i.env[envVar]; ok {
+		if devs == "copy" {
+			return false
+		}
+	}
+	return true
+}
+
+func (i DTK) IsLegacy() bool {
+	rocm_version := i.env[EnvROCmVersion]
+	return len(rocm_version) > 0
+}
+
+// Getenv returns the value of the specified environment variable.
+// If the environment variable is not specified, an empty string is returned.
+func (i DTK) Getenv(key string) string {
+	return i.env[key]
+}
+
+// DevicesFromEnvvars returns the devices requested by the image through environment variables
+func (i DTK) DevicesFromEnvvars(envVars ...string) VisibleDevices {
+	// We concantenate all the devices from the specified env.
+	var isSet bool
+	var devices []string
+	requested := make(map[string]bool)
+	for _, envVar := range envVars {
+		if devs, ok := i.env[envVar]; ok {
+			isSet = true
+			for _, d := range strings.Split(devs, ",") {
+				trimmed := strings.TrimSpace(d)
+				if len(trimmed) == 0 || requested[trimmed] {
+					continue
+				}
+				devices = append(devices, trimmed)
+				requested[trimmed] = true
+			}
+		}
+	}
+
+	// Environment variable unset with legacy image: default to "all".
+	if !isSet && len(devices) == 0 && i.IsLegacy() {
+		return NewVisibleDevices("all")
+	}
+
+	// Environment variable unset or empty or "void": return nil
+	if len(devices) == 0 || requested["void"] {
+		return NewVisibleDevices("void")
+	}
+
+	return NewVisibleDevices(devices...)
+}
+
+// OnlyFullyQualifiedCDIDevices returns true if all devices requested in the image are requested as CDI devices/
+func (i DTK) OnlyFullyQualifiedCDIDevices() bool {
+	var hasCDIdevice bool
+	for _, device := range i.DevicesFromEnvvars(EnvVarDTKVisibleDevices, EnvVarNvidiaVisibleDevices).List() {
+		if !parser.IsQualifiedName(device) {
+			return false
+		}
+		hasCDIdevice = true
+	}
+
+	for _, device := range i.DevicesFromMounts() {
+		if !strings.HasPrefix(device, "cdi/") {
+			return false
+		}
+		hasCDIdevice = true
+	}
+	return hasCDIdevice
+}
+
+const (
+	deviceListAsVolumeMountsRoot = "/var/run/dtk-container-devices"
+)
+
+// DevicesFromMounts returns a list of device specified as mounts.
+// TODO: This should be merged with getDevicesFromMounts used in the DTK Container Runtime
+func (i DTK) DevicesFromMounts() []string {
+	root := filepath.Clean(deviceListAsVolumeMountsRoot)
+	seen := make(map[string]bool)
+	var devices []string
+	for _, m := range i.mounts {
+		source := filepath.Clean(m.Source)
+		// Only consider mounts who's host volume is /dev/null
+		if source != "/dev/null" {
+			continue
+		}
+
+		destination := filepath.Clean(m.Destination)
+		if seen[destination] {
+			continue
+		}
+		seen[destination] = true
+
+		// Only consider container mount points that begin with 'root'
+		if !strings.HasPrefix(destination, root) {
+			continue
+		}
+
+		// Grab the full path beyond 'root' and add it to the list of devices
+		device := strings.Trim(strings.TrimPrefix(destination, root), "/")
+		if len(device) == 0 {
+			continue
+		}
+		devices = append(devices, device)
+	}
+	return devices
+}
+
+// CDIDevicesFromMounts returns a list of CDI devices specified as mounts on the image.
+func (i DTK) CDIDevicesFromMounts() []string {
+	var devices []string
+	for _, mountDevice := range i.DevicesFromMounts() {
+		if !strings.HasPrefix(mountDevice, "cdi/") {
+			continue
+		}
+		parts := strings.SplitN(strings.TrimPrefix(mountDevice, "cdi/"), "/", 3)
+		if len(parts) != 3 {
+			continue
+		}
+		vendor := parts[0]
+		class := parts[1]
+		device := parts[2]
+		devices = append(devices, fmt.Sprintf("%s/%s=%s", vendor, class, device))
+	}
+	return devices
+}
+
+// VisibleDevicesFromEnvVar returns the set of visible devices requested through
+// the DTK_VISIBLE_DEVICES environment variable.
+func (i DTK) VisibleDevicesFromEnvVar() []string {
+	return i.DevicesFromEnvvars(EnvVarDTKVisibleDevices, EnvVarNvidiaVisibleDevices).List()
+}

internal/config/image/envvars.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package image
+
+const (
+	EnvVarDTKVisibleDevices    = "DTK_VISIBLE_DEVICES"
+	EnvVarNvidiaVisibleDevices = "NVIDIA_VISIBLE_DEVICES"
+	EnvROCmVersion             = "ROCM_VERSION"
+)

internal/config/image/privileged.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package image
+
+import "github.com/opencontainers/runtime-spec/specs-go"
+
+const (
+	capSysAdmin = "CAP_SYS_ADMIN"
+)
+
+// IsPrivileged returns true if the container is a privileged container.
+func IsPrivileged(s *specs.Spec) bool {
+	if s.Process.Capabilities == nil {
+		return false
+	}
+
+	// We only make sure that the bounding capabibility set has
+	// CAP_SYS_ADMIN. This allows us to make sure that the container was
+	// actually started as '--privileged', but also allow non-root users to
+	// access the privileged DTK capabilities.
+	for _, c := range s.Process.Capabilities.Bounding {
+		if c == capSysAdmin {
+			return true
+		}
+	}
+	return false
+}

internal/config/runtime.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package config
+
+// RuntimeConfig stores the config options for the DTK Container Runtime
+type RuntimeConfig struct {
+	DebugFilePath string `toml:"debug"`
+	// LogLevel defines the logging level for the application
+	LogLevel string `toml:"log-level"`
+	// Runtimes defines the candidates for the low-level runtime
+	Runtimes []string    `toml:"runtimes"`
+	Mode     string      `toml:"mode"`
+	Modes    modesConfig `toml:"modes"`
+}
+
+// modesConfig defines (optional) per-mode configs
+type modesConfig struct {
+	CSV csvModeConfig `toml:"csv"`
+	CDI cdiModeConfig `toml:"cdi"`
+}
+
+type cdiModeConfig struct {
+	// SpecDirs allows for the default spec dirs for CDI to be overridden
+	SpecDirs []string `toml:"spec-dirs"`
+	// DefaultKind sets the default kind to be used when constructing fully-qualified CDI device names
+	DefaultKind string `toml:"default-kind"`
+	// AnnotationPrefixes sets the allowed prefixes for CDI annotation-based device injection
+	AnnotationPrefixes []string `toml:"annotation-prefixes"`
+}
+
+type csvModeConfig struct {
+	MountSpecPath string `toml:"mount-spec-path"`
+}
+
+// GetDefaultRuntimeConfig defines the default values for the config
+func GetDefaultRuntimeConfig() (*RuntimeConfig, error) {
+	cfg, err := GetDefault()
+	if err != nil {
+		return nil, err
+	}
+
+	return &cfg.DTKContainerRuntimeConfig, nil
+}

internal/config/toml.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package config
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os"
+	"regexp"
+
+	"github.com/pelletier/go-toml"
+)
+
+// Toml is a type for the TOML representation of a config.
+type Toml toml.Tree
+
+type options struct {
+	configFile string
+	required   bool
+}
+
+// Option is a functional option for loading TOML config files.
+type Option func(*options)
+
+// WithConfigFile sets the config file option.
+func WithConfigFile(configFile string) Option {
+	return func(o *options) {
+		o.configFile = configFile
+	}
+}
+
+// WithRequired sets the required option.
+// If this is set to true, a failure to open the specified file is treated as an error
+func WithRequired(required bool) Option {
+	return func(o *options) {
+		o.required = required
+	}
+}
+
+// New creates a new toml tree based on the provided options
+func New(opts ...Option) (*Toml, error) {
+	o := &options{}
+	for _, opt := range opts {
+		opt(o)
+	}
+
+	return o.loadConfigToml()
+}
+
+func (o options) loadConfigToml() (*Toml, error) {
+	filename := o.configFile
+	if filename == "" {
+		return defaultToml()
+	}
+
+	_, err := os.Stat(filename)
+	if os.IsNotExist(err) && o.required {
+		return nil, os.ErrNotExist
+	}
+
+	tomlFile, err := os.Open(filename)
+	if os.IsNotExist(err) {
+		return defaultToml()
+	} else if err != nil {
+		return nil, fmt.Errorf("failed to load specified config file: %w", err)
+	}
+	defer tomlFile.Close()
+
+	return loadConfigTomlFrom(tomlFile)
+
+}
+
+func defaultToml() (*Toml, error) {
+	cfg, err := GetDefault()
+	if err != nil {
+		return nil, err
+	}
+	contents, err := toml.Marshal(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return loadConfigTomlFrom(bytes.NewReader(contents))
+}
+
+func loadConfigTomlFrom(reader io.Reader) (*Toml, error) {
+	tree, err := toml.LoadReader(reader)
+	if err != nil {
+		return nil, err
+	}
+	return (*Toml)(tree), nil
+}
+
+// Config returns the typed config associated with the toml tree.
+func (t *Toml) Config() (*Config, error) {
+	cfg, err := GetDefault()
+	if err != nil {
+		return nil, err
+	}
+	if t == nil {
+		return cfg, nil
+	}
+	if err := t.Unmarshal(cfg); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal config: %v", err)
+	}
+	return cfg, nil
+}
+
+// Unmarshal wraps the toml.Tree Unmarshal function.
+func (t *Toml) Unmarshal(v interface{}) error {
+	return (*toml.Tree)(t).Unmarshal(v)
+}
+
+// Save saves the config to the specified Writer.
+func (t *Toml) Save(w io.Writer) (int64, error) {
+	contents, err := t.contents()
+	if err != nil {
+		return 0, err
+	}
+
+	n, err := w.Write(contents)
+	return int64(n), err
+}
+
+// contents returns the config TOML as a byte slice.
+// Any required formatting is applied.
+func (t Toml) contents() ([]byte, error) {
+	commented := t.commentDefaults()
+
+	buffer := bytes.NewBuffer(nil)
+
+	enc := toml.NewEncoder(buffer).Indentation("")
+	if err := enc.Encode((*toml.Tree)(commented)); err != nil {
+		return nil, fmt.Errorf("invalid config: %v", err)
+	}
+	return t.format(buffer.Bytes())
+}
+
+// format fixes the comments for the config to ensure that they start in column
+// 1 and are not followed by a space.
+func (t Toml) format(contents []byte) ([]byte, error) {
+	r := regexp.MustCompile(`(\n*)\s*?#\s*(\S.*)`)
+	replaced := r.ReplaceAll(contents, []byte("$1#$2"))
+
+	return replaced, nil
+}
+
+// Delete deletes the specified key from the TOML config.
+func (t *Toml) Delete(key string) error {
+	return (*toml.Tree)(t).Delete(key)
+}
+
+// Get returns the value for the specified key.
+func (t *Toml) Get(key string) interface{} {
+	return (*toml.Tree)(t).Get(key)
+}
+
+// Set sets the specified key to the specified value in the TOML config.
+func (t *Toml) Set(key string, value interface{}) {
+	(*toml.Tree)(t).Set(key, value)
+}
+
+// commentDefaults applies the required comments for default values to the Toml.
+func (t *Toml) commentDefaults() *Toml {
+	asToml := (*toml.Tree)(t)
+	commentedDefaults := map[string]interface{}{
+		"swarm-resource": "DOCKER_RESOURCE_HCU",
+		"accept-dtk-visible-devices-envvar-when-unprivileged": true,
+		"accept-dtk-visible-devices-as-volume-mounts":         false,
+		"dtk-container-cli.root":                              "/run/dtk/driver",
+		"dtk-container-cli.path":                              "/usr/bin/dtk-container-cli",
+		"dtk-container-cli.debug":                             "/var/log/dtk-container-toolkit.log",
+		"dtk-container-cli.ldcache":                           "/etc/ld.so.cache",
+		"dtk-container-cli.no-cgroups":                        false,
+		"dtk-container-cli.user":                              "root:video",
+		"dtk-container-runtime.debug":                         "/var/log/dtk-container-runtime.log",
+	}
+	for k, v := range commentedDefaults {
+		set := asToml.Get(k)
+		if !shouldComment(k, v, set) {
+			continue
+		}
+		asToml.SetWithComment(k, "", true, v)
+	}
+	return (*Toml)(asToml)
+}
+
+func shouldComment(key string, defaultValue interface{}, setTo interface{}) bool {
+	if key == "dtk-container-cli.user" && defaultValue == setTo && isSuse() {
+		return false
+	}
+	if key == "dtk-container-runtime.debug" && setTo == "/dev/null" {
+		return true
+	}
+	if setTo == nil || defaultValue == setTo || setTo == "" {
+		return true
+	}
+	return false
+}

internal/config/toolkit-cli.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package config
+
+// CTKConfig stores the config options for the DTK Container Toolkit CLI (dtk-ctk)
+type CTKConfig struct {
+	Path string `toml:"path"`
+}

internal/discover/addtional_gids.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package discover
+
+import (
+	"dtk-container-toolkit/internal/info"
+	"dtk-container-toolkit/internal/logger"
+	"os/user"
+	"strconv"
+)
+
+type userGroup struct {
+	None
+	logger logger.Interface
+	groups []string
+}
+
+var _ Discover = (*userGroup)(nil)
+
+// NewUserGroupDiscover
+func NewUserGroupDiscover(logger logger.Interface, groups ...string) Discover {
+	if len(groups) == 0 {
+		grps, err := info.GetAdditionalGroups()
+		if err != nil {
+			logger.Warningf("failed to get groups: %v", err)
+		}
+		groups = append(groups, grps...)
+	}
+	return &userGroup{
+		logger: logger,
+		groups: groups,
+	}
+}
+
+func (g *userGroup) AdditionalGIDs() ([]uint32, error) {
+	var gids []uint32
+
+	for _, group := range g.groups {
+		gid, err := GetGid(group)
+		if err != nil {
+			g.logger.Warningf("Failed to get group id: %s, %v", gid, err)
+			continue
+		}
+		gids = append(gids, uint32(gid))
+	}
+
+	return gids, nil
+}
+
+func GetGid(group string) (int, error) {
+	g, err := user.LookupGroup(group)
+	if err != nil {
+		return -1, err
+	}
+
+	return strconv.Atoi(g.Gid)
+}

internal/discover/cache.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package discover
+
+import "sync"
+
+type cache struct {
+	d Discover
+
+	sync.Mutex
+	devices      []Device
+	hooks        []Hook
+	mounts       []Mount
+	additionGids []uint32
+}
+
+var _ Discover = (*cache)(nil)
+
+// WithCache decorates the specified disoverer with a cache.
+func WithCache(d Discover) Discover {
+	if d == nil {
+		return None{}
+	}
+	return &cache{d: d}
+}
+
+func (c *cache) Devices() ([]Device, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	if c.devices == nil {
+		devices, err := c.d.Devices()
+		if err != nil {
+			return nil, err
+		}
+		c.devices = devices
+	}
+	return c.devices, nil
+}
+
+func (c *cache) Hooks() ([]Hook, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	if c.hooks == nil {
+		hooks, err := c.d.Hooks()
+		if err != nil {
+			return nil, err
+		}
+		c.hooks = hooks
+	}
+	return c.hooks, nil
+}
+
+func (c *cache) Mounts() ([]Mount, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	if c.mounts == nil {
+		mounts, err := c.d.Mounts()
+		if err != nil {
+			return nil, err
+		}
+		c.mounts = mounts
+	}
+	return c.mounts, nil
+}
+
+func (c *cache) AdditionalGIDs() ([]uint32, error) {
+	c.Lock()
+	defer c.Unlock()
+
+	if c.additionGids == nil {
+		additionGids, err := c.d.AdditionalGIDs()
+		if err != nil {
+			return nil, err
+		}
+		c.additionGids = additionGids
+	}
+	return c.additionGids, nil
+}

internal/discover/char_devices.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package discover
+
+import (
+	"dtk-container-toolkit/internal/logger"
+	"dtk-container-toolkit/internal/lookup"
+)
+
+// charDevices is a discover for a list of character devices
+type charDevices mounts
+
+var _ Discover = (*charDevices)(nil)
+
+// NewCharDeviceDiscoverer creates a discoverer which locates the specified set of device nodes.
+func NewCharDeviceDiscoverer(logger logger.Interface, devRoot string, devices []string) Discover {
+	locator := lookup.NewCharDeviceLocator(
+		lookup.WithLogger(logger),
+		lookup.WithRoot(devRoot),
+	)
+
+	return (*charDevices)(newMounts(logger, locator, devRoot, devices))
+}
+
+// Mounts returns the discovered mounts for the charDevices.
+// Since this explicitly specifies a device list, the mounts are nil.
+func (d *charDevices) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// Devices returns the discovered devices for the charDevices.
+// Here the device nodes are first discovered as mounts and these are converted to devices.
+func (d *charDevices) Devices() ([]Device, error) {
+	deviceAsMounts, err := (*mounts)(d).Mounts()
+	if err != nil {
+		return nil, err
+	}
+	var devices []Device
+	for _, mount := range deviceAsMounts {
+		device := Device{
+			HostPath: mount.HostPath,
+			Path:     mount.Path,
+		}
+		devices = append(devices, device)
+		d.logger.Infof("charDevices: %v %v", mount.HostPath, mount.Path)
+	}
+
+	return devices, nil
+}

internal/discover/discover.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package discover
+
+// Device represents a discovered character device.
+type Device struct {
+	HostPath string
+	Path     string
+}
+
+// Mount represents a discovered mount.
+type Mount struct {
+	HostPath string
+	Path     string
+	Options  []string
+}
+
+// Hook represents a discovered hook.
+type Hook struct {
+	Lifecycle string
+	Path      string
+	Args      []string
+}
+
+// Discover defines an interface for discovering the devices, mounts, and hooks available on a system
+//
+//go:generate moq -stub -out discover_mock.go . Discover
+type Discover interface {
+	Devices() ([]Device, error)
+	Mounts() ([]Mount, error)
+	Hooks() ([]Hook, error)
+	AdditionalGIDs() ([]uint32, error)
+}

internal/discover/drm_bypath_hooks.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package discover
+
+import (
+	"dtk-container-toolkit/internal/logger"
+	"dtk-container-toolkit/internal/lookup"
+	"fmt"
+	"os"
+	"path/filepath"
+)
+
+type drmDevicesByPath struct {
+	None
+	logger         logger.Interface
+	dtkCDIHookPath string
+	devRoot        string
+	devicesFrom    Discover
+}
+
+// NewCreateDRMByPathSymlinks creates a discoverer for a hook to create the by-path symlinks for DRM devices discovered by the specified devices discoverer
+func NewCreateDRMByPathSymlinks(logger logger.Interface, devices Discover, devRoot string, dtkCDIHookPath string) Discover {
+	d := drmDevicesByPath{
+		logger:         logger,
+		dtkCDIHookPath: dtkCDIHookPath,
+		devRoot:        devRoot,
+		devicesFrom:    devices,
+	}
+
+	return &d
+}
+
+// Hooks returns a hook to create the symlinks from the required CSV files
+func (d drmDevicesByPath) Hooks() ([]Hook, error) {
+	devices, err := d.devicesFrom.Devices()
+	if err != nil {
+		return nil, fmt.Errorf("failed to discover devices for by-path symlinks: %v", err)
+	}
+	if len(devices) == 0 {
+		return nil, nil
+	}
+	links, err := d.getSpecificLinkArgs(devices)
+	if err != nil {
+		return nil, fmt.Errorf("failed to determine specific links: %v", err)
+	}
+	if len(links) == 0 {
+		return nil, nil
+	}
+
+	var hooks []Hook
+
+	hook := CreateSymlinkHook(d.dtkCDIHookPath, links)
+	hooks = append(hooks, hook)
+
+	return hooks, nil
+}
+
+// getSpecificLinkArgs returns the required specific links that need to be created
+func (d drmDevicesByPath) getSpecificLinkArgs(devices []Device) ([]string, error) {
+	selectedDevices := make(map[string]bool)
+	for _, d := range devices {
+		selectedDevices[filepath.Base(d.HostPath)] = true
+	}
+
+	linkLocator := lookup.NewFileLocator(
+		lookup.WithLogger(d.logger),
+		lookup.WithRoot(d.devRoot),
+	)
+	candidates, err := linkLocator.Locate("/dev/dri/by-path/pci-*-*")
+	if err != nil {
+		d.logger.Warningf("Failed to locate by-path links: %v; ignoring", err)
+		return nil, nil
+	}
+
+	var links []string
+	for _, c := range candidates {
+		device, err := os.Readlink(c)
+		if err != nil {
+			d.logger.Warningf("Failed to evaluate symlink %v; ignoring", c)
+			continue
+		}
+
+		if selectedDevices[filepath.Base(device)] {
+			d.logger.Debugf("adding device symlink %v -> %v", c, device)
+			links = append(links, fmt.Sprintf("%v::%v", device, c))
+		}
+	}
+
+	return links, nil
+}

internal/discover/filter.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package discover
+
+import (
+	"dtk-container-toolkit/internal/logger"
+)
+
+// Filter defines an interface for filtering discovered entities
+type Filter interface {
+	DeviceIsSelected(device Device) bool
+}
+
+// filtered represents a filtered discoverer
+type filtered struct {
+	Discover
+	logger logger.Interface
+	filter Filter
+}
+
+// newFilteredDiscoverer creates a discoverer that applies the specified filter to the returned entities of the discoverer
+func newFilteredDiscoverer(logger logger.Interface, applyTo Discover, filter Filter) Discover {
+	return filtered{
+		Discover: applyTo,
+		logger:   logger,
+		filter:   filter,
+	}
+}
+
+// Devices returns a filtered list of devices based on the specified filter.
+func (d filtered) Devices() ([]Device, error) {
+	devices, err := d.Discover.Devices()
+	if err != nil {
+		return nil, err
+	}
+
+	if d.filter == nil {
+		return devices, nil
+	}
+
+	var selected []Device
+	for _, device := range devices {
+		if d.filter.DeviceIsSelected(device) {
+			selected = append(selected, device)
+		} else {
+			d.logger.Infof("Skipping device %v", device)
+		}
+	}
+
+	return selected, nil
+}

internal/discover/first-valid.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package discover
+
+import "errors"
+
+type firstOf []Discover
+
+// FirstValid returns a discoverer that returns the first non-error result from a list of discoverers.
+func FirstValid(discoverers ...Discover) Discover {
+	var f firstOf
+	for _, d := range discoverers {
+		if d == nil {
+			continue
+		}
+		f = append(f, d)
+	}
+	return f
+}
+
+func (f firstOf) Devices() ([]Device, error) {
+	var errs error
+	for _, d := range f {
+		devices, err := d.Devices()
+		if err != nil {
+			errs = errors.Join(errs, err)
+			continue
+		}
+		return devices, nil
+	}
+	return nil, errs
+}
+
+func (f firstOf) Hooks() ([]Hook, error) {
+	var errs error
+	for _, d := range f {
+		hooks, err := d.Hooks()
+		if err != nil {
+			errs = errors.Join(errs, err)
+			continue
+		}
+		return hooks, nil
+	}
+	return nil, errs
+}
+
+func (f firstOf) Mounts() ([]Mount, error) {
+	var errs error
+	for _, d := range f {
+		mounts, err := d.Mounts()
+		if err != nil {
+			errs = errors.Join(errs, err)
+			continue
+		}
+		return mounts, nil
+	}
+	return nil, nil
+}
+
+func (f firstOf) AdditionalGIDs() ([]uint32, error) {
+	var errs error
+	for _, d := range f {
+		additionalGids, err := d.AdditionalGIDs()
+		if err != nil {
+			errs = errors.Join(errs, err)
+			continue
+		}
+		return additionalGids, nil
+	}
+	return nil, nil
+}

internal/discover/graphics.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package discover
+
+import (
+	"dtk-container-toolkit/internal/info/drm"
+	"dtk-container-toolkit/internal/logger"
+	"dtk-container-toolkit/internal/lookup"
+	"dtk-container-toolkit/internal/lookup/root"
+	"fmt"
+	"os"
+)
+
+// NewDRMNodesDiscoverer returns a discoverer for the DRM device nodes associated with the specified visible devices.
+//
+// TODO: The logic for creating DRM devices should be consolidated between this
+// and the logic for generating CDI specs for a single device. This is only used
+// when applying OCI spec modifications to an incoming spec in "legacy" mode.
+func NewDRMNodesDiscoverer(logger logger.Interface, busIds []string, requestBusIds []string, devRoot string) (Discover, error) {
+	drmDeviceNodes, err := newDRMDeviceDiscoverer(logger, busIds, requestBusIds, devRoot)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create DRM device discoverer: %v", err)
+	}
+
+	return drmDeviceNodes, nil
+}
+
+// newDRMDeviceDiscoverer creates a discoverer for the DRM devices associated with the requested devices.
+func newDRMDeviceDiscoverer(logger logger.Interface, busIds []string, requestBusIds []string, devRoot string) (Discover, error) {
+	allDevices := NewCharDeviceDiscoverer(
+		logger,
+		devRoot,
+		[]string{
+			"/dev/dri/card*",
+			"/dev/dri/renderD*",
+		},
+	)
+
+	filter := make(selectDeviceByPath)
+	for _, busId := range requestBusIds {
+		drmDeviceNodes, err := drm.GetDeviceNodesByBusID(busId)
+		if err != nil {
+			return nil, fmt.Errorf("failed to determine DRM devices for %v: %v", busId, err)
+		}
+		logger.Infof("selected drm nodes from bus %s: %v", busId, drmDeviceNodes)
+		for _, drmDeviceNode := range drmDeviceNodes {
+			filter[drmDeviceNode] = true
+		}
+	}
+
+	// We return a discoverer that applies the DRM device filter created above to all discovered DRM device nodes.
+	d := newFilteredDiscoverer(
+		logger,
+		allDevices,
+		filter,
+	)
+
+	return d, nil
+}
+
+// selectDeviceByPath is a filter that allows devices to be selected by the path
+type selectDeviceByPath map[string]bool
+
+var _ Filter = (*selectDeviceByPath)(nil)
+
+// DeviceIsSelected determines whether the device's path has been selected
+func (s selectDeviceByPath) DeviceIsSelected(device Device) bool {
+	return s[device.Path]
+}
+
+// MountIsSelected is always true
+func (s selectDeviceByPath) MountIsSelected(Mount) bool {
+	return true
+}
+
+// HookIsSelected is always true
+func (s selectDeviceByPath) HookIsSelected(Hook) bool {
+	return true
+}
+
+// NewCommonHCUDiscoverer creates a discoverer for the mounts required by HCU.
+func NewCommonHCUDiscoverer(logger logger.Interface, dtkCDIHookPath string, driver *root.Driver, isMount bool) (Discover, error) {
+	metaDevices := NewCharDeviceDiscoverer(
+		logger,
+		driver.Root,
+		[]string{
+			"/dev/kfd",
+			"/dev/mkfd",
+			"/dev/mem",
+		},
+	)
+
+	var directory []string
+	if isMount {
+		directory = append(directory, "hyhal")
+	}
+	libraries := NewMounts(
+		logger,
+		lookup.NewDirectoryLocator(
+			lookup.WithLogger(logger),
+			lookup.WithCount(1),
+			lookup.WithSearchPaths("/usr/local", "/opt"),
+		),
+		driver.Root,
+		directory,
+	)
+
+	var linkHook Hook
+
+	info, err := os.Stat("/usr/local/hyhal")
+	if err == nil && info.IsDir() {
+		linkHook = CreateSymlinkHook(dtkCDIHookPath, []string{"/usr/local/hyhal::/opt/hyhal"})
+	}
+
+	d := Merge(
+		metaDevices,
+		libraries,
+		NewUserGroupDiscover(logger),
+		linkHook,
+	)
+	return d, nil
+}

internal/discover/hooks.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package discover
+
+import (
+	"path/filepath"
+
+	"tags.cncf.io/container-device-interface/pkg/cdi"
+)
+
+var _ Discover = (*Hook)(nil)
+
+// Devices returns an empty list of devices for a Hook discoverer.
+func (h Hook) Devices() ([]Device, error) {
+	return nil, nil
+}
+
+// Mounts returns an empty list of mounts for a Hook discoverer.
+func (h Hook) Mounts() ([]Mount, error) {
+	return nil, nil
+}
+
+// Hooks allows the Hook type to also implement the Discoverer interface.
+// It returns a single hook
+func (h Hook) Hooks() ([]Hook, error) {
+	return []Hook{h}, nil
+}
+
+func (h Hook) AdditionalGIDs() ([]uint32, error) {
+	return []uint32{}, nil
+}
+
+// CreateSymlinkHook creates a hook which creates a symlink from link -> target.
+func CreateSymlinkHook(dtkCDIHookPath string, links []string) Hook {
+	if len(links) == 0 {
+		return Hook{}
+	}
+
+	var args []string
+	for _, link := range links {
+		args = append(args, "--link", link)
+	}
+	return CreateDtkCDIHook(
+		dtkCDIHookPath,
+		"create-symlinks",
+		args...,
+	)
+}
+
+// CreateDtkCDIHook creates a hook which invokes the DTK Container CLI hook subcommand.
+func CreateDtkCDIHook(dtkCDIHookPath string, hookName string, additionalArgs ...string) Hook {
+	return cdiHook(dtkCDIHookPath).Create(hookName, additionalArgs...)
+}
+
+type cdiHook string
+
+func (c cdiHook) Create(name string, args ...string) Hook {
+	return Hook{
+		Lifecycle: cdi.CreateContainerHook,
+		Path:      string(c),
+		Args:      append(c.requiredArgs(name), args...),
+	}
+}
+func (c cdiHook) requiredArgs(name string) []string {
+	base := filepath.Base(string(c))
+	if base == "dtk-ctk" {
+		return []string{base, "hook", name}
+	}
+	return []string{base, name}
+}

internal/discover/list.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package discover
+
+import "fmt"
+
+// list is a discoverer that contains a list of Discoverers. The output of the
+// Mounts functions is the concatenation of the output for each of the
+// elements in the list.
+type list struct {
+	discoverers []Discover
+}
+
+var _ Discover = (*list)(nil)
+
+// Merge creates a discoverer that is the composite of a list of discoverers.
+func Merge(d ...Discover) Discover {
+	l := list{
+		discoverers: d,
+	}
+
+	return &l
+}
+
+// Devices returns all devices from the included discoverers
+func (d list) Devices() ([]Device, error) {
+	var allDevices []Device
+
+	for i, di := range d.discoverers {
+		devices, err := di.Devices()
+		if err != nil {
+			return nil, fmt.Errorf("error discovering devices for discoverer %v: %v", i, err)
+		}
+		allDevices = append(allDevices, devices...)
+	}
+
+	return allDevices, nil
+}
+
+// Mounts returns all mounts from the included discoverers
+func (d list) Mounts() ([]Mount, error) {
+	var allMounts []Mount
+
+	for i, di := range d.discoverers {
+		mounts, err := di.Mounts()
+		if err != nil {
+			return nil, fmt.Errorf("error discovering mounts for discoverer %v: %v", i, err)
+		}
+		allMounts = append(allMounts, mounts...)
+	}
+
+	return allMounts, nil
+}
+
+// Hooks returns all Hooks from the included discoverers
+func (d list) Hooks() ([]Hook, error) {
+	var allHooks []Hook
+
+	for i, di := range d.discoverers {
+		hooks, err := di.Hooks()
+		if err != nil {
+			return nil, fmt.Errorf("error discovering hooks for discoverer %v: %v", i, err)
+		}
+		allHooks = append(allHooks, hooks...)
+	}
+
+	return allHooks, nil
+}
+
+func (d list) AdditionalGIDs() ([]uint32, error) {
+	var additionalGIDs []uint32
+
+	for i, di := range d.discoverers {
+		gids, err := di.AdditionalGIDs()
+		if err != nil {
+			return nil, fmt.Errorf("error discovering AdditionalGIDs for discoverer %v: %v", i, err)
+		}
+		additionalGIDs = append(additionalGIDs, gids...)
+	}
+
+	return additionalGIDs, nil
+}

internal/discover/mofed.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package discover
+
+import "dtk-container-toolkit/internal/logger"
+
+// NewMOFEDDiscoverer creates a discoverer for MOFED devices.
+func NewMOFEDDiscoverer(logger logger.Interface, devRoot string) (Discover, error) {
+	devices := NewCharDeviceDiscoverer(
+		logger,
+		devRoot,
+		[]string{
+			"/dev/infiniband/uverbs*",
+			"/dev/infiniband/rdma_cm",
+		},
+	)
+
+	return devices, nil
+}

internal/discover/mounts-to-container-path.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package discover
+
+import (
+	"dtk-container-toolkit/internal/logger"
+	"dtk-container-toolkit/internal/lookup"
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+// mountsToContainerPath defines a Discoverer for a required set of mounts.
+// When these are discovered by a locator the specified container root is used
+// to construct the container path for the mount returned.
+type mountsToContainerPath struct {
+	None
+	logger        logger.Interface
+	locator       lookup.Locator
+	required      []string
+	containerRoot string
+}
+
+func (d *mountsToContainerPath) Mounts() ([]Mount, error) {
+	seen := make(map[string]bool)
+	var mounts []Mount
+	for _, target := range d.required {
+		if strings.Contains(target, "*") {
+			// TODO: We could relax this condition.
+			return nil, fmt.Errorf("wildcard patterns are not supported: %s", target)
+		}
+		candidates, err := d.locator.Locate(target)
+		if err != nil {
+			d.logger.Warningf("Could not locate %v: %v", target, err)
+			continue
+		}
+		if len(candidates) == 0 {
+			d.logger.Warningf("Missing %v", target)
+			continue
+		}
+		hostPath := candidates[0]
+		if seen[hostPath] {
+			d.logger.Debugf("Skipping duplicate mount %v", hostPath)
+			continue
+		}
+		seen[hostPath] = true
+		d.logger.Debugf("Located %v as %v", target, hostPath)
+
+		containerPath := filepath.Join(d.containerRoot, target)
+		d.logger.Infof("Selecting %v as %v", hostPath, containerPath)
+
+		mount := Mount{
+			HostPath: hostPath,
+			Path:     containerPath,
+			Options: []string{
+				"ro",
+				"nosuid",
+				"nodev",
+				"bind",
+			},
+		}
+		mounts = append(mounts, mount)
+	}
+
+	return mounts, nil
+}

internal/discover/mounts.go

+/**
+# Copyright (c) 2024, HCUOpt CORPORATION.  All rights reserved.
+**/
+
+package discover
+
+import (
+	"dtk-container-toolkit/internal/logger"
+	"dtk-container-toolkit/internal/lookup"
+	"fmt"
+	"path/filepath"
+	"strings"
+	"sync"
+)
+
+// mounts is a generic discoverer for Mounts. It is customized by specifying the
+// required entities as a list and a Locator that is used to find the target mounts
+// based on the entry in the list.
+type mounts struct {
+	None
+	logger   logger.Interface
+	lookup   lookup.Locator
+	root     string
+	required []string
+	sync.Mutex
+	cache []Mount
+}
+
+var _ Discover = (*mounts)(nil)
+
+// NewMounts creates a discoverer for the required mounts using the specified locator.
+func NewMounts(logger logger.Interface, lookup lookup.Locator, root string, required []string) Discover {
+	return newMounts(logger, lookup, root, required)
+}
+
+// newMounts creates a discoverer for the required mounts using the specified locator.
+func newMounts(logger logger.Interface, lookup lookup.Locator, root string, required []string) *mounts {
+	return &mounts{
+		logger:   logger,
+		lookup:   lookup,
+		root:     filepath.Join("/", root),
+		required: required,
+	}
+}
+
+func (d *mounts) Mounts() ([]Mount, error) {
+	if d.lookup == nil {
+		return nil, fmt.Errorf("no lookup defined")
+	}
+
+	if d.cache != nil {
+		d.logger.Debugf("returning cached mounts")
+		return d.cache, nil
+	}
+
+	d.Lock()
+	defer d.Unlock()
+
+	d.logger.Infof("Locating %v in %v", d.required, d.root)
+
+	uniqueMounts := make(map[string]Mount)
+
+	for _, candidate := range d.required {
+		d.logger.Infof("Locating %v", candidate)
+		located, err := d.lookup.Locate(candidate)
+		if err != nil {
+			d.logger.Warningf("Could not locate %v: %v", candidate, err)
+			continue
+		}
+		if len(located) == 0 {
+			d.logger.Warningf("Missing %v", candidate)
+			continue
+		}
+		d.logger.Debugf("Located %v as %v", candidate, located)
+		for _, p := range located {
+			if _, ok := uniqueMounts[p]; ok {
+				d.logger.Debugf("Skipping duplicate mount %v", p)
+				continue
+			}
+
+			r := d.relativeTo(p)
+			if r == "" {
+				r = p
+			}
+
+			d.logger.Infof("Selecting %v as %v", p, r)
+			uniqueMounts[p] = Mount{
+				HostPath: p,
+				Path:     r,
+				Options: []string{
+					"ro",
+					"nosuid",
+					"nodev",
+					"bind",
+				},
+			}
+		}
+	}
+
+	var mounts []Mount
+	for _, m := range uniqueMounts {
+		mounts = append(mounts, m)
+	}
+	d.cache = mounts
+
+	return d.cache, nil
+}
+
+// relativeTo returns the path relative to the root for the file locator
+func (d *mounts) relativeTo(path string) string {
+	if d.root == "/" {
+		return path
+	}
+
+	return strings.TrimPrefix(path, d.root)
+}