feat: moved compoundAI operator, APIserver, and examples (#10)

deploy/compoundai/operator/internal/controller_common/resource_test.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package controller_common
+
+import (
+	"testing"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func TestIsSpecChanged(t *testing.T) {
+	tests := []struct {
+		name     string
+		current  client.Object
+		desired  client.Object
+		expected bool
+	}{
+		{
+			name: "no change in hash with deployment spec and env variables",
+			current: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "apps/v1",
+					"kind":       "Deployment",
+					"metadata": map[string]interface{}{
+						"name":      "nim-deployment",
+						"namespace": "default",
+					},
+					"spec": map[string]interface{}{
+						"replicas": 2,
+						"selector": map[string]interface{}{
+							"matchLabels": map[string]interface{}{
+								"app": "nim",
+							},
+						},
+						"template": map[string]interface{}{
+							"metadata": map[string]interface{}{
+								"labels": map[string]interface{}{
+									"app": "nim",
+								},
+							},
+							"spec": map[string]interface{}{
+								"containers": []interface{}{
+									map[string]interface{}{
+										"name":  "nim",
+										"image": "nim:v0.1.0",
+										"ports": []interface{}{
+											map[string]interface{}{
+												"containerPort": 80,
+											},
+										},
+										"env": []interface{}{
+											map[string]interface{}{"name": "ENV_VAR1", "value": "value1"},
+											map[string]interface{}{"name": "ENV_VAR2", "value": "value2"},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			desired: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "apps/v1",
+					"kind":       "Deployment",
+					"metadata": map[string]interface{}{
+						"name":      "nim-deployment",
+						"namespace": "default",
+					},
+					"spec": map[string]interface{}{
+						"replicas": 2,
+						"selector": map[string]interface{}{
+							"matchLabels": map[string]interface{}{
+								"app": "nim",
+							},
+						},
+						"template": map[string]interface{}{
+							"metadata": map[string]interface{}{
+								"labels": map[string]interface{}{
+									"app": "nim",
+								},
+							},
+							"spec": map[string]interface{}{
+								"containers": []interface{}{
+									map[string]interface{}{
+										"name":  "nim",
+										"image": "nim:v0.1.0",
+										"ports": []interface{}{
+											map[string]interface{}{
+												"containerPort": 80,
+											},
+										},
+										"env": []interface{}{
+											map[string]interface{}{"name": "ENV_VAR1", "value": "value1"},
+											map[string]interface{}{"name": "ENV_VAR2", "value": "value2"},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "no change in hash with change in order of elements",
+			current: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "apps/v1",
+					"kind":       "Deployment",
+					"metadata": map[string]interface{}{
+						"name":      "nim-deployment",
+						"namespace": "default",
+					},
+					"spec": map[string]interface{}{
+						"replicas": 2,
+						"selector": map[string]interface{}{
+							"matchLabels": map[string]interface{}{
+								"app": "nim",
+							},
+						},
+						"template": map[string]interface{}{
+							"metadata": map[string]interface{}{
+								"labels": map[string]interface{}{
+									"app": "nim",
+								},
+							},
+							"spec": map[string]interface{}{
+								"containers": []interface{}{
+									map[string]interface{}{
+										"name":  "nim",
+										"image": "nim:v0.1.0",
+										"ports": []interface{}{
+											map[string]interface{}{
+												"containerPort": 80,
+											},
+										}, // switch order of env
+										"env": []interface{}{
+											map[string]interface{}{"name": "ENV_VAR2", "value": "value2"},
+											map[string]interface{}{"name": "ENV_VAR1", "value": "value1"},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			desired: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "apps/v1",
+					"kind":       "Deployment",
+					"metadata": map[string]interface{}{
+						"name":      "nim-deployment",
+						"namespace": "default",
+					},
+					"spec": map[string]interface{}{
+						"replicas": 2,
+						"selector": map[string]interface{}{
+							"matchLabels": map[string]interface{}{
+								"app": "nim",
+							},
+						},
+						"template": map[string]interface{}{
+							"metadata": map[string]interface{}{
+								"labels": map[string]interface{}{
+									"app": "nim",
+								},
+							},
+							"spec": map[string]interface{}{
+								"containers": []interface{}{
+									map[string]interface{}{
+										"name":  "nim",
+										"image": "nim:v0.1.0",
+										"ports": []interface{}{
+											map[string]interface{}{
+												"containerPort": 80,
+											},
+										},
+										"env": []interface{}{
+											map[string]interface{}{"name": "ENV_VAR1", "value": "value1"},
+											map[string]interface{}{"name": "ENV_VAR2", "value": "value2"},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			expected: false,
+		},
+		{
+			name: "change in hash with change in value of elements",
+			current: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "apps/v1",
+					"kind":       "Deployment",
+					"metadata": map[string]interface{}{
+						"name":      "nim-deployment",
+						"namespace": "default",
+					},
+					"spec": map[string]interface{}{
+						"replicas": 2,
+						"selector": map[string]interface{}{
+							"matchLabels": map[string]interface{}{
+								"app": "nim",
+							},
+						},
+						"template": map[string]interface{}{
+							"metadata": map[string]interface{}{
+								"labels": map[string]interface{}{
+									"app": "nim",
+								},
+							},
+							"spec": map[string]interface{}{
+								"containers": []interface{}{
+									map[string]interface{}{
+										"name":  "nim",
+										"image": "nim:v0.1.0",
+										"ports": []interface{}{
+											map[string]interface{}{
+												"containerPort": 80,
+											},
+										},
+										"env": []interface{}{
+											map[string]interface{}{"name": "ENV_VAR1", "value": "value2"},
+											map[string]interface{}{"name": "ENV_VAR2", "value": "value1"},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			desired: &unstructured.Unstructured{
+				Object: map[string]interface{}{
+					"apiVersion": "apps/v1",
+					"kind":       "Deployment",
+					"metadata": map[string]interface{}{
+						"name":      "nim-deployment",
+						"namespace": "default",
+					},
+					"spec": map[string]interface{}{
+						"replicas": 3,
+						"selector": map[string]interface{}{
+							"matchLabels": map[string]interface{}{
+								"app": "nim",
+							},
+						},
+						"template": map[string]interface{}{
+							"metadata": map[string]interface{}{
+								"labels": map[string]interface{}{
+									"app": "nim",
+								},
+							},
+							"spec": map[string]interface{}{
+								"containers": []interface{}{
+									map[string]interface{}{
+										"name":  "nim",
+										"image": "nim:v0.1.0",
+										"ports": []interface{}{
+											map[string]interface{}{
+												"containerPort": 80,
+											},
+										},
+										"env": []interface{}{
+											map[string]interface{}{"name": "ENV_VAR1", "value": "asdf"},
+											map[string]interface{}{"name": "ENV_VAR2", "value": "jljl"},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			expected: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tt.current.SetAnnotations(map[string]string{
+				NvidiaAnnotationHashKey: GetResourceHash(tt.current),
+			})
+			if got := IsSpecChanged(tt.current, tt.desired); got != tt.expected {
+				t.Errorf("IsSpecChanged() = %v, want %v, hash current %s vs desired %s", got, tt.expected, GetResourceHash(tt.current), GetResourceHash(tt.desired))
+			}
+		})
+	}
+}

deploy/compoundai/operator/internal/envoy/envoy.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package envoy
+
+import (
+	"bytes"
+	"strings"
+	"text/template"
+)
+
+const (
+	EnvoyAdminPort = 9901
+)
+
+type CreateEnvoyConfig struct {
+	ListenPort              int
+	DebugHeaderName         string
+	DebugHeaderValue        string
+	DebugServerAddress      string
+	DebugServerPort         int
+	ProductionServerAddress string
+	ProductionServerPort    int
+}
+
+const configTemplate = `
+static_resources:
+  listeners:
+  - name: listener_0
+    address:
+      socket_address:
+        address: 0.0.0.0
+        port_value: {{ .Config.ListenPort }}
+
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: ingress_http
+          access_log:
+          - name: envoy.access_loggers.stdout
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.access_loggers.stream.v3.StdoutAccessLog
+          http_filters:
+          - name: envoy.filters.http.router
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: backend
+              domains: ["*"]
+              routes:
+              - match:
+                  prefix: "/"
+                  headers:
+                    - name: "{{ .Config.DebugHeaderName }}"
+                      exact_match: "{{ .Config.DebugHeaderValue }}"
+                route:
+                  cluster: service_debug
+              - match:
+                  prefix: "/"
+                route:
+                  cluster: service_production
+
+  clusters:
+  - name: service_debug
+    connect_timeout: 0.25s
+    type: strict_dns
+    dns_lookup_family: v4_only
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: service_debug
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: {{ .Config.DebugServerAddress }}
+                port_value: {{ .Config.DebugServerPort }}
+
+  - name: service_production
+    connect_timeout: 0.25s
+    type: strict_dns
+    dns_lookup_family: v4_only
+    lb_policy: round_robin
+    load_assignment:
+      cluster_name: service_production
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: {{ .Config.ProductionServerAddress }}
+                port_value: {{ .Config.ProductionServerPort }}
+
+admin:
+    access_log_path: /dev/null
+    address:
+        socket_address:
+            address: 127.0.0.1
+            port_value: {{ .AdminPort }}
+`
+
+func GenerateEnvoyConfigurationContent(config CreateEnvoyConfig) (string, error) {
+	t := template.Must(template.New("envoy").Parse(configTemplate))
+	buf := new(bytes.Buffer)
+	err := t.Execute(buf, map[string]interface{}{
+		"Config":    config,
+		"AdminPort": EnvoyAdminPort,
+	})
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(buf.String()), nil
+}

deploy/compoundai/operator/internal/nim/nim.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nim
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+
+	"emperror.dev/errors"
+	compounaiCommon "github.com/dynemo-ai/dynemo/deploy/compoundai/operator/api/compoundai/common"
+	"github.com/dynemo-ai/dynemo/deploy/compoundai/operator/api/compoundai/modelschemas"
+	"github.com/dynemo-ai/dynemo/deploy/compoundai/operator/api/compoundai/schemasv1"
+	yataiclient "github.com/dynemo-ai/dynemo/deploy/compoundai/operator/api/compoundai/yatai-client"
+	"github.com/dynemo-ai/dynemo/deploy/compoundai/operator/api/v1alpha1"
+	commonconfig "github.com/dynemo-ai/dynemo/deploy/compoundai/operator/pkg/compoundai/config"
+	commonconsts "github.com/dynemo-ai/dynemo/deploy/compoundai/operator/pkg/compoundai/consts"
+	"github.com/huandu/xstrings"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	"github.com/dynemo-ai/dynemo/deploy/compoundai/operator/internal/archive"
+	"gopkg.in/yaml.v2"
+)
+
+// ServiceConfig represents the YAML configuration structure for a service
+type NovaConfig struct {
+	Enabled   bool   `yaml:"enabled"`
+	Namespace string `yaml:"namespace"`
+	Name      string `yaml:"name"`
+}
+
+type Resources struct {
+	CPU    string            `yaml:"cpu,omitempty"`
+	Memory string            `yaml:"memory,omitempty"`
+	GPU    string            `yaml:"gpu,omitempty"`
+	Custom map[string]string `yaml:"custom,omitempty"`
+}
+
+type Traffic struct {
+	Timeout int `yaml:"timeout"`
+}
+
+type Autoscaling struct {
+	MinReplicas int `yaml:"min_replicas"`
+	MaxReplicas int `yaml:"max_replicas"`
+}
+
+type Config struct {
+	Nova        *NovaConfig  `yaml:"nova,omitempty"`
+	Resources   *Resources   `yaml:"resources,omitempty"`
+	Traffic     *Traffic     `yaml:"traffic,omitempty"`
+	Autoscaling *Autoscaling `yaml:"autoscaling,omitempty"`
+}
+
+type ServiceConfig struct {
+	Name         string              `yaml:"name"`
+	Dependencies []map[string]string `yaml:"dependencies,omitempty"`
+	Config       Config              `yaml:"config"`
+}
+
+func RetrieveCompoundAINimDownloadURL(ctx context.Context, compoundAIDeployment *v1alpha1.CompoundAIDeployment, secretGetter SecretGetter, recorder EventRecorder) (*string, *string, error) {
+	compoundAINimDownloadURL := ""
+	compoundAINimApiToken := ""
+	var compoundAINim *schemasv1.BentoFullSchema
+	compoundAINimRepositoryName, _, compoundAINimVersion := xstrings.Partition(compoundAIDeployment.Spec.CompoundAINim, ":")
+
+	var err error
+	var yataiClient_ **yataiclient.YataiClient
+	var yataiConf_ **commonconfig.YataiConfig
+
+	yataiClient_, yataiConf_, err = GetYataiClient(ctx, secretGetter)
+	if err != nil {
+		err = errors.Wrap(err, "get yatai client")
+		return nil, nil, err
+	}
+
+	if yataiClient_ == nil || yataiConf_ == nil {
+		err = errors.New("can't get yatai client, please check yatai configuration")
+		return nil, nil, err
+	}
+
+	yataiClient := *yataiClient_
+	yataiConf := *yataiConf_
+
+	recorder.Eventf(compoundAIDeployment, corev1.EventTypeNormal, "GenerateImageBuilderPod", "Getting compoundAINim %s from yatai service", compoundAIDeployment.Spec.CompoundAINim)
+	compoundAINim, err = yataiClient.GetBento(ctx, compoundAINimRepositoryName, compoundAINimVersion)
+	if err != nil {
+		err = errors.Wrap(err, "get compoundAINim")
+		return nil, nil, err
+	}
+	recorder.Eventf(compoundAIDeployment, corev1.EventTypeNormal, "GenerateImageBuilderPod", "Got compoundAINim %s from yatai service", compoundAIDeployment.Spec.CompoundAINim)
+
+	if compoundAINim.TransmissionStrategy != nil && *compoundAINim.TransmissionStrategy == modelschemas.TransmissionStrategyPresignedURL {
+		var compoundAINim_ *schemasv1.BentoSchema
+		recorder.Eventf(compoundAIDeployment, corev1.EventTypeNormal, "GenerateImageBuilderPod", "Getting presigned url for compoundAINim %s from yatai service", compoundAIDeployment.Spec.CompoundAINim)
+		compoundAINim_, err = yataiClient.PresignBentoDownloadURL(ctx, compoundAINimRepositoryName, compoundAINimVersion)
+		if err != nil {
+			err = errors.Wrap(err, "presign compoundAINim download url")
+			return nil, nil, err
+		}
+		recorder.Eventf(compoundAIDeployment, corev1.EventTypeNormal, "GenerateImageBuilderPod", "Got presigned url for compoundAINim %s from yatai service", compoundAIDeployment.Spec.CompoundAINim)
+		compoundAINimDownloadURL = compoundAINim_.PresignedDownloadUrl
+	} else {
+		compoundAINimDownloadURL = fmt.Sprintf("%s/api/v1/bento_repositories/%s/bentos/%s/download", yataiConf.Endpoint, compoundAINimRepositoryName, compoundAINimVersion)
+		compoundAINimApiToken = fmt.Sprintf("%s:%s:$%s", commonconsts.YataiImageBuilderComponentName, yataiConf.ClusterName, commonconsts.EnvYataiApiToken)
+	}
+
+	return &compoundAINimDownloadURL, &compoundAINimApiToken, nil
+}
+
+// ServicesConfig represents the top-level YAML structure of a compoundAINim yaml file stored in a compoundAINim tar file
+type CompoundAINIMConfig struct {
+	Services []ServiceConfig `yaml:"services"`
+}
+
+type EventRecorder interface {
+	Eventf(obj runtime.Object, eventtype string, reason string, message string, args ...interface{})
+}
+
+func RetrieveCompoundAINIMConfigurationFile(ctx context.Context, url string, yataiApiToken string) (*bytes.Buffer, error) {
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set(commonconsts.YataiApiTokenHeaderName, yataiApiToken)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			logger := log.FromContext(ctx)
+			logger.Error(err, "error closing response body")
+		}
+	}()
+
+	// Read the tar file into memory
+	tarData, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	// Extract the YAML file
+	yamlFileName := "bento.yaml"
+	yamlContent, err := archive.ExtractFileFromTar(tarData, yamlFileName)
+	if err != nil {
+		return nil, err
+	}
+
+	return yamlContent, nil
+}
+
+func GetYataiClientWithAuth(ctx context.Context, compoundAINimRequest *v1alpha1.CompoundAINimRequest, secretGetter SecretGetter) (**yataiclient.YataiClient, **commonconfig.YataiConfig, error) {
+	orgId, ok := compoundAINimRequest.Labels[commonconsts.NgcOrganizationHeaderName]
+	if !ok {
+		orgId = commonconsts.DefaultOrgId
+	}
+
+	userId, ok := compoundAINimRequest.Labels[commonconsts.NgcUserHeaderName]
+	if !ok {
+		userId = commonconsts.DefaultUserId
+	}
+
+	auth := yataiclient.CompoundAIAuthHeaders{
+		OrgId:  orgId,
+		UserId: userId,
+	}
+
+	client, yataiConf, err := GetYataiClient(ctx, secretGetter)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	(*client).SetAuth(auth)
+	return client, yataiConf, err
+}
+
+type SecretGetter func(ctx context.Context, namespace, name string) (*corev1.Secret, error)
+
+func GetYataiClient(ctx context.Context, secretGetter SecretGetter) (yataiClient **yataiclient.YataiClient, yataiConf **commonconfig.YataiConfig, err error) {
+	yataiConf_, err := commonconfig.GetYataiConfig(ctx, secretGetter, commonconsts.YataiImageBuilderComponentName, false)
+	isNotFound := k8serrors.IsNotFound(err)
+	if err != nil && !isNotFound {
+		err = errors.Wrap(err, "get yatai config")
+		return
+	}
+
+	if isNotFound {
+		return
+	}
+
+	if yataiConf_.Endpoint == "" {
+		return
+	}
+
+	if yataiConf_.ClusterName == "" {
+		yataiConf_.ClusterName = "default"
+	}
+
+	yataiClient_ := yataiclient.NewYataiClient(yataiConf_.Endpoint, fmt.Sprintf("%s:%s:%s", commonconsts.YataiImageBuilderComponentName, yataiConf_.ClusterName, yataiConf_.ApiToken))
+
+	yataiClient = &yataiClient_
+	yataiConf = &yataiConf_
+	return
+}
+
+func ParseCompoundAINIMConfig(ctx context.Context, yamlContent *bytes.Buffer) (*CompoundAINIMConfig, error) {
+	var config CompoundAINIMConfig
+	logger := log.FromContext(ctx)
+	logger.Info("trying to parse compoundAINim config", "yamlContent", yamlContent.String())
+	err := yaml.Unmarshal(yamlContent.Bytes(), &config)
+	return &config, err
+}
+
+func GetCompoundAINIMConfig(ctx context.Context, compoundAIDeployment *v1alpha1.CompoundAIDeployment, secretGetter SecretGetter, recorder EventRecorder) (*CompoundAINIMConfig, error) {
+	compoundAINimDownloadURL, compoundAINimApiToken, err := RetrieveCompoundAINimDownloadURL(ctx, compoundAIDeployment, secretGetter, recorder)
+	if err != nil {
+		return nil, err
+	}
+	yamlContent, err := RetrieveCompoundAINIMConfigurationFile(ctx, *compoundAINimDownloadURL, *compoundAINimApiToken)
+	if err != nil {
+		return nil, err
+	}
+	return ParseCompoundAINIMConfig(ctx, yamlContent)
+}
+
+// generate CompoundAINIMDeployment from config
+func GenerateCompoundAINIMDeployments(parentCompoundAIDeployment *v1alpha1.CompoundAIDeployment, config *CompoundAINIMConfig) (map[string]*v1alpha1.CompoundAINimDeployment, error) {
+	novaServices := make(map[string]string)
+	deployments := make(map[string]*v1alpha1.CompoundAINimDeployment)
+	for _, service := range config.Services {
+		deployment := &v1alpha1.CompoundAINimDeployment{}
+		deployment.Name = fmt.Sprintf("%s-%s", parentCompoundAIDeployment.Name, strings.ToLower(service.Name))
+		deployment.Namespace = parentCompoundAIDeployment.Namespace
+		deployment.Spec.CompoundAINim = strings.Split(parentCompoundAIDeployment.Spec.CompoundAINim, ":")[0]
+		deployment.Spec.ServiceName = service.Name
+		if service.Config.Nova != nil && service.Config.Nova.Enabled {
+			novaServices[service.Name] = fmt.Sprintf("%s/%s", service.Config.Nova.Name, service.Config.Nova.Namespace)
+		}
+		if service.Config.Resources != nil {
+			deployment.Spec.Resources = &compounaiCommon.Resources{
+				Requests: &compounaiCommon.ResourceItem{
+					CPU:    service.Config.Resources.CPU,
+					Memory: service.Config.Resources.Memory,
+					GPU:    service.Config.Resources.GPU,
+					Custom: service.Config.Resources.Custom,
+				},
+				Limits: &compounaiCommon.ResourceItem{
+					CPU:    service.Config.Resources.CPU,
+					Memory: service.Config.Resources.Memory,
+					GPU:    service.Config.Resources.GPU,
+					Custom: service.Config.Resources.Custom,
+				},
+			}
+		}
+		if service.Config.Autoscaling != nil {
+			deployment.Spec.Autoscaling = &v1alpha1.Autoscaling{
+				MinReplicas: service.Config.Autoscaling.MinReplicas,
+				MaxReplicas: service.Config.Autoscaling.MaxReplicas,
+			}
+		}
+		deployments[service.Name] = deployment
+	}
+	for _, service := range config.Services {
+		deployment := deployments[service.Name]
+		// generate external services
+		for _, dependency := range service.Dependencies {
+			dependentServiceName := dependency["service"]
+			if deployment.Spec.ExternalServices == nil {
+				deployment.Spec.ExternalServices = make(map[string]v1alpha1.ExternalService)
+			}
+			dependencyDeployment := deployments[dependentServiceName]
+			if dependencyDeployment == nil {
+				return nil, fmt.Errorf("dependency %s not found", dependentServiceName)
+			}
+			if novaService, ok := novaServices[dependentServiceName]; ok {
+				deployment.Spec.ExternalServices[dependentServiceName] = v1alpha1.ExternalService{
+					DeploymentSelectorKey:   "nova",
+					DeploymentSelectorValue: novaService,
+				}
+			} else {
+				deployment.Spec.ExternalServices[dependentServiceName] = v1alpha1.ExternalService{
+					DeploymentSelectorKey:   "name",
+					DeploymentSelectorValue: dependentServiceName,
+				}
+			}
+		}
+	}
+	return deployments, nil
+}

deploy/compoundai/operator/internal/nim/nim_test.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package nim
+
+import (
+	"testing"
+
+	compounaiCommon "github.com/dynemo-ai/dynemo/deploy/compoundai/operator/api/compoundai/common"
+	"github.com/dynemo-ai/dynemo/deploy/compoundai/operator/api/v1alpha1"
+	"github.com/onsi/gomega"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+func TestGenerateCompoundAINIMDeployments(t *testing.T) {
+	type args struct {
+		parentCompoundAIDeployment *v1alpha1.CompoundAIDeployment
+		config                     *CompoundAINIMConfig
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    map[string]*v1alpha1.CompoundAINimDeployment
+		wantErr bool
+	}{
+		{
+			name: "Test GenerateCompoundAINIMDeployments http dependency",
+			args: args{
+				parentCompoundAIDeployment: &v1alpha1.CompoundAIDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test-compoundaideployment",
+						Namespace: "default",
+					},
+					Spec: v1alpha1.CompoundAIDeploymentSpec{
+						CompoundAINim: "compoundainim:ac4e234",
+					},
+				},
+				config: &CompoundAINIMConfig{
+					Services: []ServiceConfig{
+						{
+							Name:         "service1",
+							Dependencies: []map[string]string{{"service": "service2"}},
+							Config: Config{
+								Nova: &NovaConfig{
+									Enabled:   true,
+									Namespace: "default",
+									Name:      "service1",
+								},
+								Resources: &Resources{
+									CPU:    "1",
+									Memory: "1Gi",
+									GPU:    "0",
+									Custom: map[string]string{},
+								},
+								Autoscaling: &Autoscaling{
+									MinReplicas: 1,
+									MaxReplicas: 5,
+								},
+							},
+						},
+						{
+							Name:         "service2",
+							Dependencies: []map[string]string{},
+							Config: Config{
+								Nova: &NovaConfig{
+									Enabled: false,
+								},
+							},
+						},
+					},
+				},
+			},
+			want: map[string]*v1alpha1.CompoundAINimDeployment{
+				"service1": {
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test-compoundaideployment-service1",
+						Namespace: "default",
+					},
+					Spec: v1alpha1.CompoundAINimDeploymentSpec{
+						CompoundAINim: "compoundainim",
+						ServiceName:   "service1",
+						Resources: &compounaiCommon.Resources{
+							Requests: &compounaiCommon.ResourceItem{
+								CPU:    "1",
+								Memory: "1Gi",
+								GPU:    "0",
+								Custom: map[string]string{},
+							},
+							Limits: &compounaiCommon.ResourceItem{
+								CPU:    "1",
+								Memory: "1Gi",
+								GPU:    "0",
+								Custom: map[string]string{},
+							},
+						},
+						Autoscaling: &v1alpha1.Autoscaling{
+							MinReplicas: 1,
+							MaxReplicas: 5,
+						},
+						ExternalServices: map[string]v1alpha1.ExternalService{
+							"service2": {
+								DeploymentSelectorKey:   "name",
+								DeploymentSelectorValue: "service2",
+							},
+						},
+					},
+				},
+				"service2": {
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test-compoundaideployment-service2",
+						Namespace: "default",
+					},
+					Spec: v1alpha1.CompoundAINimDeploymentSpec{
+						ServiceName:   "service2",
+						CompoundAINim: "compoundainim",
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test GenerateCompoundAINIMDeployments nova dependency",
+			args: args{
+				parentCompoundAIDeployment: &v1alpha1.CompoundAIDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test-compoundaideployment",
+						Namespace: "default",
+					},
+					Spec: v1alpha1.CompoundAIDeploymentSpec{
+						CompoundAINim: "compoundainim:ac4e234",
+					},
+				},
+				config: &CompoundAINIMConfig{
+					Services: []ServiceConfig{
+						{
+							Name:         "service1",
+							Dependencies: []map[string]string{{"service": "service2"}},
+							Config: Config{
+								Nova: &NovaConfig{
+									Enabled:   true,
+									Namespace: "default",
+									Name:      "service1",
+								},
+								Resources: &Resources{
+									CPU:    "1",
+									Memory: "1Gi",
+									GPU:    "0",
+									Custom: map[string]string{},
+								},
+								Autoscaling: &Autoscaling{
+									MinReplicas: 1,
+									MaxReplicas: 5,
+								},
+							},
+						},
+						{
+							Name:         "service2",
+							Dependencies: []map[string]string{},
+							Config: Config{
+								Nova: &NovaConfig{
+									Enabled:   true,
+									Namespace: "default",
+									Name:      "service2",
+								},
+							},
+						},
+					},
+				},
+			},
+			want: map[string]*v1alpha1.CompoundAINimDeployment{
+				"service1": {
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test-compoundaideployment-service1",
+						Namespace: "default",
+					},
+					Spec: v1alpha1.CompoundAINimDeploymentSpec{
+						CompoundAINim: "compoundainim",
+						ServiceName:   "service1",
+						Resources: &compounaiCommon.Resources{
+							Requests: &compounaiCommon.ResourceItem{
+								CPU:    "1",
+								Memory: "1Gi",
+								GPU:    "0",
+								Custom: map[string]string{},
+							},
+							Limits: &compounaiCommon.ResourceItem{
+								CPU:    "1",
+								Memory: "1Gi",
+								GPU:    "0",
+								Custom: map[string]string{},
+							},
+						},
+						Autoscaling: &v1alpha1.Autoscaling{
+							MinReplicas: 1,
+							MaxReplicas: 5,
+						},
+						ExternalServices: map[string]v1alpha1.ExternalService{
+							"service2": {
+								DeploymentSelectorKey:   "nova",
+								DeploymentSelectorValue: "service2/default",
+							},
+						},
+					},
+				},
+				"service2": {
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test-compoundaideployment-service2",
+						Namespace: "default",
+					},
+					Spec: v1alpha1.CompoundAINimDeploymentSpec{
+						CompoundAINim: "compoundainim",
+						ServiceName:   "service2",
+					},
+				},
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test GenerateCompoundAINIMDeployments dependency not found",
+			args: args{
+				parentCompoundAIDeployment: &v1alpha1.CompoundAIDeployment{
+					ObjectMeta: metav1.ObjectMeta{
+						Name:      "test-compoundaideployment",
+						Namespace: "default",
+					},
+					Spec: v1alpha1.CompoundAIDeploymentSpec{
+						CompoundAINim: "compoundainim:ac4e234",
+					},
+				},
+				config: &CompoundAINIMConfig{
+					Services: []ServiceConfig{
+						{
+							Name:         "service1",
+							Dependencies: []map[string]string{{"service": "service2"}},
+							Config: Config{
+								Nova: &NovaConfig{
+									Enabled:   true,
+									Namespace: "default",
+									Name:      "service1",
+								},
+								Resources: &Resources{
+									CPU:    "1",
+									Memory: "1Gi",
+									GPU:    "0",
+									Custom: map[string]string{},
+								},
+								Autoscaling: &Autoscaling{
+									MinReplicas: 1,
+									MaxReplicas: 5,
+								},
+							},
+						},
+						{
+							Name:         "service3",
+							Dependencies: []map[string]string{},
+							Config: Config{
+								Nova: &NovaConfig{
+									Enabled:   true,
+									Namespace: "default",
+									Name:      "service3",
+								},
+							},
+						},
+					},
+				},
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := gomega.NewGomegaWithT(t)
+			got, err := GenerateCompoundAINIMDeployments(tt.args.parentCompoundAIDeployment, tt.args.config)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("GenerateCompoundAINIMDeployments() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			g.Expect(got).To(gomega.Equal(tt.want))
+		})
+	}
+}

deploy/compoundai/operator/pkg/compoundai/README.md

+all these packages are from https://github.com/bentoml/yatai-common
\ No newline at end of file

deploy/compoundai/operator/pkg/compoundai/config/config.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package config
+
+import (
+	"context"
+	"os"
+	"strings"
+
+	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/dynemo-ai/dynemo/deploy/compoundai/operator/pkg/compoundai/consts"
+)
+
+func GetYataiSystemNamespaceFromEnv() string {
+	return getEnv(consts.EnvYataiSystemNamespace, consts.DefaultKubeNamespaceYataiSystem)
+}
+
+func GetYataiImageBuilderNamespace(ctx context.Context, secretGetter func(ctx context.Context, namespace, name string) (*corev1.Secret, error)) (namespace string, err error) {
+	namespace = os.Getenv(consts.EnvYataiImageBuilderNamespace)
+	if namespace != "" {
+		return
+	}
+
+	yataiSystemNamespace := GetYataiSystemNamespaceFromEnv()
+	yataiImageBuilderSharedEnvSecretName := consts.KubeSecretNameYataiImageBuilderSharedEnv
+
+	secret, err := secretGetter(ctx, yataiSystemNamespace, yataiImageBuilderSharedEnvSecretName)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			err = errors.Wrapf(err, "secret %s not found in namespace %s", yataiImageBuilderSharedEnvSecretName, yataiSystemNamespace)
+		}
+		return
+	}
+
+	namespace = string(secret.Data[consts.EnvYataiImageBuilderNamespace])
+	if namespace == "" {
+		namespace = consts.DefaultKubeNamespaceYataiImageBuilderComponent
+	}
+
+	return
+}
+
+func GetYataiDeploymentNamespace(ctx context.Context, secretGetter func(ctx context.Context, namespace, name string) (*corev1.Secret, error)) (namespace string, err error) {
+	namespace = os.Getenv(consts.EnvYataiDeploymentNamespace)
+	if namespace != "" {
+		return
+	}
+
+	yataiSystemNamespace := GetYataiSystemNamespaceFromEnv()
+	yataiDeploymentSharedEnvSecretName := consts.KubeSecretNameYataiDeploymentSharedEnv
+
+	secret, err := secretGetter(ctx, yataiSystemNamespace, yataiDeploymentSharedEnvSecretName)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			err = errors.Wrapf(err, "secret %s not found in namespace %s", yataiDeploymentSharedEnvSecretName, yataiSystemNamespace)
+		}
+		return
+	}
+
+	namespace = string(secret.Data[consts.EnvYataiDeploymentNamespace])
+	if namespace == "" {
+		namespace = consts.DefaultKubeNamespaceYataiDeploymentComponent
+	}
+
+	return
+}
+
+func GetImageBuildersNamespace(ctx context.Context, cliset *kubernetes.Clientset) (namespace string, err error) {
+	namespace = os.Getenv(consts.EnvImageBuildersNamespace)
+	if namespace != "" {
+		return
+	}
+
+	yataiSystemNamespace := GetYataiSystemNamespaceFromEnv()
+	yataiImageBuilderSharedEnvSecretName := consts.KubeSecretNameYataiImageBuilderSharedEnv
+
+	secret, err := cliset.CoreV1().Secrets(yataiSystemNamespace).Get(ctx, yataiImageBuilderSharedEnvSecretName, metav1.GetOptions{})
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			err = errors.Wrapf(err, "secret %s not found in namespace %s", yataiImageBuilderSharedEnvSecretName, yataiSystemNamespace)
+		}
+		return
+	}
+
+	namespace = string(secret.Data[consts.EnvImageBuildersNamespace])
+	if namespace == "" {
+		namespace = consts.DefaultKubeNamespaceImageBuilders
+	}
+
+	return
+}
+
+func GetBentoDeploymentNamespaces(ctx context.Context, secretGetter func(ctx context.Context, namespace, name string) (*corev1.Secret, error)) (namespaces []string, err error) {
+	namespaces_ := os.Getenv(consts.EnvBentoDeploymentNamespaces)
+	if namespaces_ != "" {
+		namespaces = strings.Split(namespaces_, ",")
+		return
+	}
+
+	yataiSystemNamespace := GetYataiSystemNamespaceFromEnv()
+	yataiDeploymentSharedEnvSecretName := consts.KubeSecretNameYataiDeploymentSharedEnv
+
+	secret, err := secretGetter(ctx, yataiSystemNamespace, yataiDeploymentSharedEnvSecretName)
+	if err != nil {
+		if k8serrors.IsNotFound(err) {
+			err = errors.Wrapf(err, "secret %s not found in namespace %s", yataiDeploymentSharedEnvSecretName, yataiSystemNamespace)
+		}
+		return
+	}
+
+	namespaces_ = string(secret.Data[consts.EnvBentoDeploymentNamespaces])
+	if namespaces_ == "" {
+		namespaces = []string{consts.DefaultKubeNamespaceBentoDeployment}
+	} else {
+		namespaces = strings.Split(namespaces_, ",")
+	}
+
+	return
+}
+
+type DockerRegistryConfig struct {
+	BentoRepositoryName string `yaml:"bento_repository_name"`
+	ModelRepositoryName string `yaml:"model_repository_name"`
+	Server              string `yaml:"server"`
+	InClusterServer     string `yaml:"in_cluster_server"`
+	Username            string `yaml:"username"`
+	Password            string `yaml:"password"`
+	Secure              bool   `yaml:"secure"`
+}
+
+func GetDockerRegistryConfig(ctx context.Context, secretGetter func(ctx context.Context, namespace, name string) (*corev1.Secret, error)) (conf *DockerRegistryConfig, err error) {
+	conf = &DockerRegistryConfig{}
+	conf.BentoRepositoryName = os.Getenv(consts.EnvDockerRegistryBentoRepositoryName)
+	conf.ModelRepositoryName = os.Getenv(consts.EnvDockerRegistryModelRepositoryName)
+	conf.Server = os.Getenv(consts.EnvDockerRegistryServer)
+	conf.InClusterServer = os.Getenv(consts.EnvDockerRegistryInClusterServer)
+	conf.Username = os.Getenv(consts.EnvDockerRegistryUsername)
+	conf.Password = os.Getenv(consts.EnvDockerRegistryPassword)
+	conf.Secure = os.Getenv(consts.EnvDockerRegistrySecure) == "true"
+
+	if conf.Server == "" {
+		yataiSystemNamespace := GetYataiSystemNamespaceFromEnv()
+		yataiImageBuilderSharedEnvSecretName := consts.KubeSecretNameYataiImageBuilderSharedEnv
+
+		var secret *corev1.Secret
+
+		secret, err = secretGetter(ctx, yataiSystemNamespace, yataiImageBuilderSharedEnvSecretName)
+		if err != nil {
+			if k8serrors.IsNotFound(err) {
+				err = errors.Wrapf(err, "secret %s not found in namespace %s", yataiImageBuilderSharedEnvSecretName, yataiSystemNamespace)
+			}
+			return
+		}
+
+		conf.BentoRepositoryName = string(secret.Data[consts.EnvDockerRegistryBentoRepositoryName])
+		conf.ModelRepositoryName = string(secret.Data[consts.EnvDockerRegistryModelRepositoryName])
+		conf.Server = string(secret.Data[consts.EnvDockerRegistryServer])
+		conf.InClusterServer = string(secret.Data[consts.EnvDockerRegistryInClusterServer])
+		conf.Username = string(secret.Data[consts.EnvDockerRegistryUsername])
+		conf.Password = string(secret.Data[consts.EnvDockerRegistryPassword])
+		conf.Secure = string(secret.Data[consts.EnvDockerRegistrySecure]) == "true"
+
+	}
+
+	if conf.Server == "" {
+		err = errors.Wrapf(errors.New("not found"), "the environment variable %s is not set", consts.EnvDockerRegistryServer)
+	}
+
+	return
+}
+
+type YataiConfig struct {
+	Endpoint    string `yaml:"endpoint"`
+	ClusterName string `yaml:"cluster_name"`
+	ApiToken    string `yaml:"api_token"`
+}
+
+func GetYataiConfig(ctx context.Context, secretGetter func(ctx context.Context, namespace, name string) (*corev1.Secret, error), yataiComponentName string, ignoreEnv bool) (conf *YataiConfig, err error) {
+	conf = &YataiConfig{}
+	if !ignoreEnv {
+		conf.Endpoint = os.Getenv(consts.EnvYataiEndpoint)
+		conf.ClusterName = os.Getenv(consts.EnvYataiClusterName)
+		conf.ApiToken = os.Getenv(consts.EnvYataiApiToken)
+	}
+
+	yataiSystemNamespace := GetYataiSystemNamespaceFromEnv()
+
+	if conf.Endpoint == "" {
+		var secret *corev1.Secret
+		secret, err = secretGetter(ctx, yataiSystemNamespace, consts.KubeSecretNameYataiCommonEnv)
+		if err != nil {
+			if k8serrors.IsNotFound(err) {
+				err = errors.Wrapf(err, "secret %s not found in namespace %s", consts.KubeSecretNameYataiCommonEnv, yataiSystemNamespace)
+			}
+			return
+		}
+		conf.Endpoint = string(secret.Data[consts.EnvYataiEndpoint])
+		conf.ClusterName = string(secret.Data[consts.EnvYataiClusterName])
+	}
+
+	if conf.ApiToken == "" {
+		var secret *corev1.Secret
+		var secretName string
+		var secretNamespace string
+		if yataiComponentName == consts.YataiImageBuilderComponentName {
+			secretName = consts.KubeSecretNameYataiImageBuilderEnv
+			secretNamespace, err = GetYataiImageBuilderNamespace(ctx, secretGetter)
+			if err != nil {
+				err = errors.Wrapf(err, "failed to get namespace for %s", yataiComponentName)
+				return
+			}
+		} else if yataiComponentName == consts.YataiDeploymentComponentName {
+			secretName = consts.KubeSecretNameYataiDeploymentEnv
+			secretNamespace, err = GetYataiDeploymentNamespace(ctx, secretGetter)
+			if err != nil {
+				err = errors.Wrapf(err, "failed to get namespace for %s", yataiComponentName)
+				return
+			}
+		} else {
+			err = errors.Errorf("invalid yatai component name %s", yataiComponentName)
+			return
+		}
+		secret, err = secretGetter(ctx, secretNamespace, secretName)
+		if err != nil {
+			if k8serrors.IsNotFound(err) {
+				err = errors.Errorf("the secret %s in namespace %s does not exist", secretName, secretNamespace)
+			} else {
+				err = errors.Wrapf(err, "failed to get secret %s in namespace %s", secretName, secretNamespace)
+			}
+			return
+		}
+		conf.ApiToken = string(secret.Data[consts.EnvYataiApiToken])
+	}
+
+	return
+}
+
+// if key found in environ return value else return fallback
+func getEnv(key, fallback string) string {
+	if value, ok := os.LookupEnv(key); ok {
+		return value
+	}
+	return fallback
+}
+
+type InternalImages struct {
+	BentoDownloader    string
+	Curl               string
+	Kaniko             string
+	MetricsTransformer string
+	Buildkit           string
+	BuildkitRootless   string
+	Buildah            string
+}
+
+func GetInternalImages() (conf *InternalImages) {
+	conf = &InternalImages{}
+	conf.BentoDownloader = getEnv(consts.EnvInternalImagesBentoDownloader, consts.InternalImagesBentoDownloaderDefault)
+	conf.Curl = getEnv(consts.EnvInternalImagesCurl, consts.InternalImagesCurlDefault)
+	conf.Kaniko = getEnv(consts.EnvInternalImagesKaniko, consts.InternalImagesKanikoDefault)
+	conf.MetricsTransformer = getEnv(consts.EnvInternalImagesMetricsTransformer, consts.InternalImagesMetricsTransformerDefault)
+	conf.Buildkit = getEnv(consts.EnvInternalImagesBuildkit, consts.InternalImagesBuildkitDefault)
+	conf.BuildkitRootless = getEnv(consts.EnvInternalImagesBuildkitRootless, consts.InternalImagesBuildkitRootlessDefault)
+	conf.Buildah = getEnv(consts.EnvInternalImagesBuildah, consts.InternalImagesBuildahDefault)
+
+	return
+}

deploy/compoundai/operator/pkg/compoundai/consts/consts.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package consts
+
+const (
+	DefaultETCDTimeoutSeconds              = 5
+	DefaultETCDDialKeepaliveTimeSeconds    = 30
+	DefaultETCDDialKeepaliveTimeoutSeconds = 10
+
+	HPADefaultMaxReplicas = 10
+
+	HPACPUDefaultAverageUtilization = 80
+
+	YataiDebugImg             = "yatai.ai/yatai-infras/debug"
+	YataiKubectlNamespace     = "default"
+	YataiKubectlContainerName = "main"
+	YataiKubectlImage         = "yatai.ai/yatai-infras/k8s"
+
+	TracingContextKey = "tracing-context"
+	// nolint: gosec
+	YataiApiTokenHeaderName = "X-YATAI-API-TOKEN"
+
+	YataiOrganizationHeaderName = "X-Yatai-Organization"
+	NgcOrganizationHeaderName   = "Nv-Ngc-Org"
+	NgcUserHeaderName           = "Nv-Actor-Id"
+
+	DefaultUserId = "default"
+	DefaultOrgId  = "default"
+
+	BentoServicePort          = 3000
+	BentoContainerDefaultPort = 3000
+	BentoServicePortName      = "http"
+	BentoContainerPortName    = "http"
+
+	NoneStr = "None"
+
+	AmazonS3Endpoint = "s3.amazonaws.com"
+
+	YataiImageBuilderComponentName = "yatai-image-builder"
+	YataiDeploymentComponentName   = "yatai-deployment"
+
+	// nolint: gosec
+	YataiK8sBotApiTokenName = "yatai-k8s-bot"
+
+	YataiBentoDeploymentComponentApiServer = "api-server"
+	YataiBentoDeploymentComponentRunner    = "runner"
+
+	InternalImagesBentoDownloaderDefault    = "quay.io/bentoml/bento-downloader:0.0.3"
+	InternalImagesCurlDefault               = "quay.io/bentoml/curl:0.0.1"
+	InternalImagesKanikoDefault             = "quay.io/bentoml/kaniko:1.9.1"
+	InternalImagesMetricsTransformerDefault = "quay.io/bentoml/yatai-bento-metrics-transformer:0.0.3"
+	InternalImagesBuildkitDefault           = "quay.io/bentoml/buildkit:master"
+	InternalImagesBuildkitRootlessDefault   = "quay.io/bentoml/buildkit:master-rootless"
+	InternalImagesBuildahDefault            = "quay.io/bentoml/bentoml-buildah:0.0.1"
+)

deploy/compoundai/operator/pkg/compoundai/consts/env.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package consts
+
+const (
+	EnvYataiEndpoint    = "YATAI_ENDPOINT"
+	EnvYataiClusterName = "YATAI_CLUSTER_NAME"
+	// nolint: gosec
+	EnvYataiApiToken = "YATAI_API_TOKEN"
+
+	EnvBentoServicePort = "PORT"
+
+	// tracking envars
+	EnvYataiVersion       = "YATAI_T_VERSION"
+	EnvYataiOrgUID        = "YATAI_T_ORG_UID"
+	EnvYataiDeploymentUID = "YATAI_T_DEPLOYMENT_UID"
+	EnvYataiClusterUID    = "YATAI_T_CLUSTER_UID"
+
+	EnvYataiBentoDeploymentName      = "YATAI_BENTO_DEPLOYMENT_NAME"
+	EnvYataiBentoDeploymentNamespace = "YATAI_BENTO_DEPLOYMENT_NAMESPACE"
+
+	EnvS3Endpoint   = "S3_ENDPOINT"
+	EnvS3Region     = "S3_REGION"
+	EnvS3BucketName = "S3_BUCKET_NAME"
+	EnvS3AccessKey  = "S3_ACCESS_KEY"
+	// nolint:gosec
+	EnvS3SecretKey = "S3_SECRET_KEY"
+	EnvS3Secure    = "S3_SECURE"
+
+	EnvDockerRegistryServer          = "DOCKER_REGISTRY_SERVER"
+	EnvDockerRegistryInClusterServer = "DOCKER_REGISTRY_IN_CLUSTER_SERVER"
+	EnvDockerRegistryUsername        = "DOCKER_REGISTRY_USERNAME"
+	// nolint:gosec
+	EnvDockerRegistryPassword            = "DOCKER_REGISTRY_PASSWORD"
+	EnvDockerRegistrySecure              = "DOCKER_REGISTRY_SECURE"
+	EnvDockerRegistryBentoRepositoryName = "DOCKER_REGISTRY_BENTO_REPOSITORY_NAME"
+	EnvDockerRegistryModelRepositoryName = "DOCKER_REGISTRY_MODEL_REPOSITORY_NAME"
+
+	EnvInternalImagesBentoDownloader    = "INTERNAL_IMAGES_BENTO_DOWNLOADER"
+	EnvInternalImagesCurl               = "INTERNAL_IMAGES_CURL"
+	EnvInternalImagesKaniko             = "INTERNAL_IMAGES_KANIKO"
+	EnvInternalImagesMetricsTransformer = "INTERNAL_IMAGES_METRICS_TRANSFORMER"
+	EnvInternalImagesBuildkit           = "INTERNAL_IMAGES_BUILDKIT"
+	EnvInternalImagesBuildkitRootless   = "INTERNAL_IMAGES_BUILDKIT_ROOTLESS"
+	EnvInternalImagesBuildah            = "INTERNAL_IMAGES_BUILDAH"
+
+	EnvYataiSystemNamespace       = "YATAI_SYSTEM_NAMESPACE"
+	EnvYataiImageBuilderNamespace = "YATAI_IMAGE_BUILDER_NAMESPACE"
+	EnvYataiDeploymentNamespace   = "YATAI_DEPLOYMENT_NAMESPACE"
+	EnvBentoDeploymentNamespaces  = "BENTO_DEPLOYMENT_NAMESPACES"
+	EnvImageBuildersNamespace     = "IMAGE_BUILDERS_NAMESPACE"
+
+	EnvAWSAccessKeyID     = "AWS_ACCESS_KEY_ID"
+	EnvGCPAccessKeyID     = "GCP_ACCESS_KEY_ID"
+	EnvAWSSecretAccessKey = "AWS_SECRET_ACCESS_KEY"
+	EnvGCPSecretAccessKey = "GCP_SECRET_ACCESS_KEY"
+
+	EnvAWSECRWithIAMRole = "AWS_ECR_WITH_IAM_ROLE"
+	EnvAWSECRRegion      = "AWS_ECR_REGION"
+)

deploy/compoundai/operator/pkg/compoundai/consts/kube.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package consts
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	"k8s.io/apimachinery/pkg/labels"
+)
+
+const (
+	KubeIngressClassName = "yatai-ingress"
+
+	KubeLabelYataiSelector        = "yatai.ai/selector"
+	KubeLabelYataiBentoRepository = "yatai.ai/bento-repository"
+	KubeLabelYataiBento           = "yatai.ai/bento"
+	KubeLabelYataiModelRepository = "yatai.ai/model-repository"
+	KubeLabelYataiModel           = "yatai.ai/model"
+
+	KubeHPAQPSMetric = "http_request"
+	KubeHPAGPUMetric = "container_accelerator_duty_cycle"
+
+	DefaultKubeNamespaceBentoDeployment            = "yatai"
+	DefaultKubeNamespaceImageBuilders              = "yatai-builders"
+	DefaultKubeNamespaceYataiSystem                = "yatai-system"
+	DefaultKubeNamespaceYataiImageBuilderComponent = "yatai-image-builder"
+	DefaultKubeNamespaceYataiDeploymentComponent   = "yatai-deployment"
+
+	KubeLabelYataiBentoDeployment              = "yatai.ai/bento-deployment"
+	KubeLabelYataiBentoDeploymentComponentType = "yatai.ai/bento-deployment-component-type"
+	KubeLabelYataiBentoDeploymentComponentName = "yatai.ai/bento-deployment-component-name"
+	KubeLabelYataiBentoDeploymentTargetType    = "yatai.ai/bento-deployment-target-type"
+	KubeLabelYataiBentoDeploymentRunner        = "yatai.ai/bento-deployment-runner"
+	KubeLabelBentoRepository                   = "yatai.ai/bento-repository"
+	KubeLabelBentoVersion                      = "yatai.ai/bento-version"
+	KubeLabelCreator                           = "yatai.ai/creator"
+	// nolint: gosec
+	KubeLabelYataiDeployToken    = "yatai.ai/deploy-token"
+	KubeLabelIsBentoImageBuilder = "yatai.ai/is-bento-image-builder"
+	KubeLabelIsModelSeeder       = "yatai.ai/is-model-seeder"
+	KubeLabelBentoRequest        = "yatai.ai/bento-request"
+
+	KubeLabelYataiOwnerReference = "yatai.ai/owner-reference"
+
+	KubeLabelGPUAccelerator = "gpu-accelerator"
+
+	KubeLabelHostName = "kubernetes.io/hostname"
+	KubeLabelArch     = "kubernetes.io/arch"
+
+	KubeLabelValueFalse = "false"
+	KubeLabelValueTrue  = "true"
+
+	KubeLabelYataiImageBuilderPod = "yatai.ai/yatai-image-builder-pod"
+	KubeLabelBentoDeploymentPod   = "yatai.ai/bento-deployment-pod"
+
+	KubeLabelManagedBy    = "app.kubernetes.io/managed-by"
+	KubeLabelHelmHeritage = "heritage"
+	KubeLabelHelmRelease  = "release"
+
+	KubeAnnotationBentoRepository        = "yatai.ai/bento-repository"
+	KubeAnnotationBentoVersion           = "yatai.ai/bento-version"
+	KubeAnnotationYataiDeploymentId      = "yatai.ai/deployment-id"
+	KubeAnnotationDockerRegistryInsecure = "yatai.ai/docker-registry-insecure"
+	KubeAnnotationHelmReleaseName        = "meta.helm.sh/release-name"
+
+	KubeAnnotationPrometheusScrape = "prometheus.io/scrape"
+	KubeAnnotationPrometheusPort   = "prometheus.io/port"
+	KubeAnnotationPrometheusPath   = "prometheus.io/path"
+
+	KubeAnnotationARMSAutoEnable                  = "armsPilotAutoEnable"
+	KubeAnnotationARMSAppName                     = "armsPilotCreateAppName"
+	KubeAnnotationYataiImageBuilderSeparateModels = "yatai.ai/yatai-image-builder-separate-models"
+	KubeAnnotationAWSAccessKeySecretName          = "yatai.ai/aws-access-key-secret-name"
+	KubeAnnotationGCPAccessKeySecretName          = "yatai.ai/gcp-access-key-secret"
+	KubeAnnotationIsMultiTenancy                  = "yatai.ai/is-multi-tenancy"
+
+	KubeCreator = "yatai"
+
+	KubeResourceGPUNvidia = "nvidia.com/gpu"
+
+	KubeEventResourceKindPod        = "Pod"
+	KubeEventResourceKindHPA        = "HorizontalPodAutoscaler"
+	KubeEventResourceKindReplicaSet = "ReplicaSet"
+
+	KubeTaintKeyDedicatedNodeGroup = "mcd.io/dedicated-node-group"
+	KubeLabelDedicatedNodeGroup    = "mcd.io/dedicated-node-group"
+
+	KubeImageCSIDriver          = "image.csi.k8s.io"
+	KubeImageCSIDriverWarmMetal = "csi-image.warm-metal.tech"
+
+	KubeConfigMapNameNetworkConfig = "network"
+
+	KubeConfigMapKeyNetworkConfigDomainSuffix       = "domain-suffix"
+	KubeConfigMapKeyNetworkConfigIngressClass       = "ingress-class"
+	KubeConfigMapKeyNetworkConfigIngressAnnotations = "ingress-annotations"
+	KubeConfigMapKeyNetworkConfigIngressPath        = "ingress-path"
+	KubeConfigMapKeyNetworkConfigIngressPathType    = "ingress-path-type"
+
+	KubeConfigMapNameYataiConfig = "yatai"
+
+	KubeConfigMapKeyYataiConfigEndpoint           = "endpoint"
+	KubeConfigMapKeyYataiConfigClusterName        = "cluster-name"
+	KubeConfigMapKeyYataiConfigApiTokenSecretName = "api-token-secret-name"
+	KubeConfigMapKeyYataiConfigApiTokenSecretKey  = "api-token-secret-key"
+
+	// nolint: gosec
+	KubeSecretNameRegcred = "yatai-regcred"
+
+	KubeSecretNameYataiCommonEnv = "yatai-common-env"
+
+	KubeSecretNameYataiImageBuilderSharedEnv = "yatai-image-builder-shared-env"
+	KubeSecretNameYataiDeploymentSharedEnv   = "compoundai-deployment-shared-env"
+
+	KubeSecretNameYataiImageBuilderEnv = "yatai-image-builder-env"
+	KubeSecretNameYataiDeploymentEnv   = "compoundai-deployment-env"
+)
+
+var KubeListEverything = metav1.ListOptions{
+	LabelSelector: labels.Everything().String(),
+	FieldSelector: fields.Everything().String(),
+}

deploy/compoundai/operator/pkg/compoundai/reqcli/http.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package reqcli
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	dftHttpCli  *http.Client
+	cliLoadOnce sync.Once
+)
+
+var httpTimeout = 90 * time.Second
+
+func getDefaultTransPort() *http.Transport {
+	return &http.Transport{
+		Proxy: http.ProxyFromEnvironment,
+		DialContext: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 60 * time.Second,
+			DualStack: true,
+		}).DialContext,
+		ForceAttemptHTTP2:     true,
+		MaxIdleConns:          100,
+		IdleConnTimeout:       180 * time.Second,
+		TLSHandshakeTimeout:   10 * time.Second,
+		ExpectContinueTimeout: 1 * time.Second,
+		// nolint: gosec
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+}
+
+func GetDefaultHttpClient() *http.Client {
+	cliLoadOnce.Do(func() {
+		dftHttpCli = &http.Client{
+			Timeout:   httpTimeout,
+			Transport: getDefaultTransPort(),
+		}
+	})
+
+	return dftHttpCli
+}
+
+func NewHttpCli() (*http.Client, error) {
+	return GetDefaultHttpClient(), nil
+}
+
+func NewHttpCliWithTimeout(timeout time.Duration) (*http.Client, error) {
+	httpCli := &http.Client{
+		Timeout:   timeout,
+		Transport: getDefaultTransPort(),
+	}
+	return httpCli, nil
+}
+
+type JsonRequestBuilder struct {
+	timeout       *time.Duration
+	method        string
+	url           string
+	query         map[string]string
+	headers       map[string]string
+	payload       interface{}
+	result        interface{}
+	reqProcessors []func(req *http.Request)
+}
+
+func NewJsonRequestBuilder() *JsonRequestBuilder {
+	builder := JsonRequestBuilder{}
+	return &builder
+}
+
+func (b *JsonRequestBuilder) Timeout(timeout time.Duration) *JsonRequestBuilder {
+	b.timeout = &timeout
+	return b
+}
+
+func (b *JsonRequestBuilder) Method(method string) *JsonRequestBuilder {
+	b.method = method
+	return b
+}
+
+func (b *JsonRequestBuilder) Url(url string) *JsonRequestBuilder {
+	b.url = url
+	return b
+}
+
+func (b *JsonRequestBuilder) Query(query map[string]string) *JsonRequestBuilder {
+	b.query = query
+	return b
+}
+
+func (b *JsonRequestBuilder) Headers(headers map[string]string) *JsonRequestBuilder {
+	b.headers = headers
+	return b
+}
+
+func (b *JsonRequestBuilder) Payload(payload interface{}) *JsonRequestBuilder {
+	b.payload = payload
+	return b
+}
+
+func (b *JsonRequestBuilder) Result(result interface{}) *JsonRequestBuilder {
+	b.result = result
+	return b
+}
+
+func (b *JsonRequestBuilder) ProcessReq(processor func(req *http.Request)) *JsonRequestBuilder {
+	b.reqProcessors = append(b.reqProcessors, processor)
+	return b
+}
+
+func (b *JsonRequestBuilder) Do(ctx context.Context) (statusCode int, err error) {
+	var req *http.Request
+
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "DoJsonRequest Error: [%s]%s", b.method, b.url)
+		}
+	}()
+
+	if b.payload == nil {
+		req, err = http.NewRequestWithContext(ctx, b.method, b.url, nil)
+	} else {
+		switch p := b.payload.(type) {
+		case io.Reader:
+			req, err = http.NewRequestWithContext(ctx, b.method, b.url, p)
+		default:
+			var data []byte
+			data, err = json.Marshal(b.payload)
+			if err != nil {
+				return
+			}
+
+			req, err = http.NewRequestWithContext(ctx, b.method, b.url, bytes.NewBuffer(data))
+		}
+	}
+
+	if err != nil {
+		return
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+
+	if b.headers != nil {
+		for k, v := range b.headers {
+			req.Header.Set(k, v)
+		}
+	}
+
+	for _, reqProcessor := range b.reqProcessors {
+		reqProcessor(req)
+	}
+
+	q := req.URL.Query()
+	for key, value := range b.query {
+		q.Add(key, value)
+	}
+	req.URL.RawQuery = q.Encode()
+
+	cli := GetDefaultHttpClient()
+	if b.timeout != nil {
+		cli.Timeout = *b.timeout
+	}
+	logrus.Debugf("http %s %s", b.method, b.url)
+	var resp *http.Response
+	resp, err = cli.Do(req)
+	if err != nil {
+		return
+	}
+	statusCode = resp.StatusCode
+	defer func() {
+		_ = resp.Body.Close()
+	}()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		logrus.Errorf("resp.Body error, %s", err)
+		return
+	}
+
+	if resp.StatusCode != 200 {
+		msg := fmt.Sprintf("%s %s status=%d, %s", b.method, b.url, resp.StatusCode, body)
+		logrus.Error(msg)
+		err = errors.New(msg)
+		return
+	}
+
+	if b.result != nil {
+		err = errors.Wrap(json.Unmarshal(body, b.result), "json unmarshal")
+	}
+
+	return
+}
+
+func DoJsonRequest(ctx context.Context, method string, url string, headers map[string]string, payload, result interface{}) (err error) {
+	_, err = NewJsonRequestBuilder().Method(method).Url(url).Headers(headers).Payload(payload).Result(result).Do(ctx)
+	return
+}

deploy/compoundai/operator/pkg/compoundai/system/env.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package system
+
+import (
+	"os"
+	"sync"
+
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	// NamespaceEnvKey is the environment variable that specifies the system namespace.
+	NamespaceEnvKey = "SYSTEM_NAMESPACE"
+	// ResourceLabelEnvKey is the environment variable that specifies the system resource
+	// label.
+	ResourceLabelEnvKey = "SYSTEM_RESOURCE_LABEL"
+
+	DefaultNamespace = "yatai-deployment"
+	MagicDNSEnvKey   = "MAGIC_DNS"
+	DefaultMagicDNS  = "sslip.io"
+)
+
+var (
+	once sync.Once
+)
+
+// GetNamespace returns the name of the K8s namespace where our system components
+// run.
+func GetNamespace() string {
+	if ns := os.Getenv(NamespaceEnvKey); ns != "" {
+		return ns
+	}
+
+	once.Do(func() {
+		logrus.Infof("%s environment variable not set, using default namespace %s", NamespaceEnvKey, DefaultNamespace)
+	})
+	return DefaultNamespace
+}
+
+// GetResourceLabel returns the label key identifying K8s objects our system
+// components source their configuration from.
+func GetResourceLabel() string {
+	return os.Getenv(ResourceLabelEnvKey)
+}
+
+func GetMagicDNS() string {
+	magicDNS := os.Getenv(MagicDNSEnvKey)
+	if magicDNS == "" {
+		magicDNS = DefaultMagicDNS
+	}
+	return magicDNS
+}

deploy/compoundai/operator/pkg/compoundai/system/ingress.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package system
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/pkg/errors"
+	"github.com/rs/xid"
+	"github.com/sirupsen/logrus"
+	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+
+	"github.com/dynemo-ai/dynemo/deploy/compoundai/operator/pkg/compoundai/consts"
+)
+
+type IngressConfig struct {
+	ClassName   *string
+	Annotations map[string]string
+	Path        string
+	PathType    networkingv1.PathType
+}
+
+func GetIngressConfig(ctx context.Context, configmapGetter func(ctx context.Context, namespace, name string) (*corev1.ConfigMap, error)) (ingressConfig *IngressConfig, err error) {
+	configMap, err := GetNetworkConfigConfigMap(ctx, configmapGetter)
+	if err != nil {
+		err = errors.Wrapf(err, "failed to get configmap %s", consts.KubeConfigMapNameNetworkConfig)
+		return
+	}
+
+	var className *string
+
+	className_ := strings.TrimSpace(configMap.Data[consts.KubeConfigMapKeyNetworkConfigIngressClass])
+	if className_ != "" {
+		className = &className_
+	}
+
+	annotations := make(map[string]string)
+
+	annotations_ := strings.TrimSpace(configMap.Data[consts.KubeConfigMapKeyNetworkConfigIngressAnnotations])
+	if annotations_ != "" {
+		err = json.Unmarshal([]byte(annotations_), &annotations)
+		if err != nil {
+			err = errors.Wrapf(err, "failed to json unmarshal %s in configmap %s: %s", consts.KubeConfigMapKeyNetworkConfigIngressAnnotations, consts.KubeConfigMapNameNetworkConfig, annotations_)
+			return
+		}
+	}
+
+	path := strings.TrimSpace(configMap.Data[consts.KubeConfigMapKeyNetworkConfigIngressPath])
+	if path == "" {
+		path = "/"
+	}
+
+	pathType := networkingv1.PathTypeImplementationSpecific
+
+	pathType_ := strings.TrimSpace(configMap.Data[consts.KubeConfigMapKeyNetworkConfigIngressPathType])
+	if pathType_ != "" {
+		pathType = networkingv1.PathType(pathType_)
+	}
+
+	ingressConfig = &IngressConfig{
+		ClassName:   className,
+		Annotations: annotations,
+		Path:        path,
+		PathType:    pathType,
+	}
+
+	return
+}
+
+func GetIngressIP(ctx context.Context, configmapGetter func(ctx context.Context, namespace, name string) (*corev1.ConfigMap, error), cliset *kubernetes.Clientset) (ip string, err error) {
+	ingressConfig, err := GetIngressConfig(ctx, configmapGetter)
+	if err != nil {
+		err = errors.Wrapf(err, "failed to get ingress config")
+		return
+	}
+
+	ingressClassName := ingressConfig.ClassName
+	ingressAnnotations := ingressConfig.Annotations
+
+	ingressCli := cliset.NetworkingV1().Ingresses(GetNamespace())
+
+	ingName := "default-domain-"
+	pathType := networkingv1.PathTypeImplementationSpecific
+
+	podName := os.Getenv("POD_NAME")
+	if podName == "" {
+		// random string
+		guid := xid.New()
+		podName = fmt.Sprintf("a%s", strings.ToLower(guid.String()))
+	}
+
+	logrus.Infof("Creating ingress %s to get a ingress IP automatically", ingName)
+	ing, err := ingressCli.Create(ctx, &networkingv1.Ingress{
+		ObjectMeta: metav1.ObjectMeta{
+			GenerateName: ingName,
+			Namespace:    GetNamespace(),
+			Annotations:  ingressAnnotations,
+		},
+		Spec: networkingv1.IngressSpec{
+			IngressClassName: ingressClassName,
+			Rules: []networkingv1.IngressRule{{
+				Host: fmt.Sprintf("%s.this-is-yatai-in-order-to-generate-the-default-domain-suffix.yeah", podName),
+				IngressRuleValue: networkingv1.IngressRuleValue{
+					HTTP: &networkingv1.HTTPIngressRuleValue{
+						Paths: []networkingv1.HTTPIngressPath{
+							{
+								Path:     "/",
+								PathType: &pathType,
+								Backend: networkingv1.IngressBackend{
+									Service: &networkingv1.IngressServiceBackend{
+										Name: "default-domain-service",
+										Port: networkingv1.ServiceBackendPort{
+											Number: consts.BentoServicePort,
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			}},
+		},
+	}, metav1.CreateOptions{})
+	if err != nil && !k8serrors.IsAlreadyExists(err) {
+		err = errors.Wrapf(err, "failed to create ingress %s", ingName)
+		return
+	}
+	defer func() {
+		_ = ingressCli.Delete(ctx, ing.Name, metav1.DeleteOptions{})
+	}()
+
+	// Interval to poll for objects.
+	pollInterval := 10 * time.Second
+	// How long to wait for objects.
+	waitTimeout := 20 * time.Minute
+
+	logrus.Infof("Waiting for ingress %s to be ready", ing.Name)
+	// Wait for the Ingress to be Ready.
+	if err = wait.PollUntilContextTimeout(ctx, pollInterval, waitTimeout, false, func(ctx context.Context) (done bool, err error) {
+		ing, err = ingressCli.Get(
+			ctx, ing.Name, metav1.GetOptions{})
+		if err != nil {
+			return true, err
+		}
+		return len(ing.Status.LoadBalancer.Ingress) > 0, nil
+	}); err != nil {
+		err = errors.Wrapf(err, "failed to wait for ingress %s to be ready", ing.Name)
+		return
+	}
+	logrus.Infof("Ingress %s is ready", ing.Name)
+
+	address := ing.Status.LoadBalancer.Ingress[0]
+
+	ip = address.IP
+	if ip == "" {
+		if address.Hostname == "" {
+			err = errors.Errorf("the ingress %s status has no IP or hostname", ing.Name)
+			return
+		}
+		var ipAddr *net.IPAddr
+		ipAddr, err = net.ResolveIPAddr("ip4", address.Hostname)
+		if err != nil {
+			err = errors.Wrapf(err, "failed to resolve ip address for hostname %s", address.Hostname)
+			return
+		}
+		ip = ipAddr.String()
+	}
+
+	return
+}
+
+func GetDomainSuffix(ctx context.Context, configmapGetter func(ctx context.Context, namespace, name string) (*corev1.ConfigMap, error), cliset *kubernetes.Clientset) (domainSuffix string, err error) {
+	configMap, err := GetNetworkConfigConfigMap(ctx, configmapGetter)
+	if err != nil {
+		err = errors.Wrapf(err, "failed to get configmap %s", consts.KubeConfigMapNameNetworkConfig)
+		return
+	}
+
+	domainSuffix = strings.TrimSpace(configMap.Data[consts.KubeConfigMapKeyNetworkConfigDomainSuffix])
+	if domainSuffix != "" {
+		logrus.Infof("The %s in the network config has already set to `%s`", consts.KubeConfigMapKeyNetworkConfigDomainSuffix, domainSuffix)
+		return
+	}
+
+	magicDNS := GetMagicDNS()
+
+	var ip string
+
+	ip, err = GetIngressIP(ctx, configmapGetter, cliset)
+	if err != nil {
+		return
+	}
+
+	domainSuffix = fmt.Sprintf("%s.%s", ip, magicDNS)
+
+	logrus.Infof("you have not set the %s in the network config, so use magic DNS to generate a domain suffix automatically: `%s`, and set it to the network config", consts.KubeConfigMapKeyNetworkConfigDomainSuffix, domainSuffix)
+
+	configMapCli := cliset.CoreV1().ConfigMaps(configMap.Namespace)
+	_, err = configMapCli.Patch(ctx, configMap.Name, types.MergePatchType, []byte(fmt.Sprintf(`{"data":{"%s":"%s"}}`, consts.KubeConfigMapKeyNetworkConfigDomainSuffix, domainSuffix)), metav1.PatchOptions{})
+	if err != nil {
+		err = errors.Wrapf(err, "failed to patch configmap %s", consts.KubeConfigMapNameNetworkConfig)
+		return
+	}
+
+	return
+}

deploy/compoundai/operator/pkg/compoundai/system/network.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package system
+
+import (
+	"context"
+
+	corev1 "k8s.io/api/core/v1"
+
+	"github.com/dynemo-ai/dynemo/deploy/compoundai/operator/pkg/compoundai/consts"
+)
+
+func GetNetworkConfigConfigMap(ctx context.Context, configmapGetter func(ctx context.Context, namespace, name string) (*corev1.ConfigMap, error)) (configMap *corev1.ConfigMap, err error) {
+	configMap, err = configmapGetter(ctx, GetNamespace(), consts.KubeConfigMapNameNetworkConfig)
+	return
+}

deploy/compoundai/operator/pkg/compoundai/utils/url.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package utils
+
+import (
+	"net/url"
+	"path"
+)
+
+func UrlJoin(baseUrl string, extra string, params ...map[string]string) string {
+	u, err := url.Parse(baseUrl)
+	if err != nil {
+		return baseUrl
+	}
+	u.Path = path.Join(u.Path, extra)
+	q := u.Query()
+	for _, p := range params {
+		for k, v := range p {
+			q.Add(k, v)
+		}
+	}
+	u.RawQuery = q.Encode()
+	return u.String()
+}
+
+func UrlJoinWithQuery(baseUrl string, extra string, query url.Values) string {
+	u, err := url.Parse(baseUrl)
+	if err != nil {
+		return baseUrl
+	}
+	u.Path = path.Join(u.Path, extra)
+	u.RawQuery = query.Encode()
+	return u.String()
+}

deploy/compoundai/operator/test/e2e/e2e_suite_test.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package e2e
+
+import (
+	"fmt"
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// Run e2e tests using the Ginkgo runner.
+func TestE2E(t *testing.T) {
+	RegisterFailHandler(Fail)
+	_, _ = fmt.Fprintf(GinkgoWriter, "Starting compoundai-kubernetes-operator suite\n")
+	RunSpecs(t, "e2e suite")
+}

deploy/compoundai/operator/test/e2e/e2e_test.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package e2e
+
+import (
+	"fmt"
+	"os/exec"
+	"time"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/dynemo-ai/dynemo/deploy/compoundai/operator/test/utils"
+)
+
+const namespace = "compoundai-kubernetes-operator-system"
+
+var _ = Describe("controller", Ordered, func() {
+	BeforeAll(func() {
+		By("installing prometheus operator")
+		Expect(utils.InstallPrometheusOperator()).To(Succeed())
+
+		By("installing the cert-manager")
+		Expect(utils.InstallCertManager()).To(Succeed())
+
+		By("creating manager namespace")
+		cmd := exec.Command("kubectl", "create", "ns", namespace)
+		_, _ = utils.Run(cmd)
+	})
+
+	AfterAll(func() {
+		By("uninstalling the Prometheus manager bundle")
+		utils.UninstallPrometheusOperator()
+
+		By("uninstalling the cert-manager bundle")
+		utils.UninstallCertManager()
+
+		By("removing manager namespace")
+		cmd := exec.Command("kubectl", "delete", "ns", namespace)
+		_, _ = utils.Run(cmd)
+	})
+
+	Context("Operator", func() {
+		It("should run successfully", func() {
+			var controllerPodName string
+			var err error
+
+			// projectimage stores the name of the image used in the example
+			var projectimage = "example.com/compoundai-kubernetes-operator:v0.0.1"
+
+			By("building the manager(Operator) image")
+			cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectimage))
+			_, err = utils.Run(cmd)
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+			By("loading the the manager(Operator) image on Kind")
+			err = utils.LoadImageToKindClusterWithName(projectimage)
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+			By("installing CRDs")
+			cmd = exec.Command("make", "install")
+			_, err = utils.Run(cmd)
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+			By("deploying the controller-manager")
+			cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectimage))
+			_, err = utils.Run(cmd)
+			ExpectWithOffset(1, err).NotTo(HaveOccurred())
+
+			By("validating that the controller-manager pod is running as expected")
+			verifyControllerUp := func() error {
+				// Get pod name
+
+				cmd = exec.Command("kubectl", "get",
+					"pods", "-l", "control-plane=controller-manager",
+					"-o", "go-template={{ range .items }}"+
+						"{{ if not .metadata.deletionTimestamp }}"+
+						"{{ .metadata.name }}"+
+						"{{ \"\\n\" }}{{ end }}{{ end }}",
+					"-n", namespace,
+				)
+
+				podOutput, err := utils.Run(cmd)
+				ExpectWithOffset(2, err).NotTo(HaveOccurred())
+				podNames := utils.GetNonEmptyLines(string(podOutput))
+				if len(podNames) != 1 {
+					return fmt.Errorf("expect 1 controller pods running, but got %d", len(podNames))
+				}
+				controllerPodName = podNames[0]
+				ExpectWithOffset(2, controllerPodName).Should(ContainSubstring("controller-manager"))
+
+				// Validate pod status
+				cmd = exec.Command("kubectl", "get",
+					"pods", controllerPodName, "-o", "jsonpath={.status.phase}",
+					"-n", namespace,
+				)
+				status, err := utils.Run(cmd)
+				ExpectWithOffset(2, err).NotTo(HaveOccurred())
+				if string(status) != "Running" {
+					return fmt.Errorf("controller pod in %s status", status)
+				}
+				return nil
+			}
+			EventuallyWithOffset(1, verifyControllerUp, time.Minute, time.Second).Should(Succeed())
+
+		})
+	})
+})

deploy/compoundai/operator/test/utils/utils.go

+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package utils
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2" //nolint:golint,revive
+)
+
+const (
+	prometheusOperatorVersion = "v0.68.0"
+	prometheusOperatorURL     = "https://github.com/prometheus-operator/prometheus-operator/" +
+		"releases/download/%s/bundle.yaml"
+
+	certmanagerVersion = "v1.5.3"
+	certmanagerURLTmpl = "https://github.com/jetstack/cert-manager/releases/download/%s/cert-manager.yaml"
+)
+
+func warnError(err error) {
+	_, _ = fmt.Fprintf(GinkgoWriter, "warning: %v\n", err)
+}
+
+// InstallPrometheusOperator installs the prometheus Operator to be used to export the enabled metrics.
+func InstallPrometheusOperator() error {
+	url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion)
+	cmd := exec.Command("kubectl", "create", "-f", url)
+	_, err := Run(cmd)
+	return err
+}
+
+// Run executes the provided command within this context
+func Run(cmd *exec.Cmd) ([]byte, error) {
+	dir, _ := GetProjectDir()
+	cmd.Dir = dir
+
+	if err := os.Chdir(cmd.Dir); err != nil {
+		_, _ = fmt.Fprintf(GinkgoWriter, "chdir dir: %s\n", err)
+	}
+
+	cmd.Env = append(os.Environ(), "GO111MODULE=on")
+	command := strings.Join(cmd.Args, " ")
+	_, _ = fmt.Fprintf(GinkgoWriter, "running: %s\n", command)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		return output, fmt.Errorf("%s failed with error: (%v) %s", command, err, string(output))
+	}
+
+	return output, nil
+}
+
+// UninstallPrometheusOperator uninstalls the prometheus
+func UninstallPrometheusOperator() {
+	url := fmt.Sprintf(prometheusOperatorURL, prometheusOperatorVersion)
+	cmd := exec.Command("kubectl", "delete", "-f", url)
+	if _, err := Run(cmd); err != nil {
+		warnError(err)
+	}
+}
+
+// UninstallCertManager uninstalls the cert manager
+func UninstallCertManager() {
+	url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion)
+	cmd := exec.Command("kubectl", "delete", "-f", url)
+	if _, err := Run(cmd); err != nil {
+		warnError(err)
+	}
+}
+
+// InstallCertManager installs the cert manager bundle.
+func InstallCertManager() error {
+	url := fmt.Sprintf(certmanagerURLTmpl, certmanagerVersion)
+	cmd := exec.Command("kubectl", "apply", "-f", url)
+	if _, err := Run(cmd); err != nil {
+		return err
+	}
+	// Wait for cert-manager-webhook to be ready, which can take time if cert-manager
+	// was re-installed after uninstalling on a cluster.
+	cmd = exec.Command("kubectl", "wait", "deployment.apps/cert-manager-webhook",
+		"--for", "condition=Available",
+		"--namespace", "cert-manager",
+		"--timeout", "5m",
+	)
+
+	_, err := Run(cmd)
+	return err
+}
+
+// LoadImageToKindCluster loads a local docker image to the kind cluster
+func LoadImageToKindClusterWithName(name string) error {
+	cluster := "kind"
+	if v, ok := os.LookupEnv("KIND_CLUSTER"); ok {
+		cluster = v
+	}
+	kindOptions := []string{"load", "docker-image", name, "--name", cluster}
+	cmd := exec.Command("kind", kindOptions...)
+	_, err := Run(cmd)
+	return err
+}
+
+// GetNonEmptyLines converts given command output string into individual objects
+// according to line breakers, and ignores the empty elements in it.
+func GetNonEmptyLines(output string) []string {
+	var res []string
+	elements := strings.Split(output, "\n")
+	for _, element := range elements {
+		if element != "" {
+			res = append(res, element)
+		}
+	}
+
+	return res
+}
+
+// GetProjectDir will return the directory where the project is
+func GetProjectDir() (string, error) {
+	wd, err := os.Getwd()
+	if err != nil {
+		return wd, err
+	}
+	wd = strings.Replace(wd, "/test/e2e", "", -1)
+	return wd, nil
+}

deploy/compoundai/sdk/uv.lock

@@ -146,7 +146,7 @@ wheels = [
 
 [[package]]
 name = "bentoml"
-version = "1.4.0"
+version = "1.4.2"
 source = { registry = "https://pypi.org/simple" }
 dependencies = [
     { name = "a2wsgi" },
@@ -157,11 +157,9 @@ dependencies = [
     { name = "click" },
     { name = "click-option-group" },
     { name = "cloudpickle" },
-    { name = "deepmerge" },
     { name = "fs" },
     { name = "httpx" },
     { name = "httpx-ws" },
-    { name = "inflection" },
     { name = "jinja2" },
     { name = "kantoku" },
     { name = "numpy" },
@@ -191,13 +189,12 @@ dependencies = [
     { name = "starlette" },
     { name = "tomli" },
     { name = "tomli-w" },
-    { name = "uv" },
     { name = "uvicorn" },
     { name = "watchfiles" },
 ]
-sdist = { url = "https://files.pythonhosted.org/packages/ab/3d/cb8f05176431e2711ea546439618a74ab95cc8ae34ade617277cf9274da8/bentoml-1.4.0.tar.gz", hash = "sha256:2e6b9d37c7af19112f1772c2383659412a324237703c64fb806ff8738ad60d86", size = 968421 }
+sdist = { url = "https://files.pythonhosted.org/packages/ce/ff/17a161f61d0aae6eba7a801323da4b8de0c98b1ce31c5d5370c5d0cf5997/bentoml-1.4.2.tar.gz", hash = "sha256:d83e2b138cee1f8ac758ede69c413bdb309c2f1df0d71176547bb4604c8ca903", size = 966659 }
 wheels = [
-    { url = "https://files.pythonhosted.org/packages/5f/e1/cc8c93e017188009ed08122c0f63fc6f9bcce218e7be312fb9f674c227b8/bentoml-1.4.0-py3-none-any.whl", hash = "sha256:0bb7e3d676dac499929dd0317c6c0612b45264e407108917dfbc74042f837194", size = 1148696 },
+    { url = "https://files.pythonhosted.org/packages/d1/bb/54cc91840209e155c6e7092dc865bad8df5dc1ecaf24cc54f8bf32cca03f/bentoml-1.4.2-py3-none-any.whl", hash = "sha256:bec93162c1115461b9f60147d0dbb370a467b26c723f58556c023b4b7d96be8c", size = 1146823 },
 ]
 
 [[package]]
@@ -294,18 +291,13 @@ version = "0.1.0"
 source = { editable = "." }
 dependencies = [
     { name = "bentoml" },
+    { name = "types-psutil" },
 ]
 
 [package.metadata]
-requires-dist = [{ name = "bentoml", specifier = ">=1.4.0" }]
-
-[[package]]
-name = "deepmerge"
-version = "2.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/a8/3a/b0ba594708f1ad0bc735884b3ad854d3ca3bdc1d741e56e40bbda6263499/deepmerge-2.0.tar.gz", hash = "sha256:5c3d86081fbebd04dd5de03626a0607b809a98fb6ccba5770b62466fe940ff20", size = 19890 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/2d/82/e5d2c1c67d19841e9edc74954c827444ae826978499bde3dfc1d007c8c11/deepmerge-2.0-py3-none-any.whl", hash = "sha256:6de9ce507115cff0bed95ff0ce9ecc31088ef50cbdf09bc90a09349a318b3d00", size = 13475 },
+requires-dist = [
+    { name = "bentoml", specifier = ">=1.4.1" },
+    { name = "types-psutil", specifier = "==7.0.0.20250218" },
 ]
 
 [[package]]
@@ -440,15 +432,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/a0/d9/a1e041c5e7caa9a05c925f4bdbdfb7f006d1f74996af53467bc394c97be7/importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b", size = 26514 },
 ]
 
-[[package]]
-name = "inflection"
-version = "0.5.1"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/e1/7e/691d061b7329bc8d54edbf0ec22fbfb2afe61facb681f9aaa9bff7a27d04/inflection-0.5.1.tar.gz", hash = "sha256:1a29730d366e996aaacffb2f1f1cb9593dc38e2ddd30c91250c6dde09ea9b417", size = 15091 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/59/91/aa6bde563e0085a02a435aa99b49ef75b0a4b062635e606dab23ce18d720/inflection-0.5.1-py2.py3-none-any.whl", hash = "sha256:f38b2b640938a4f35ade69ac3d053042959b62a0f1076a5bbaa1b9526605a8a2", size = 9454 },
-]
-
 [[package]]
 name = "jinja2"
 version = "3.1.5"
@@ -1037,6 +1020,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/61/cc/58b1adeb1bb46228442081e746fcdbc4540905c87e8add7c277540934edb/tornado-6.4.2-cp38-abi3-win_amd64.whl", hash = "sha256:908b71bf3ff37d81073356a5fadcc660eb10c1476ee6e2725588626ce7e5ca38", size = 438907 },
 ]
 
+[[package]]
+name = "types-psutil"
+version = "7.0.0.20250218"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/b5/7c/145600d30456e7ccbb499abcf718aab2bd830e604a0ae8eb32b67cd346a6/types_psutil-7.0.0.20250218.tar.gz", hash = "sha256:1e642cdafe837b240295b23b1cbd4691d80b08a07d29932143cbbae30eb0db9c", size = 19828 }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/50/c8/f4365293408da4a9bcb1849d3efd8c60427cffff68cbb98ab1b81851d8bb/types_psutil-7.0.0.20250218-py3-none-any.whl", hash = "sha256:1447a30c282aafefcf8941ece854e1100eee7b0296a9d9be9977292f0269b121", size = 22763 },
+]
+
 [[package]]
 name = "typing-extensions"
 version = "4.12.2"
@@ -1046,31 +1038,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/26/9f/ad63fc0248c5379346306f8668cda6e2e2e9c95e01216d2b8ffd9ff037d0/typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d", size = 37438 },
 ]
 
-[[package]]
-name = "uv"
-version = "0.6.2"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/6d/05/118e10d91981b85f47b27d089782a6598a9584ff607bffb8e2f6be1f1245/uv-0.6.2.tar.gz", hash = "sha256:d696a4f3d4a3ac1b305255e8814ae3a147ea3428a977bb3b4335a339941799bc", size = 3066291 }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/a1/cf/9c3c9a427c7ecc37be238c4433188614b3d342191c0299c632f512d493ff/uv-0.6.2-py3-none-linux_armv6l.whl", hash = "sha256:d501ae16fb33969b12a64ac7b9c49d672b8c3964026c5dcaee3b1dcd50a6a22c", size = 15513992 },
-    { url = "https://files.pythonhosted.org/packages/86/01/1e1f88826d92d11f2232f96eef190574a4edb470546a141bba652cd37240/uv-0.6.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:2c13ca920d87dc00721a86ac3d19667cff5435b369d21e3d6df76b373d8fa8df", size = 15659547 },
-    { url = "https://files.pythonhosted.org/packages/ee/40/59e9c03431d4c82420e081f92719e5784db8f1c92a25b2abdfe6ac645b7e/uv-0.6.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:f24e119d338bae32b5a604585b7b518036fba556e2c2d9dbd2d7cf1411213b57", size = 14589044 },
-    { url = "https://files.pythonhosted.org/packages/11/8b/5d9f9f4e3969d6a2c9ce9a0b4a85ecb8ca89bf5c00e9ec097cf472abb2a2/uv-0.6.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.musllinux_1_1_aarch64.whl", hash = "sha256:1db90b728a173926e2018b89df776a373b1e50520466f61e0dbf05f9a64a6db5", size = 15034328 },
-    { url = "https://files.pythonhosted.org/packages/f3/ba/f31fd6af8f70b21d9e0b7cca0241a8f10e03d24862f49f93fbc5ff1e4fce/uv-0.6.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d23fb9cd41aecb31845e884d0bfde243e04e763abeab3532138321b4ebe7437c", size = 15275180 },
-    { url = "https://files.pythonhosted.org/packages/aa/3b/358cfea4265a0966fafa7934ed0f9f1fb031d7ebbe8a15e02a308afff6ad/uv-0.6.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:df0a1d95fd1539c05de434259fafcee0b6852900d4178e94b3b6b6b06438b60c", size = 15969503 },
-    { url = "https://files.pythonhosted.org/packages/57/f5/840d8fb46c1cf723e1b7168832de52e58d86764aa625c2100b35a27261af/uv-0.6.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:f2f0dc9a0564b31d4efdee317c176a23bbe7e61aec6d281a331ba6ae32f828ff", size = 16950563 },
-    { url = "https://files.pythonhosted.org/packages/f6/37/75c5ff09db56c34f0f5d3d55dd4188e52d09219ef76bfe176dae58ed5f4a/uv-0.6.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:326aff8c4fb8153e2384e79904c27b1c9d4c3a5879b53a6fbc2da3283fda321d", size = 16631562 },
-    { url = "https://files.pythonhosted.org/packages/9d/5f/91bfae5ecf9f6c5f4754aa794159acc77245a53233a966865ae4974e5cdf/uv-0.6.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b8763f310a473f46c0226f5e08a876bd34de121ac370cc7294a5397a13a18d8a", size = 20994598 },
-    { url = "https://files.pythonhosted.org/packages/8d/39/17f77b4b5f1a1e579d9ce94859aada9418c9ebcaa227b54b10648218bafa/uv-0.6.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e2e421947ef889e6c8913992c560d611826464eabc78f8f702a5eff824aabc7", size = 16367280 },
-    { url = "https://files.pythonhosted.org/packages/a7/6b/fbd9794e1344b299e02993322f44b500f4d66ecdb83860e2fcf35d8cac2c/uv-0.6.2-py3-none-manylinux_2_28_aarch64.whl", hash = "sha256:7dd26dabd918e5648ecf94fb7c0787db954237e34ea3bdd944b98d007b44c3a5", size = 15317824 },
-    { url = "https://files.pythonhosted.org/packages/51/a0/9249a55365c2f9781243a7f35c3a01864b19aa9a62b1fc50b7231793346e/uv-0.6.2-py3-none-musllinux_1_1_armv7l.whl", hash = "sha256:f3719da2e59403783eab634a6238b90051fc65379e02c10b9ca1b32b26d35f77", size = 15228644 },
-    { url = "https://files.pythonhosted.org/packages/27/76/790b3d9c0b9ecd9ab6c1b7e904c36d470685c70d0b21a134b026452e0fcc/uv-0.6.2-py3-none-musllinux_1_1_i686.whl", hash = "sha256:b435687e5c26a64858ea842fbb4b35ced8e8741a99d1b75d0c0143462e956db9", size = 15608612 },
-    { url = "https://files.pythonhosted.org/packages/05/b6/79961374b2318461b4dfc0e565d63281bf788fea93fc81b2d1738847aec2/uv-0.6.2-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:0f1e8e15c92607862e72e0467a31947af7b9aef93924072e9b4d5dcb5633d374", size = 16480962 },
-    { url = "https://files.pythonhosted.org/packages/68/20/df7788bde9d114c501cd8ebb60235be07ff0fb0dc26fa1e7e99ada251d73/uv-0.6.2-py3-none-win32.whl", hash = "sha256:52b7452f4c523b9875de53ba73df87acd1cdea36640281d0d80c8074eda42f16", size = 15717804 },
-    { url = "https://files.pythonhosted.org/packages/e1/0a/fc966f859b6252050c71e1afcdce116c8ef3513f8b423bb3ca05fb13485d/uv-0.6.2-py3-none-win_amd64.whl", hash = "sha256:5337cdb6ecc604d0cf36fe6799dd0479111b606009e6c29685d213c74eb40373", size = 17017798 },
-    { url = "https://files.pythonhosted.org/packages/03/82/4318c4874c8dd59a0386e2bf0f4d09fc5bb4900349238828153235d387eb/uv-0.6.2-py3-none-win_arm64.whl", hash = "sha256:27ecb8f6ef796220062f31a12e2dc5dc7a14704aa1df0da2dfa3530346c7e3cc", size = 15923484 },
-]
-
 [[package]]
 name = "uvicorn"
 version = "0.34.0"

pyproject.toml

@@ -58,7 +58,7 @@ skip = "./.git,./.github,./lib/llm/tests/data"
 ignore-regex = "\\b(.{1,4}|[A-Z]\\w*T)\\b"
 # ignore allowed words
 # ignoring atleast to avoid testing::AtLeast from getting flagged
-ignore-words-list = "atleast"
+ignore-words-list = "atleast,afterall"
 # use the 'clear' dictionary for unambiguous spelling mistakes
 builtin = "clear"
 # disable warnings about binary files and wrong encoding