Unverified Commit f11ea8f7 authored by julienmancuso's avatar julienmancuso Committed by GitHub
Browse files

feat: remove bento/yatai references (#782)

parent ea84ab11
......@@ -48,14 +48,6 @@ Create chart name and version as used by the chart label.
{{ include "dynamo-operator.fullname" . }}-dynamo-env
{{- end }}
{{/*
Generate k8s robot token
*/}}
{{- define "dynamo-operator.yataiApiToken" -}}
{{- $secretObj := (lookup "v1" "Secret" .Release.Namespace (include "dynamo-operator.dynamo.envname" .)) | default dict }}
{{- $secretData := (get $secretObj "data") | default dict }}
{{- (get $secretData "YATAI_API_TOKEN") | default (randAlphaNum 16 | nospace | b64enc) | b64dec }}
{{- end -}}
{{/*
Common labels
......
......@@ -12,7 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
{{- if .Values.dynamo.bentoImageBuildEngine | eq "buildkit" }}
{{- if .Values.dynamo.imageBuildEngine | eq "buildkit" }}
---
apiVersion: apps/v1
kind: StatefulSet
......
......@@ -15,7 +15,7 @@
apiVersion: v1
kind: Secret
metadata:
name: yatai-regcred
name: dynamo-regcred
labels:
{{- include "dynamo-operator.labels" . | nindent 4 }}
type: kubernetes.io/dockerconfigjson
......
......@@ -21,23 +21,17 @@ metadata:
{{- include "dynamo-operator.labels" . | nindent 4 }}
type: Opaque
stringData:
YATAI_ENDPOINT: {{ .Values.dynamo.yatai.endpoint | quote }}
YATAI_CLUSTER_NAME: {{ .Values.dynamo.yatai.clusterName | quote }}
YATAI_SYSTEM_NAMESPACE: {{ default .Release.Namespace .Values.dynamo.yataiSystem.namespace }}
YATAI_DEPLOYMENT_NAMESPACE: {{ .Release.Namespace }}
YATAI_IMAGE_BUILDER_NAMESPACE: {{ .Release.Namespace }}
YATAI_API_TOKEN: {{ include "dynamo-operator.yataiApiToken" . | quote }}
API_STORE_ENDPOINT : {{ .Values.dynamo.apiStore.endpoint | quote }}
API_STORE_CLUSTER_NAME: {{ .Values.dynamo.apiStore.clusterName | quote }}
DYNAMO_SYSTEM_NAMESPACE: {{ .Release.Namespace }}
DYNAMO_DEPLOYMENT_NAMESPACE: {{ .Release.Namespace }}
DYNAMO_IMAGE_BUILDER_NAMESPACE: {{ .Release.Namespace }}
INTERNAL_IMAGES_METRICS_TRANSFORMER: {{ .Values.dynamo.internalImages.metricsTransformer | quote }}
INTERNAL_IMAGES_DEBUGGER: {{ .Values.dynamo.internalImages.debugger | quote }}
INTERNAL_IMAGES_MONITOR_EXPORTER: {{ .Values.dynamo.internalImages.monitorExporter | quote }}
INTERNAL_IMAGES_PROXY: {{ .Values.dynamo.internalImages.proxy | quote }}
{{- if .Values.dynamo.disableAutomateBentoImageBuilder }}
DISABLE_AUTOMATE_BENTO_IMAGE_BUILDER: "true"
{{- end }}
{{- if .Values.dynamo.enableRestrictedSecurityContext }}
ENABLE_RESTRICTED_SECURITY_CONTEXT: "true"
{{- end }}
......@@ -53,17 +47,15 @@ stringData:
DOCKER_REGISTRY_PASSWORD: {{ .password | quote }}
{{- end }}
DOCKER_REGISTRY_SECURE: {{ .Values.dynamo.dockerRegistry.secure | quote }}
DOCKER_REGISTRY_BENTO_REPOSITORY_NAME: {{ .Values.dynamo.dockerRegistry.bentoRepositoryName | quote }}
DOCKER_REGISTRY_DYNAMO_COMPONENTS_REPOSITORY_NAME: {{ .Values.dynamo.dockerRegistry.dynamoComponentsRepositoryName | quote }}
INTERNAL_IMAGES_BENTO_DOWNLOADER: {{ .Values.dynamo.internalImages.bentoDownloader | quote }}
INTERNAL_IMAGES_DYNAMO_COMPONENTS_DOWNLOADER: {{ .Values.dynamo.internalImages.dynamoComponentsDownloader | quote }}
INTERNAL_IMAGES_KANIKO: {{ .Values.dynamo.internalImages.kaniko | quote }}
INTERNAL_IMAGES_BUILDKIT: {{ .Values.dynamo.internalImages.buildkit | quote }}
INTERNAL_IMAGES_BUILDKIT_ROOTLESS: {{ .Values.dynamo.internalImages.buildkitRootless | quote }}
BUILDKIT_URL: tcp://{{ include "dynamo-operator.fullname" . }}-buildkitd:1234
BENTO_IMAGE_BUILD_ENGINE: {{ .Values.dynamo.bentoImageBuildEngine | quote }}
DISABLE_YATAI_COMPONENT_REGISTRATION: {{ .Values.dynamo.disableYataiComponentRegistration | quote }}
DYNAMO_IMAGE_BUILD_ENGINE: {{ .Values.dynamo.imageBuildEngine | quote }}
ADD_NAMESPACE_PREFIX_TO_IMAGE_NAME: {{ .Values.dynamo.addNamespacePrefixToImageName | quote }}
......
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
apiVersion: v1
kind: Secret
metadata:
name: dynamo-deployment-shared-env
namespace: {{ .Release.Namespace }}
labels:
{{- include "dynamo-operator.labels" . | nindent 4 }}
type: Opaque
stringData:
BENTO_DEPLOYMENT_ALL_NAMESPACES: "false"
BENTO_DEPLOYMENT_NAMESPACES: {{ default .Release.Namespace .Values.dynamo.yataiSystem.namespace }}
YATAI_DEPLOYMENT_NAMESPACE: {{ default .Release.Namespace .Values.dynamo.yataiSystem.namespace }}
\ No newline at end of file
......@@ -73,27 +73,21 @@ controllerManager:
annotations: {}
dynamo:
yatai:
apiStore:
endpoint: http://dynamo-server.dynamo-system.svc.cluster.local
clusterName: default
yataiSystem:
# If left blank, will default to the installation namespace
namespace: ""
internalImages:
bentoDownloader: quay.io/bentoml/bento-downloader:0.0.5
kaniko: quay.io/bentoml/kaniko:debug
buildkit: moby/buildkit:latest
buildkitRootless: quay.io/bentoml/buildkit:master-rootless
dynamoComponentsDownloader: quay.io/bentoml/bento-downloader:0.0.5
kaniko: gcr.io/kaniko-project/executor:debug
buildkit: moby/buildkit:v0.20.2
buildkitRootless: moby/buildkit:v0.20.2-rootless
metricsTransformer: quay.io/bentoml/yatai-bento-metrics-transformer:0.0.4
debugger: quay.io/bentoml/bento-debugger:0.0.8
monitorExporter: quay.io/bentoml/bentoml-monitor-exporter:0.0.3
proxy: quay.io/bentoml/bentoml-proxy:0.0.1
disableAutomateBentoImageBuilder: false
enableRestrictedSecurityContext: false
disableYataiComponentRegistration: false
dockerRegistry:
server: 'nvcr.io/nvidian/nim-llm-dev'
......@@ -103,9 +97,9 @@ dynamo:
passwordExistingSecretName: ''
passwordExistingSecretKey: ''
secure: true
bentoRepositoryName: dynamo-pipelines
dynamoComponentsRepositoryName: dynamo-pipelines
bentoImageBuildEngine: buildkit # options: kaniko, buildkit, buildkit-rootless
imageBuildEngine: buildkit # options: kaniko, buildkit, buildkit-rootless
addNamespacePrefixToImageName: false
estargz:
......
......@@ -37,31 +37,27 @@ dynamo-operator:
- --health-probe-bind-address=:8081
- --metrics-bind-address=127.0.0.1:8080
dynamo:
yatai:
apiStore:
endpoint: http://dynamo-store
clusterName: default
yataiSystem:
namespace: ""
internalImages:
bentoDownloader: quay.io/bentoml/bento-downloader:0.0.5
kaniko: quay.io/bentoml/kaniko:debug
buildkit: quay.io/bentoml/buildkit:master
buildkitRootless: quay.io/bentoml/buildkit:master-rootless
dynamoComponentsDownloader: quay.io/bentoml/bento-downloader:0.0.5
kaniko: gcr.io/kaniko-project/executor:debug
buildkit: moby/buildkit:v0.20.2
buildkitRootless: moby/buildkit:v0.20.2-rootless
metricsTransformer: quay.io/bentoml/yatai-bento-metrics-transformer:0.0.4
debugger: quay.io/bentoml/bento-debugger:0.0.8
monitorExporter: quay.io/bentoml/bentoml-monitor-exporter:0.0.3
proxy: quay.io/bentoml/bentoml-proxy:0.0.1
disableAutomateBentoImageBuilder: false
enableRestrictedSecurityContext: false
disableYataiComponentRegistration: false
dockerRegistry:
server: ""
inClusterServer: ""
username: ""
password: ""
secure: true
bentoRepositoryName: dynamo-pipelines
bentoImageBuildEngine: buildkit
dynamoComponentsRepositoryName: dynamo-pipelines
imageBuildEngine: buildkit
addNamespacePrefixToImageName: false
estargz:
enabled: false
......
......@@ -17,7 +17,7 @@
* Modifications Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES
*/
package yataiclient
package api_store_client
import (
"context"
......@@ -27,53 +27,27 @@ import (
"github.com/ai-dynamo/dynamo/deploy/dynamo/operator/api/dynamo/schemas"
)
const (
YataiApiTokenHeaderName = "X-YATAI-API-TOKEN"
NgcOrganizationHeaderName = "Nv-Ngc-Org"
NgcUserHeaderName = "Nv-Actor-Id"
)
type DynamoAuthHeaders struct {
OrgId string
UserId string
}
type YataiClient struct {
type ApiStoreClient struct {
endpoint string
apiToken string
headers DynamoAuthHeaders
}
func NewYataiClient(endpoint, apiToken string) *YataiClient {
return &YataiClient{
func NewApiStoreClient(endpoint string) *ApiStoreClient {
return &ApiStoreClient{
endpoint: endpoint,
apiToken: apiToken,
}
}
func (c *YataiClient) SetAuth(headers DynamoAuthHeaders) {
c.headers = headers
}
func (c *YataiClient) getHeaders() map[string]string {
return map[string]string{
YataiApiTokenHeaderName: c.apiToken,
NgcOrganizationHeaderName: c.headers.OrgId,
NgcUserHeaderName: c.headers.UserId,
}
}
func (c *YataiClient) GetBento(ctx context.Context, bentoRepositoryName, bentoVersion string) (bento *schemas.DynamoNIM, err error) {
url_ := urlJoin(c.endpoint, fmt.Sprintf("/api/v1/bento_repositories/%s/bentos/%s", bentoRepositoryName, bentoVersion))
bento = &schemas.DynamoNIM{}
_, err = DoJsonRequest(ctx, "GET", url_, c.getHeaders(), nil, nil, bento, nil)
func (c *ApiStoreClient) GetDynamoComponent(ctx context.Context, name, version string) (component *schemas.DynamoComponent, err error) {
url_ := urlJoin(c.endpoint, fmt.Sprintf("/api/v1/dynamo_nims/%s/versions/%s", name, version))
component = &schemas.DynamoComponent{}
_, err = DoJsonRequest(ctx, "GET", url_, nil, nil, nil, component, nil)
return
}
func (c *YataiClient) PresignBentoDownloadURL(ctx context.Context, bentoRepositoryName, bentoVersion string) (bento *schemas.DynamoNIM, err error) {
url_ := urlJoin(c.endpoint, fmt.Sprintf("/api/v1/dynamo_nims/%s/versions/%s/presign_download_url", bentoRepositoryName, bentoVersion))
bento = &schemas.DynamoNIM{}
_, err = DoJsonRequest(ctx, "PATCH", url_, c.getHeaders(), nil, nil, bento, nil)
func (c *ApiStoreClient) PresignDynamoComponentDownloadURL(ctx context.Context, name, version string) (component *schemas.DynamoComponent, err error) {
url_ := urlJoin(c.endpoint, fmt.Sprintf("/api/v1/dynamo_nims/%s/versions/%s/presign_download_url", name, version))
component = &schemas.DynamoComponent{}
_, err = DoJsonRequest(ctx, "PATCH", url_, nil, nil, nil, component, nil)
return
}
......
......@@ -15,7 +15,7 @@
* limitations under the License.
*/
package yataiclient
package api_store_client
import (
"context"
......
......@@ -23,10 +23,10 @@ import (
"time"
)
type DynamoNIM struct {
PresignedDownloadUrl string `json:"presigned_download_url"`
TransmissionStrategy *TransmissionStrategy `json:"transmission_strategy"`
Manifest *DynamoNIMManifest `json:"manifest"`
type DynamoComponent struct {
PresignedDownloadUrl string `json:"presigned_download_url"`
TransmissionStrategy *TransmissionStrategy `json:"transmission_strategy"`
Manifest *DynamoComponentManifest `json:"manifest"`
}
type TransmissionStrategy string
......@@ -36,7 +36,7 @@ const (
TransmissionStrategyProxy TransmissionStrategy = "proxy"
)
type DynamoNIMManifest struct {
type DynamoComponentManifest struct {
BentomlVersion string `json:"bentoml_version"`
Models []string `json:"models"`
}
......@@ -77,10 +77,8 @@ const (
)
type DockerRegistrySchema struct {
BentosRepositoryURI string `json:"bentosRepositoryURI"`
ModelsRepositoryURI string `json:"modelsRepositoryURI"`
BentosRepositoryURIInCluster string `json:"bentosRepositoryURIInCluster"`
ModelsRepositoryURIInCluster string `json:"modelsRepositoryURIInCluster"`
DynamoRepositoryURI string `json:"dynamoRepositoryURI"`
DynamoRepositoryURIInCluster string `json:"dynamoRepositoryURIInCluster"`
Server string `json:"server"`
Username string `json:"username"`
Password string `json:"password"`
......
......@@ -179,7 +179,7 @@ func main() {
if err = (&controller.DynamoNimDeploymentReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("yatai-deployment"),
Recorder: mgr.GetEventRecorderFor("dynamo-deployment"),
Config: ctrlConfig,
NatsAddr: natsAddr,
EtcdAddr: etcdAddr,
......@@ -192,7 +192,7 @@ func main() {
if err = (&controller.DynamoNimRequestReconciler{
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("yatai-image-builder"),
Recorder: mgr.GetEventRecorderFor("dynamo-image-builder"),
Config: ctrlConfig,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "DynamoNimRequest")
......
......@@ -24,43 +24,41 @@ import (
"github.com/ai-dynamo/dynamo/deploy/dynamo/operator/internal/consts"
)
func GetYataiImageBuilderNamespace(ctx context.Context) (namespace string, err error) {
return os.Getenv(consts.EnvYataiImageBuilderNamespace), nil
func GetDynamoImageBuilderNamespace(ctx context.Context) (namespace string, err error) {
return os.Getenv(consts.EnvDynamoImageBuilderNamespace), nil
}
type DockerRegistryConfig struct {
BentoRepositoryName string `yaml:"bento_repository_name"`
ModelRepositoryName string `yaml:"model_repository_name"`
Server string `yaml:"server"`
InClusterServer string `yaml:"in_cluster_server"`
Username string `yaml:"username"`
Password string `yaml:"password"`
Secure bool `yaml:"secure"`
DynamoComponentsRepositoryName string `yaml:"dynamo_components_repository_name"`
Server string `yaml:"server"`
InClusterServer string `yaml:"in_cluster_server"`
Username string `yaml:"username"`
Password string `yaml:"password"`
Secure bool `yaml:"secure"`
}
func GetDockerRegistryConfig() (conf *DockerRegistryConfig, err error) {
return &DockerRegistryConfig{
BentoRepositoryName: os.Getenv(consts.EnvDockerRegistryBentoRepositoryName),
ModelRepositoryName: os.Getenv(consts.EnvDockerRegistryModelRepositoryName),
Server: os.Getenv(consts.EnvDockerRegistryServer),
InClusterServer: os.Getenv(consts.EnvDockerRegistryInClusterServer),
Username: os.Getenv(consts.EnvDockerRegistryUsername),
Password: os.Getenv(consts.EnvDockerRegistryPassword),
Secure: os.Getenv(consts.EnvDockerRegistrySecure) == "true",
DynamoComponentsRepositoryName: os.Getenv(consts.EnvDockerRegistryDynamoComponentsRepositoryName),
Server: os.Getenv(consts.EnvDockerRegistryServer),
InClusterServer: os.Getenv(consts.EnvDockerRegistryInClusterServer),
Username: os.Getenv(consts.EnvDockerRegistryUsername),
Password: os.Getenv(consts.EnvDockerRegistryPassword),
Secure: os.Getenv(consts.EnvDockerRegistrySecure) == "true",
}, nil
}
type YataiConfig struct {
type ApiStoreConfig struct {
Endpoint string `yaml:"endpoint"`
ClusterName string `yaml:"cluster_name"`
ApiToken string `yaml:"api_token"`
}
func GetYataiConfig(ctx context.Context) (conf *YataiConfig, err error) {
return &YataiConfig{
Endpoint: os.Getenv(consts.EnvYataiEndpoint),
ClusterName: os.Getenv(consts.EnvYataiClusterName),
ApiToken: os.Getenv(consts.EnvYataiApiToken),
func GetApiStoreConfig(ctx context.Context) (conf *ApiStoreConfig, err error) {
return &ApiStoreConfig{
Endpoint: os.Getenv(consts.EnvApiStoreEndpoint),
ClusterName: os.Getenv(consts.EnvApiStoreClusterName),
ApiToken: os.Getenv(consts.EnvApiStoreApiToken),
}, nil
}
......@@ -72,16 +70,16 @@ func getEnv(key, fallback string) string {
}
type InternalImages struct {
BentoDownloader string
Kaniko string
MetricsTransformer string
Buildkit string
BuildkitRootless string
DynamoComponentsDownloader string
Kaniko string
MetricsTransformer string
Buildkit string
BuildkitRootless string
}
func GetInternalImages() (conf *InternalImages) {
conf = &InternalImages{}
conf.BentoDownloader = getEnv(consts.EnvInternalImagesBentoDownloader, consts.InternalImagesBentoDownloaderDefault)
conf.DynamoComponentsDownloader = getEnv(consts.EnvInternalImagesDynamoComponentsDownloader, consts.InternalImagesDynamoComponentsDownloaderDefault)
conf.Kaniko = getEnv(consts.EnvInternalImagesKaniko, consts.InternalImagesKanikoDefault)
conf.MetricsTransformer = getEnv(consts.EnvInternalImagesMetricsTransformer, consts.InternalImagesMetricsTransformerDefault)
conf.Buildkit = getEnv(consts.EnvInternalImagesBuildkit, consts.InternalImagesBuildkitDefault)
......
......@@ -3,96 +3,81 @@ package consts
const (
HPACPUDefaultAverageUtilization = 80
// nolint: gosec
YataiApiTokenHeaderName = "X-YATAI-API-TOKEN"
NgcOrganizationHeaderName = "Nv-Ngc-Org"
NgcUserHeaderName = "Nv-Actor-Id"
DefaultUserId = "default"
DefaultOrgId = "default"
BentoServicePort = 3000
BentoServicePortName = "http"
BentoContainerPortName = "http"
DynamoServicePort = 3000
DynamoServicePortName = "http"
DynamoContainerPortName = "http"
YataiImageBuilderComponentName = "yatai-image-builder"
YataiDeploymentComponentName = "yatai-deployment"
DynamoImageBuilderComponentName = "dynamo-image-builder"
YataiBentoDeploymentComponentApiServer = "api-server"
DynamoDeploymentComponentApiServer = "api-server"
InternalImagesBentoDownloaderDefault = "quay.io/bentoml/bento-downloader:0.0.3"
InternalImagesKanikoDefault = "quay.io/bentoml/kaniko:1.9.1"
InternalImagesMetricsTransformerDefault = "quay.io/bentoml/yatai-bento-metrics-transformer:0.0.3"
InternalImagesBuildkitDefault = "quay.io/bentoml/buildkit:master"
InternalImagesBuildkitRootlessDefault = "quay.io/bentoml/buildkit:master-rootless"
InternalImagesDynamoComponentsDownloaderDefault = "quay.io/bentoml/bento-downloader:0.0.3"
InternalImagesKanikoDefault = "gcr.io/kaniko-project/executor:debug"
InternalImagesMetricsTransformerDefault = "quay.io/bentoml/yatai-bento-metrics-transformer:0.0.3"
InternalImagesBuildkitDefault = "moby/buildkit:v0.20.2"
InternalImagesBuildkitRootlessDefault = "moby/buildkit:v0.20.2-rootless"
EnvYataiEndpoint = "YATAI_ENDPOINT"
EnvYataiClusterName = "YATAI_CLUSTER_NAME"
EnvApiStoreEndpoint = "API_STORE_ENDPOINT"
EnvApiStoreClusterName = "API_STORE_CLUSTER_NAME"
// nolint: gosec
EnvYataiApiToken = "YATAI_API_TOKEN"
EnvApiStoreApiToken = "API_STORE_API_TOKEN"
EnvBentoServicePort = "PORT"
EnvDynamoServicePort = "PORT"
// tracking envars
EnvYataiDeploymentUID = "YATAI_T_DEPLOYMENT_UID"
EnvDynamoDeploymentUID = "DYNAMO_DEPLOYMENT_UID"
EnvYataiBentoDeploymentName = "YATAI_BENTO_DEPLOYMENT_NAME"
EnvYataiBentoDeploymentNamespace = "YATAI_BENTO_DEPLOYMENT_NAMESPACE"
EnvDynamoDeploymentName = "DYNAMO_DEPLOYMENT_NAME"
EnvDynamoDeploymentNamespace = "DYNAMO_DEPLOYMENT_NAMESPACE"
EnvDockerRegistryServer = "DOCKER_REGISTRY_SERVER"
EnvDockerRegistryInClusterServer = "DOCKER_REGISTRY_IN_CLUSTER_SERVER"
EnvDockerRegistryUsername = "DOCKER_REGISTRY_USERNAME"
// nolint:gosec
EnvDockerRegistryPassword = "DOCKER_REGISTRY_PASSWORD"
EnvDockerRegistrySecure = "DOCKER_REGISTRY_SECURE"
EnvDockerRegistryBentoRepositoryName = "DOCKER_REGISTRY_BENTO_REPOSITORY_NAME"
EnvDockerRegistryModelRepositoryName = "DOCKER_REGISTRY_MODEL_REPOSITORY_NAME"
EnvInternalImagesBentoDownloader = "INTERNAL_IMAGES_BENTO_DOWNLOADER"
EnvInternalImagesKaniko = "INTERNAL_IMAGES_KANIKO"
EnvInternalImagesMetricsTransformer = "INTERNAL_IMAGES_METRICS_TRANSFORMER"
EnvInternalImagesBuildkit = "INTERNAL_IMAGES_BUILDKIT"
EnvInternalImagesBuildkitRootless = "INTERNAL_IMAGES_BUILDKIT_ROOTLESS"
EnvYataiSystemNamespace = "YATAI_SYSTEM_NAMESPACE"
EnvYataiImageBuilderNamespace = "YATAI_IMAGE_BUILDER_NAMESPACE"
EnvYataiDeploymentNamespace = "YATAI_DEPLOYMENT_NAMESPACE"
EnvBentoDeploymentNamespaces = "BENTO_DEPLOYMENT_NAMESPACES"
EnvImageBuildersNamespace = "IMAGE_BUILDERS_NAMESPACE"
KubeLabelYataiSelector = "yatai.ai/selector"
KubeLabelYataiBentoRepository = "yatai.ai/bento-repository"
KubeLabelYataiBento = "yatai.ai/bento"
KubeLabelYataiModelRepository = "yatai.ai/model-repository"
KubeLabelYataiModel = "yatai.ai/model"
KubeLabelYataiBentoDeployment = "yatai.ai/bento-deployment"
KubeLabelYataiBentoDeploymentComponentType = "yatai.ai/bento-deployment-component-type"
KubeLabelYataiBentoDeploymentTargetType = "yatai.ai/bento-deployment-target-type"
KubeLabelBentoRepository = "yatai.ai/bento-repository"
KubeLabelBentoVersion = "yatai.ai/bento-version"
KubeLabelCreator = "yatai.ai/creator"
KubeLabelIsBentoImageBuilder = "yatai.ai/is-bento-image-builder"
KubeLabelIsModelSeeder = "yatai.ai/is-model-seeder"
KubeLabelBentoRequest = "yatai.ai/bento-request"
EnvDockerRegistryPassword = "DOCKER_REGISTRY_PASSWORD"
EnvDockerRegistrySecure = "DOCKER_REGISTRY_SECURE"
EnvDockerRegistryDynamoComponentsRepositoryName = "DOCKER_REGISTRY_DYNAMO_COMPONENTS_REPOSITORY_NAME"
EnvInternalImagesDynamoComponentsDownloader = "INTERNAL_IMAGES_DYNAMO_COMPONENTS_DOWNLOADER"
EnvInternalImagesKaniko = "INTERNAL_IMAGES_KANIKO"
EnvInternalImagesMetricsTransformer = "INTERNAL_IMAGES_METRICS_TRANSFORMER"
EnvInternalImagesBuildkit = "INTERNAL_IMAGES_BUILDKIT"
EnvInternalImagesBuildkitRootless = "INTERNAL_IMAGES_BUILDKIT_ROOTLESS"
EnvDynamoSystemNamespace = "DYNAMO_SYSTEM_NAMESPACE"
EnvDynamoImageBuilderNamespace = "DYNAMO_IMAGE_BUILDER_NAMESPACE"
KubeLabelDynamoSelector = "nvidia.com/selector"
KubeLabelDynamoRepository = "nvidia.com/dynamo-repository"
KubeLabelDynamoVersion = "nvidia.com/dynamo-version"
KubeLabelDynamoDeployment = "nvidia.com/dynamo-deployment"
KubeLabelDynamoDeploymentComponentType = "nvidia.com/dynamo-deployment-component-type"
KubeLabelDynamoDeploymentTargetType = "nvidia.com/dynamo-deployment-target-type"
KubeLabelDynamoCreator = "nvidia.com/dynamo-creator"
KubeLabelIsDynamoImageBuilder = "nvidia.com/is-dynamo-image-builder"
KubeLabelDynamoRequest = "nvidia.com/dynamo-request"
KubeLabelValueFalse = "false"
KubeLabelValueTrue = "true"
KubeLabelYataiImageBuilderPod = "yatai.ai/yatai-image-builder-pod"
KubeLabelBentoDeploymentPod = "yatai.ai/bento-deployment-pod"
KubeLabelDynamoImageBuilderPod = "nvidia.com/dynamo-image-builder-pod"
KubeLabelDynamoDeploymentPod = "nvidia.com/dynamo-deployment-pod"
KubeAnnotationBentoRepository = "yatai.ai/bento-repository"
KubeAnnotationBentoVersion = "yatai.ai/bento-version"
KubeAnnotationDockerRegistryInsecure = "yatai.ai/docker-registry-insecure"
KubeAnnotationYataiImageBuilderSeparateModels = "yatai.ai/yatai-image-builder-separate-models"
KubeAnnotationIsMultiTenancy = "yatai.ai/is-multi-tenancy"
KubeAnnotationDynamoRepository = "nvidia.com/dynamo-repository"
KubeAnnotationDynamoVersion = "nvidia.com/dynamo-version"
KubeAnnotationDynamoDockerRegistryInsecure = "nvidia.com/docker-registry-insecure"
KubeResourceGPUNvidia = "nvidia.com/gpu"
// nolint: gosec
KubeSecretNameRegcred = "yatai-regcred"
KubeSecretNameRegcred = "dynamo-regcred"
KubeAnnotationDynamoNimRequestHash = "nvidia.com/dynamo-request-hash"
KubeAnnotationDynamoNimRequestImageBuiderHash = "nvidia.com/dynamo-request-image-builder-hash"
KubeAnnotationDynamoNimStorageNS = "nvidia.com/dynamo-storage-namespace"
)
......@@ -37,7 +37,7 @@ import (
nvidiacomv1alpha1 "github.com/ai-dynamo/dynamo/deploy/dynamo/operator/api/v1alpha1"
commonController "github.com/ai-dynamo/dynamo/deploy/dynamo/operator/internal/controller_common"
"github.com/ai-dynamo/dynamo/deploy/dynamo/operator/internal/nim"
"github.com/ai-dynamo/dynamo/deploy/dynamo/operator/internal/dynamo"
)
const (
......@@ -121,26 +121,26 @@ func (r *DynamoDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
return ctrl.Result{}, nil
}
// fetch the DynamoNIMConfig
dynamoNIMConfig, err := nim.GetDynamoNIMConfig(ctx, dynamoDeployment, r.Recorder)
// fetch the dynamoGraphConfig
dynamoGraphConfig, err := dynamo.GetDynamoGraphConfig(ctx, dynamoDeployment, r.Recorder)
if err != nil {
reason = "failed_to_get_the_DynamoNIMConfig"
reason = "failed_to_get_the_DynamoGraphConfig"
return ctrl.Result{}, err
}
// generate the DynamoNimDeployments from the config
dynamoNimDeployments, err := nim.GenerateDynamoNIMDeployments(ctx, dynamoDeployment, dynamoNIMConfig, r.generateDefaultIngressSpec(dynamoDeployment))
// generate the dynamoComponentsDeployments from the config
dynamoComponentsDeployments, err := dynamo.GenerateDynamoComponentsDeployments(ctx, dynamoDeployment, dynamoGraphConfig, r.generateDefaultIngressSpec(dynamoDeployment))
if err != nil {
reason = "failed_to_generate_the_DynamoNimDeployments"
reason = "failed_to_generate_the_DynamoComponentsDeployments"
return ctrl.Result{}, err
}
// merge the DynamoNimDeployments with the DynamoNimDeployments from the CRD
for serviceName, deployment := range dynamoNimDeployments {
// merge the dynamoComponentsDeployments with the dynamoComponentsDeployments from the CRD
for serviceName, deployment := range dynamoComponentsDeployments {
if _, ok := dynamoDeployment.Spec.Services[serviceName]; ok {
err := mergo.Merge(&deployment.Spec.DynamoNimDeploymentSharedSpec, dynamoDeployment.Spec.Services[serviceName].DynamoNimDeploymentSharedSpec, mergo.WithOverride)
if err != nil {
reason = "failed_to_merge_the_DynamoNimDeployments"
reason = "failed_to_merge_the_DynamoComponentsDeployments"
return ctrl.Result{}, err
}
}
......@@ -149,8 +149,8 @@ func (r *DynamoDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
}
}
// Set common env vars on each of the dynamoNimDeployments
for _, deployment := range dynamoNimDeployments {
// Set common env vars on each of the dynamoComponentsDeployments
for _, deployment := range dynamoComponentsDeployments {
if len(dynamoDeployment.Spec.Envs) > 0 {
deployment.Spec.Envs = mergeEnvs(dynamoDeployment.Spec.Envs, deployment.Spec.Envs)
}
......@@ -177,20 +177,20 @@ func (r *DynamoDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Req
}
notReadyDeployments := []string{}
// reconcile the DynamoNimDeployments
for serviceName, dynamoNimDeployment := range dynamoNimDeployments {
logger.Info("Reconciling the DynamoNimDeployment", "serviceName", serviceName, "dynamoNimDeployment", dynamoNimDeployment)
if err := ctrl.SetControllerReference(dynamoDeployment, dynamoNimDeployment, r.Scheme); err != nil {
reason = "failed_to_set_the_controller_reference_for_the_DynamoNimDeployment"
// reconcile the dynamoComponentsDeployments
for serviceName, dynamoComponentDeployment := range dynamoComponentsDeployments {
logger.Info("Reconciling the DynamoNimDeployment", "serviceName", serviceName, "dynamoComponentDeployment", dynamoComponentDeployment)
if err := ctrl.SetControllerReference(dynamoDeployment, dynamoComponentDeployment, r.Scheme); err != nil {
reason = "failed_to_set_the_controller_reference_for_the_DynamoComponentDeployment"
return ctrl.Result{}, err
}
dynamoNimDeployment, err = commonController.SyncResource(ctx, r.Client, dynamoNimDeployment, types.NamespacedName{Name: dynamoNimDeployment.Name, Namespace: dynamoNimDeployment.Namespace}, false)
dynamoComponentDeployment, err = commonController.SyncResource(ctx, r.Client, dynamoComponentDeployment, types.NamespacedName{Name: dynamoComponentDeployment.Name, Namespace: dynamoComponentDeployment.Namespace}, false)
if err != nil {
reason = "failed_to_sync_the_DynamoNimDeployment"
return ctrl.Result{}, err
}
if !dynamoNimDeployment.Status.IsReady() {
notReadyDeployments = append(notReadyDeployments, dynamoNimDeployment.Name)
if !dynamoComponentDeployment.Status.IsReady() {
notReadyDeployments = append(notReadyDeployments, dynamoComponentDeployment.Name)
}
}
if len(notReadyDeployments) == 0 {
......
......@@ -15,7 +15,7 @@
* limitations under the License.
*/
package nim
package dynamo
import (
"bytes"
......@@ -26,12 +26,11 @@ import (
"strings"
"emperror.dev/errors"
apiStoreClient "github.com/ai-dynamo/dynamo/deploy/dynamo/operator/api/dynamo/api_store_client"
compounaiCommon "github.com/ai-dynamo/dynamo/deploy/dynamo/operator/api/dynamo/common"
"github.com/ai-dynamo/dynamo/deploy/dynamo/operator/api/dynamo/schemas"
yataiclient "github.com/ai-dynamo/dynamo/deploy/dynamo/operator/api/dynamo/yatai-client"
"github.com/ai-dynamo/dynamo/deploy/dynamo/operator/api/v1alpha1"
commonconfig "github.com/ai-dynamo/dynamo/deploy/dynamo/operator/internal/config"
commonconsts "github.com/ai-dynamo/dynamo/deploy/dynamo/operator/internal/consts"
"github.com/huandu/xstrings"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
......@@ -89,58 +88,53 @@ func GetDefaultDynamoNamespace(ctx context.Context, dynamoDeployment *v1alpha1.D
return fmt.Sprintf("dynamo-%s", dynamoDeployment.Name)
}
func RetrieveDynamoNimDownloadURL(ctx context.Context, dynamoDeployment *v1alpha1.DynamoDeployment, recorder EventRecorder) (*string, *string, error) {
dynamoNimDownloadURL := ""
dynamoNimApiToken := ""
var dynamoNim *schemas.DynamoNIM
dynamoNimRepositoryName, _, dynamoNimVersion := xstrings.Partition(dynamoDeployment.Spec.DynamoNim, ":")
func RetrieveDynamoGraphDownloadURL(ctx context.Context, dynamoDeployment *v1alpha1.DynamoDeployment, recorder EventRecorder) (*string, error) {
dynamoGraphDownloadURL := ""
var dynamoComponent *schemas.DynamoComponent
dynamoComponentRepositoryName, _, dynamoComponentVersion := xstrings.Partition(dynamoDeployment.Spec.DynamoNim, ":")
var err error
var yataiClient_ **yataiclient.YataiClient
var yataiConf_ **commonconfig.YataiConfig
var apiStoreClient *apiStoreClient.ApiStoreClient
var apiStoreConf *commonconfig.ApiStoreConfig
yataiClient_, yataiConf_, err = GetYataiClient(ctx)
apiStoreClient, apiStoreConf, err = GetApiStoreClient(ctx)
if err != nil {
err = errors.Wrap(err, "get yatai client")
return nil, nil, err
err = errors.Wrap(err, "get api store client")
return nil, err
}
if yataiClient_ == nil || yataiConf_ == nil {
err = errors.New("can't get yatai client, please check yatai configuration")
return nil, nil, err
if apiStoreClient == nil || apiStoreConf == nil {
err = errors.New("can't get api store client, please check api store configuration")
return nil, err
}
yataiClient := *yataiClient_
yataiConf := *yataiConf_
recorder.Eventf(dynamoDeployment, corev1.EventTypeNormal, "GenerateImageBuilderPod", "Getting dynamoNim %s from yatai service", dynamoDeployment.Spec.DynamoNim)
dynamoNim, err = yataiClient.GetBento(ctx, dynamoNimRepositoryName, dynamoNimVersion)
recorder.Eventf(dynamoDeployment, corev1.EventTypeNormal, "GenerateImageBuilderPod", "Getting dynamo graph %s from api store service", dynamoDeployment.Spec.DynamoNim)
dynamoComponent, err = apiStoreClient.GetDynamoComponent(ctx, dynamoComponentRepositoryName, dynamoComponentVersion)
if err != nil {
err = errors.Wrap(err, "get dynamoNim")
return nil, nil, err
err = errors.Wrap(err, "get dynamo component")
return nil, err
}
recorder.Eventf(dynamoDeployment, corev1.EventTypeNormal, "GenerateImageBuilderPod", "Got dynamoNim %s from yatai service", dynamoDeployment.Spec.DynamoNim)
recorder.Eventf(dynamoDeployment, corev1.EventTypeNormal, "GenerateImageBuilderPod", "Got dynamo graph %s from api store service", dynamoDeployment.Spec.DynamoNim)
if dynamoNim.TransmissionStrategy != nil && *dynamoNim.TransmissionStrategy == schemas.TransmissionStrategyPresignedURL {
var dynamoNim_ *schemas.DynamoNIM
recorder.Eventf(dynamoDeployment, corev1.EventTypeNormal, "GenerateImageBuilderPod", "Getting presigned url for dynamoNim %s from yatai service", dynamoDeployment.Spec.DynamoNim)
dynamoNim_, err = yataiClient.PresignBentoDownloadURL(ctx, dynamoNimRepositoryName, dynamoNimVersion)
if dynamoComponent.TransmissionStrategy != nil && *dynamoComponent.TransmissionStrategy == schemas.TransmissionStrategyPresignedURL {
var dynamoComponent_ *schemas.DynamoComponent
recorder.Eventf(dynamoDeployment, corev1.EventTypeNormal, "GenerateImageBuilderPod", "Getting presigned url for dynamo graph %s from api store service", dynamoDeployment.Spec.DynamoNim)
dynamoComponent_, err = apiStoreClient.PresignDynamoComponentDownloadURL(ctx, dynamoComponentRepositoryName, dynamoComponentVersion)
if err != nil {
err = errors.Wrap(err, "presign dynamoNim download url")
return nil, nil, err
err = errors.Wrap(err, "presign dynamo component download url")
return nil, err
}
recorder.Eventf(dynamoDeployment, corev1.EventTypeNormal, "GenerateImageBuilderPod", "Got presigned url for dynamoNim %s from yatai service", dynamoDeployment.Spec.DynamoNim)
dynamoNimDownloadURL = dynamoNim_.PresignedDownloadUrl
recorder.Eventf(dynamoDeployment, corev1.EventTypeNormal, "GenerateImageBuilderPod", "Got presigned url for dynamo graph %s from api store service", dynamoDeployment.Spec.DynamoNim)
dynamoGraphDownloadURL = dynamoComponent_.PresignedDownloadUrl
} else {
dynamoNimDownloadURL = fmt.Sprintf("%s/api/v1/dynamo_nims/%s/versions/%s/download", yataiConf.Endpoint, dynamoNimRepositoryName, dynamoNimVersion)
dynamoNimApiToken = fmt.Sprintf("%s:%s:$%s", commonconsts.YataiImageBuilderComponentName, yataiConf.ClusterName, commonconsts.EnvYataiApiToken)
dynamoGraphDownloadURL = fmt.Sprintf("%s/api/v1/dynamo_nims/%s/versions/%s/download", apiStoreConf.Endpoint, dynamoComponentRepositoryName, dynamoComponentVersion)
}
return &dynamoNimDownloadURL, &dynamoNimApiToken, nil
return &dynamoGraphDownloadURL, nil
}
// ServicesConfig represents the top-level YAML structure of a dynamoNim yaml file stored in a dynamoNim tar file
type DynamoNIMConfig struct {
type DynamoGraphConfig struct {
DynamoTag string `yaml:"service"`
Services []ServiceConfig `yaml:"services"`
EntryService string `yaml:"entry_service"`
......@@ -150,12 +144,11 @@ type EventRecorder interface {
Eventf(obj runtime.Object, eventtype string, reason string, message string, args ...interface{})
}
func RetrieveDynamoNIMConfigurationFile(ctx context.Context, url string, yataiApiToken string) (*bytes.Buffer, error) {
func RetrieveDynamoGraphConfigurationFile(ctx context.Context, url string) (*bytes.Buffer, error) {
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Header.Set(commonconsts.YataiApiTokenHeaderName, yataiApiToken)
client := &http.Client{}
resp, err := client.Do(req)
......@@ -185,55 +178,53 @@ func RetrieveDynamoNIMConfigurationFile(ctx context.Context, url string, yataiAp
return yamlContent, nil
}
func GetYataiClient(ctx context.Context) (yataiClient **yataiclient.YataiClient, yataiConf **commonconfig.YataiConfig, err error) {
yataiConf_, err := commonconfig.GetYataiConfig(ctx)
func GetApiStoreClient(ctx context.Context) (*apiStoreClient.ApiStoreClient, *commonconfig.ApiStoreConfig, error) {
apiStoreConf, err := commonconfig.GetApiStoreConfig(ctx)
isNotFound := k8serrors.IsNotFound(err)
if err != nil && !isNotFound {
err = errors.Wrap(err, "get yatai config")
return
err = errors.Wrap(err, "get api store config")
return nil, nil, err
}
if isNotFound {
return
return nil, nil, errors.New("endpoint config not found")
}
if yataiConf_.Endpoint == "" {
return
if apiStoreConf.Endpoint == "" {
return nil, nil, errors.New("endpoint is empty")
}
if yataiConf_.ClusterName == "" {
yataiConf_.ClusterName = "default"
if apiStoreConf.ClusterName == "" {
apiStoreConf.ClusterName = "default"
}
yataiClient_ := yataiclient.NewYataiClient(yataiConf_.Endpoint, fmt.Sprintf("%s:%s:%s", commonconsts.YataiImageBuilderComponentName, yataiConf_.ClusterName, yataiConf_.ApiToken))
apiStoreClient := apiStoreClient.NewApiStoreClient(apiStoreConf.Endpoint)
yataiClient = &yataiClient_
yataiConf = &yataiConf_
return
return apiStoreClient, apiStoreConf, nil
}
func ParseDynamoNIMConfig(ctx context.Context, yamlContent *bytes.Buffer) (*DynamoNIMConfig, error) {
var config DynamoNIMConfig
func ParseDynamoGraphConfig(ctx context.Context, yamlContent *bytes.Buffer) (*DynamoGraphConfig, error) {
var config DynamoGraphConfig
logger := log.FromContext(ctx)
logger.Info("trying to parse dynamoNim config", "yamlContent", yamlContent.String())
logger.Info("trying to parse dynamo graph config", "yamlContent", yamlContent.String())
err := yaml.Unmarshal(yamlContent.Bytes(), &config)
return &config, err
}
func GetDynamoNIMConfig(ctx context.Context, dynamoDeployment *v1alpha1.DynamoDeployment, recorder EventRecorder) (*DynamoNIMConfig, error) {
dynamoNimDownloadURL, dynamoNimApiToken, err := RetrieveDynamoNimDownloadURL(ctx, dynamoDeployment, recorder)
func GetDynamoGraphConfig(ctx context.Context, dynamoDeployment *v1alpha1.DynamoDeployment, recorder EventRecorder) (*DynamoGraphConfig, error) {
dynamoGraphDownloadURL, err := RetrieveDynamoGraphDownloadURL(ctx, dynamoDeployment, recorder)
if err != nil {
return nil, err
}
yamlContent, err := RetrieveDynamoNIMConfigurationFile(ctx, *dynamoNimDownloadURL, *dynamoNimApiToken)
yamlContent, err := RetrieveDynamoGraphConfigurationFile(ctx, *dynamoGraphDownloadURL)
if err != nil {
return nil, err
}
return ParseDynamoNIMConfig(ctx, yamlContent)
return ParseDynamoGraphConfig(ctx, yamlContent)
}
// generate DynamoNIMDeployment from config
func GenerateDynamoNIMDeployments(ctx context.Context, parentDynamoDeployment *v1alpha1.DynamoDeployment, config *DynamoNIMConfig, ingressSpec *v1alpha1.IngressSpec) (map[string]*v1alpha1.DynamoNimDeployment, error) {
// GenerateDynamoComponentsDeployments generates a map of DynamoComponentDeployments from a DynamoGraphConfig
func GenerateDynamoComponentsDeployments(ctx context.Context, parentDynamoDeployment *v1alpha1.DynamoDeployment, config *DynamoGraphConfig, ingressSpec *v1alpha1.IngressSpec) (map[string]*v1alpha1.DynamoNimDeployment, error) {
dynamoServices := make(map[string]string)
deployments := make(map[string]*v1alpha1.DynamoNimDeployment)
for _, service := range config.Services {
......
......@@ -15,7 +15,7 @@
* limitations under the License.
*/
package nim
package dynamo
import (
"context"
......@@ -30,7 +30,7 @@ import (
func TestGenerateDynamoNIMDeployments(t *testing.T) {
type args struct {
parentDynamoDeployment *v1alpha1.DynamoDeployment
config *DynamoNIMConfig
config *DynamoGraphConfig
ingressSpec *v1alpha1.IngressSpec
}
tests := []struct {
......@@ -51,7 +51,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
DynamoNim: "dynamonim:ac4e234",
},
},
config: &DynamoNIMConfig{
config: &DynamoGraphConfig{
DynamoTag: "dynamonim:MyService1",
Services: []ServiceConfig{
{
......@@ -159,7 +159,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
DynamoNim: "dynamonim:ac4e234",
},
},
config: &DynamoNIMConfig{
config: &DynamoGraphConfig{
DynamoTag: "dynamonim:MyService2",
EntryService: "service1",
Services: []ServiceConfig{
......@@ -272,7 +272,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
DynamoNim: "dynamonim:ac4e234",
},
},
config: &DynamoNIMConfig{
config: &DynamoGraphConfig{
DynamoTag: "dynamonim:MyService2",
EntryService: "service1",
Services: []ServiceConfig{
......@@ -378,7 +378,7 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
DynamoNim: "dynamonim:ac4e234",
},
},
config: &DynamoNIMConfig{
config: &DynamoGraphConfig{
DynamoTag: "dynamonim:MyService3",
Services: []ServiceConfig{
{
......@@ -423,9 +423,9 @@ func TestGenerateDynamoNIMDeployments(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := gomega.NewGomegaWithT(t)
got, err := GenerateDynamoNIMDeployments(context.Background(), tt.args.parentDynamoDeployment, tt.args.config, tt.args.ingressSpec)
got, err := GenerateDynamoComponentsDeployments(context.Background(), tt.args.parentDynamoDeployment, tt.args.config, tt.args.ingressSpec)
if (err != nil) != tt.wantErr {
t.Errorf("GenerateDynamoNIMDeployments() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("GenerateDynamoComponentsDeployments() error = %v, wantErr %v", err, tt.wantErr)
return
}
g.Expect(got).To(gomega.Equal(tt.want))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment