logs.Info(fmt.Sprintf("Getting Compound AI NIM %s",compoundAINimDeployment.Spec.CompoundAINim))
r.Recorder.Eventf(compoundAINimDeployment,corev1.EventTypeNormal,"GetCompoundAINim","Getting Compound AI NIM %s",compoundAINimDeployment.Spec.CompoundAINim)
logs.Info(fmt.Sprintf("CompoundAINim %s not found",compoundAINimDeployment.Spec.CompoundAINim))
r.Recorder.Eventf(compoundAINimDeployment,corev1.EventTypeNormal,"GetCompoundAINim","CompoundAINim %s not found",compoundAINimDeployment.Spec.CompoundAINim)
r.Recorder.Eventf(compoundAINimDeployment,corev1.EventTypeNormal,"GetCompoundAINimRequest","CompoundAINimRequest %s is found and waiting for its compoundAINim to be provided",compoundAINimDeployment.Spec.CompoundAINim)
err=errors.Errorf("CompoundAINimRequest %s/%s is not available: %s",compoundAINimRequest.Namespace,compoundAINimRequest.Name,compoundAINimRequestAvailableCondition.Message)
r.Recorder.Eventf(compoundAINimDeployment,corev1.EventTypeNormal,"GetCompoundAINim","CompoundAINim %s is found",compoundAINimDeployment.Spec.CompoundAINim)
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeWarning,"GetDeployment","Failed to get Deployment %s: %s",deploymentNamespacedName,err)
logs.Error(err,"Failed to get Deployment.")
return
}
ifoldDeploymentIsNotFound{
logs.Info("Deployment not found. Creating a new one.")
err=errors.Wrapf(patch.DefaultAnnotator.SetLastAppliedAnnotation(deployment),"set last applied annotation for deployment %s",deployment.Name)
iferr!=nil{
logs.Error(err,"Failed to set last applied annotation.")
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeWarning,"SetLastAppliedAnnotation","Failed to set last applied annotation for Deployment %s: %s",deploymentNamespacedName,err)
return
}
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeNormal,"CreateDeployment","Creating a new Deployment %s",deploymentNamespacedName)
err=r.Create(ctx,deployment)
iferr!=nil{
logs.Error(err,"Failed to create Deployment.")
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeWarning,"CreateDeployment","Failed to create Deployment %s: %s",deploymentNamespacedName,err)
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeWarning,"CalculatePatch","Failed to calculate patch for Deployment %s: %s",deploymentNamespacedName,err)
return
}
if!patchResult.IsEmpty(){
logs.Info("Deployment spec is different. Updating Deployment.")
err=errors.Wrapf(patch.DefaultAnnotator.SetLastAppliedAnnotation(deployment),"set last applied annotation for deployment %s",deployment.Name)
iferr!=nil{
logs.Error(err,"Failed to set last applied annotation.")
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeWarning,"SetLastAppliedAnnotation","Failed to set last applied annotation for Deployment %s: %s",deploymentNamespacedName,err)
r.Recorder.Eventf(compoundAINimDeployment,corev1.EventTypeWarning,"GetHPA","Failed to get HPA %s: %s",hpaNamespacedName,err)
logs.Error(err,"Failed to get HPA.")
return
}
ifoldHPAIsNotFound{
logs.Info("HPA not found. Creating a new one.")
err=errors.Wrapf(patch.DefaultAnnotator.SetLastAppliedAnnotation(hpa),"set last applied annotation for hpa %s",hpa.Name)
iferr!=nil{
logs.Error(err,"Failed to set last applied annotation.")
r.Recorder.Eventf(compoundAINimDeployment,corev1.EventTypeWarning,"SetLastAppliedAnnotation","Failed to set last applied annotation for HPA %s: %s",hpaNamespacedName,err)
return
}
r.Recorder.Eventf(compoundAINimDeployment,corev1.EventTypeNormal,"CreateHPA","Creating a new HPA %s",hpaNamespacedName)
err=r.Create(ctx,hpa)
iferr!=nil{
logs.Error(err,"Failed to create HPA.")
r.Recorder.Eventf(compoundAINimDeployment,corev1.EventTypeWarning,"CreateHPA","Failed to create HPA %s: %s",hpaNamespacedName,err)
r.Recorder.Eventf(compoundAINimDeployment,corev1.EventTypeWarning,"CalculatePatch","Failed to calculate patch for HPA %s: %s",hpaNamespacedName,err)
return
}
if!patchResult.IsEmpty(){
logs.Info(fmt.Sprintf("HPA spec is different. Updating HPA. The patch result is: %s",patchResult.String()))
err=errors.Wrapf(patch.DefaultAnnotator.SetLastAppliedAnnotation(hpa),"set last applied annotation for hpa %s",hpa.Name)
iferr!=nil{
logs.Error(err,"Failed to set last applied annotation.")
r.Recorder.Eventf(compoundAINimDeployment,corev1.EventTypeWarning,"SetLastAppliedAnnotation","Failed to set last applied annotation for HPA %s: %s",hpaNamespacedName,err)
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeWarning,"GetService","Failed to get Service %s: %s",serviceNamespacedName,err)
logs.Error(err,"Failed to get Service.")
return
}
ifoldServiceIsNotFound{
logs.Info("Service not found. Creating a new one.")
err=errors.Wrapf(patch.DefaultAnnotator.SetLastAppliedAnnotation(service),"set last applied annotation for service %s",service.Name)
iferr!=nil{
logs.Error(err,"Failed to set last applied annotation.")
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeWarning,"SetLastAppliedAnnotation","Failed to set last applied annotation for Service %s: %s",serviceNamespacedName,err)
return
}
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeNormal,"CreateService","Creating a new Service %s",serviceNamespacedName)
err=r.Create(ctx,service)
iferr!=nil{
logs.Error(err,"Failed to create Service.")
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeWarning,"CreateService","Failed to create Service %s: %s",serviceNamespacedName,err)
return
}
logs.Info("Service created.")
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeNormal,"CreateService","Created Service %s",serviceNamespacedName)
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeWarning,"CalculatePatch","Failed to calculate patch for Service %s: %s",serviceNamespacedName,err)
return
}
if!patchResult.IsEmpty(){
logs.Info("Service spec is different. Updating Service.")
err=errors.Wrapf(patch.DefaultAnnotator.SetLastAppliedAnnotation(service),"set last applied annotation for service %s",service.Name)
iferr!=nil{
logs.Error(err,"Failed to set last applied annotation.")
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeWarning,"SetLastAppliedAnnotation","Failed to set last applied annotation for Service %s: %s",serviceNamespacedName,err)
return
}
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeNormal,"UpdateService","Updating Service %s",serviceNamespacedName)
oldService.Annotations=service.Annotations
oldService.Labels=service.Labels
oldService.Spec=service.Spec
err=r.Update(ctx,oldService)
iferr!=nil{
logs.Error(err,"Failed to update Service.")
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeWarning,"UpdateService","Failed to update Service %s: %s",serviceNamespacedName,err)
return
}
logs.Info("Service updated.")
r.Recorder.Eventf(opt.compoundAINimDeployment,corev1.EventTypeNormal,"UpdateService","Updated Service %s",serviceNamespacedName)
logs.Info("Ingress not found. Creating a new one.")
err=errors.Wrapf(patch.DefaultAnnotator.SetLastAppliedAnnotation(ingress),"set last applied annotation for ingress %s",ingress.Name)
iferr!=nil{
logs.Error(err,"Failed to set last applied annotation.")
r.Recorder.Eventf(compoundAINimDeployment,corev1.EventTypeWarning,"SetLastAppliedAnnotation","Failed to set last applied annotation for Ingress %s: %s",ingressNamespacedName,err)
return
}
r.Recorder.Eventf(compoundAINimDeployment,corev1.EventTypeNormal,"CreateIngress","Creating a new Ingress %s",ingressNamespacedName)
err=r.Create(ctx,ingress)
iferr!=nil{
logs.Error(err,"Failed to create Ingress.")
r.Recorder.Eventf(compoundAINimDeployment,corev1.EventTypeWarning,"CreateIngress","Failed to create Ingress %s: %s",ingressNamespacedName,err)
r.Recorder.Eventf(compoundAINimDeployment,corev1.EventTypeWarning,"CalculatePatch","Failed to calculate patch for Ingress %s: %s",ingressNamespacedName,err)
return
}
if!patchResult.IsEmpty(){
logs.Info("Ingress spec is different. Updating Ingress.")
err=errors.Wrapf(patch.DefaultAnnotator.SetLastAppliedAnnotation(ingress),"set last applied annotation for ingress %s",ingress.Name)
iferr!=nil{
logs.Error(err,"Failed to set last applied annotation.")
r.Recorder.Eventf(compoundAINimDeployment,corev1.EventTypeWarning,"SetLastAppliedAnnotation","Failed to set last applied annotation for Ingress %s: %s",ingressNamespacedName,err)
err=errors.Wrapf(err,"failed to json unmarshal %s in configmap %s: %s",commonconsts.KubeConfigMapKeyNetworkConfigIngressAnnotations,commonconsts.KubeConfigMapNameNetworkConfig,annotations_)
logs.Info("Triggering reconciliation for CompoundAINimRequest","CompoundAINimRequestName",compoundAINimRequest.GetName(),"Namespace",compoundAINimRequest.GetNamespace())
r.Recorder.Eventf(compoundAINimRequest,corev1.EventTypeNormal,"DeleteJob","Due to the nonexistence of the model %s, job %s has been deleted.",modelTag,job_.Name)
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Docker config secret %s in namespace %s is ready",commonconsts.KubeSecretNameRegcred,opt.CompoundAINimRequest.Namespace)
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Getting presigned url for model %s from yatai service",model.Tag)
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Presigned url for model %s is got from yatai service",model.Tag)
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateModelSeederPod","Configmap %s is got from namespace %s",configCmName,configNamespace)
err=errors.Wrapf(err,"failed to yaml unmarshal extra_pod_spec, please check the configmap %s in namespace %s",configCmName,configNamespace)
return
}
}
}else{
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateModelSeederPod","Configmap %s is not found in namespace %s",configCmName,configNamespace)
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Getting presigned url for compoundAINim %s from yatai service",opt.CompoundAINimRequest.Spec.BentoTag)
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Got presigned url for compoundAINim %s from yatai service",opt.CompoundAINimRequest.Spec.BentoTag)
err=errors.Wrapf(err,"failed to get secret %s",yataiAPITokenSecretName)
return
}
ifisNotFound{
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Secret %s is not found, so creating it in namespace %s",yataiAPITokenSecretName,opt.CompoundAINimRequest.Namespace)
err=r.Create(ctx,yataiAPITokenSecret)
isExists:=k8serrors.IsAlreadyExists(err)
iferr!=nil&&!isExists{
err=errors.Wrapf(err,"failed to create secret %s",yataiAPITokenSecretName)
return
}
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Secret %s is created in namespace %s",yataiAPITokenSecretName,opt.CompoundAINimRequest.Namespace)
}else{
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Secret %s is found in namespace %s, so updating it",yataiAPITokenSecretName,opt.CompoundAINimRequest.Namespace)
err=r.Update(ctx,yataiAPITokenSecret)
iferr!=nil{
err=errors.Wrapf(err,"failed to update secret %s",yataiAPITokenSecretName)
return
}
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Secret %s is updated in namespace %s",yataiAPITokenSecretName,opt.CompoundAINimRequest.Namespace)
}
}
internalImages:=commonconfig.GetInternalImages()
logrus.Infof("Image builder is using the images %v",*internalImages)
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Getting presigned url for model %s from yatai service",model.Tag)
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Presigned url for model %s is got from yatai service",model.Tag)
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Configmap %s is got from namespace %s",configCmName,configNamespace)
err=errors.Wrapf(err,"failed to yaml unmarshal default_image_builder_container_resources, please check the configmap %s in namespace %s",configCmName,configNamespace)
return
}
}
buildArgs=[]string{}
ifval,ok:=configCm.Data["build_args"];ok{
err=yaml.Unmarshal([]byte(val),&buildArgs)
iferr!=nil{
err=errors.Wrapf(err,"failed to yaml unmarshal build_args, please check the configmap %s in namespace %s",configCmName,configNamespace)
return
}
}
builderArgs=[]string{}
ifval,ok:=configCm.Data["builder_args"];ok{
err=yaml.Unmarshal([]byte(val),&builderArgs)
iferr!=nil{
err=errors.Wrapf(err,"failed to yaml unmarshal builder_args, please check the configmap %s in namespace %s",configCmName,configNamespace)
return
}
}
logrus.Info("passed in builder args: ",builderArgs)
}else{
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Configmap %s is not found in namespace %s",configCmName,configNamespace)
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Secret %s is got from namespace %s",buildArgsSecretName,configNamespace)
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Secret %s is in namespace %s, but CompoundAINimRequest is in namespace %s, so we need to copy the secret to CompoundAINimRequest namespace",buildArgsSecretName,configNamespace,opt.CompoundAINimRequest.Namespace)
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Getting secret %s in namespace %s",buildArgsSecretName,opt.CompoundAINimRequest.Namespace)
err=errors.Wrapf(err,"failed to get secret %s from namespace %s",buildArgsSecretName,opt.CompoundAINimRequest.Namespace)
return
}
iflocalBuildArgsSecretIsNotFound{
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Copying secret %s from namespace %s to namespace %s",buildArgsSecretName,configNamespace,opt.CompoundAINimRequest.Namespace)
err=r.Create(ctx,&corev1.Secret{
ObjectMeta:metav1.ObjectMeta{
Name:buildArgsSecretName,
Namespace:opt.CompoundAINimRequest.Namespace,
},
Data:buildArgsSecret.Data,
})
iferr!=nil{
err=errors.Wrapf(err,"failed to create secret %s in namespace %s",buildArgsSecretName,opt.CompoundAINimRequest.Namespace)
return
}
}else{
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Secret %s is already in namespace %s",buildArgsSecretName,opt.CompoundAINimRequest.Namespace)
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Updating secret %s in namespace %s",buildArgsSecretName,opt.CompoundAINimRequest.Namespace)
err=r.Update(ctx,&corev1.Secret{
ObjectMeta:metav1.ObjectMeta{
Name:buildArgsSecretName,
Namespace:opt.CompoundAINimRequest.Namespace,
},
Data:buildArgsSecret.Data,
})
iferr!=nil{
err=errors.Wrapf(err,"failed to update secret %s in namespace %s",buildArgsSecretName,opt.CompoundAINimRequest.Namespace)
r.Recorder.Eventf(opt.CompoundAINimRequest,corev1.EventTypeNormal,"GenerateImageBuilderPod","Secret %s is not found in namespace %s",buildArgsSecretName,configNamespace)
}
builderContainerArgs:=[]string{
"-c",
fmt.Sprintf("sleep 15; %s && exit 0 || exit %d",shquot.POSIXShell(append(command,args...)),BuilderJobFailedExitCode),// TODO: remove once functionality exists to wait for istio sidecar.
Limit the number of alerts an alerting rule and series a recording
rule can produce.
Limit is supported starting with Prometheus >= 2.31 and Thanos Ruler >= 0.24.
type:integer
name:
description:Name of the rule group.
minLength:1
type:string
partial_response_strategy:
description:|-
PartialResponseStrategy is only used by ThanosRuler and will
be ignored by Prometheus instances.
More info: https://github.com/thanos-io/thanos/blob/main/docs/components/rule.md#partial-response
pattern:^(?i)(abort|warn)?$
type:string
rules:
description:List of alerting and recording rules.
items:
description:|-
Rule describes an alerting or recording rule
See Prometheus documentation: [alerting](https://www.prometheus.io/docs/prometheus/latest/configuration/alerting_rules/) or [recording](https://www.prometheus.io/docs/prometheus/latest/configuration/recording_rules/#recording-rules) rule
properties:
alert:
description:|-
Name of the alert. Must be a valid label value.
Only one of `record` and `alert` must be set.
type:string
annotations:
additionalProperties:
type:string
description:|-
Annotations to add to each alert.
Only valid for alerting rules.
type:object
expr:
anyOf:
-type:integer
-type:string
description:PromQL expression to evaluate.
x-kubernetes-int-or-string:true
for:
description:Alerts are considered firing once they have
