Skip to content

GitLab

Unverified Commit 94876e58 authored by julienmancuso's avatar julienmancuso Committed by GitHub
Browse files

Fix: Canonicalize PodGangSet to prevent spurious updates (#2469)

parent 78c7e352
Show whitespace changes
Inline Side-by-side
Showing with 1302 additions and 48 deletions
deploy/cloud/operator/go.mod
View file @ 94876e58
......@@ -13,6 +13,7 @@ require (
github.com/onsi/ginkgo/v2 v2.23.4
github.com/onsi/gomega v1.37.0
github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.71.2
github.com/stretchr/testify v1.10.0
go.etcd.io/etcd/client/v3 v3.5.21
istio.io/api v1.23.1
istio.io/client-go v1.23.1
......@@ -57,6 +58,7 @@ require (
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/prometheus/client_golang v1.22.0 // indirect
github.com/prometheus/client_model v0.6.1 // indirect
github.com/prometheus/common v0.62.0 // indirect
......
deploy/cloud/operator/internal/controller/dynamocomponentdeployment_controller.go
View file @ 94876e58
......@@ -1217,7 +1217,7 @@ func (r *DynamoComponentDeploymentReconciler) generatePodTemplateSpec(ctx contex
podLabels[commonconsts.KubeLabelDynamoSelector] = kubeName
podSpec := &basePodSpec
podSpec := basePodSpec
podSpec.Containers = containers
extraPodMetadata := opt.dynamoComponentDeployment.Spec.ExtraPodMetadata
......
deploy/cloud/operator/internal/controller/dynamocomponentdeployment_controller_test.go
View file @ 94876e58
......@@ -796,6 +796,7 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
},
},
},
RestartPolicy: corev1.RestartPolicyAlways,
Containers: []corev1.Container{
{
Name: "main",
......@@ -895,6 +896,7 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
},
},
},
RestartPolicy: corev1.RestartPolicyAlways,
Containers: []corev1.Container{
{
Name: "main",
......
deploy/cloud/operator/internal/controller_common/pod.go 0 → 100644
View file @ 94876e58
package controller_common
import (
"sort"
corev1 "k8s.io/api/core/v1"
)
// CanonicalizePodSpec sorts the pod spec in a way that is deterministic and easy to reason about.
//
//nolint:gocyclo
func CanonicalizePodSpec(podSpec *corev1.PodSpec) *corev1.PodSpec {
// Helper function to get EnvFromSource sort key
envFromKey := func(e corev1.EnvFromSource) string {
if e.ConfigMapRef != nil {
return "cm:" + e.ConfigMapRef.Name + ":" + e.Prefix
}
if e.SecretRef != nil {
return "sec:" + e.SecretRef.Name + ":" + e.Prefix
}
return "other:" + e.Prefix
}
// Helper function to sort container-like fields (works for both Container and EphemeralContainer)
sortContainerFields := func(env []corev1.EnvVar, envFrom []corev1.EnvFromSource, ports []corev1.ContainerPort, volumeMounts []corev1.VolumeMount, securityContext *corev1.SecurityContext) {
// Sort env vars by name
if len(env) > 1 {
sort.Slice(env, func(i, j int) bool { return env[i].Name < env[j].Name })
}
// Sort envFrom by referenced source and prefix
if len(envFrom) > 1 {
sort.Slice(envFrom, func(i, j int) bool {
return envFromKey(envFrom[i]) < envFromKey(envFrom[j])
})
}
// Sort ports by name then port number
if len(ports) > 1 {
sort.Slice(ports, func(i, j int) bool {
if ports[i].Name == ports[j].Name {
return ports[i].ContainerPort < ports[j].ContainerPort
}
return ports[i].Name < ports[j].Name
})
}
// Sort volume mounts by name then mount path
if len(volumeMounts) > 1 {
sort.Slice(volumeMounts, func(i, j int) bool {
if volumeMounts[i].Name == volumeMounts[j].Name {
return volumeMounts[i].MountPath < volumeMounts[j].MountPath
}
return volumeMounts[i].Name < volumeMounts[j].Name
})
}
// Sort security context capability lists
if securityContext != nil && securityContext.Capabilities != nil {
if caps := securityContext.Capabilities.Add; len(caps) > 1 {
sort.Slice(caps, func(i, j int) bool { return string(caps[i]) < string(caps[j]) })
}
if caps := securityContext.Capabilities.Drop; len(caps) > 1 {
sort.Slice(caps, func(i, j int) bool { return string(caps[i]) < string(caps[j]) })
}
}
}
// Sort regular containers
for i := range podSpec.Containers {
c := &podSpec.Containers[i]
sortContainerFields(c.Env, c.EnvFrom, c.Ports, c.VolumeMounts, c.SecurityContext)
}
if len(podSpec.Containers) > 1 {
sort.Slice(podSpec.Containers, func(i, j int) bool {
return podSpec.Containers[i].Name < podSpec.Containers[j].Name
})
}
// Sort init containers
for i := range podSpec.InitContainers {
c := &podSpec.InitContainers[i]
sortContainerFields(c.Env, c.EnvFrom, c.Ports, c.VolumeMounts, c.SecurityContext)
}
if len(podSpec.InitContainers) > 1 {
sort.Slice(podSpec.InitContainers, func(i, j int) bool {
return podSpec.InitContainers[i].Name < podSpec.InitContainers[j].Name
})
}
// Sort ephemeral containers
for i := range podSpec.EphemeralContainers {
ec := &podSpec.EphemeralContainers[i]
sortContainerFields(ec.Env, ec.EnvFrom, ec.Ports, ec.VolumeMounts, ec.SecurityContext)
}
if len(podSpec.EphemeralContainers) > 1 {
sort.Slice(podSpec.EphemeralContainers, func(i, j int) bool {
return podSpec.EphemeralContainers[i].Name < podSpec.EphemeralContainers[j].Name
})
}
// Sort image pull secrets
if len(podSpec.ImagePullSecrets) > 1 {
sort.Slice(podSpec.ImagePullSecrets, func(i, j int) bool {
return podSpec.ImagePullSecrets[i].Name < podSpec.ImagePullSecrets[j].Name
})
}
// Sort volumes and their nested items
sortKeyToPathItems := func(items []corev1.KeyToPath) {
if len(items) > 1 {
sort.Slice(items, func(i, j int) bool {
if items[i].Key == items[j].Key {
return items[i].Path < items[j].Path
}
return items[i].Key < items[j].Key
})
}
}
for i := range podSpec.Volumes {
v := &podSpec.Volumes[i]
// ConfigMap items
if v.ConfigMap != nil {
sortKeyToPathItems(v.ConfigMap.Items)
}
// Secret items
if v.Secret != nil {
sortKeyToPathItems(v.Secret.Items)
}
// DownwardAPI items
if v.DownwardAPI != nil && len(v.DownwardAPI.Items) > 1 {
sort.Slice(v.DownwardAPI.Items, func(i, j int) bool {
return v.DownwardAPI.Items[i].Path < v.DownwardAPI.Items[j].Path
})
}
// Projected sources
if v.Projected != nil {
// Sort projected sources
if len(v.Projected.Sources) > 1 {
sort.Slice(v.Projected.Sources, func(i, j int) bool {
getProjectionKey := func(p corev1.VolumeProjection) string {
if p.ConfigMap != nil {
return "cm:" + p.ConfigMap.Name
}
if p.Secret != nil {
return "sec:" + p.Secret.Name
}
if p.DownwardAPI != nil {
return "downward:"
}
if p.ServiceAccountToken != nil {
return "sat:" + p.ServiceAccountToken.Audience
}
return "z:other"
}
return getProjectionKey(v.Projected.Sources[i]) < getProjectionKey(v.Projected.Sources[j])
})
}
// Sort nested items for each projection
for j := range v.Projected.Sources {
p := &v.Projected.Sources[j]
if p.ConfigMap != nil {
sortKeyToPathItems(p.ConfigMap.Items)
}
if p.Secret != nil {
sortKeyToPathItems(p.Secret.Items)
}
if p.DownwardAPI != nil && len(p.DownwardAPI.Items) > 1 {
sort.Slice(p.DownwardAPI.Items, func(i, j int) bool {
return p.DownwardAPI.Items[i].Path < p.DownwardAPI.Items[j].Path
})
}
}
}
}
// Sort volumes by name
if len(podSpec.Volumes) > 1 {
sort.Slice(podSpec.Volumes, func(i, j int) bool {
return podSpec.Volumes[i].Name < podSpec.Volumes[j].Name
})
}
// Sort tolerations
if len(podSpec.Tolerations) > 1 {
sort.Slice(podSpec.Tolerations, func(i, j int) bool {
a, b := podSpec.Tolerations[i], podSpec.Tolerations[j]
if a.Key != b.Key {
return a.Key < b.Key
}
if string(a.Operator) != string(b.Operator) {
return string(a.Operator) < string(b.Operator)
}
if a.Value != b.Value {
return a.Value < b.Value
}
if string(a.Effect) != string(b.Effect) {
return string(a.Effect) < string(b.Effect)
}
// Handle TolerationSeconds (could be nil)
aSec, bSec := int64(0), int64(0)
if a.TolerationSeconds != nil {
aSec = *a.TolerationSeconds
}
if b.TolerationSeconds != nil {
bSec = *b.TolerationSeconds
}
return aSec < bSec
})
}
// Sort topology spread constraints
if len(podSpec.TopologySpreadConstraints) > 1 {
sort.Slice(podSpec.TopologySpreadConstraints, func(i, j int) bool {
a, b := podSpec.TopologySpreadConstraints[i], podSpec.TopologySpreadConstraints[j]
if a.TopologyKey != b.TopologyKey {
return a.TopologyKey < b.TopologyKey
}
if string(a.WhenUnsatisfiable) != string(b.WhenUnsatisfiable) {
return string(a.WhenUnsatisfiable) < string(b.WhenUnsatisfiable)
}
return a.MaxSkew < b.MaxSkew
})
}
// Sort host aliases
if len(podSpec.HostAliases) > 1 {
// First sort hostnames within each alias
for i := range podSpec.HostAliases {
if len(podSpec.HostAliases[i].Hostnames) > 1 {
sort.Strings(podSpec.HostAliases[i].Hostnames)
}
}
// Then sort aliases by IP
sort.Slice(podSpec.HostAliases, func(i, j int) bool {
return podSpec.HostAliases[i].IP < podSpec.HostAliases[j].IP
})
}
// Sort DNS config
if podSpec.DNSConfig != nil {
// Sort DNS options
if len(podSpec.DNSConfig.Options) > 1 {
sort.Slice(podSpec.DNSConfig.Options, func(i, j int) bool {
if podSpec.DNSConfig.Options[i].Name == podSpec.DNSConfig.Options[j].Name {
vi, vj := "", ""
if podSpec.DNSConfig.Options[i].Value != nil {
vi = *podSpec.DNSConfig.Options[i].Value
}
if podSpec.DNSConfig.Options[j].Value != nil {
vj = *podSpec.DNSConfig.Options[j].Value
}
return vi < vj
}
return podSpec.DNSConfig.Options[i].Name < podSpec.DNSConfig.Options[j].Name
})
}
// Sort nameservers and search domains
if len(podSpec.DNSConfig.Nameservers) > 1 {
sort.Strings(podSpec.DNSConfig.Nameservers)
}
if len(podSpec.DNSConfig.Searches) > 1 {
sort.Strings(podSpec.DNSConfig.Searches)
}
}
return podSpec
}
deploy/cloud/operator/internal/controller_common/pod_test.go 0 → 100644
View file @ 94876e58
package controller_common
import (
"testing"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
)
func TestCanonicalizePodSpec(t *testing.T) {
tests := []struct {
name string
input *corev1.PodSpec
expected *corev1.PodSpec
}{
{
name: "sorts containers by name",
input: &corev1.PodSpec{
Containers: []corev1.Container{
{Name: "zebra"},
{Name: "alpha"},
{Name: "beta"},
},
},
expected: &corev1.PodSpec{
Containers: []corev1.Container{
{Name: "alpha"},
{Name: "beta"},
{Name: "zebra"},
},
},
},
{
name: "sorts init containers by name",
input: &corev1.PodSpec{
InitContainers: []corev1.Container{
{Name: "init-zebra"},
{Name: "init-alpha"},
},
},
expected: &corev1.PodSpec{
InitContainers: []corev1.Container{
{Name: "init-alpha"},
{Name: "init-zebra"},
},
},
},
{
name: "sorts ephemeral containers by name",
input: &corev1.PodSpec{
EphemeralContainers: []corev1.EphemeralContainer{
{EphemeralContainerCommon: corev1.EphemeralContainerCommon{Name: "debug-zebra"}},
{EphemeralContainerCommon: corev1.EphemeralContainerCommon{Name: "debug-alpha"}},
},
},
expected: &corev1.PodSpec{
EphemeralContainers: []corev1.EphemeralContainer{
{EphemeralContainerCommon: corev1.EphemeralContainerCommon{Name: "debug-alpha"}},
{EphemeralContainerCommon: corev1.EphemeralContainerCommon{Name: "debug-zebra"}},
},
},
},
{
name: "sorts environment variables by name",
input: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "test",
Env: []corev1.EnvVar{
{Name: "ZOO", Value: "zebra"},
{Name: "ALPHA", Value: "apple"},
{Name: "BETA", Value: "banana"},
},
},
},
},
expected: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "test",
Env: []corev1.EnvVar{
{Name: "ALPHA", Value: "apple"},
{Name: "BETA", Value: "banana"},
{Name: "ZOO", Value: "zebra"},
},
},
},
},
},
{
name: "sorts envFrom by source type and name",
input: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "test",
EnvFrom: []corev1.EnvFromSource{
{SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "secret-z"}}},
{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "config-a"}}},
{SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "secret-a"}}},
{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "config-z"}}},
},
},
},
},
expected: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "test",
EnvFrom: []corev1.EnvFromSource{
{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "config-a"}}},
{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "config-z"}}},
{SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "secret-a"}}},
{SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "secret-z"}}},
},
},
},
},
},
{
name: "sorts container ports by name then port number",
input: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "test",
Ports: []corev1.ContainerPort{
{Name: "http", ContainerPort: 8080},
{Name: "grpc", ContainerPort: 9090},
{Name: "grpc", ContainerPort: 8080},
{Name: "debug", ContainerPort: 8080},
},
},
},
},
expected: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "test",
Ports: []corev1.ContainerPort{
{Name: "debug", ContainerPort: 8080},
{Name: "grpc", ContainerPort: 8080},
{Name: "grpc", ContainerPort: 9090},
{Name: "http", ContainerPort: 8080},
},
},
},
},
},
{
name: "sorts volume mounts by name then mount path",
input: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "test",
VolumeMounts: []corev1.VolumeMount{
{Name: "vol1", MountPath: "/data2"},
{Name: "vol2", MountPath: "/data1"},
{Name: "vol1", MountPath: "/data1"},
},
},
},
},
expected: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "test",
VolumeMounts: []corev1.VolumeMount{
{Name: "vol1", MountPath: "/data1"},
{Name: "vol1", MountPath: "/data2"},
{Name: "vol2", MountPath: "/data1"},
},
},
},
},
},
{
name: "sorts security context capabilities",
input: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "test",
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"SYS_ADMIN", "NET_ADMIN", "CHOWN"},
Drop: []corev1.Capability{"ALL", "SETUID", "KILL"},
},
},
},
},
},
expected: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "test",
SecurityContext: &corev1.SecurityContext{
Capabilities: &corev1.Capabilities{
Add: []corev1.Capability{"CHOWN", "NET_ADMIN", "SYS_ADMIN"},
Drop: []corev1.Capability{"ALL", "KILL", "SETUID"},
},
},
},
},
},
},
{
name: "sorts image pull secrets by name",
input: &corev1.PodSpec{
ImagePullSecrets: []corev1.LocalObjectReference{
{Name: "registry-z"},
{Name: "registry-a"},
{Name: "registry-b"},
},
},
expected: &corev1.PodSpec{
ImagePullSecrets: []corev1.LocalObjectReference{
{Name: "registry-a"},
{Name: "registry-b"},
{Name: "registry-z"},
},
},
},
{
name: "sorts volumes by name",
input: &corev1.PodSpec{
Volumes: []corev1.Volume{
{Name: "vol-z"},
{Name: "vol-a"},
{Name: "vol-b"},
},
},
expected: &corev1.PodSpec{
Volumes: []corev1.Volume{
{Name: "vol-a"},
{Name: "vol-b"},
{Name: "vol-z"},
},
},
},
{
name: "sorts configmap volume items by key then path",
input: &corev1.PodSpec{
Volumes: []corev1.Volume{
{
Name: "config",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
Items: []corev1.KeyToPath{
{Key: "app.conf", Path: "config/app.conf"},
{Key: "db.conf", Path: "config/db.conf"},
{Key: "app.conf", Path: "backup/app.conf"},
},
},
},
},
},
},
expected: &corev1.PodSpec{
Volumes: []corev1.Volume{
{
Name: "config",
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
Items: []corev1.KeyToPath{
{Key: "app.conf", Path: "backup/app.conf"},
{Key: "app.conf", Path: "config/app.conf"},
{Key: "db.conf", Path: "config/db.conf"},
},
},
},
},
},
},
},
{
name: "sorts secret volume items by key then path",
input: &corev1.PodSpec{
Volumes: []corev1.Volume{
{
Name: "secret",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
Items: []corev1.KeyToPath{
{Key: "tls.key", Path: "tls/server.key"},
{Key: "tls.crt", Path: "tls/server.crt"},
{Key: "tls.key", Path: "backup/server.key"},
},
},
},
},
},
},
expected: &corev1.PodSpec{
Volumes: []corev1.Volume{
{
Name: "secret",
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
Items: []corev1.KeyToPath{
{Key: "tls.crt", Path: "tls/server.crt"},
{Key: "tls.key", Path: "backup/server.key"},
{Key: "tls.key", Path: "tls/server.key"},
},
},
},
},
},
},
},
{
name: "sorts downward API items by path",
input: &corev1.PodSpec{
Volumes: []corev1.Volume{
{
Name: "downward",
VolumeSource: corev1.VolumeSource{
DownwardAPI: &corev1.DownwardAPIVolumeSource{
Items: []corev1.DownwardAPIVolumeFile{
{Path: "metadata/name"},
{Path: "metadata/annotations"},
{Path: "limits/cpu"},
},
},
},
},
},
},
expected: &corev1.PodSpec{
Volumes: []corev1.Volume{
{
Name: "downward",
VolumeSource: corev1.VolumeSource{
DownwardAPI: &corev1.DownwardAPIVolumeSource{
Items: []corev1.DownwardAPIVolumeFile{
{Path: "limits/cpu"},
{Path: "metadata/annotations"},
{Path: "metadata/name"},
},
},
},
},
},
},
},
{
name: "sorts projected volume sources and their items",
input: &corev1.PodSpec{
Volumes: []corev1.Volume{
{
Name: "projected",
VolumeSource: corev1.VolumeSource{
Projected: &corev1.ProjectedVolumeSource{
Sources: []corev1.VolumeProjection{
{
Secret: &corev1.SecretProjection{
LocalObjectReference: corev1.LocalObjectReference{Name: "secret-z"},
Items: []corev1.KeyToPath{
{Key: "password", Path: "auth/password"},
{Key: "username", Path: "auth/username"},
},
},
},
{
ConfigMap: &corev1.ConfigMapProjection{
LocalObjectReference: corev1.LocalObjectReference{Name: "config-a"},
Items: []corev1.KeyToPath{
{Key: "db.conf", Path: "config/db.conf"},
{Key: "app.conf", Path: "config/app.conf"},
},
},
},
{
DownwardAPI: &corev1.DownwardAPIProjection{
Items: []corev1.DownwardAPIVolumeFile{
{Path: "metadata/name"},
{Path: "limits/cpu"},
},
},
},
{
ServiceAccountToken: &corev1.ServiceAccountTokenProjection{
Audience: "api.example.com",
Path: "tokens/api",
},
},
},
},
},
},
},
},
expected: &corev1.PodSpec{
Volumes: []corev1.Volume{
{
Name: "projected",
VolumeSource: corev1.VolumeSource{
Projected: &corev1.ProjectedVolumeSource{
Sources: []corev1.VolumeProjection{
{
ConfigMap: &corev1.ConfigMapProjection{
LocalObjectReference: corev1.LocalObjectReference{Name: "config-a"},
Items: []corev1.KeyToPath{
{Key: "app.conf", Path: "config/app.conf"},
{Key: "db.conf", Path: "config/db.conf"},
},
},
},
{
DownwardAPI: &corev1.DownwardAPIProjection{
Items: []corev1.DownwardAPIVolumeFile{
{Path: "limits/cpu"},
{Path: "metadata/name"},
},
},
},
{
ServiceAccountToken: &corev1.ServiceAccountTokenProjection{
Audience: "api.example.com",
Path: "tokens/api",
},
},
{
Secret: &corev1.SecretProjection{
LocalObjectReference: corev1.LocalObjectReference{Name: "secret-z"},
Items: []corev1.KeyToPath{
{Key: "password", Path: "auth/password"},
{Key: "username", Path: "auth/username"},
},
},
},
},
},
},
},
},
},
},
{
name: "sorts tolerations by key, operator, value, effect, seconds",
input: &corev1.PodSpec{
Tolerations: []corev1.Toleration{
{
Key: "node-type",
Operator: corev1.TolerationOpEqual,
Value: "gpu",
Effect: corev1.TaintEffectNoSchedule,
},
{
Key: "node-role",
Operator: corev1.TolerationOpEqual,
Value: "master",
Effect: corev1.TaintEffectNoSchedule,
},
{
Key: "node-role",
Operator: corev1.TolerationOpExists,
Effect: corev1.TaintEffectNoSchedule,
},
},
},
expected: &corev1.PodSpec{
Tolerations: []corev1.Toleration{
{
Key: "node-role",
Operator: corev1.TolerationOpEqual,
Value: "master",
Effect: corev1.TaintEffectNoSchedule,
},
{
Key: "node-role",
Operator: corev1.TolerationOpExists,
Effect: corev1.TaintEffectNoSchedule,
},
{
Key: "node-type",
Operator: corev1.TolerationOpEqual,
Value: "gpu",
Effect: corev1.TaintEffectNoSchedule,
},
},
},
},
{
name: "sorts topology spread constraints by topology key, when unsatisfiable, max skew",
input: &corev1.PodSpec{
TopologySpreadConstraints: []corev1.TopologySpreadConstraint{
{
TopologyKey: "kubernetes.io/zone",
WhenUnsatisfiable: corev1.DoNotSchedule,
MaxSkew: 2,
},
{
TopologyKey: "kubernetes.io/hostname",
WhenUnsatisfiable: corev1.DoNotSchedule,
MaxSkew: 1,
},
{
TopologyKey: "kubernetes.io/hostname",
WhenUnsatisfiable: corev1.ScheduleAnyway,
MaxSkew: 1,
},
},
},
expected: &corev1.PodSpec{
TopologySpreadConstraints: []corev1.TopologySpreadConstraint{
{
TopologyKey: "kubernetes.io/hostname",
WhenUnsatisfiable: corev1.DoNotSchedule,
MaxSkew: 1,
},
{
TopologyKey: "kubernetes.io/hostname",
WhenUnsatisfiable: corev1.ScheduleAnyway,
MaxSkew: 1,
},
{
TopologyKey: "kubernetes.io/zone",
WhenUnsatisfiable: corev1.DoNotSchedule,
MaxSkew: 2,
},
},
},
},
{
name: "sorts host aliases by IP and hostnames within each alias",
input: &corev1.PodSpec{
HostAliases: []corev1.HostAlias{
{
IP: "192.168.1.2",
Hostnames: []string{"web2.example.com", "api2.example.com"},
},
{
IP: "192.168.1.1",
Hostnames: []string{"web1.example.com", "api1.example.com", "admin1.example.com"},
},
},
},
expected: &corev1.PodSpec{
HostAliases: []corev1.HostAlias{
{
IP: "192.168.1.1",
Hostnames: []string{"admin1.example.com", "api1.example.com", "web1.example.com"},
},
{
IP: "192.168.1.2",
Hostnames: []string{"api2.example.com", "web2.example.com"},
},
},
},
},
{
name: "sorts DNS config options, nameservers, and searches",
input: &corev1.PodSpec{
DNSConfig: &corev1.PodDNSConfig{
Options: []corev1.PodDNSConfigOption{
{Name: "timeout", Value: func() *string { s := "5"; return &s }()},
{Name: "attempts", Value: func() *string { s := "3"; return &s }()},
{Name: "ndots", Value: func() *string { s := "2"; return &s }()},
},
Nameservers: []string{"8.8.8.8", "1.1.1.1", "8.8.4.4"},
Searches: []string{"example.com", "cluster.local", "app.local"},
},
},
expected: &corev1.PodSpec{
DNSConfig: &corev1.PodDNSConfig{
Options: []corev1.PodDNSConfigOption{
{Name: "attempts", Value: func() *string { s := "3"; return &s }()},
{Name: "ndots", Value: func() *string { s := "2"; return &s }()},
{Name: "timeout", Value: func() *string { s := "5"; return &s }()},
},
Nameservers: []string{"1.1.1.1", "8.8.4.4", "8.8.8.8"},
Searches: []string{"app.local", "cluster.local", "example.com"},
},
},
},
{
name: "handles nil pointer values gracefully",
input: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "test",
SecurityContext: &corev1.SecurityContext{
Capabilities: nil,
},
},
},
DNSConfig: &corev1.PodDNSConfig{
Options: []corev1.PodDNSConfigOption{
{Name: "timeout", Value: nil},
{Name: "attempts", Value: func() *string { s := "3"; return &s }()},
},
},
Tolerations: []corev1.Toleration{
{
Key: "test",
TolerationSeconds: nil,
},
{
Key: "test2",
TolerationSeconds: func() *int64 { s := int64(300); return &s }(),
},
},
},
expected: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "test",
SecurityContext: &corev1.SecurityContext{
Capabilities: nil,
},
},
},
DNSConfig: &corev1.PodDNSConfig{
Options: []corev1.PodDNSConfigOption{
{Name: "attempts", Value: func() *string { s := "3"; return &s }()},
{Name: "timeout", Value: nil},
},
},
Tolerations: []corev1.Toleration{
{
Key: "test",
TolerationSeconds: nil,
},
{
Key: "test2",
TolerationSeconds: func() *int64 { s := int64(300); return &s }(),
},
},
},
},
{
name: "returns original podspec when already sorted",
input: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "alpha",
Env: []corev1.EnvVar{
{Name: "A", Value: "1"},
{Name: "B", Value: "2"},
},
},
{Name: "beta"},
},
ImagePullSecrets: []corev1.LocalObjectReference{
{Name: "secret-a"},
{Name: "secret-b"},
},
},
expected: &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "alpha",
Env: []corev1.EnvVar{
{Name: "A", Value: "1"},
{Name: "B", Value: "2"},
},
},
{Name: "beta"},
},
ImagePullSecrets: []corev1.LocalObjectReference{
{Name: "secret-a"},
{Name: "secret-b"},
},
},
},
{
name: "handles empty slices gracefully",
input: &corev1.PodSpec{
Containers: []corev1.Container{},
InitContainers: []corev1.Container{},
ImagePullSecrets: []corev1.LocalObjectReference{},
Volumes: []corev1.Volume{},
Tolerations: []corev1.Toleration{},
},
expected: &corev1.PodSpec{
Containers: []corev1.Container{},
InitContainers: []corev1.Container{},
ImagePullSecrets: []corev1.LocalObjectReference{},
Volumes: []corev1.Volume{},
Tolerations: []corev1.Toleration{},
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := CanonicalizePodSpec(tt.input)
// Verify the function returns the same instance
assert.Same(t, tt.input, result, "function should return the same PodSpec instance")
// Verify the sorting is correct
assert.Equal(t, tt.expected, result, "PodSpec should be sorted correctly")
})
}
}
func TestCanonicalizePodSpec_Idempotent(t *testing.T) {
// Create a complex, unsorted PodSpec
podSpec := &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "zebra",
Env: []corev1.EnvVar{
{Name: "Z_VAR", Value: "z"},
{Name: "A_VAR", Value: "a"},
},
Ports: []corev1.ContainerPort{
{Name: "http", ContainerPort: 8080},
{Name: "grpc", ContainerPort: 9090},
},
VolumeMounts: []corev1.VolumeMount{
{Name: "vol2", MountPath: "/data2"},
{Name: "vol1", MountPath: "/data1"},
},
},
{Name: "alpha"},
},
InitContainers: []corev1.Container{
{Name: "init-zebra"},
{Name: "init-alpha"},
},
ImagePullSecrets: []corev1.LocalObjectReference{
{Name: "secret-z"},
{Name: "secret-a"},
},
Volumes: []corev1.Volume{
{Name: "vol-z"},
{Name: "vol-a"},
},
Tolerations: []corev1.Toleration{
{Key: "node-z"},
{Key: "node-a"},
},
}
// First canonicalization
result1 := CanonicalizePodSpec(podSpec)
// Second canonicalization on the same object
result2 := CanonicalizePodSpec(result1)
// Should be identical after second canonicalization
assert.Equal(t, result1, result2, "CanonicalizePodSpec should be idempotent")
// Verify containers are sorted
assert.Equal(t, "alpha", result2.Containers[0].Name)
assert.Equal(t, "zebra", result2.Containers[1].Name)
// Verify env vars within containers are sorted
assert.Equal(t, "A_VAR", result2.Containers[1].Env[0].Name)
assert.Equal(t, "Z_VAR", result2.Containers[1].Env[1].Name)
// Verify ports are sorted
assert.Equal(t, "grpc", result2.Containers[1].Ports[0].Name)
assert.Equal(t, "http", result2.Containers[1].Ports[1].Name)
// Verify volume mounts are sorted
assert.Equal(t, "vol1", result2.Containers[1].VolumeMounts[0].Name)
assert.Equal(t, "vol2", result2.Containers[1].VolumeMounts[1].Name)
}
func TestCanonicalizePodSpec_EnvFromSortPriority(t *testing.T) {
podSpec := &corev1.PodSpec{
Containers: []corev1.Container{
{
Name: "test",
EnvFrom: []corev1.EnvFromSource{
{SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "secret-b"}}},
{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "config-b"}}},
{SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "secret-a"}}},
{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "config-a"}}},
// Test duplicate names for secondary sort
{ConfigMapRef: &corev1.ConfigMapEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "config-a"}}},
{SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{Name: "secret-a"}}},
},
},
},
}
result := CanonicalizePodSpec(podSpec)
// ConfigMaps should come before Secrets (cm: < sec:)
// Within each type, sorted by name
expected := []string{
"cm:config-a:", // ConfigMap config-a
"cm:config-a:", // ConfigMap config-a (duplicate)
"cm:config-b:", // ConfigMap config-b
"sec:secret-a:", // Secret secret-a
"sec:secret-a:", // Secret secret-a (duplicate)
"sec:secret-b:", // Secret secret-b
}
envFromKey := func(e corev1.EnvFromSource) string {
if e.ConfigMapRef != nil {
return "cm:" + e.ConfigMapRef.Name + ":"
}
if e.SecretRef != nil {
return "sec:" + e.SecretRef.Name + ":"
}
return "other:"
}
for i, envFrom := range result.Containers[0].EnvFrom {
assert.Equal(t, expected[i], envFromKey(envFrom), "EnvFrom at index %d should match expected sort order", i)
}
}
func TestCanonicalizePodSpec_TolerationSecondsHandling(t *testing.T) {
sec300 := int64(300)
sec600 := int64(600)
podSpec := &corev1.PodSpec{
Tolerations: []corev1.Toleration{
{Key: "key1", TolerationSeconds: &sec600},
{Key: "key1", TolerationSeconds: nil},
{Key: "key1", TolerationSeconds: &sec300},
},
}
result := CanonicalizePodSpec(podSpec)
// Should be sorted by TolerationSeconds: nil (0) < 300 < 600
assert.Nil(t, result.Tolerations[0].TolerationSeconds)
assert.Equal(t, int64(300), *result.Tolerations[1].TolerationSeconds)
assert.Equal(t, int64(600), *result.Tolerations[2].TolerationSeconds)
}
func TestCanonicalizePodSpec_ProjectedVolumeSourcePriority(t *testing.T) {
podSpec := &corev1.PodSpec{
Volumes: []corev1.Volume{
{
Name: "projected",
VolumeSource: corev1.VolumeSource{
Projected: &corev1.ProjectedVolumeSource{
Sources: []corev1.VolumeProjection{
{Secret: &corev1.SecretProjection{LocalObjectReference: corev1.LocalObjectReference{Name: "secret-a"}}},
{ServiceAccountToken: &corev1.ServiceAccountTokenProjection{Audience: "zz.example.com"}},
{DownwardAPI: &corev1.DownwardAPIProjection{}},
{ConfigMap: &corev1.ConfigMapProjection{LocalObjectReference: corev1.LocalObjectReference{Name: "config-z"}}},
{ServiceAccountToken: &corev1.ServiceAccountTokenProjection{Audience: "aa.example.com"}},
{ConfigMap: &corev1.ConfigMapProjection{LocalObjectReference: corev1.LocalObjectReference{Name: "config-a"}}},
},
},
},
},
},
}
result := CanonicalizePodSpec(podSpec)
// Expected sort order: cm: < downward: < sat: < sec:
// Within same type, sorted by name/audience
getProjectionKey := func(p corev1.VolumeProjection) string {
if p.ConfigMap != nil {
return "cm:" + p.ConfigMap.Name
}
if p.Secret != nil {
return "sec:" + p.Secret.Name
}
if p.DownwardAPI != nil {
return "downward:"
}
if p.ServiceAccountToken != nil {
return "sat:" + p.ServiceAccountToken.Audience
}
return "z:other"
}
expected := []string{
"cm:config-a",
"cm:config-z",
"downward:",
"sat:aa.example.com",
"sat:zz.example.com",
"sec:secret-a",
}
sources := result.Volumes[0].VolumeSource.Projected.Sources
for i, source := range sources {
assert.Equal(t, expected[i], getProjectionKey(source), "Projected source at index %d should match expected sort order", i)
}
}
deploy/cloud/operator/internal/controller_common/podgangset.go 0 → 100644
View file @ 94876e58
package controller_common
import (
"sort"
grovev1alpha1 "github.com/NVIDIA/grove/operator/api/core/v1alpha1"
)
func CanonicalizePodGangSet(gangSet *grovev1alpha1.PodGangSet) *grovev1alpha1.PodGangSet {
// sort cliques by name
sort.Slice(gangSet.Spec.Template.Cliques, func(i, j int) bool {
return gangSet.Spec.Template.Cliques[i].Name < gangSet.Spec.Template.Cliques[j].Name
})
// sort scaling groups by name
sort.Slice(gangSet.Spec.Template.PodCliqueScalingGroupConfigs, func(i, j int) bool {
return gangSet.Spec.Template.PodCliqueScalingGroupConfigs[i].Name < gangSet.Spec.Template.PodCliqueScalingGroupConfigs[j].Name
})
return gangSet
}
deploy/cloud/operator/internal/controller_common/resource.go
View file @ 94876e58
......@@ -27,6 +27,7 @@ import (
"github.com/ai-dynamo/dynamo/deploy/cloud/operator/api/dynamo/common"
"github.com/ai-dynamo/dynamo/deploy/cloud/operator/internal/consts"
"github.com/google/go-cmp/cmp"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
......@@ -156,6 +157,14 @@ func SyncResource[T client.Object](ctx context.Context, r Reconciler, parentReso
return false, resource, fmt.Errorf("failed to check if spec has changed: %w", err)
}
if newHash != nil {
// Generate and log diff before updating
diff, diffErr := generateSpecDiff(oldResource, resource)
if diffErr != nil {
logs.V(1).Info(fmt.Sprintf("Failed to generate diff for %s: %v", resourceType, diffErr))
} else if diff != "" {
logs.Info(fmt.Sprintf("%s spec changes detected", resourceType), "diff", diff)
}
// update the spec of the current object with the desired spec
err = CopySpec(resource, oldResource)
if err != nil {
......@@ -252,6 +261,27 @@ func IsSpecChanged(current client.Object, desired client.Object) (*string, error
return &hashStr, nil
}
// generateSpecDiff creates a unified diff showing changes between old and new resource specs
func generateSpecDiff(oldResource, newResource client.Object) (string, error) {
oldSpec, err := getSpec(oldResource)
if err != nil {
return "", fmt.Errorf("failed to get old spec: %w", err)
}
newSpec, err := getSpec(newResource)
if err != nil {
return "", fmt.Errorf("failed to get new spec: %w", err)
}
// Generate diff using cmp
diff := cmp.Diff(oldSpec, newSpec)
if diff == "" {
return "", nil
}
return diff, nil
}
func GetSpecHash(obj client.Object) (string, error) {
spec, err := getSpec(obj)
if err != nil {
......
deploy/cloud/operator/internal/dynamo/component_common.go
View file @ 94876e58
......@@ -8,6 +8,7 @@ package dynamo
import (
commonconsts "github.com/ai-dynamo/dynamo/deploy/cloud/operator/internal/consts"
corev1 "k8s.io/api/core/v1"
"k8s.io/utils/ptr"
)
// ComponentDefaults interface defines how defaults should be provided
......@@ -50,7 +51,14 @@ func (b *BaseComponentDefaults) GetBaseContainer(context ComponentContext) (core
}
func (b *BaseComponentDefaults) GetBasePodSpec(context ComponentContext) (corev1.PodSpec, error) {
return corev1.PodSpec{}, nil
return b.getCommonPodSpec(), nil
}
func (b *BaseComponentDefaults) getCommonPodSpec() corev1.PodSpec {
return corev1.PodSpec{
TerminationGracePeriodSeconds: ptr.To(int64(60)),
RestartPolicy: corev1.RestartPolicyAlways,
}
}
func (b *BaseComponentDefaults) getCommonContainer(context ComponentContext) corev1.Container {
......
deploy/cloud/operator/internal/dynamo/component_planner.go
View file @ 94876e58
......@@ -39,8 +39,7 @@ func (p *PlannerDefaults) GetBaseContainer(context ComponentContext) (corev1.Con
}
func (p *PlannerDefaults) GetBasePodSpec(context ComponentContext) (corev1.PodSpec, error) {
podSpec := corev1.PodSpec{
ServiceAccountName: commonconsts.PlannerServiceAccountName,
}
podSpec := p.getCommonPodSpec()
podSpec.ServiceAccountName = commonconsts.PlannerServiceAccountName
return podSpec, nil
}
deploy/cloud/operator/internal/dynamo/graph.go
View file @ 94876e58
......@@ -31,6 +31,7 @@ import (
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/utils/ptr"
grovev1alpha1 "github.com/NVIDIA/grove/operator/api/core/v1alpha1"
"github.com/ai-dynamo/dynamo/deploy/cloud/operator/api/dynamo/common"
......@@ -687,13 +688,13 @@ func GenerateBasePodSpec(
controllerConfig controller_common.Config,
multinodeDeploymentType commonconsts.MultinodeDeploymentType,
serviceName string,
) (corev1.PodSpec, error) {
) (*corev1.PodSpec, error) {
// Start with base container generated per component type
componentContext := generateComponentContext(component, parentGraphDeploymentName, namespace, numberOfNodes)
componentDefaults := ComponentDefaultsFactory(component.ComponentType)
container, err := componentDefaults.GetBaseContainer(componentContext)
if err != nil {
return corev1.PodSpec{}, fmt.Errorf("failed to get base container: %w", err)
return nil, fmt.Errorf("failed to get base container: %w", err)
}
if component.ExtraPodSpec != nil && component.ExtraPodSpec.MainContainer != nil {
......@@ -702,7 +703,7 @@ func GenerateBasePodSpec(
// merge the extraPodSpec from the parent deployment with the extraPodSpec from the service
err = mergo.Merge(&container, *main, mergo.WithOverride)
if err != nil {
return corev1.PodSpec{}, fmt.Errorf("failed to merge extraPodSpec: %w", err)
return nil, fmt.Errorf("failed to merge extraPodSpec: %w", err)
}
// main container fields that require special handling
......@@ -725,7 +726,7 @@ func GenerateBasePodSpec(
overrideResources, err := controller_common.GetResourcesConfig(component.Resources)
if err != nil {
return corev1.PodSpec{}, fmt.Errorf("failed to get resources config: %w", err)
return nil, fmt.Errorf("failed to get resources config: %w", err)
}
// Requests
if overrideResources != nil && len(overrideResources.Requests) > 0 {
......@@ -784,32 +785,32 @@ func GenerateBasePodSpec(
// Apply backend-specific container modifications
multinodeDeployer := MultinodeDeployerFactory(multinodeDeploymentType)
if multinodeDeployer == nil {
return corev1.PodSpec{}, fmt.Errorf("unsupported multinode deployment type: %s", multinodeDeploymentType)
return nil, fmt.Errorf("unsupported multinode deployment type: %s", multinodeDeploymentType)
}
backend := BackendFactory(backendFramework)
if backend == nil {
return corev1.PodSpec{}, fmt.Errorf("unsupported backend framework: %s", backendFramework)
return nil, fmt.Errorf("unsupported backend framework: %s", backendFramework)
}
backend.UpdateContainer(&container, numberOfNodes, role, component, serviceName, multinodeDeployer)
// get base podspec from component
podSpec, err := componentDefaults.GetBasePodSpec(componentContext)
if err != nil {
return corev1.PodSpec{}, fmt.Errorf("failed to get base podspec: %w", err)
return nil, fmt.Errorf("failed to get base podspec: %w", err)
}
if component.ExtraPodSpec != nil && component.ExtraPodSpec.PodSpec != nil {
// merge extraPodSpec PodSpec with base podspec
err := mergo.Merge(&podSpec, component.ExtraPodSpec.PodSpec.DeepCopy(), mergo.WithOverride)
if err != nil {
return corev1.PodSpec{}, fmt.Errorf("failed to merge extraPodSpec: %w", err)
return nil, fmt.Errorf("failed to merge extraPodSpec: %w", err)
}
}
podSpec.Containers = append(podSpec.Containers, container)
podSpec.Volumes = append(podSpec.Volumes, volumes...)
podSpec.ImagePullSecrets = append(podSpec.ImagePullSecrets, imagePullSecrets...)
backend.UpdatePodSpec(&podSpec, numberOfNodes, role, component, serviceName)
return podSpec, nil
return controller_common.CanonicalizePodSpec(&podSpec), nil
}
func setMetricsLabels(labels map[string]string, dynamoGraphDeployment *v1alpha1.DynamoGraphDeployment) {
......@@ -846,13 +847,13 @@ func GeneratePodSpecForComponent(
controllerConfig controller_common.Config,
multinodeDeploymentType commonconsts.MultinodeDeploymentType,
serviceName string,
) (corev1.PodSpec, error) {
) (*corev1.PodSpec, error) {
if len(dynamoDeployment.Spec.Envs) > 0 {
component.Envs = MergeEnvs(dynamoDeployment.Spec.Envs, component.Envs)
}
podSpec, err := GenerateBasePodSpec(component, backendFramework, secretsRetriever, dynamoDeployment.Name, dynamoDeployment.Namespace, role, numberOfNodes, controllerConfig, multinodeDeploymentType, serviceName)
if err != nil {
return corev1.PodSpec{}, err
return nil, err
}
return podSpec, nil
}
......@@ -871,6 +872,7 @@ func GenerateGrovePodGangSet(
gangSet.Spec.Template.HeadlessServiceConfig = &grovev1alpha1.HeadlessServiceConfig{
PublishNotReadyAddresses: true,
}
gangSet.Spec.Template.StartupType = ptr.To(grovev1alpha1.CliqueStartupTypeAnyOrder)
if controllerConfig.Grove.TerminationDelay > 0 {
gangSet.Spec.Template.TerminationDelay = &metav1.Duration{Duration: controllerConfig.Grove.TerminationDelay}
}
......@@ -913,7 +915,8 @@ func GenerateGrovePodGangSet(
Spec: grovev1alpha1.PodCliqueSpec{
RoleName: strings.ToLower(r.Name),
Replicas: r.Replicas,
PodSpec: podSpec,
MinAvailable: ptr.To(int32(1)),
PodSpec: *podSpec,
},
}
labels, err := generateLabels(component, dynamoDeployment, r.Name)
......@@ -938,6 +941,7 @@ func GenerateGrovePodGangSet(
Name: strings.ToLower(serviceName),
CliqueNames: cliqueNames,
Replicas: component.Replicas,
MinAvailable: ptr.To(int32(1)),
})
}
}
......@@ -945,7 +949,7 @@ func GenerateGrovePodGangSet(
gangSet.Spec.Template.PodCliqueScalingGroupConfigs = scalingGroups
}
return gangSet, nil
return controller_common.CanonicalizePodGangSet(gangSet), nil
}
func generateLabels(component *v1alpha1.DynamoComponentDeploymentOverridesSpec, dynamoDeployment *v1alpha1.DynamoGraphDeployment, componentName string) (map[string]string, error) {
......@@ -1141,7 +1145,7 @@ func GenerateBasePodSpecForController(
controllerConfig controller_common.Config,
role Role,
multinodeDeploymentType commonconsts.MultinodeDeploymentType,
) (corev1.PodSpec, error) {
) (*corev1.PodSpec, error) {
// Convert to our interface
componentSpec := ConvertDynamoComponentDeploymentToSpec(dynComponent)
......@@ -1150,7 +1154,7 @@ func GenerateBasePodSpecForController(
// Determine backend framework using hybrid approach
backendFramework, err := getBackendFrameworkFromDynamoComponent(dynComponent)
if err != nil {
return corev1.PodSpec{}, fmt.Errorf("failed to determine backend framework: %w", err)
return nil, fmt.Errorf("failed to determine backend framework: %w", err)
}
// Generate base PodSpec with standard env vars using merged component envs
......@@ -1169,7 +1173,7 @@ func GenerateBasePodSpecForController(
serviceName,
)
if err != nil {
return corev1.PodSpec{}, err
return nil, err
}
return podSpec, nil
......
deploy/cloud/operator/internal/dynamo/graph_test.go
View file @ 94876e58
......@@ -1205,6 +1205,7 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
Spec: grovev1alpha1.PodGangSetSpec{
Replicas: 1,
Template: grovev1alpha1.PodGangSetTemplateSpec{
StartupType: ptr.To(grovev1alpha1.CliqueStartupTypeAnyOrder),
HeadlessServiceConfig: &grovev1alpha1.HeadlessServiceConfig{
PublishNotReadyAddresses: true,
},
......@@ -1226,6 +1227,7 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
Spec: grovev1alpha1.PodCliqueSpec{
RoleName: "frontend",
Replicas: 1,
MinAvailable: ptr.To(int32(1)),
PodSpec: corev1.PodSpec{
Volumes: []corev1.Volume{
{
......@@ -1244,6 +1246,7 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
Name: "frontend-secret",
},
},
RestartPolicy: corev1.RestartPolicyAlways,
Containers: []corev1.Container{
{
Name: "main",
......@@ -1355,6 +1358,7 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
Spec: grovev1alpha1.PodCliqueSpec{
RoleName: "planner",
Replicas: 2,
MinAvailable: ptr.To(int32(1)),
PodSpec: corev1.PodSpec{
Volumes: []corev1.Volume{
{
......@@ -1375,6 +1379,8 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
},
},
},
TerminationGracePeriodSeconds: ptr.To(int64(60)),
RestartPolicy: corev1.RestartPolicyAlways,
Containers: []corev1.Container{
{
Name: "main",
......@@ -1691,9 +1697,11 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
"worker-wkr",
},
Replicas: ptr.To(int32(5)),
MinAvailable: ptr.To(int32(1)),
},
},
// StartupType: ptr.To(grovev1alpha1.CliqueStartupTypeExplicit),
StartupType: ptr.To(grovev1alpha1.CliqueStartupTypeAnyOrder),
Cliques: []*grovev1alpha1.PodCliqueTemplateSpec{
{
Name: "worker-ldr",
......@@ -1711,7 +1719,10 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
Spec: grovev1alpha1.PodCliqueSpec{
RoleName: "worker-ldr",
Replicas: 1,
MinAvailable: ptr.To(int32(1)),
PodSpec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyAlways,
TerminationGracePeriodSeconds: ptr.To(int64(60)),
Volumes: []corev1.Volume{
{
Name: "shared-memory",
......@@ -1857,8 +1868,11 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
Spec: grovev1alpha1.PodCliqueSpec{
RoleName: "worker-wkr",
Replicas: 2,
MinAvailable: ptr.To(int32(1)),
// StartsAfter: []string{"worker-ldr"},
PodSpec: corev1.PodSpec{
RestartPolicy: corev1.RestartPolicyAlways,
TerminationGracePeriodSeconds: ptr.To(int64(60)),
Volumes: []corev1.Volume{
{
Name: "shared-memory",
......@@ -1963,6 +1977,7 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
Spec: grovev1alpha1.PodCliqueSpec{
RoleName: "frontend",
Replicas: 1,
MinAvailable: ptr.To(int32(1)),
PodSpec: corev1.PodSpec{
Volumes: []corev1.Volume{
{
......@@ -1981,6 +1996,7 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
},
},
TerminationGracePeriodSeconds: ptr.To(int64(10)),
RestartPolicy: corev1.RestartPolicyAlways,
Containers: []corev1.Container{
{
Name: "main",
......@@ -2092,7 +2108,10 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
Spec: grovev1alpha1.PodCliqueSpec{
RoleName: "planner",
Replicas: 2,
MinAvailable: ptr.To(int32(1)),
PodSpec: corev1.PodSpec{
TerminationGracePeriodSeconds: ptr.To(int64(60)),
RestartPolicy: corev1.RestartPolicyAlways,
Volumes: []corev1.Volume{
{
Name: "planner-pvc",
......@@ -2440,6 +2459,7 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
Spec: grovev1alpha1.PodGangSetSpec{
Replicas: 1,
Template: grovev1alpha1.PodGangSetTemplateSpec{
StartupType: ptr.To(grovev1alpha1.CliqueStartupTypeAnyOrder),
HeadlessServiceConfig: &grovev1alpha1.HeadlessServiceConfig{
PublishNotReadyAddresses: true,
},
......@@ -2452,6 +2472,7 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
"worker-wkr",
},
Replicas: ptr.To(int32(5)),
MinAvailable: ptr.To(int32(1)),
},
},
// StartupType: ptr.To(grovev1alpha1.CliqueStartupTypeExplicit),
......@@ -2472,6 +2493,7 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
Spec: grovev1alpha1.PodCliqueSpec{
RoleName: "worker-ldr",
Replicas: 1,
MinAvailable: ptr.To(int32(1)),
PodSpec: corev1.PodSpec{
Volumes: []corev1.Volume{
{
......@@ -2484,6 +2506,8 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
},
},
},
TerminationGracePeriodSeconds: ptr.To(int64(60)),
RestartPolicy: corev1.RestartPolicyAlways,
Containers: []corev1.Container{
{
Name: "main",
......@@ -2606,8 +2630,10 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
Spec: grovev1alpha1.PodCliqueSpec{
RoleName: "worker-wkr",
Replicas: 2,
MinAvailable: ptr.To(int32(1)),
// StartsAfter: []string{"worker-ldr"},
PodSpec: corev1.PodSpec{
TerminationGracePeriodSeconds: ptr.To(int64(60)),
Volumes: []corev1.Volume{
{
Name: "shared-memory",
......@@ -2619,6 +2645,7 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
},
},
},
RestartPolicy: corev1.RestartPolicyAlways,
Containers: []corev1.Container{
{
Name: "main",
......@@ -2712,6 +2739,7 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
Spec: grovev1alpha1.PodCliqueSpec{
RoleName: "frontend",
Replicas: 1,
MinAvailable: ptr.To(int32(1)),
PodSpec: corev1.PodSpec{
Volumes: []corev1.Volume{
{
......@@ -2730,6 +2758,7 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
},
},
TerminationGracePeriodSeconds: ptr.To(int64(10)),
RestartPolicy: corev1.RestartPolicyAlways,
Containers: []corev1.Container{
{
Name: "main",
......@@ -2841,7 +2870,9 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
Spec: grovev1alpha1.PodCliqueSpec{
RoleName: "planner",
Replicas: 2,
MinAvailable: ptr.To(int32(1)),
PodSpec: corev1.PodSpec{
TerminationGracePeriodSeconds: ptr.To(int64(60)),
Volumes: []corev1.Volume{
{
Name: "planner-pvc",
......@@ -2861,6 +2892,7 @@ func TestGenerateGrovePodGangSet(t *testing.T) {
},
},
},
RestartPolicy: corev1.RestartPolicyAlways,
Containers: []corev1.Container{
{
Name: "main",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment