dynamocomponentdeployment_controller.go 55.4 KB
Newer Older
1
/*
2
 * SPDX-FileCopyrightText: Copyright (c) 2022 Atalaya Tech. Inc
3
 * SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
4
5
6
7
8
9
10
11
12
13
14
15
16
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
17
 * Modifications Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES
18
19
20
21
22
23
24
 */

package controller

import (
	"context"
	"fmt"
25
	"maps"
26
27
28
29
30
31
32
33
34
35
	"os"
	"time"

	appsv1 "k8s.io/api/apps/v1"
	autoscalingv2 "k8s.io/api/autoscaling/v2"
	corev1 "k8s.io/api/core/v1"
	networkingv1 "k8s.io/api/networking/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"emperror.dev/errors"
36
37
38
39
40
41
	"github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1"
	"github.com/ai-dynamo/dynamo/deploy/operator/internal/common"
	commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts"
	commonController "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common"
	"github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo"
	webhookvalidation "github.com/ai-dynamo/dynamo/deploy/operator/internal/webhook/validation"
42
43
44
45
46
47
48
49
50
51
	networkingv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1"
	k8serrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/intstr"
	"k8s.io/client-go/tools/record"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/builder"
	"sigs.k8s.io/controller-runtime/pkg/client"
52
	"sigs.k8s.io/controller-runtime/pkg/event"
53
54
	"sigs.k8s.io/controller-runtime/pkg/log"
	"sigs.k8s.io/controller-runtime/pkg/predicate"
55
56
57

	leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1"
	volcanov1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
58
59
60
)

const (
61
62
63
	DefaultClusterName                                   = "default"
	DefaultServiceAccountName                            = "default"
	KubeAnnotationDeploymentStrategy                     = "nvidia.com/deployment-strategy"
64
65
	KubeAnnotationDeploymentRollingUpdateMaxSurge        = "nvidia.com/deployment-rolling-update-max-surge"
	KubeAnnotationDeploymentRollingUpdateMaxUnavailable  = "nvidia.com/deployment-rolling-update-max-unavailable"
66
67
68
69
	KubeAnnotationEnableStealingTrafficDebugMode         = "nvidia.com/enable-stealing-traffic-debug-mode"
	KubeAnnotationEnableDebugMode                        = "nvidia.com/enable-debug-mode"
	KubeAnnotationEnableDebugPodReceiveProductionTraffic = "nvidia.com/enable-debug-pod-receive-production-traffic"
	DeploymentTargetTypeDebug                            = "debug"
70
71
)

72
73
// DynamoComponentDeploymentReconciler reconciles a DynamoComponentDeployment object
type DynamoComponentDeploymentReconciler struct {
74
	client.Client
75
	Recorder              record.EventRecorder
76
	Config                commonController.Config
77
78
	EtcdStorage           etcdStorage
	DockerSecretRetriever dockerSecretRetriever
79
80
}

81
82
83
// +kubebuilder:rbac:groups=nvidia.com,resources=dynamocomponentdeployments,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=nvidia.com,resources=dynamocomponentdeployments/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=nvidia.com,resources=dynamocomponentdeployments/finalizers,verbs=update
84
85
86
87
88
89
90
91
92
93
94
95
96
97

//+kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
//+kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=core,resources=events,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=networking.k8s.io,resources=ingressclasses,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=networking.istio.io,resources=virtualservices,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;create;delete

98
99
100
// +kubebuilder:rbac:groups=scheduling.volcano.sh,resources=podgroups,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=leaderworkerset.x-k8s.io,resources=leaderworkersets,verbs=get;list;watch;create;update;patch;delete

101
102
103
// Reconcile is part of the main kubernetes reconciliation loop which aims to
// move the current state of the cluster closer to the desired state.
// TODO(user): Modify the Reconcile function to compare the state specified by
104
// the DynamoComponentDeployment object against the actual cluster state, and then
105
106
107
108
109
110
111
// perform operations to make the cluster state reflect the state specified by
// the user.
//
// For more details, check Reconcile and its Result here:
// - https://pkg.go.dev/sigs.k8s.io/controller-runtime@v0.18.2/pkg/reconcile
//
//nolint:gocyclo,nakedret
112
func (r *DynamoComponentDeploymentReconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ctrl.Result, err error) {
113
114
	logs := log.FromContext(ctx)

115
116
	dynamoComponentDeployment := &v1alpha1.DynamoComponentDeployment{}
	err = r.Get(ctx, req.NamespacedName, dynamoComponentDeployment)
117
118
119
120
	if err != nil {
		if k8serrors.IsNotFound(err) {
			// Object not found, return.  Created objects are automatically garbage collected.
			// For additional cleanup logic use finalizers.
121
			logs.Info("DynamoComponentDeployment resource not found. Ignoring since object must be deleted.")
122
123
124
125
			err = nil
			return
		}
		// Error reading the object - requeue the request.
126
		logs.Error(err, "Failed to get DynamoComponentDeployment.")
127
128
129
		return
	}

130
	logs = logs.WithValues("dynamoComponentDeployment", dynamoComponentDeployment.Name, "namespace", dynamoComponentDeployment.Namespace)
131

132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
	// Setup defer to handle errors and update status
	defer func() {
		if err == nil {
			return
		}
		reconcileErr := err
		logs.Error(reconcileErr, "Failed to reconcile DynamoComponentDeployment.")
		r.Recorder.Eventf(dynamoComponentDeployment, corev1.EventTypeWarning, "ReconcileError",
			"Failed to reconcile DynamoComponentDeployment: %v", reconcileErr)
		if _, statusErr := r.setStatusConditions(ctx, req,
			metav1.Condition{
				Type:    v1alpha1.DynamoGraphDeploymentConditionTypeAvailable,
				Status:  metav1.ConditionFalse,
				Reason:  "Reconciling",
				Message: fmt.Sprintf("Failed to reconcile DynamoComponentDeployment: %v", reconcileErr),
			},
		); statusErr != nil {
			logs.Error(statusErr, "Failed to update DynamoComponentDeployment status after reconcile error")
		}
	}()

	// Validate the DynamoComponentDeployment spec (defense in depth - only when webhooks are disabled)
	if !r.Config.WebhooksEnabled {
		validator := webhookvalidation.NewDynamoComponentDeploymentValidator(dynamoComponentDeployment)
		if _, validationErr := validator.Validate(); validationErr != nil {
			logs.Error(validationErr, "DynamoComponentDeployment validation failed, refusing to reconcile")

			// Set validation error condition
			meta.SetStatusCondition(&dynamoComponentDeployment.Status.Conditions, metav1.Condition{
				Type:               "Valid",
				Status:             metav1.ConditionFalse,
				ObservedGeneration: dynamoComponentDeployment.Generation,
				Reason:             "ValidationFailed",
				Message:            fmt.Sprintf("Validation failed: %v", validationErr),
			})

			// Update status and don't requeue (user must fix the spec)
			if statusErr := r.Status().Update(ctx, dynamoComponentDeployment); statusErr != nil {
				logs.Error(statusErr, "Failed to update DynamoComponentDeployment status with validation error")
				err = statusErr
				return ctrl.Result{}, err
			}

			// Record event for visibility
			r.Recorder.Event(dynamoComponentDeployment, corev1.EventTypeWarning, "ValidationFailed", validationErr.Error())

			// Don't requeue - user must fix the spec
			logs.Info("DynamoComponentDeployment is invalid, not reconciling until spec is fixed")
			err = nil
			return ctrl.Result{}, nil
		}

		// Set Valid condition to True and persist it
		meta.SetStatusCondition(&dynamoComponentDeployment.Status.Conditions, metav1.Condition{
			Type:               "Valid",
			Status:             metav1.ConditionTrue,
			ObservedGeneration: dynamoComponentDeployment.Generation,
			Reason:             "ValidationPassed",
			Message:            "DynamoComponentDeployment spec is valid",
		})
		if statusErr := r.Status().Update(ctx, dynamoComponentDeployment); statusErr != nil {
			logs.Error(statusErr, "Failed to update DynamoComponentDeployment status with validation success")
			err = statusErr
			return ctrl.Result{}, err
		}
	}

199
	deleted, err := commonController.HandleFinalizer(ctx, dynamoComponentDeployment, r.Client, r)
200
201
202
203
204
205
206
207
	if err != nil {
		logs.Error(err, "Failed to handle finalizer")
		return ctrl.Result{}, err
	}
	if deleted {
		return ctrl.Result{}, nil
	}

208
209
210
211
212
	if len(dynamoComponentDeployment.Status.Conditions) == 0 {
		logs.Info("Starting to reconcile DynamoComponentDeployment")
		logs.Info("Initializing DynamoComponentDeployment status")
		r.Recorder.Event(dynamoComponentDeployment, corev1.EventTypeNormal, "Reconciling", "Starting to reconcile DynamoComponentDeployment")
		dynamoComponentDeployment, err = r.setStatusConditions(ctx, req,
213
			metav1.Condition{
214
				Type:    v1alpha1.DynamoGraphDeploymentConditionTypeAvailable,
215
216
				Status:  metav1.ConditionUnknown,
				Reason:  "Reconciling",
217
				Message: "Starting to reconcile DynamoComponentDeployment",
218
219
			},
			metav1.Condition{
220
				Type:    v1alpha1.DynamoGraphDeploymentConditionTypeDynamoComponentReady,
221
222
				Status:  metav1.ConditionUnknown,
				Reason:  "Reconciling",
223
				Message: "Starting to reconcile DynamoComponentDeployment",
224
225
226
227
228
229
230
			},
		)
		if err != nil {
			return
		}
	}

231
	// Create the appropriate workload resource based on deployment type
232
	var componentReconcileResult ComponentReconcileResult
233
	if r.Config.LWS.Enabled && dynamoComponentDeployment.IsMultinode() {
234
		componentReconcileResult, err = r.reconcileLeaderWorkerSetResources(ctx, dynamoComponentDeployment)
235
	} else {
236
		componentReconcileResult, err = r.reconcileDeploymentResources(ctx, dynamoComponentDeployment)
237
	}
238
239
240
241
	if err != nil {
		return ctrl.Result{}, fmt.Errorf("failed to reconcile the resources: %w", err)
	}
	modified := componentReconcileResult.modified
242
243

	// create or update api-server service
244
	serviceModified, err := r.createOrUpdateOrDeleteServices(ctx, generateResourceOption{
245
		dynamoComponentDeployment: dynamoComponentDeployment,
246
247
	})
	if err != nil {
248
		return ctrl.Result{}, fmt.Errorf("failed to create or update the service: %w", err)
249
250
	}

251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
	// create or update headless service for model endpoint discovery
	componentMap := map[string]*v1alpha1.DynamoComponentDeploymentSharedSpec{
		dynamoComponentDeployment.Name: &dynamoComponentDeployment.Spec.DynamoComponentDeploymentSharedSpec,
	}
	if err := dynamo.ReconcileModelServicesForComponents(
		ctx,
		r,
		dynamoComponentDeployment,
		componentMap,
		dynamoComponentDeployment.Namespace,
	); err != nil {
		logs.Error(err, "Failed to reconcile model service")
		return ctrl.Result{}, err
	}

266
	// create or update api-server ingresses
267
	ingressModified, err := r.createOrUpdateOrDeleteIngress(ctx, generateResourceOption{
268
		dynamoComponentDeployment: dynamoComponentDeployment,
269
	})
270
	if err != nil {
271
		return ctrl.Result{}, fmt.Errorf("failed to create or update the ingress: %w", err)
272
273
	}

274
	if serviceModified || ingressModified {
275
276
277
278
		modified = true
	}

	if !modified {
279
		r.Recorder.Eventf(dynamoComponentDeployment, corev1.EventTypeNormal, "UpdateDynamoGraphDeployment", "No changes to dynamo deployment %s", dynamoComponentDeployment.Name)
280
281
282
	}

	logs.Info("Finished reconciling.")
283
	r.Recorder.Eventf(dynamoComponentDeployment, corev1.EventTypeNormal, "Update", "All resources updated!")
284

285
286
287
	err = r.setStatusConditionAndServiceReplicaStatus(ctx, dynamoComponentDeployment, componentReconcileResult)
	if err != nil {
		return ctrl.Result{}, fmt.Errorf("failed to set status condition and service replica status: %w", err)
288
289
	}

290
291
292
	return
}

293
294
295
296
297
type ComponentReconcileResult struct {
	modified             bool
	status               metav1.ConditionStatus
	reason               string
	message              string
298
	serviceReplicaStatus *v1alpha1.ServiceReplicaStatus
299
300
301
302
303
304
305
306
307
308
309
}

func (r *DynamoComponentDeploymentReconciler) reconcileDeploymentResources(ctx context.Context, dynamoComponentDeployment *v1alpha1.DynamoComponentDeployment) (ComponentReconcileResult, error) {
	logger := log.FromContext(ctx)
	deploymentModified, deployment, err := r.createOrUpdateOrDeleteDeployments(ctx, generateResourceOption{
		dynamoComponentDeployment: dynamoComponentDeployment,
	})
	if err != nil {
		return ComponentReconcileResult{}, fmt.Errorf("failed to create or update the deployment: %w", err)
	}

310
311
312
313
314
315
316
317
318
319
	logger.V(1).Info("Deployment sync completed",
		"deploymentModified", deploymentModified,
		"deploymentName", deployment.Name,
		"deploymentGeneration", deployment.Generation,
		"deploymentObservedGeneration", deployment.Status.ObservedGeneration,
		"deploymentReplicas", deployment.Status.Replicas,
		"deploymentUpdatedReplicas", deployment.Status.UpdatedReplicas,
		"deploymentAvailableReplicas", deployment.Status.AvailableReplicas,
		"deploymentReadyReplicas", deployment.Status.ReadyReplicas)

320
	serviceReplicaStatus := &v1alpha1.ServiceReplicaStatus{
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
		ComponentKind:     v1alpha1.ComponentKindDeployment,
		ComponentName:     deployment.Name,
		Replicas:          deployment.Status.Replicas,
		UpdatedReplicas:   deployment.Status.UpdatedReplicas,
		ReadyReplicas:     &deployment.Status.ReadyReplicas,
		AvailableReplicas: &deployment.Status.AvailableReplicas,
	}

	if IsDeploymentReady(deployment) {
		return ComponentReconcileResult{
			modified:             deploymentModified,
			status:               metav1.ConditionTrue,
			reason:               "DeploymentReady",
			message:              "Deployment is ready",
			serviceReplicaStatus: serviceReplicaStatus,
		}, nil
	}
	return ComponentReconcileResult{
		modified:             deploymentModified,
		status:               metav1.ConditionFalse,
		reason:               "DeploymentNotReady",
		message:              "Deployment is not ready",
		serviceReplicaStatus: serviceReplicaStatus,
	}, nil
}

func (r *DynamoComponentDeploymentReconciler) reconcileLeaderWorkerSetResources(ctx context.Context, dynamoComponentDeployment *v1alpha1.DynamoComponentDeployment) (ComponentReconcileResult, error) {
	logger := log.FromContext(ctx)

	desiredReplicas := int32(1)
	if dynamoComponentDeployment.Spec.Replicas != nil {
		desiredReplicas = *dynamoComponentDeployment.Spec.Replicas
	}

	anyModified := false
	leaderWorkerSets := make([]*leaderworkersetv1.LeaderWorkerSet, 0, desiredReplicas)
	for i := range int(desiredReplicas) {
		volcanoPodGroupModified, _, err := commonController.SyncResource(ctx, r, dynamoComponentDeployment, func(ctx context.Context) (*volcanov1beta1.PodGroup, bool, error) {
			return r.generateVolcanoPodGroup(ctx, generateResourceOption{
				dynamoComponentDeployment:               dynamoComponentDeployment,
				isStealingTrafficDebugModeEnabled:       false,
				containsStealingTrafficDebugModeEnabled: false,
				instanceID:                              &i,
			})
		})
		if err != nil {
			return ComponentReconcileResult{}, fmt.Errorf("failed to sync the PodGroup: %w", err)
		}

		leaderWorkerSetModified, lwsObj, err := commonController.SyncResource(ctx, r, dynamoComponentDeployment, func(ctx context.Context) (*leaderworkersetv1.LeaderWorkerSet, bool, error) {
			return r.generateLeaderWorkerSet(ctx, generateResourceOption{
				dynamoComponentDeployment:               dynamoComponentDeployment,
				isStealingTrafficDebugModeEnabled:       false,
				containsStealingTrafficDebugModeEnabled: false,
				instanceID:                              &i,
			})
		})
		if err != nil {
			return ComponentReconcileResult{}, fmt.Errorf("failed to sync the LeaderWorkerSet: %w", err)
		}

		if leaderWorkerSetModified || volcanoPodGroupModified {
			anyModified = true
		}
		leaderWorkerSets = append(leaderWorkerSets, lwsObj)
	}

	// Clean up any excess LeaderWorkerSets (if replicas were decreased)
	baseKubeName := r.getKubeName(dynamoComponentDeployment, false)
	for i := int(desiredReplicas); ; i++ {
		// Try to find a LeaderWorkerSet with the next index
		nextLWSName := fmt.Sprintf("%s-%d", baseKubeName, i)
		lwsToDelete := &leaderworkersetv1.LeaderWorkerSet{}
		err := r.Get(ctx, types.NamespacedName{
			Name:      nextLWSName,
			Namespace: dynamoComponentDeployment.Namespace,
		}, lwsToDelete)

		if err != nil {
			if k8serrors.IsNotFound(err) {
				break
			}
			return ComponentReconcileResult{}, fmt.Errorf("failed to get the LeaderWorkerSet for deletion: %w", err)
		}

		err = r.Delete(ctx, lwsToDelete)
		if err != nil {
			return ComponentReconcileResult{}, fmt.Errorf("failed to delete the LeaderWorkerSet: %w", err)
		}

		podGroupName := nextLWSName
		podGroupToDelete := &volcanov1beta1.PodGroup{}
		err = r.Get(ctx, types.NamespacedName{
			Name:      podGroupName,
			Namespace: dynamoComponentDeployment.Namespace,
		}, podGroupToDelete)

		if err != nil {
			if !k8serrors.IsNotFound(err) {
				logger.Error(err, "Failed to get PodGroup for deletion", "podGroupName", podGroupName)
			}
		} else {
			err = r.Delete(ctx, podGroupToDelete)
			if err != nil {
				logger.Error(err, "Failed to delete PodGroup", "podGroupName", podGroupName)
			}
		}

		anyModified = true
	}
431
432

	allReady := true
433
	lwsReplicaStatuses := []v1alpha1.ServiceReplicaStatus{}
434
435
436
437
	for _, leaderWorkerSet := range leaderWorkerSets {
		if !IsLeaderWorkerSetReady(leaderWorkerSet) {
			allReady = false
		}
438
		lwsReplicaStatuses = append(lwsReplicaStatuses, getLeaderWorkerSetReplicasStatus(leaderWorkerSet))
439
440
441
	}

	if allReady {
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
		return ComponentReconcileResult{
			modified:             anyModified,
			status:               metav1.ConditionTrue,
			reason:               "AllLeaderWorkerSetsReady",
			message:              "All LeaderWorkerSets are ready",
			serviceReplicaStatus: combineLWSReplicaStatuses(lwsReplicaStatuses),
		}, nil
	}
	return ComponentReconcileResult{
		modified:             anyModified,
		status:               metav1.ConditionFalse,
		reason:               "SomeLeaderWorkerSetsNotReady",
		message:              "Some LeaderWorkerSets are not ready",
		serviceReplicaStatus: combineLWSReplicaStatuses(lwsReplicaStatuses),
	}, nil

}

func (r *DynamoComponentDeploymentReconciler) setStatusConditionAndServiceReplicaStatus(ctx context.Context, dynamoComponentDeployment *v1alpha1.DynamoComponentDeployment, componentReconcileResult ComponentReconcileResult) error {
	condition := metav1.Condition{
		Type:    v1alpha1.DynamoGraphDeploymentConditionTypeAvailable,
		Status:  componentReconcileResult.status,
		Reason:  componentReconcileResult.reason,
		Message: componentReconcileResult.message,
	}

	meta.SetStatusCondition(&dynamoComponentDeployment.Status.Conditions, condition)
	dynamoComponentDeployment.Status.Service = componentReconcileResult.serviceReplicaStatus
470
	dynamoComponentDeployment.Status.ObservedGeneration = dynamoComponentDeployment.Generation
471
472
473
474

	err := r.Status().Update(ctx, dynamoComponentDeployment)
	if err != nil {
		return fmt.Errorf("failed to update DynamoComponentDeployment status: %w", err)
475
	}
476
477
478
479
480
481
482
483
484
485
486
487
488
	return nil
}

func getLeaderWorkerSetReplicasStatus(leaderWorkerSet *leaderworkersetv1.LeaderWorkerSet) v1alpha1.ServiceReplicaStatus {
	return v1alpha1.ServiceReplicaStatus{
		ComponentKind:   v1alpha1.ComponentKindLeaderWorkerSet,
		ComponentName:   leaderWorkerSet.Name,
		Replicas:        leaderWorkerSet.Status.Replicas,
		UpdatedReplicas: leaderWorkerSet.Status.UpdatedReplicas,
		ReadyReplicas:   &leaderWorkerSet.Status.ReadyReplicas,
	}
}

489
func combineLWSReplicaStatuses(serviceReplicaStatuses []v1alpha1.ServiceReplicaStatus) *v1alpha1.ServiceReplicaStatus {
490
	if len(serviceReplicaStatuses) == 0 {
491
		return nil
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
	}

	firstServiceStatus := serviceReplicaStatuses[0]
	var readyReplicas int32 = 0
	if firstServiceStatus.ReadyReplicas != nil {
		readyReplicas = *firstServiceStatus.ReadyReplicas
	}
	for _, serviceReplicaStatus := range serviceReplicaStatuses[1:] {
		firstServiceStatus.Replicas += serviceReplicaStatus.Replicas
		firstServiceStatus.UpdatedReplicas += serviceReplicaStatus.UpdatedReplicas
		if serviceReplicaStatus.ReadyReplicas != nil {
			readyReplicas += *serviceReplicaStatus.ReadyReplicas
		}
	}

	firstServiceStatus.ReadyReplicas = &readyReplicas
508
	return &firstServiceStatus
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
}

// IsLeaderWorkerSetReady determines if a LeaderWorkerSet is fully ready and available
func IsLeaderWorkerSetReady(leaderWorkerSet *leaderworkersetv1.LeaderWorkerSet) bool {
	if leaderWorkerSet == nil {
		return false
	}

	desiredReplicas := int32(1)
	if leaderWorkerSet.Spec.Replicas != nil {
		desiredReplicas = *leaderWorkerSet.Spec.Replicas
	}

	// Special case: if no replicas are desired, the LeaderWorkerSet is considered ready
	if desiredReplicas == 0 {
		return true
	}

	status := leaderWorkerSet.Status

	if status.ReadyReplicas < desiredReplicas {
		return false
	}

	// Look for the Available condition specifically - this is defined in the CRD for LeaderWorkerSet
	for _, cond := range leaderWorkerSet.Status.Conditions {
		if cond.Type == string(leaderworkersetv1.LeaderWorkerSetAvailable) {
			return cond.Status == metav1.ConditionTrue
		}
	}

	return false
}

func (r *DynamoComponentDeploymentReconciler) generateVolcanoPodGroup(ctx context.Context, opt generateResourceOption) (*volcanov1beta1.PodGroup, bool, error) {
	logs := log.FromContext(ctx)
	logs.Info("Generating Volcano PodGroup")

	if opt.instanceID == nil {
		return nil, false, errors.New("generateVolcanoPodGroup: instanceID cannot be nil")
	}
	instanceID := *opt.instanceID

	if instanceID < 0 {
		return nil, false, fmt.Errorf("generateVolcanoPodGroup: instanceID cannot be negative, got %d", instanceID)
	}

556
	podGroupName := r.getKubeName(opt.dynamoComponentDeployment, opt.isStealingTrafficDebugModeEnabled)
557
558
559
560
561
562
563
	podGroupName = fmt.Sprintf("%s-%d", podGroupName, instanceID)

	kubeNs := opt.dynamoComponentDeployment.Namespace

	labels := make(map[string]string)
	labels["instance-id"] = fmt.Sprintf("%d", instanceID)

564
	minMember := opt.dynamoComponentDeployment.GetNumberOfNodes()
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580

	podGroup := &volcanov1beta1.PodGroup{
		ObjectMeta: metav1.ObjectMeta{
			Name:      podGroupName,
			Namespace: kubeNs,
			Labels:    labels,
		},
		Spec: volcanov1beta1.PodGroupSpec{
			MinMember: minMember,
		},
	}

	return podGroup, false, nil
}

func (r *DynamoComponentDeploymentReconciler) generateLeaderPodTemplateSpec(ctx context.Context, opt generateResourceOption, kubeName string, labels map[string]string, instanceID int) (*corev1.PodTemplateSpec, error) {
581
	leaderPodTemplateSpec, err := r.generatePodTemplateSpec(ctx, opt, dynamo.RoleLeader)
582
583
584
585
	if err != nil {
		return nil, errors.Wrap(err, "failed to generate leader pod template")
	}

586
	maps.Copy(leaderPodTemplateSpec.ObjectMeta.Labels, labels)
587
588
589
590
591
592
593
594
595
596
597
	leaderPodTemplateSpec.ObjectMeta.Labels["role"] = "leader"
	leaderPodTemplateSpec.ObjectMeta.Labels["instance-id"] = fmt.Sprintf("%d", instanceID)
	delete(leaderPodTemplateSpec.ObjectMeta.Labels, commonconsts.KubeLabelDynamoSelector)

	if leaderPodTemplateSpec.ObjectMeta.Annotations == nil {
		leaderPodTemplateSpec.ObjectMeta.Annotations = make(map[string]string)
	}
	leaderPodTemplateSpec.ObjectMeta.Annotations["scheduling.k8s.io/group-name"] = kubeName

	leaderPodTemplateSpec.Spec.SchedulerName = "volcano"

598
	err = checkMainContainer(&leaderPodTemplateSpec.Spec)
599

600
601
	if err != nil {
		return nil, errors.Wrap(err, "generateLeaderPodTemplateSpec: failed to check main container")
602
603
604
605
606
	}

	return leaderPodTemplateSpec, nil
}

607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
func checkMainContainer(spec *corev1.PodSpec) error {

	if len(spec.Containers) == 0 {
		return errors.New("No containers found in pod spec")
	}

	mainContainerFound := false
	for _, container := range spec.Containers {
		if container.Name != commonconsts.MainContainerName {
			continue
		}

		if len(container.Command) == 0 {
			return errors.New("container Command cannot be nil for LWS pod")
		}

		if len(container.Args) == 0 {
			return errors.New("container Args cannot be empty for LWS pod")
		}

		mainContainerFound = true
		break
	}

	if !mainContainerFound {
		return errors.New("main container not found in pod spec")
	}

	return nil
}

638
func (r *DynamoComponentDeploymentReconciler) generateWorkerPodTemplateSpec(ctx context.Context, opt generateResourceOption, kubeName string, labels map[string]string, instanceID int) (*corev1.PodTemplateSpec, error) {
639
	workerPodTemplateSpec, err := r.generatePodTemplateSpec(ctx, opt, dynamo.RoleWorker)
640
641
642
643
	if err != nil {
		return nil, errors.Wrap(err, "failed to generate worker pod template")
	}

644
	maps.Copy(workerPodTemplateSpec.ObjectMeta.Labels, labels)
645
646
647
648
649
650
651
652
653
654
655
	workerPodTemplateSpec.ObjectMeta.Labels["role"] = "worker"
	workerPodTemplateSpec.ObjectMeta.Labels["instance-id"] = fmt.Sprintf("%d", instanceID)
	delete(workerPodTemplateSpec.ObjectMeta.Labels, commonconsts.KubeLabelDynamoSelector)

	workerPodTemplateSpec.Spec.SchedulerName = "volcano"

	if workerPodTemplateSpec.ObjectMeta.Annotations == nil {
		workerPodTemplateSpec.ObjectMeta.Annotations = make(map[string]string)
	}
	workerPodTemplateSpec.ObjectMeta.Annotations["scheduling.k8s.io/group-name"] = kubeName

656
	err = checkMainContainer(&workerPodTemplateSpec.Spec)
657

658
659
	if err != nil {
		return nil, errors.Wrap(err, "generateWorkerPodTemplateSpec: failed to check LWS worker main container")
660
661
662
	}

	if opt.dynamoComponentDeployment.Spec.Resources == nil || opt.dynamoComponentDeployment.Spec.Resources.Limits == nil || opt.dynamoComponentDeployment.Spec.Resources.Limits.GPU == "" {
663
		return nil, fmt.Errorf("generateWorkerPodTemplateSpec: GPU limit is not set for LWS worker pod")
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
	}

	return workerPodTemplateSpec, nil
}

// generateLeaderWorkerSet creates a LeaderWorkerSet resource from the DynamoComponentDeployment
func (r *DynamoComponentDeploymentReconciler) generateLeaderWorkerSet(ctx context.Context, opt generateResourceOption) (*leaderworkersetv1.LeaderWorkerSet, bool, error) {
	logs := log.FromContext(ctx)
	logs.Info("Generating LeaderWorkerSet")

	if opt.instanceID == nil {
		return nil, false, errors.New("generateLeaderWorkerSet: instanceID cannot be nil")
	}
	instanceID := *opt.instanceID

	if instanceID < 0 {
		return nil, false, fmt.Errorf("generateLeaderWorkerSet: instanceID cannot be negative, got %d", instanceID)
	}

683
	kubeName := r.getKubeName(opt.dynamoComponentDeployment, opt.isStealingTrafficDebugModeEnabled)
684
685
686
	kubeName = fmt.Sprintf("%s-%d", kubeName, instanceID)

	kubeNs := opt.dynamoComponentDeployment.Namespace
687
	labels := r.getKubeLabels(opt.dynamoComponentDeployment)
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721

	if labels == nil {
		labels = make(map[string]string)
	}
	labels["instance-id"] = fmt.Sprintf("%d", instanceID)

	leaderWorkerSet := &leaderworkersetv1.LeaderWorkerSet{
		ObjectMeta: metav1.ObjectMeta{
			Name:      kubeName,
			Namespace: kubeNs,
			Labels:    labels,
		},
	}

	leaderPodLabels := make(map[string]string)
	for k, v := range labels {
		leaderPodLabels[k] = v
	}
	leaderPodTemplateSpec, err := r.generateLeaderPodTemplateSpec(ctx, opt, kubeName, leaderPodLabels, instanceID)
	if err != nil {
		return nil, false, errors.Wrap(err, "generateLeaderWorkerSet: failed to generate leader pod template")
	}

	workerPodLabels := make(map[string]string)
	for k, v := range labels {
		workerPodLabels[k] = v
	}
	workerPodTemplateSpec, err := r.generateWorkerPodTemplateSpec(ctx, opt, kubeName, workerPodLabels, instanceID)
	if err != nil {
		return nil, false, errors.Wrap(err, "generateLeaderWorkerSet: failed to generate worker pod template")
	}

	// Each individual LeaderWorkerSet always has exactly 1 replica
	singleReplica := int32(1)
722
	groupSize := opt.dynamoComponentDeployment.GetNumberOfNodes()
723
724
725
726
727
728
729
730
731
732
733
734
735
736

	leaderWorkerSet.Spec = leaderworkersetv1.LeaderWorkerSetSpec{
		Replicas:      &singleReplica,
		StartupPolicy: leaderworkersetv1.LeaderCreatedStartupPolicy,
		LeaderWorkerTemplate: leaderworkersetv1.LeaderWorkerTemplate{
			LeaderTemplate: leaderPodTemplateSpec,
			WorkerTemplate: *workerPodTemplateSpec,
			Size:           &groupSize,
		},
	}

	return leaderWorkerSet, false, nil
}

737
func (r *DynamoComponentDeploymentReconciler) FinalizeResource(ctx context.Context, dynamoComponentDeployment *v1alpha1.DynamoComponentDeployment) error {
738
	logger := log.FromContext(ctx)
739
740
741
742
	logger.Info("Finalizing the DynamoComponentDeployment", "dynamoComponentDeployment", dynamoComponentDeployment)
	if dynamoComponentDeployment.Spec.ServiceName != "" && dynamoComponentDeployment.Spec.DynamoNamespace != nil && *dynamoComponentDeployment.Spec.DynamoNamespace != "" {
		logger.Info("Deleting the etcd keys for the service", "service", dynamoComponentDeployment.Spec.ServiceName, "dynamoNamespace", *dynamoComponentDeployment.Spec.DynamoNamespace)
		err := r.EtcdStorage.DeleteKeys(ctx, fmt.Sprintf("/%s/components/%s", *dynamoComponentDeployment.Spec.DynamoNamespace, dynamoComponentDeployment.Spec.ServiceName))
743
		if err != nil {
744
			logger.Error(err, "Failed to delete the etcd keys for the service", "service", dynamoComponentDeployment.Spec.ServiceName, "dynamoNamespace", *dynamoComponentDeployment.Spec.DynamoNamespace)
745
746
747
748
749
750
			return err
		}
	}
	return nil
}

751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
// IsDeploymentReady determines if a Kubernetes Deployment is fully ready and available.
// It checks various status fields to ensure all replicas are available and the deployment
// configuration has been fully applied.
func IsDeploymentReady(deployment *appsv1.Deployment) bool {
	if deployment == nil {
		return false
	}
	// Paused deployments should not be considered ready
	if deployment.Spec.Paused {
		return false
	}
	// Default to 1 replica if not specified
	desiredReplicas := int32(1)
	if deployment.Spec.Replicas != nil {
		desiredReplicas = *deployment.Spec.Replicas
	}
	// Special case: if no replicas are desired, the deployment is considered ready
	if desiredReplicas == 0 {
		return true
	}
	status := deployment.Status
	// Check all basic status requirements:
	// 1. ObservedGeneration: Deployment controller has observed the latest configuration
	// 2. UpdatedReplicas: All replicas have been updated to the latest version
	// 3. AvailableReplicas: All desired replicas are available (schedulable and healthy)
776
	// 4. Replicas: Total replicas equals desired (no surge pods remaining from rolling update)
777
778
	if status.ObservedGeneration < deployment.Generation ||
		status.UpdatedReplicas < desiredReplicas ||
779
780
		status.AvailableReplicas < desiredReplicas ||
		status.Replicas != desiredReplicas {
781
782
783
784
785
786
787
788
789
790
791
792
793
		return false
	}
	// Finally, check for the DeploymentAvailable condition
	// This is Kubernetes' own assessment that the deployment is available
	for _, cond := range deployment.Status.Conditions {
		if cond.Type == appsv1.DeploymentAvailable && cond.Status == corev1.ConditionTrue {
			return true
		}
	}
	// If we get here, the basic checks passed but the Available condition wasn't found
	return false
}

794
795
func (r *DynamoComponentDeploymentReconciler) setStatusConditions(ctx context.Context, req ctrl.Request, conditions ...metav1.Condition) (dynamoComponentDeployment *v1alpha1.DynamoComponentDeployment, err error) {
	dynamoComponentDeployment = &v1alpha1.DynamoComponentDeployment{}
796
797
	maxRetries := 3
	for range maxRetries - 1 {
798
799
		if err = r.Get(ctx, req.NamespacedName, dynamoComponentDeployment); err != nil {
			err = errors.Wrap(err, "Failed to re-fetch DynamoComponentDeployment")
800
801
802
			return
		}
		for _, condition := range conditions {
803
			meta.SetStatusCondition(&dynamoComponentDeployment.Status.Conditions, condition)
804
		}
805
		if err = r.Status().Update(ctx, dynamoComponentDeployment); err != nil {
806
807
808
809
810
			if k8serrors.IsConflict(err) {
				time.Sleep(100 * time.Millisecond)
				continue
			}
			break
811
812
813
814
815
		} else {
			break
		}
	}
	if err != nil {
816
		err = errors.Wrap(err, "Failed to update DynamoComponentDeployment status")
817
818
		return
	}
819
820
	if err = r.Get(ctx, req.NamespacedName, dynamoComponentDeployment); err != nil {
		err = errors.Wrap(err, "Failed to re-fetch DynamoComponentDeployment")
821
822
823
824
825
826
		return
	}
	return
}

//nolint:nakedret
827
828
func (r *DynamoComponentDeploymentReconciler) createOrUpdateOrDeleteDeployments(ctx context.Context, opt generateResourceOption) (modified bool, depl *appsv1.Deployment, err error) {
	containsStealingTrafficDebugModeEnabled := checkIfContainsStealingTrafficDebugModeEnabled(opt.dynamoComponentDeployment)
829
	// create the main deployment
830
831
832
833
834
835
836
	modified, depl, err = commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*appsv1.Deployment, bool, error) {
		return r.generateDeployment(ctx, generateResourceOption{
			dynamoComponentDeployment:               opt.dynamoComponentDeployment,
			isStealingTrafficDebugModeEnabled:       false,
			containsStealingTrafficDebugModeEnabled: containsStealingTrafficDebugModeEnabled,
		})
	})
837
838
839
840
	if err != nil {
		err = errors.Wrap(err, "create or update deployment")
		return
	}
841
	// create the debug deployment
842
843
844
845
846
847
848
	modified2, _, err := commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*appsv1.Deployment, bool, error) {
		return r.generateDeployment(ctx, generateResourceOption{
			dynamoComponentDeployment:               opt.dynamoComponentDeployment,
			isStealingTrafficDebugModeEnabled:       true,
			containsStealingTrafficDebugModeEnabled: containsStealingTrafficDebugModeEnabled,
		})
	})
849
850
	if err != nil {
		err = errors.Wrap(err, "create or update debug deployment")
851
	}
852
	modified = modified || modified2
853
854
855
	return
}

856
857
func getResourceAnnotations(dynamoComponentDeployment *v1alpha1.DynamoComponentDeployment) map[string]string {
	resourceAnnotations := dynamoComponentDeployment.Spec.Annotations
858
859
860
861
862
863
864
865
866
867
868
869
	if resourceAnnotations == nil {
		resourceAnnotations = map[string]string{}
	}

	return resourceAnnotations
}

func checkIfIsDebugModeEnabled(annotations map[string]string) bool {
	if annotations == nil {
		return false
	}

870
	return annotations[KubeAnnotationEnableDebugMode] == commonconsts.KubeLabelValueTrue
871
872
873
874
875
876
877
}

func checkIfIsStealingTrafficDebugModeEnabled(annotations map[string]string) bool {
	if annotations == nil {
		return false
	}

878
	return annotations[KubeAnnotationEnableStealingTrafficDebugMode] == commonconsts.KubeLabelValueTrue
879
880
881
882
883
884
885
}

func checkIfIsDebugPodReceiveProductionTrafficEnabled(annotations map[string]string) bool {
	if annotations == nil {
		return false
	}

886
	return annotations[KubeAnnotationEnableDebugPodReceiveProductionTraffic] == commonconsts.KubeLabelValueTrue
887
888
}

889
890
func checkIfContainsStealingTrafficDebugModeEnabled(dynamoComponentDeployment *v1alpha1.DynamoComponentDeployment) bool {
	return checkIfIsStealingTrafficDebugModeEnabled(dynamoComponentDeployment.Spec.Annotations)
891
892
893
}

//nolint:nakedret
894
895
func (r *DynamoComponentDeploymentReconciler) createOrUpdateOrDeleteServices(ctx context.Context, opt generateResourceOption) (modified bool, err error) {
	resourceAnnotations := getResourceAnnotations(opt.dynamoComponentDeployment)
896
	isDebugPodReceiveProductionTrafficEnabled := checkIfIsDebugPodReceiveProductionTrafficEnabled(resourceAnnotations)
897
	containsStealingTrafficDebugModeEnabled := checkIfContainsStealingTrafficDebugModeEnabled(opt.dynamoComponentDeployment)
898
	// main generic service
899
	modified, _, err = commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*corev1.Service, bool, error) {
900
		return r.generateService(generateResourceOption{
901
902
903
904
905
906
907
			dynamoComponentDeployment:               opt.dynamoComponentDeployment,
			isStealingTrafficDebugModeEnabled:       false,
			isDebugPodReceiveProductionTraffic:      isDebugPodReceiveProductionTrafficEnabled,
			containsStealingTrafficDebugModeEnabled: containsStealingTrafficDebugModeEnabled,
			isGenericService:                        true,
		})
	})
908
909
910
911
	if err != nil {
		return
	}

912
	// debug production service (if enabled)
913
	modified_, _, err := commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*corev1.Service, bool, error) {
914
		return r.generateService(generateResourceOption{
915
916
917
918
919
920
921
			dynamoComponentDeployment:               opt.dynamoComponentDeployment,
			isStealingTrafficDebugModeEnabled:       false,
			isDebugPodReceiveProductionTraffic:      isDebugPodReceiveProductionTrafficEnabled,
			containsStealingTrafficDebugModeEnabled: containsStealingTrafficDebugModeEnabled,
			isGenericService:                        false,
		})
	})
922
923
924
	if err != nil {
		return
	}
925
926
	modified = modified || modified_
	// debug service (if enabled)
927
	modified_, _, err = commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*corev1.Service, bool, error) {
928
		return r.generateService(generateResourceOption{
929
930
931
932
933
934
935
			dynamoComponentDeployment:               opt.dynamoComponentDeployment,
			isStealingTrafficDebugModeEnabled:       true,
			isDebugPodReceiveProductionTraffic:      isDebugPodReceiveProductionTrafficEnabled,
			containsStealingTrafficDebugModeEnabled: containsStealingTrafficDebugModeEnabled,
			isGenericService:                        false,
		})
	})
936
	if err != nil {
937
938
		return
	}
939
	modified = modified || modified_
940
941
942
	return
}

943
944
func (r *DynamoComponentDeploymentReconciler) createOrUpdateOrDeleteIngress(ctx context.Context, opt generateResourceOption) (bool, error) {
	modified, _, err := commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*networkingv1.Ingress, bool, error) {
945
946
		return r.generateIngress(ctx, opt)
	})
947
	if err != nil {
948
		return false, err
949
	}
950
	if r.Config.IngressConfig.UseVirtualService() {
951
952
953
954
955
956
957
		modified_, _, err := commonController.SyncResource(ctx, r, opt.dynamoComponentDeployment, func(ctx context.Context) (*networkingv1beta1.VirtualService, bool, error) {
			return r.generateVirtualService(ctx, opt)
		})
		if err != nil {
			return false, err
		}
		return modified || modified_, nil
958
	}
959
	return modified, nil
960
961
}

962
func (r *DynamoComponentDeploymentReconciler) generateIngress(ctx context.Context, opt generateResourceOption) (*networkingv1.Ingress, bool, error) {
963
	log := log.FromContext(ctx)
964
965
966
967
	log.Info("Starting generateIngress")

	ingress := &networkingv1.Ingress{
		ObjectMeta: metav1.ObjectMeta{
968
969
			Name:      opt.dynamoComponentDeployment.Name,
			Namespace: opt.dynamoComponentDeployment.Namespace,
970
971
		},
	}
972

973
	if opt.dynamoComponentDeployment.Spec.Ingress == nil || !opt.dynamoComponentDeployment.Spec.Ingress.Enabled || opt.dynamoComponentDeployment.Spec.Ingress.IngressControllerClassName == nil {
974
975
		log.Info("Ingress is not enabled")
		return ingress, true, nil
976
	}
977
	return dynamo.GenerateComponentIngress(ctx, opt.dynamoComponentDeployment.Name, opt.dynamoComponentDeployment.Namespace, *opt.dynamoComponentDeployment.Spec.Ingress), false, nil
978
979
}

980
func (r *DynamoComponentDeploymentReconciler) generateVirtualService(ctx context.Context, opt generateResourceOption) (*networkingv1beta1.VirtualService, bool, error) {
981
982
983
	log := log.FromContext(ctx)
	log.Info("Starting generateVirtualService")

984
985
	vs := &networkingv1beta1.VirtualService{
		ObjectMeta: metav1.ObjectMeta{
986
987
			Name:      opt.dynamoComponentDeployment.Name,
			Namespace: opt.dynamoComponentDeployment.Namespace,
988
		},
989
990
	}

991
	if !opt.dynamoComponentDeployment.Spec.Ingress.IsVirtualServiceEnabled() {
992
993
994
		log.Info("VirtualService is not enabled")
		return vs, true, nil
	}
995
	return dynamo.GenerateComponentVirtualService(ctx, opt.dynamoComponentDeployment.Name, opt.dynamoComponentDeployment.Namespace, *opt.dynamoComponentDeployment.Spec.Ingress), false, nil
996
997
}

998
func (r *DynamoComponentDeploymentReconciler) getKubeName(dynamoComponentDeployment *v1alpha1.DynamoComponentDeployment, debug bool) string {
999
	if debug {
1000
		return fmt.Sprintf("%s-d", dynamoComponentDeployment.Name)
1001
	}
1002
	return dynamoComponentDeployment.Name
1003
1004
}

1005
func (r *DynamoComponentDeploymentReconciler) getServiceName(dynamoComponentDeployment *v1alpha1.DynamoComponentDeployment, debug bool) string {
1006
1007
	var kubeName string
	if debug {
1008
		kubeName = fmt.Sprintf("%s-d", dynamoComponentDeployment.Name)
1009
	} else {
1010
		kubeName = fmt.Sprintf("%s-p", dynamoComponentDeployment.Name)
1011
1012
1013
1014
	}
	return kubeName
}

1015
1016
func (r *DynamoComponentDeploymentReconciler) getGenericServiceName(dynamoComponentDeployment *v1alpha1.DynamoComponentDeployment) string {
	return r.getKubeName(dynamoComponentDeployment, false)
1017
1018
}

1019
func (r *DynamoComponentDeploymentReconciler) getKubeLabels(dynamoComponentDeployment *v1alpha1.DynamoComponentDeployment) map[string]string {
1020
1021
1022
1023
1024
1025
1026
1027
1028
	labels := map[string]string{}
	if dynamoComponentDeployment != nil {
		if dynamoComponentDeployment.Spec.Labels != nil {
			maps.Copy(labels, dynamoComponentDeployment.Spec.Labels)
		}
		if dynamoComponentDeployment.Labels != nil {
			maps.Copy(labels, dynamoComponentDeployment.Labels)
		}
		dynamo.AddBaseModelLabel(labels, dynamoComponentDeployment.Spec.ModelRef)
1029
	}
1030
	return labels
1031
1032
}

1033
1034
func (r *DynamoComponentDeploymentReconciler) getKubeAnnotations(dynamoComponentDeployment *v1alpha1.DynamoComponentDeployment) map[string]string {
	annotations := map[string]string{}
1035
1036
1037
1038
1039
1040
1041
1042
	if dynamoComponentDeployment != nil {
		if dynamoComponentDeployment.Spec.Annotations != nil {
			maps.Copy(annotations, dynamoComponentDeployment.Spec.Annotations)
		}
		if dynamoComponentDeployment.Spec.ExtraPodMetadata != nil && dynamoComponentDeployment.Spec.ExtraPodMetadata.Annotations != nil {
			maps.Copy(annotations, dynamoComponentDeployment.Spec.ExtraPodMetadata.Annotations)
		}
		dynamo.AddBaseModelAnnotation(annotations, dynamoComponentDeployment.Spec.ModelRef)
1043
1044
1045
1046
1047
	}
	return annotations
}

//nolint:nakedret
1048
1049
func (r *DynamoComponentDeploymentReconciler) generateDeployment(ctx context.Context, opt generateResourceOption) (kubeDeployment *appsv1.Deployment, toDelete bool, err error) {
	kubeNs := opt.dynamoComponentDeployment.Namespace
1050

1051
	labels := r.getKubeLabels(opt.dynamoComponentDeployment)
1052

1053
	annotations := r.getKubeAnnotations(opt.dynamoComponentDeployment)
1054

1055
	kubeName := r.getKubeName(opt.dynamoComponentDeployment, opt.isStealingTrafficDebugModeEnabled)
1056

1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
	kubeDeployment = &appsv1.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			Name:        kubeName,
			Namespace:   kubeNs,
			Labels:      labels,
			Annotations: annotations,
		},
	}

	if opt.isStealingTrafficDebugModeEnabled && !opt.containsStealingTrafficDebugModeEnabled {
		// if stealing traffic debug mode is enabked but disabled in the deployment, we need to delete the deployment
		return kubeDeployment, true, nil
	}

	// nolint: gosimple
1072
	podTemplateSpec, err := r.generatePodTemplateSpec(ctx, opt, dynamo.RoleMain)
1073
1074
1075
1076
	if err != nil {
		return
	}

1077
	maxSurge, maxUnavailable := getDeploymentRollingUpdateMaxSurgeAndMaxUnavailable(annotations)
1078
1079
1080
1081

	strategy := appsv1.DeploymentStrategy{
		Type: appsv1.RollingUpdateDeploymentStrategyType,
		RollingUpdate: &appsv1.RollingUpdateDeployment{
1082
1083
			MaxSurge:       &maxSurge,
			MaxUnavailable: &maxUnavailable,
1084
1085
1086
		},
	}

1087
	resourceAnnotations := getResourceAnnotations(opt.dynamoComponentDeployment)
1088
1089
	strategyStr := resourceAnnotations[KubeAnnotationDeploymentStrategy]
	if strategyStr != "" {
1090
		strategyType := common.DeploymentStrategy(strategyStr)
1091
		switch strategyType {
1092
		case common.DeploymentStrategyRollingUpdate:
1093
1094
1095
			strategy = appsv1.DeploymentStrategy{
				Type: appsv1.RollingUpdateDeploymentStrategyType,
				RollingUpdate: &appsv1.RollingUpdateDeployment{
1096
1097
					MaxSurge:       &maxSurge,
					MaxUnavailable: &maxUnavailable,
1098
1099
				},
			}
1100
		case common.DeploymentStrategyRecreate:
1101
1102
1103
1104
1105
1106
1107
			strategy = appsv1.DeploymentStrategy{
				Type: appsv1.RecreateDeploymentStrategyType,
			}
		}
	}

	var replicas *int32
1108
	replicas = opt.dynamoComponentDeployment.Spec.Replicas
1109
1110
1111
1112
	if opt.isStealingTrafficDebugModeEnabled {
		replicas = &[]int32{int32(1)}[0]
	}

1113
1114
1115
1116
	kubeDeployment.Spec = appsv1.DeploymentSpec{
		Replicas: replicas,
		Selector: &metav1.LabelSelector{
			MatchLabels: map[string]string{
1117
				commonconsts.KubeLabelDynamoSelector: kubeName,
1118
1119
			},
		},
1120
1121
		Template: *podTemplateSpec,
		Strategy: strategy,
1122
1123
1124
1125
1126
	}

	return
}

1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
func getDeploymentRollingUpdateMaxSurgeAndMaxUnavailable(annotations map[string]string) (intstr.IntOrString, intstr.IntOrString) {
	maxSurge := intstr.FromString("25%")
	maxUnavailable := intstr.FromString("25%")

	if annotations[KubeAnnotationDeploymentRollingUpdateMaxSurge] != "" {
		maxSurge = intstr.Parse(annotations[KubeAnnotationDeploymentRollingUpdateMaxSurge])
	}
	if annotations[KubeAnnotationDeploymentRollingUpdateMaxUnavailable] != "" {
		maxUnavailable = intstr.Parse(annotations[KubeAnnotationDeploymentRollingUpdateMaxUnavailable])
	}

	return maxSurge, maxUnavailable
}

1141
type generateResourceOption struct {
1142
	dynamoComponentDeployment               *v1alpha1.DynamoComponentDeployment
1143
1144
1145
1146
	isStealingTrafficDebugModeEnabled       bool
	containsStealingTrafficDebugModeEnabled bool
	isDebugPodReceiveProductionTraffic      bool
	isGenericService                        bool
1147
	instanceID                              *int
1148
}
1149
1150

//nolint:gocyclo,nakedret
1151
func (r *DynamoComponentDeploymentReconciler) generatePodTemplateSpec(ctx context.Context, opt generateResourceOption, role dynamo.Role) (podTemplateSpec *corev1.PodTemplateSpec, err error) {
1152
	podLabels := r.getKubeLabels(opt.dynamoComponentDeployment)
1153
	if opt.isStealingTrafficDebugModeEnabled {
1154
		podLabels[commonconsts.KubeLabelDynamoDeploymentTargetType] = DeploymentTargetTypeDebug
1155
1156
	}

1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
	// Convert user-provided metrics annotation into controller-managed label
	// By default (no annotation), metrics are enabled
	metricsAnnotationValue := ""
	if opt.dynamoComponentDeployment.Spec.Annotations != nil {
		metricsAnnotationValue = opt.dynamoComponentDeployment.Spec.Annotations[commonconsts.KubeAnnotationEnableMetrics]
	}
	switch metricsAnnotationValue {
	case commonconsts.KubeLabelValueFalse:
		// Explicitly disabled, don't add the label
	default:
		// Any other value (including empty) enables metrics
		podLabels[commonconsts.KubeLabelMetricsEnabled] = commonconsts.KubeLabelValueTrue
	}

1171
1172
1173
	// Add label for the dynamo graph deployment on the pods themselves
	podLabels[commonconsts.KubeLabelDynamoGraphDeploymentName] = opt.dynamoComponentDeployment.Spec.Labels[commonconsts.KubeLabelDynamoGraphDeploymentName]

1174
1175
1176
1177
1178
	// Add component type label if specified
	if opt.dynamoComponentDeployment.Spec.ComponentType != "" {
		podLabels[commonconsts.KubeLabelDynamoComponentType] = opt.dynamoComponentDeployment.Spec.ComponentType
	}

1179
1180
1181
1182
	if opt.dynamoComponentDeployment.Spec.SubComponentType != "" {
		podLabels[commonconsts.KubeLabelDynamoSubComponentType] = opt.dynamoComponentDeployment.Spec.SubComponentType
	}

1183
	podAnnotations := make(map[string]string)
1184

1185
	kubeName := r.getKubeName(opt.dynamoComponentDeployment, opt.isStealingTrafficDebugModeEnabled)
1186

1187
	resourceAnnotations := opt.dynamoComponentDeployment.Spec.Annotations
1188
1189
1190
1191
1192
1193
1194

	if resourceAnnotations == nil {
		resourceAnnotations = make(map[string]string)
	}

	isDebugModeEnabled := checkIfIsDebugModeEnabled(resourceAnnotations)

1195
	podSpec, err := dynamo.GenerateBasePodSpecForController(opt.dynamoComponentDeployment, r.DockerSecretRetriever, r.Config, role, commonconsts.MultinodeDeploymentTypeLWS)
1196
	if err != nil {
1197
		err = errors.Wrap(err, "failed to generate base pod spec")
1198
1199
1200
		return nil, err
	}

1201
	// Ensure we have at least one container (the main container should be there from GenerateBasePodSpec)
1202
	if len(podSpec.Containers) == 0 {
1203
		return nil, errors.New("no containers found in base pod spec")
1204
1205
	}

1206
	debuggerImage := "python:3.12-slim"
1207
1208
1209
1210
1211
1212
	debuggerImage_ := os.Getenv("INTERNAL_IMAGES_DEBUGGER")
	if debuggerImage_ != "" {
		debuggerImage = debuggerImage_
	}

	if opt.isStealingTrafficDebugModeEnabled || isDebugModeEnabled {
1213
		podSpec.Containers = append(podSpec.Containers, corev1.Container{
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
			Name:  "debugger",
			Image: debuggerImage,
			Command: []string{
				"sleep",
				"infinity",
			},
			SecurityContext: &corev1.SecurityContext{
				Capabilities: &corev1.Capabilities{
					Add: []corev1.Capability{"SYS_PTRACE"},
				},
			},
			Resources: corev1.ResourceRequirements{
				Requests: corev1.ResourceList{
					corev1.ResourceCPU:    resource.MustParse("100m"),
					corev1.ResourceMemory: resource.MustParse("100Mi"),
				},
				Limits: corev1.ResourceList{
					corev1.ResourceCPU:    resource.MustParse("1000m"),
					corev1.ResourceMemory: resource.MustParse("1000Mi"),
				},
			},
			Stdin: true,
			TTY:   true,
		})
	}

1240
	podLabels[commonconsts.KubeLabelDynamoSelector] = kubeName
1241

1242
	extraPodMetadata := opt.dynamoComponentDeployment.Spec.ExtraPodMetadata
1243
1244

	if extraPodMetadata != nil {
1245
1246
		maps.Copy(podAnnotations, extraPodMetadata.Annotations)
		maps.Copy(podLabels, extraPodMetadata.Labels)
1247
1248
	}

1249
1250
1251
1252
1253
1254
	// Propagate restart annotation to pod template to trigger rolling restart
	// This is the same mechanism used by kubectl rollout restart
	if restartAt, exists := resourceAnnotations[commonconsts.RestartAnnotation]; exists {
		podAnnotations[commonconsts.RestartAnnotation] = restartAt
	}

1255
1256
	if podSpec.ServiceAccountName == "" {
		serviceAccounts := &corev1.ServiceAccountList{}
1257
		err = r.List(ctx, serviceAccounts, client.InNamespace(opt.dynamoComponentDeployment.Namespace), client.MatchingLabels{
1258
			commonconsts.KubeLabelDynamoComponentPod: commonconsts.KubeLabelValueTrue,
1259
1260
		})
		if err != nil {
1261
			err = errors.Wrapf(err, "failed to list service accounts in namespace %s", opt.dynamoComponentDeployment.Namespace)
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
			return
		}
		if len(serviceAccounts.Items) > 0 {
			podSpec.ServiceAccountName = serviceAccounts.Items[0].Name
		} else {
			podSpec.ServiceAccountName = DefaultServiceAccountName
		}
	}

	if opt.isStealingTrafficDebugModeEnabled || isDebugModeEnabled {
		podSpec.ShareProcessNamespace = &[]bool{true}[0]
	}

	podTemplateSpec = &corev1.PodTemplateSpec{
		ObjectMeta: metav1.ObjectMeta{
			Labels:      podLabels,
			Annotations: podAnnotations,
		},
1280
		Spec: *podSpec,
1281
1282
1283
1284
1285
	}

	return
}

1286
func (r *DynamoComponentDeploymentReconciler) generateService(opt generateResourceOption) (*corev1.Service, bool, error) {
1287
1288
	var kubeName string
	if opt.isGenericService {
1289
		kubeName = r.getGenericServiceName(opt.dynamoComponentDeployment)
1290
	} else {
1291
		kubeName = r.getServiceName(opt.dynamoComponentDeployment, opt.isStealingTrafficDebugModeEnabled)
1292
1293
	}

1294
	kubeNs := opt.dynamoComponentDeployment.Namespace
1295

1296
	kubeService := &corev1.Service{
1297
1298
1299
1300
1301
1302
		ObjectMeta: metav1.ObjectMeta{
			Name:      kubeName,
			Namespace: kubeNs,
		},
	}

1303
1304
1305
1306
1307
	isK8sDiscovery := r.Config.IsK8sDiscoveryEnabled(opt.dynamoComponentDeployment.Spec.Annotations)

	// if discovery backend is k8s we want to create a service for each component
	// else, only create for the frontend component
	if !opt.isGenericService && !opt.containsStealingTrafficDebugModeEnabled && !(isK8sDiscovery || opt.dynamoComponentDeployment.IsFrontendComponent()) {
1308
		// if it's not the main component or if it's not a generic service and not contains stealing traffic debug mode enabled, we don't need to create the service
1309
1310
1311
		return kubeService, true, nil
	}

1312
	labels := r.getKubeLabels(opt.dynamoComponentDeployment)
1313

1314
1315
	if opt.dynamoComponentDeployment.Spec.DynamoNamespace == nil {
		return nil, false, fmt.Errorf("expected DynamoComponentDeployment %s to have a dynamoNamespace", opt.dynamoComponentDeployment.Name)
1316
1317
	}

1318
1319
1320
1321
1322
	selector := map[string]string{
		commonconsts.KubeLabelDynamoComponentType: opt.dynamoComponentDeployment.Spec.ComponentType,
		commonconsts.KubeLabelDynamoNamespace:     *opt.dynamoComponentDeployment.Spec.DynamoNamespace,
	}
	// // If using LeaderWorkerSet, modify selector to only target leaders
1323
	if opt.dynamoComponentDeployment.IsMultinode() {
1324
1325
		selector["role"] = "leader"
	}
1326
	if opt.isStealingTrafficDebugModeEnabled {
1327
		selector[commonconsts.KubeLabelDynamoDeploymentTargetType] = DeploymentTargetTypeDebug
1328
	}
1329
1330
1331
	if isK8sDiscovery {
		labels[commonconsts.KubeLabelDynamoDiscoveryBackend] = "kubernetes"
	}
1332
1333
1334
1335
	// Discovery is enabled for non frontend components
	if isK8sDiscovery && !opt.dynamoComponentDeployment.IsFrontendComponent() {
		labels[commonconsts.KubeLabelDynamoDiscoveryEnabled] = commonconsts.KubeLabelValueTrue
	}
1336

1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
	var servicePort corev1.ServicePort
	if opt.dynamoComponentDeployment.IsFrontendComponent() {
		servicePort = corev1.ServicePort{
			Name:       commonconsts.DynamoServicePortName,
			Port:       commonconsts.DynamoServicePort,
			TargetPort: intstr.FromString(commonconsts.DynamoContainerPortName),
			Protocol:   corev1.ProtocolTCP,
		}
	} else { // TODO: only for worker components
		servicePort = corev1.ServicePort{
			Name:       commonconsts.DynamoSystemPortName,
			Port:       commonconsts.DynamoSystemPort,
			TargetPort: intstr.FromString(commonconsts.DynamoSystemPortName),
			Protocol:   corev1.ProtocolTCP,
		}
	}
1353
1354
1355

	spec := corev1.ServiceSpec{
		Selector: selector,
1356
		Ports:    []corev1.ServicePort{servicePort},
1357
1358
	}

1359
	annotations := r.getKubeAnnotations(opt.dynamoComponentDeployment)
1360

1361
1362
1363
	kubeService.ObjectMeta.Annotations = annotations
	kubeService.ObjectMeta.Labels = labels
	kubeService.Spec = spec
1364

1365
	return kubeService, false, nil
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
}

type TLSModeOpt string

const (
	TLSModeNone   TLSModeOpt = "none"
	TLSModeAuto   TLSModeOpt = "auto"
	TLSModeStatic TLSModeOpt = "static"
)

type IngressConfig struct {
	ClassName           *string
	Annotations         map[string]string
	Path                string
	PathType            networkingv1.PathType
	TLSMode             TLSModeOpt
	StaticTLSSecretName string
}

// SetupWithManager sets up the controller with the Manager.
1386
func (r *DynamoComponentDeploymentReconciler) SetupWithManager(mgr ctrl.Manager) error {
1387
	m := ctrl.NewControllerManagedBy(mgr).
1388
		For(&v1alpha1.DynamoComponentDeployment{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
1389
1390
1391
1392
1393
1394
1395
		Owns(&appsv1.Deployment{}, builder.WithPredicates(predicate.Funcs{
			// ignore creation cause we don't want to be called again after we create the deployment
			CreateFunc:  func(ce event.CreateEvent) bool { return false },
			DeleteFunc:  func(de event.DeleteEvent) bool { return true },
			UpdateFunc:  func(de event.UpdateEvent) bool { return true },
			GenericFunc: func(ge event.GenericEvent) bool { return true },
		})).
1396
1397
1398
		Owns(&corev1.Service{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
		Owns(&networkingv1.Ingress{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
		Owns(&corev1.PersistentVolumeClaim{}, builder.WithPredicates(predicate.GenerationChangedPredicate{})).
1399
		WithEventFilter(commonController.EphemeralDeploymentEventFilter(r.Config))
1400

1401
	if r.Config.LWS.Enabled {
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
		m.Owns(&leaderworkersetv1.LeaderWorkerSet{}, builder.WithPredicates(predicate.Funcs{
			// ignore creation cause we don't want to be called again after we create the LeaderWorkerSet
			CreateFunc:  func(ce event.CreateEvent) bool { return false },
			DeleteFunc:  func(de event.DeleteEvent) bool { return true },
			UpdateFunc:  func(de event.UpdateEvent) bool { return true },
			GenericFunc: func(ge event.GenericEvent) bool { return true },
		})).
			Owns(&volcanov1beta1.PodGroup{}, builder.WithPredicates(predicate.Funcs{
				// ignore creation cause we don't want to be called again after we create the LeaderWorkerSet
				CreateFunc:  func(ce event.CreateEvent) bool { return false },
				DeleteFunc:  func(de event.DeleteEvent) bool { return true },
				UpdateFunc:  func(de event.UpdateEvent) bool { return true },
				GenericFunc: func(ge event.GenericEvent) bool { return true },
			}))
	}

1418
	if r.Config.IngressConfig.UseVirtualService() {
1419
1420
		m.Owns(&networkingv1beta1.VirtualService{}, builder.WithPredicates(predicate.GenerationChangedPredicate{}))
	}
1421
1422
1423
	m.Owns(&autoscalingv2.HorizontalPodAutoscaler{})
	return m.Complete(r)
}
1424
1425
1426
1427

func (r *DynamoComponentDeploymentReconciler) GetRecorder() record.EventRecorder {
	return r.Recorder
}