ingress.go

/*
 * SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 * SPDX-License-Identifier: Apache-2.0
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package system

import (
	"context"
	"encoding/json"
	"fmt"
	"net"
	"os"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/rs/xid"
	"github.com/sirupsen/logrus"
	corev1 "k8s.io/api/core/v1"
	networkingv1 "k8s.io/api/networking/v1"
	k8serrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"

	"github.com/dynemo-ai/dynemo/deploy/compoundai/operator/pkg/compoundai/consts"
)

type IngressConfig struct {
	ClassName   *string
	Annotations map[string]string
	Path        string
	PathType    networkingv1.PathType
}

func GetIngressConfig(ctx context.Context, configmapGetter func(ctx context.Context, namespace, name string) (*corev1.ConfigMap, error)) (ingressConfig *IngressConfig, err error) {
	configMap, err := GetNetworkConfigConfigMap(ctx, configmapGetter)
	if err != nil {
		err = errors.Wrapf(err, "failed to get configmap %s", consts.KubeConfigMapNameNetworkConfig)
		return
	}

	var className *string

	className_ := strings.TrimSpace(configMap.Data[consts.KubeConfigMapKeyNetworkConfigIngressClass])
	if className_ != "" {
		className = &className_
	}

	annotations := make(map[string]string)

	annotations_ := strings.TrimSpace(configMap.Data[consts.KubeConfigMapKeyNetworkConfigIngressAnnotations])
	if annotations_ != "" {
		err = json.Unmarshal([]byte(annotations_), &annotations)
		if err != nil {
			err = errors.Wrapf(err, "failed to json unmarshal %s in configmap %s: %s", consts.KubeConfigMapKeyNetworkConfigIngressAnnotations, consts.KubeConfigMapNameNetworkConfig, annotations_)
			return
		}
	}

	path := strings.TrimSpace(configMap.Data[consts.KubeConfigMapKeyNetworkConfigIngressPath])
	if path == "" {
		path = "/"
	}

	pathType := networkingv1.PathTypeImplementationSpecific

	pathType_ := strings.TrimSpace(configMap.Data[consts.KubeConfigMapKeyNetworkConfigIngressPathType])
	if pathType_ != "" {
		pathType = networkingv1.PathType(pathType_)
	}

	ingressConfig = &IngressConfig{
		ClassName:   className,
		Annotations: annotations,
		Path:        path,
		PathType:    pathType,
	}

	return
}

func GetIngressIP(ctx context.Context, configmapGetter func(ctx context.Context, namespace, name string) (*corev1.ConfigMap, error), cliset *kubernetes.Clientset) (ip string, err error) {
	ingressConfig, err := GetIngressConfig(ctx, configmapGetter)
	if err != nil {
		err = errors.Wrapf(err, "failed to get ingress config")
		return
	}

	ingressClassName := ingressConfig.ClassName
	ingressAnnotations := ingressConfig.Annotations

	ingressCli := cliset.NetworkingV1().Ingresses(GetNamespace())

	ingName := "default-domain-"
	pathType := networkingv1.PathTypeImplementationSpecific

	podName := os.Getenv("POD_NAME")
	if podName == "" {
		// random string
		guid := xid.New()
		podName = fmt.Sprintf("a%s", strings.ToLower(guid.String()))
	}

	logrus.Infof("Creating ingress %s to get a ingress IP automatically", ingName)
	ing, err := ingressCli.Create(ctx, &networkingv1.Ingress{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: ingName,
			Namespace:    GetNamespace(),
			Annotations:  ingressAnnotations,
		},
		Spec: networkingv1.IngressSpec{
			IngressClassName: ingressClassName,
			Rules: []networkingv1.IngressRule{{
				Host: fmt.Sprintf("%s.this-is-yatai-in-order-to-generate-the-default-domain-suffix.yeah", podName),
				IngressRuleValue: networkingv1.IngressRuleValue{
					HTTP: &networkingv1.HTTPIngressRuleValue{
						Paths: []networkingv1.HTTPIngressPath{
							{
								Path:     "/",
								PathType: &pathType,
								Backend: networkingv1.IngressBackend{
									Service: &networkingv1.IngressServiceBackend{
										Name: "default-domain-service",
										Port: networkingv1.ServiceBackendPort{
											Number: consts.BentoServicePort,
										},
									},
								},
							},
						},
					},
				},
			}},
		},
	}, metav1.CreateOptions{})
	if err != nil && !k8serrors.IsAlreadyExists(err) {
		err = errors.Wrapf(err, "failed to create ingress %s", ingName)
		return
	}
	defer func() {
		_ = ingressCli.Delete(ctx, ing.Name, metav1.DeleteOptions{})
	}()

	// Interval to poll for objects.
	pollInterval := 10 * time.Second
	// How long to wait for objects.
	waitTimeout := 20 * time.Minute

	logrus.Infof("Waiting for ingress %s to be ready", ing.Name)
	// Wait for the Ingress to be Ready.
	if err = wait.PollUntilContextTimeout(ctx, pollInterval, waitTimeout, false, func(ctx context.Context) (done bool, err error) {
		ing, err = ingressCli.Get(
			ctx, ing.Name, metav1.GetOptions{})
		if err != nil {
			return true, err
		}
		return len(ing.Status.LoadBalancer.Ingress) > 0, nil
	}); err != nil {
		err = errors.Wrapf(err, "failed to wait for ingress %s to be ready", ing.Name)
		return
	}
	logrus.Infof("Ingress %s is ready", ing.Name)

	address := ing.Status.LoadBalancer.Ingress[0]

	ip = address.IP
	if ip == "" {
		if address.Hostname == "" {
			err = errors.Errorf("the ingress %s status has no IP or hostname", ing.Name)
			return
		}
		var ipAddr *net.IPAddr
		ipAddr, err = net.ResolveIPAddr("ip4", address.Hostname)
		if err != nil {
			err = errors.Wrapf(err, "failed to resolve ip address for hostname %s", address.Hostname)
			return
		}
		ip = ipAddr.String()
	}

	return
}

func GetDomainSuffix(ctx context.Context, configmapGetter func(ctx context.Context, namespace, name string) (*corev1.ConfigMap, error), cliset *kubernetes.Clientset) (domainSuffix string, err error) {
	configMap, err := GetNetworkConfigConfigMap(ctx, configmapGetter)
	if err != nil {
		err = errors.Wrapf(err, "failed to get configmap %s", consts.KubeConfigMapNameNetworkConfig)
		return
	}

	domainSuffix = strings.TrimSpace(configMap.Data[consts.KubeConfigMapKeyNetworkConfigDomainSuffix])
	if domainSuffix != "" {
		logrus.Infof("The %s in the network config has already set to `%s`", consts.KubeConfigMapKeyNetworkConfigDomainSuffix, domainSuffix)
		return
	}

	magicDNS := GetMagicDNS()

	var ip string

	ip, err = GetIngressIP(ctx, configmapGetter, cliset)
	if err != nil {
		return
	}

	domainSuffix = fmt.Sprintf("%s.%s", ip, magicDNS)

	logrus.Infof("you have not set the %s in the network config, so use magic DNS to generate a domain suffix automatically: `%s`, and set it to the network config", consts.KubeConfigMapKeyNetworkConfigDomainSuffix, domainSuffix)

	configMapCli := cliset.CoreV1().ConfigMaps(configMap.Namespace)
	_, err = configMapCli.Patch(ctx, configMap.Name, types.MergePatchType, []byte(fmt.Sprintf(`{"data":{"%s":"%s"}}`, consts.KubeConfigMapKeyNetworkConfigDomainSuffix, domainSuffix)), metav1.PatchOptions{})
	if err != nil {
		err = errors.Wrapf(err, "failed to patch configmap %s", consts.KubeConfigMapNameNetworkConfig)
		return
	}

	return
}