Validate pipeline resources with admission webhook

Tiltfile

@@ -51,10 +51,26 @@ helm_repo(
     labels=['Repositories'],
 )
 
+########### Certmanager
+# Certmanager is required for the validating webhooks in the cortex bundles, so
+# we need to deploy it before the bundles. If you don't need the webhooks locally,
+# you can disable them in the values.yaml and skip deploying certmanager.
+cache_dir = '.tilt/cert-manager'
+cert_manager_version = 'v1.19.3'
+if not os.path.exists(cache_dir):
+    local('mkdir -p ' + cache_dir)
+if not os.path.exists(cache_dir + '/cert-manager-' + cert_manager_version + '.yaml'):
+    url = 'https://github.com/cert-manager/cert-manager/releases/download/' + cert_manager_version + '/cert-manager.yaml'
+    local('curl -L ' + url + ' -o ' + cache_dir + '/cert-manager-' + cert_manager_version + '.yaml')
+k8s_yaml(cache_dir + '/cert-manager-' + cert_manager_version + '.yaml')
+k8s_resource('cert-manager', labels=['ZZ_Certmanager'])
+k8s_resource('cert-manager-webhook', labels=['ZZ_Certmanager'])
+k8s_resource('cert-manager-cainjector', labels=['ZZ_Certmanager'])
+
 ########### Dependency CRDs
 # Make sure the local cluster is running if you are running into startup issues here.
 url = 'https://raw.githubusercontent.com/cobaltcore-dev/openstack-hypervisor-operator/refs/heads/main/charts/openstack-hypervisor-operator/crds/hypervisor-crd.yaml'
-local('curl ' + url + ' | kubectl apply -f -')
+local('curl -L ' + url + ' | kubectl apply -f -')
 
 ########### Cortex Operator & CRDs
 docker_build('ghcr.io/cobaltcore-dev/cortex', '.',

cmd/main.go

@@ -289,26 +289,27 @@ func main() {
 	// The pipeline monitor is a bucket for all metrics produced during the
 	// execution of individual steps (see step monitor below) and the overall
 	// pipeline.
-	pipelineMonitor := schedulinglib.NewPipelineMonitor()
-	metrics.Registry.MustRegister(&pipelineMonitor)
+	// TODO: Only initialize me for scheduling domains that actually use pipelines.
+	filterWeigherPipelineMonitor := schedulinglib.NewPipelineMonitor()
+	metrics.Registry.MustRegister(&filterWeigherPipelineMonitor)
+	detectorPipelineMonitor := schedulinglib.NewDetectorPipelineMonitor()
+	metrics.Registry.MustRegister(&detectorPipelineMonitor)
 
-	if slices.Contains(mainConfig.EnabledControllers, "nova-decisions-pipeline-controller") {
-		decisionController := &nova.FilterWeigherPipelineController{
-			Monitor: pipelineMonitor,
+	if slices.Contains(mainConfig.EnabledControllers, "nova-pipeline-controllers") {
+		// Filter-weigher pipeline controller setup.
+		filterWeigherController := &nova.FilterWeigherPipelineController{
+			Monitor: filterWeigherPipelineMonitor,
 		}
 		// Inferred through the base controller.
-		decisionController.Client = multiclusterClient
-		if err := (decisionController).SetupWithManager(mgr, multiclusterClient); err != nil {
-			setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler")
+		filterWeigherController.Client = multiclusterClient
+		if err := filterWeigherController.SetupWithManager(mgr, multiclusterClient); err != nil {
+			setupLog.Error(err, "unable to create controller", "controller", "nova FilterWeigherPipelineController")
 			os.Exit(1)
 		}
 		httpAPIConf := conf.GetConfigOrDie[nova.HTTPAPIConfig]()
-		nova.NewAPI(httpAPIConf, decisionController).Init(mux)
-	}
-	if slices.Contains(mainConfig.EnabledControllers, "nova-deschedulings-pipeline-controller") {
-		// Deschedulings controller
-		monitor := schedulinglib.NewDetectorPipelineMonitor()
-		metrics.Registry.MustRegister(&monitor)
+		nova.NewAPI(httpAPIConf, filterWeigherController).Init(mux)
+
+		// Detector pipeline controller setup.
 		novaClient := nova.NewNovaClient()
 		novaClientConfig := conf.GetConfigOrDie[nova.NovaClientConfig]()
 		if err := mgr.Add(manager.RunnableFunc(func(ctx context.Context) error {
@@ -317,15 +318,14 @@ func main() {
 			setupLog.Error(err, "unable to initialize nova client")
 			os.Exit(1)
 		}
-		cycleBreaker := &nova.DetectorCycleBreaker{NovaClient: novaClient}
 		deschedulingsController := &nova.DetectorPipelineController{
-			Monitor: monitor,
-			Breaker: cycleBreaker,
+			Monitor: detectorPipelineMonitor,
+			Breaker: &nova.DetectorCycleBreaker{NovaClient: novaClient},
 		}
 		// Inferred through the base controller.
 		deschedulingsController.Client = multiclusterClient
 		if err := (deschedulingsController).SetupWithManager(mgr, multiclusterClient); err != nil {
-			setupLog.Error(err, "unable to create controller", "controller", "DeschedulingsReconciler")
+			setupLog.Error(err, "unable to create controller", "controller", "nova DetectorPipelineController")
 			os.Exit(1)
 		}
 		go deschedulingsController.CreateDeschedulingsPeriodically(ctx)
@@ -337,6 +337,13 @@ func main() {
 			setupLog.Error(err, "unable to create controller", "controller", "Cleanup")
 			os.Exit(1)
 		}
+
+		// Webhook that validates all pipelines.
+		novaPipelineWebhook := nova.NewPipelineWebhook()
+		if err := novaPipelineWebhook.SetupWebhookWithManager(mgr); err != nil {
+			setupLog.Error(err, "unable to setup nova pipeline webhook")
+			os.Exit(1)
+		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "nova-deschedulings-executor") {
 		executorConfig := conf.GetConfigOrDie[nova.DeschedulingsExecutorConfig]()
@@ -360,7 +367,7 @@ func main() {
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "manila-decisions-pipeline-controller") {
 		controller := &manila.FilterWeigherPipelineController{
-			Monitor: pipelineMonitor,
+			Monitor: filterWeigherPipelineMonitor,
 		}
 		// Inferred through the base controller.
 		controller.Client = multiclusterClient
@@ -369,10 +376,17 @@ func main() {
 			os.Exit(1)
 		}
 		manila.NewAPI(controller).Init(mux)
+
+		// Webhook that validates all pipelines.
+		manilaPipelineWebhook := manila.NewPipelineWebhook()
+		if err := manilaPipelineWebhook.SetupWebhookWithManager(mgr); err != nil {
+			setupLog.Error(err, "unable to setup manila pipeline webhook")
+			os.Exit(1)
+		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "cinder-decisions-pipeline-controller") {
 		controller := &cinder.FilterWeigherPipelineController{
-			Monitor: pipelineMonitor,
+			Monitor: filterWeigherPipelineMonitor,
 		}
 		// Inferred through the base controller.
 		controller.Client = multiclusterClient
@@ -381,28 +395,49 @@ func main() {
 			os.Exit(1)
 		}
 		cinder.NewAPI(controller).Init(mux)
+
+		// Webhook that validates all pipelines.
+		cinderPipelineWebhook := cinder.NewPipelineWebhook()
+		if err := cinderPipelineWebhook.SetupWebhookWithManager(mgr); err != nil {
+			setupLog.Error(err, "unable to setup cinder pipeline webhook")
+			os.Exit(1)
+		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "ironcore-decisions-pipeline-controller") {
 		controller := &machines.FilterWeigherPipelineController{
-			Monitor: pipelineMonitor,
+			Monitor: filterWeigherPipelineMonitor,
 		}
 		// Inferred through the base controller.
 		controller.Client = multiclusterClient
 		if err := (controller).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler")
 			os.Exit(1)
 		}
+
+		// Webhook that validates all pipelines.
+		ironcorePipelineWebhook := machines.NewPipelineWebhook()
+		if err := ironcorePipelineWebhook.SetupWebhookWithManager(mgr); err != nil {
+			setupLog.Error(err, "unable to setup ironcore pipeline webhook")
+			os.Exit(1)
+		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "pods-decisions-pipeline-controller") {
 		controller := &pods.FilterWeigherPipelineController{
-			Monitor: pipelineMonitor,
+			Monitor: filterWeigherPipelineMonitor,
 		}
 		// Inferred through the base controller.
 		controller.Client = multiclusterClient
 		if err := (controller).SetupWithManager(mgr, multiclusterClient); err != nil {
 			setupLog.Error(err, "unable to create controller", "controller", "DecisionReconciler")
 			os.Exit(1)
 		}
+
+		// Webhook that validates all pipelines.
+		podsPipelineWebhook := pods.NewPipelineWebhook()
+		if err := podsPipelineWebhook.SetupWebhookWithManager(mgr); err != nil {
+			setupLog.Error(err, "unable to setup pods pipeline webhook")
+			os.Exit(1)
+		}
 	}
 	if slices.Contains(mainConfig.EnabledControllers, "explanation-controller") {
 		// Setup a controller which will reconcile the history and explanation for

helm/bundles/cortex-cinder/values.yaml

@@ -85,6 +85,9 @@ cortex: &cortex
 cortex-scheduling-controllers:
   <<: *cortex
   namePrefix: cortex-cinder-scheduling
+  # Enable webhook that will validate CRDs for the scheduling controllers.
+  webhook: {enable: true}
+  certmanager: {enable: true} # Needed for the webhook TLS certificates.
   conf:
     <<: *cortexConf
     leaderElectionID: cortex-cinder-scheduling

helm/bundles/cortex-ironcore/values.yaml

@@ -24,6 +24,9 @@ cortex:
   crd: {enable: false}
   # Use this to unambiguate multiple cortex deployments in the same cluster.
   namePrefix: cortex-ironcore
+  # Enable webhook that will validate CRDs for the scheduling controllers.
+  webhook: {enable: true}
+  certmanager: {enable: true} # Needed for the webhook TLS certificates.
   conf:
     # The operator will only touch CRs with this scheduling domain name.
     schedulingDomain: machines

helm/bundles/cortex-manila/values.yaml

@@ -85,6 +85,9 @@ cortex: &cortex
 cortex-scheduling-controllers:
   <<: *cortex
   namePrefix: cortex-manila-scheduling
+  # Enable webhook that will validate CRDs for the scheduling controllers.
+  webhook: {enable: true}
+  certmanager: {enable: true} # Needed for the webhook TLS certificates.
   conf:
     <<: *cortexConf
     leaderElectionID: cortex-manila-scheduling

helm/bundles/cortex-nova/values.yaml

@@ -72,7 +72,6 @@ cortex: &cortex
   # A centralized ServiceMonitor and metrics service are defined in the cortex-nova chart.
   metrics: {enable: false}
   prometheus: {enable: false}
-  namePrefix: cortex-nova
   conf: &cortexConf
     schedulingDomain: nova
     keystoneSecretRef:
@@ -93,6 +92,9 @@ cortex-scheduling-controllers:
   rbac:
     # The cortex nova scheduling controllers need hypervisor crd access.
     hypervisor: {enable: true}
+  # Enable webhook that will validate CRDs for the scheduling controllers.
+  webhook: {enable: true}
+  certmanager: {enable: true} # Needed for the webhook TLS certificates.
   conf:
     <<: *cortexConf
     leaderElectionID: cortex-nova-scheduling
@@ -103,8 +105,7 @@ cortex-scheduling-controllers:
         <<: *cortexMonitoringLabels
         component: nova-scheduling
     enabledControllers:
-      - nova-decisions-pipeline-controller
-      - nova-deschedulings-pipeline-controller
+      - nova-pipeline-controllers
       - nova-deschedulings-executor
       - explanation-controller
     enabledTasks:

helm/bundles/cortex-pods/values.yaml

@@ -24,6 +24,9 @@ cortex:
   crd: { enable: false }
   # Use this to unambiguate multiple cortex deployments in the same cluster.
   namePrefix: cortex-pods
+  # Enable webhook that will validate CRDs for the scheduling controllers.
+  webhook: {enable: true}
+  certmanager: {enable: true} # Needed for the webhook TLS certificates.
   conf:
     # The operator will only touch CRs with this scheduling domain name.
     schedulingDomain: pods

helm/library/cortex/templates/certmanager/certificate.yaml

@@ -5,7 +5,7 @@ kind: Issuer
 metadata:
   labels:
     {{- include "chart.labels" . | nindent 4 }}
-  name: selfsigned-issuer
+  name: {{ .Values.namePrefix }}-selfsigned-issuer
   namespace: {{ .Release.Namespace }}
 spec:
   selfSigned: {}
@@ -30,7 +30,26 @@ spec:
     - cortex-metrics-service.{{ .Release.Namespace }}.svc
   issuerRef:
     kind: Issuer
-    name: selfsigned-issuer
+    name: {{ .Values.namePrefix }}-selfsigned-issuer
   secretName: metrics-server-cert
 {{- end }}
+{{- if .Values.webhook.enable }}
+---
+# Certificate for the webhook
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  labels:
+    {{- include "chart.labels" . | nindent 4 }}
+  name: {{ .Values.namePrefix }}-webhook-cert
+  namespace: {{ .Release.Namespace }}
+spec:
+  dnsNames:
+    - {{ .Values.namePrefix }}-webhook-service.{{ .Release.Namespace }}.svc
+    - {{ .Values.namePrefix }}-webhook-service.{{ .Release.Namespace }}.svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: {{ .Values.namePrefix }}-selfsigned-issuer
+  secretName: {{ .Values.namePrefix }}-webhook-server-cert
+{{- end }}
 {{- end }}

helm/library/cortex/templates/manager/manager.yaml

@@ -35,13 +35,21 @@ spec:
             {{- range .Values.controllerManager.container.args }}
             - {{ . }}
             {{- end }}
+            {{- if and .Values.webhook.enable .Values.certmanager.enable }}
+            - "--webhook-cert-path=/tmp/k8s-webhook-server/serving-certs"
+            {{- end }}
           ports:
             - name: api
               containerPort: 8080
               protocol: TCP
             - name: metrics
               containerPort: 2112
               protocol: TCP
+            {{- if .Values.webhook.enable }}
+            - name: webhook
+              containerPort: 9443
+              protocol: TCP
+            {{- end }}
           command:
             - /manager
           image: {{ .Values.controllerManager.container.image.repository }}:{{ .Values.controllerManager.container.image.tag | default .Chart.AppVersion }}
@@ -74,6 +82,11 @@ spec:
               mountPath: /tmp/k8s-metrics-server/metrics-certs
               readOnly: true
             {{- end }}
+            {{- if and .Values.webhook.enable .Values.certmanager.enable }}
+            - name: webhook-certs
+              mountPath: /tmp/k8s-webhook-server/serving-certs
+              readOnly: true
+            {{- end }}
       securityContext:
         {{- toYaml .Values.controllerManager.securityContext | nindent 8 }}
       serviceAccountName: {{ .Values.namePrefix }}-{{ .Values.controllerManager.serviceAccountName }}
@@ -91,6 +104,11 @@ spec:
           secret:
             secretName: metrics-server-cert
         {{- end }}
+        {{- if and .Values.webhook.enable .Values.certmanager.enable }}
+        - name: webhook-certs
+          secret:
+            secretName: {{ .Values.namePrefix }}-webhook-server-cert
+        {{- end }}
 ---
 apiVersion: v1
 kind: ConfigMap

helm/library/cortex/templates/webhook/validating-webhook-configuration.yaml

@@ -0,0 +1,48 @@
+{{- if .Values.webhook.enable }}
+apiVersion: admissionregistration.k8s.io/v1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: {{ .Values.namePrefix }}-validating-webhook-configuration
+  labels:
+    {{- include "chart.labels" . | nindent 4 }}
+  {{- if .Values.certmanager.enable }}
+  annotations:
+    "cert-manager.io/inject-ca-from": {{ .Release.Namespace }}/{{ .Values.namePrefix }}-webhook-cert
+  {{- end }}
+webhooks:
+  # This webhook validates the creation and update of cortex pipelines.
+  - name: {{ .Values.namePrefix }}-validate-v1alpha1-pipeline.cortex.cloud
+    admissionReviewVersions: [v1]
+    clientConfig:
+      # If we want to talk to the cortex webhook in another kubernetes cluster,
+      # we can template this out to use the ingress url provided by cortex. E.g.:
+      # url: "https://<my ingress url>/validate-cortex-cloud-v1alpha1-pipeline" for cross-cluster
+      service:
+        name: {{ .Values.namePrefix }}-webhook-service
+        namespace: {{ .Release.Namespace }}
+        # This path is managed by controller-runtime. It will route to the
+        # correct validating webhook handler based on the path.
+        path: /validate-cortex-cloud-v1alpha1-pipeline
+      {{- if not .Values.certmanager.enable }}
+      {{- if .Values.webhook.caBundle }}
+      caBundle: {{ .Values.webhook.caBundle }}
+      {{- end }}
+      {{- end }}
+    timeoutSeconds: 10
+    failurePolicy: Ignore
+    rules:
+      - apiGroups:
+          - cortex.cloud
+        apiVersions:
+          - v1alpha1
+        operations:
+          - CREATE
+          - UPDATE
+        resources:
+          - pipelines
+    sideEffects: None
+    {{- if .Values.webhook.namespaceSelector }}
+    namespaceSelector:
+      {{- toYaml .Values.webhook.namespaceSelector | nindent 6 }}
+    {{- end }}
+{{- end }}