0.26.0

cmd/operator/app/thread_keeper.go

@@ -59,7 +59,10 @@ func initKeeper(ctx context.Context) error {
 
 	err = ctrlRuntime.
 		NewControllerManagedBy(manager).
-		For(&api.ClickHouseKeeperInstallation{}, builder.WithPredicates(keeperPredicate())).
+		For(
+			&api.ClickHouseKeeperInstallation{},
+			builder.WithPredicates(keeperPredicate()),
+		).
 		Owns(&apps.StatefulSet{}).
 		Complete(
 			&controller.Controller{
@@ -88,40 +91,30 @@ func runKeeper(ctx context.Context) error {
 func keeperPredicate() predicate.Funcs {
 	return predicate.Funcs{
 		CreateFunc: func(e event.CreateEvent) bool {
-			obj, ok := e.Object.(*api.ClickHouseKeeperInstallation)
+			new, ok := e.Object.(*api.ClickHouseKeeperInstallation)
 			if !ok {
 				return false
 			}
 
-			// Check if namespace should be watched (includes deny list check)
-			if !chop.Config().IsNamespaceWatched(obj.Namespace) {
-				logger.V(2).Info("chkInformer: skip event, namespace is not watched or is in deny list", "namespace", obj.Namespace)
+			if !controller.ShouldEnqueue(new) {
 				return false
 			}
 
-			if obj.Spec.Suspend.Value() {
-				return false
-			}
 			return true
 		},
 		DeleteFunc: func(e event.DeleteEvent) bool {
 			return true
 		},
 		UpdateFunc: func(e event.UpdateEvent) bool {
-			obj, ok := e.ObjectNew.(*api.ClickHouseKeeperInstallation)
+			new, ok := e.ObjectNew.(*api.ClickHouseKeeperInstallation)
 			if !ok {
 				return false
 			}
 
-			// Check if namespace should be watched (includes deny list check)
-			if !chop.Config().IsNamespaceWatched(obj.Namespace) {
-				logger.V(2).Info("chkInformer: skip event, namespace is not watched or is in deny list", "namespace", obj.Namespace)
+			if !controller.ShouldEnqueue(new) {
 				return false
 			}
 
-			if obj.Spec.Suspend.Value() {
-				return false
-			}
 			return true
 		},
 		GenericFunc: func(e event.GenericEvent) bool {

config/config-dev.yaml

@@ -247,6 +247,10 @@ clickhouse:
       # Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle.
       # All collected metrics are returned.
       collect: 9
+    # Regexp to match tables in system database to fetch metrics from.
+    # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function.
+    # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics.
+    tablesRegexp: "^(metrics|custom_metrics)$"
 
 keeper:
   configuration:
@@ -351,6 +355,20 @@ reconcile:
       # 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet.
       onFailure: abort
 
+    # Recreate StatefulSet scenario
+    recreate:
+      # What to do in case operator is in need to recreate StatefulSet?
+      # Possible options:
+      # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is,
+      #    do not try to fix or delete or update it, just abort reconcile cycle.
+      #    Do not proceed to the next StatefulSet(s) and wait for an admin to assist.
+      # 2. recreate - proceed and recreate StatefulSet.
+
+      # Triggered when PVC data loss or missing volumes are detected
+      onDataLoss: recreate
+      # Triggered when StatefulSet update fails or StatefulSet is not ready
+      onUpdateFailure: recreate
+
   # Reconcile Host scenario
   host:
     # The operator during reconcile procedure should wait for a ClickHouse host to achieve the following conditions:

config/config.yaml

@@ -175,7 +175,7 @@ clickhouse:
     # Specified in seconds.
     timeouts:
       # Timout to setup connection from the operator to ClickHouse instances. In seconds.
-      connect: 1
+      connect: 5
       # Timout to perform SQL query from the operator to ClickHouse instances. In seconds.
       query: 4
 
@@ -216,7 +216,6 @@ clickhouse:
             quotas:
             settings:
             files:
-
       - version: ">= 23.5"
         spec:
           configuration:
@@ -245,6 +244,10 @@ clickhouse:
       # Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle.
       # All collected metrics are returned.
       collect: 9
+    # Regexp to match tables in system database to fetch metrics from.
+    # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function.
+    # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics.
+    tablesRegexp: "^(metrics|custom_metrics)$"
 
 keeper:
   configuration:
@@ -349,6 +352,20 @@ reconcile:
       # 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet.
       onFailure: abort
 
+    # Recreate StatefulSet scenario
+    recreate:
+      # What to do in case operator is in need to recreate StatefulSet?
+      # Possible options:
+      # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is,
+      #    do not try to fix or delete or update it, just abort reconcile cycle.
+      #    Do not proceed to the next StatefulSet(s) and wait for an admin to assist.
+      # 2. recreate - proceed and recreate StatefulSet.
+
+      # Triggered when PVC data loss or missing volumes are detected
+      onDataLoss: recreate
+      # Triggered when StatefulSet update fails or StatefulSet is not ready
+      onUpdateFailure: recreate
+
   # Reconcile Host scenario
   host:
     # The operator during reconcile procedure should wait for a ClickHouse host to achieve the following conditions:

deploy/builder/templates-config/config.yaml

@@ -169,7 +169,7 @@ clickhouse:
     # Specified in seconds.
     timeouts:
       # Timout to setup connection from the operator to ClickHouse instances. In seconds.
-      connect: 1
+      connect: 5
       # Timout to perform SQL query from the operator to ClickHouse instances. In seconds.
       query: 4
 
@@ -210,7 +210,6 @@ clickhouse:
             quotas:
             settings:
             files:
-
       - version: ">= 23.5"
         spec:
           configuration:
@@ -239,6 +238,10 @@ clickhouse:
       # Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle.
       # All collected metrics are returned.
       collect: 9
+    # Regexp to match tables in system database to fetch metrics from.
+    # Multiple tables can be matched using regexp. Matched tables are merged using merge() table function.
+    # Default is "^(metrics|custom_metrics)$" which fetches from both system.metrics and system.custom_metrics.
+    tablesRegexp: "^(metrics|custom_metrics)$"
 
 keeper:
   configuration:
@@ -343,6 +346,20 @@ reconcile:
       # 3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet.
       onFailure: abort
 
+    # Recreate StatefulSet scenario
+    recreate:
+      # What to do in case operator is in need to recreate StatefulSet?
+      # Possible options:
+      # 1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is,
+      #    do not try to fix or delete or update it, just abort reconcile cycle.
+      #    Do not proceed to the next StatefulSet(s) and wait for an admin to assist.
+      # 2. recreate - proceed and recreate StatefulSet.
+
+      # Triggered when PVC data loss or missing volumes are detected
+      onDataLoss: recreate
+      # Triggered when StatefulSet update fails or StatefulSet is not ready
+      onUpdateFailure: recreate
+
   # Reconcile Host scenario
   host:
     # The operator during reconcile procedure should wait for a ClickHouse host to achieve the following conditions:

deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-01-chi-chit.yaml

@@ -26,6 +26,10 @@ spec:
       served: true
       storage: true
       additionalPrinterColumns:
+        - name: status
+          type: string
+          description: Resource status
+          jsonPath: .status.status
         - name: version
           type: string
           description: Operator version
@@ -49,10 +53,6 @@ spec:
           description: TaskID
           priority: 1 # show in wide view
           jsonPath: .status.taskID
-        - name: status
-          type: string
-          description: Resource status
-          jsonPath: .status.status
         - name: hosts-completed
           type: integer
           description: Completed hosts count
@@ -497,6 +497,80 @@ spec:
                           minimum: 0
                           maximum: 100
                           description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default."
+                    statefulSet: &TypeReconcileStatefulSet
+                      type: object
+                      description: "Optional, StatefulSet reconcile behavior tuning"
+                      properties:
+                        create:
+                          type: object
+                          description: "Behavior during create StatefulSet"
+                          properties:
+                            onFailure:
+                              type: string
+                              description: |
+                                What to do in case created StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds.
+                                Possible options:
+                                1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is.
+                                2. delete - delete newly created problematic StatefulSet and follow 'abort' path afterwards.
+                                3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet.
+                              enum:
+                                - ""
+                                - "abort"
+                                - "delete"
+                                - "ignore"
+                        update:
+                          type: object
+                          description: "Behavior during update StatefulSet"
+                          properties:
+                            timeout:
+                              type: integer
+                              description: "How many seconds to wait for StatefulSet to be 'Ready' during update"
+                              minimum: 0
+                              maximum: 3600
+                            pollInterval:
+                              type: integer
+                              description: "How many seconds to wait between checks for StatefulSet status during update"
+                              minimum: 1
+                              maximum: 600
+                            onFailure:
+                              type: string
+                              description: |
+                                What to do in case updated StatefulSet is not in 'Ready' after `reconcile.statefulSet.update.timeout` seconds.
+                                Possible options:
+                                1. abort - abort the process, do nothing with the problematic StatefulSet, leave it as it is.
+                                2. rollback - delete Pod and rollback StatefulSet to previous Generation. Follow 'abort' path afterwards.
+                                3. ignore - ignore an error, pretend nothing happened, continue reconcile and move on to the next StatefulSet.
+                              enum:
+                                - ""
+                                - "abort"
+                                - "rollback"
+                                - "ignore"
+                        recreate:
+                          type: object
+                          description: "Behavior during recreate StatefulSet"
+                          properties:
+                            onDataLoss:
+                              type: string
+                              description: |
+                                What to do in case operator needs to recreate StatefulSet due to PVC data loss or missing volumes.
+                                Possible options:
+                                1. abort - abort the process, do nothing with the problematic StatefulSet.
+                                2. recreate - proceed and recreate StatefulSet.
+                              enum:
+                                - ""
+                                - "abort"
+                                - "recreate"
+                            onUpdateFailure:
+                              type: string
+                              description: |
+                                What to do in case operator needs to recreate StatefulSet due to update failure or StatefulSet not ready.
+                                Possible options:
+                                1. abort - abort the process, do nothing with the problematic StatefulSet.
+                                2. recreate - proceed and recreate StatefulSet.
+                              enum:
+                                - ""
+                                - "abort"
+                                - "recreate"
                     host: &TypeReconcileHost
                       type: object
                       description: |

deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml

@@ -240,6 +240,12 @@ spec:
                                 Timeout used to limit metrics collection request. In seconds.
                                 Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle.
                                 All collected metrics are returned.
+                        tablesRegexp:
+                          type: string
+                          description: |
+                            Regexp to match tables in system database to fetch metrics from.
+                            Multiple tables can be matched using regexp. Matched tables are merged using merge() table function.
+                            Default is "^(metrics|custom_metrics)$".
                 template:
                   type: object
                   description: "Parameters which are used if you want to generate ClickHouseInstallationTemplate custom resources from files which are stored inside clickhouse-operator deployment"
@@ -318,6 +324,24 @@ spec:
                                 1. abort - do nothing, just break the process and wait for admin.
                                 2. rollback (default) - delete Pod and rollback StatefulSet to previous Generation. Pod would be recreated by StatefulSet based on rollback-ed configuration.
                                 3. ignore - ignore error, pretend nothing happened and move on to the next StatefulSet.
+                        recreate:
+                          type: object
+                          description: "Behavior during recreate StatefulSet"
+                          properties:
+                            onDataLoss:
+                              type: string
+                              description: |
+                                What to do in case operator needs to recreate StatefulSet due to PVC data loss or missing volumes.
+                                Possible options:
+                                1. abort - abort the process, do nothing with the problematic StatefulSet.
+                                2. recreate (default) - proceed and recreate StatefulSet.
+                            onUpdateFailure:
+                              type: string
+                              description: |
+                                What to do in case operator needs to recreate StatefulSet due to update failure or StatefulSet not ready.
+                                Possible options:
+                                1. abort - abort the process, do nothing with the problematic StatefulSet.
+                                2. recreate (default) - proceed and recreate StatefulSet.
                     host:
                       type: object
                       description: |