feat: implement skip toleration mechanism for batch rolling

apis/rollout/v1alpha1/validation/rolloutrun.go

@@ -108,6 +108,7 @@ func validateRolloutRunStepTargets(targets []rolloutv1alpha1.RolloutRunStepTarge
 			allErrs = append(allErrs, field.Duplicate(fldPath.Index(i).Child("name"), target.CrossClusterObjectNameReference))
 		}
 		targetMap[target.CrossClusterObjectNameReference] = true
+		allErrs = append(allErrs, validateToleration(target.Toleration, fldPath.Index(i).Child("toleration"))...)
 	}
 
 	return allErrs

apis/rollout/v1alpha1/validation/rolloutstrategy.go

@@ -188,6 +188,24 @@ func ValidateRolloutStrategyTargets(targets *rolloutv1alpha1.RolloutStrategyTarg
 
 	allErrs = append(allErrs, appsvalidation.ValidatePositiveIntOrPercent(targets.Replicas, fldPath.Child("replicas"))...)
 	allErrs = append(allErrs, ValidateResourceMatch(targets.Match, fldPath.Child("matchTargets"))...)
+	allErrs = append(allErrs, validateToleration(targets.Toleration, fldPath.Child("toleration"))...)
+
+	return allErrs
+}
+
+func validateToleration(toleration *rolloutv1alpha1.RolloutStepTargetToleration, fldPath *field.Path) field.ErrorList {
+	if toleration == nil {
+		return nil
+	}
+
+	allErrs := field.ErrorList{}
+
+	if toleration.FailureThreshold == nil {
+		allErrs = append(allErrs, field.Required(fldPath.Child("failureThreshold"), "must be set when toleration is configured"))
+	}
+	if toleration.InitialDelaySeconds == nil {
+		allErrs = append(allErrs, field.Required(fldPath.Child("initialDelaySeconds"), "must be set when toleration is configured"))
+	}
 
 	return allErrs
 }

apis/rollout/v1alpha1/validation/rolloutstrategy_test.go

@@ -311,6 +311,52 @@ func TestValidateRolloutStrategy_V2(t *testing.T) {
 			}(),
 			wantErr: false,
 		},
+		{
+			name: "toleration without failureThreshold",
+			obj: func() *rolloutv1alpha1.RolloutStrategy {
+				obj := validV2Strategy.DeepCopy()
+				obj.BatchV2.Batches[0].Targets[0].Toleration = &rolloutv1alpha1.RolloutStepTargetToleration{
+					InitialDelaySeconds: ptr.To[int32](300),
+				}
+				return obj
+			}(),
+			wantErr: true,
+			errLen:  1,
+		},
+		{
+			name: "toleration without initialDelaySeconds",
+			obj: func() *rolloutv1alpha1.RolloutStrategy {
+				obj := validV2Strategy.DeepCopy()
+				obj.BatchV2.Batches[0].Targets[0].Toleration = &rolloutv1alpha1.RolloutStepTargetToleration{
+					FailureThreshold: ptr.To[int32](2),
+				}
+				return obj
+			}(),
+			wantErr: true,
+			errLen:  1,
+		},
+		{
+			name: "toleration with both fields set is valid",
+			obj: func() *rolloutv1alpha1.RolloutStrategy {
+				obj := validV2Strategy.DeepCopy()
+				obj.BatchV2.Batches[0].Targets[0].Toleration = &rolloutv1alpha1.RolloutStepTargetToleration{
+					FailureThreshold:    ptr.To[int32](2),
+					InitialDelaySeconds: ptr.To[int32](300),
+				}
+				return obj
+			}(),
+			wantErr: false,
+		},
+		{
+			name: "toleration with both fields nil",
+			obj: func() *rolloutv1alpha1.RolloutStrategy {
+				obj := validV2Strategy.DeepCopy()
+				obj.BatchV2.Batches[0].Targets[0].Toleration = &rolloutv1alpha1.RolloutStepTargetToleration{}
+				return obj
+			}(),
+			wantErr: true,
+			errLen:  2,
+		},
 	}
 
 	for i := range tests {

config/crd/bases/rollout.kusionstack.io_rolloutruns.yaml

@@ -2264,32 +2264,30 @@ spec:
                           - targets
                         type: object
                       type: array
-                    toleration:
-                      description: Toleration is the toleration policy of the canary strategy
-                      properties:
-                        initialDelaySeconds:
-                          description: Number of seconds after the toleration check has started before the task are initiated.
-                          format: int32
-                          type: integer
-                        taskFailureThreshold:
-                          anyOf:
-                            - type: integer
-                            - type: string
-                          description: |-
-                            FailureThreshold indicates how many failed pods can be tolerated before marking the rollout task as success
-                            If not set, the default value is 0, which means no failed pods can be tolerated
-                            This is a task level threshold.
-                          x-kubernetes-int-or-string: true
-                        workloadTotalFailureThreshold:
-                          anyOf:
-                            - type: integer
-                            - type: string
-                          description: |-
-                            WorkloadFailureThreshold indicates how many failed pods can be tolerated in all upgraded pods of one workload.
-                            The default value is 0, which means no failed pods can be tolerated.
-                            This is a workload level threshold.
-                          x-kubernetes-int-or-string: true
-                      type: object
+                    tolerations:
+                      description: |-
+                        Tolerations records the accumulated toleration from skipped batches per workload.
+                        When a batch is skipped, the gap between expected and actual replicas for each workload
+                        is accumulated into this field, allowing subsequent batches to tolerate the deficit.
+                      items:
+                        description: RolloutRunTolerationTarget records the toleration value accumulated from skipped batches for a specific workload.
+                        properties:
+                          cluster:
+                            description: Cluster defines which cluster the workload is in.
+                            type: string
+                          name:
+                            description: Name is the workload name.
+                            type: string
+                          toleration:
+                            description: |-
+                              Toleration is the accumulated toleration value from skipped batches.
+                              It represents how many replicas the workload is allowed to be short of.
+                            format: int32
+                            type: integer
+                        required:
+                          - toleration
+                        type: object
+                      type: array
                   type: object
                 canary:
                   description: Canary defines the canary strategy

go.mod

@@ -23,7 +23,7 @@ require (
 	k8s.io/klog/v2 v2.130.1
 	k8s.io/kubernetes v1.22.2
 	k8s.io/utils v0.0.0-20241210054802-24370beab758
-	kusionstack.io/kube-api v0.7.5-0.20260512114711-9570d38337c2
+	kusionstack.io/kube-api v0.7.5-0.20260901080654-b15c6deabdf8
 	kusionstack.io/kube-utils v0.2.1-0.20251125083928-1134a582b341
 	kusionstack.io/resourceconsist v0.0.4
 	sigs.k8s.io/controller-runtime v0.21.0

go.sum

@@ -1021,8 +1021,10 @@ k8s.io/sample-apiserver v0.22.2/go.mod h1:h+/DIV5EmuNq4vfPr5TSXy9mIBVXXlPAKQMPbj
 k8s.io/system-validators v1.5.0/go.mod h1:bPldcLgkIUK22ALflnsXk8pvkTEndYdNuaHH6gRrl0Q=
 k8s.io/utils v0.0.0-20240102154912-e7106e64919e h1:eQ/4ljkx21sObifjzXwlPKpdGLrCfRziVtos3ofG/sQ=
 k8s.io/utils v0.0.0-20240102154912-e7106e64919e/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
-kusionstack.io/kube-api v0.7.5-0.20260512114711-9570d38337c2 h1:jrUVO6a6/fuwdfF8NnehVXGbiYV57AfIJvqnsYRB3sI=
-kusionstack.io/kube-api v0.7.5-0.20260512114711-9570d38337c2/go.mod h1:e1jtrQH2LK5fD2nTyfIXG6nYrYbU8VXShRxTRwVPaLk=
+kusionstack.io/kube-api v0.7.5-0.20260629032255-be28009453cb h1:7lrtNodnXf7rsm3NzVhGZ4TwrM2ekJ656FN0qT9fHaI=
+kusionstack.io/kube-api v0.7.5-0.20260629032255-be28009453cb/go.mod h1:e1jtrQH2LK5fD2nTyfIXG6nYrYbU8VXShRxTRwVPaLk=
+kusionstack.io/kube-api v0.7.5-0.20260901080654-b15c6deabdf8 h1:4deV4LV41/ecKS9KgUYRnQgLeDBacy9lRjBkxW2+vzo=
+kusionstack.io/kube-api v0.7.5-0.20260901080654-b15c6deabdf8/go.mod h1:e1jtrQH2LK5fD2nTyfIXG6nYrYbU8VXShRxTRwVPaLk=
 kusionstack.io/kube-utils v0.2.1-0.20251125083928-1134a582b341 h1:dnMtHJvIpU3338WpqGiNN2qXWZFiXaoiuzR9jwhvWpg=
 kusionstack.io/kube-utils v0.2.1-0.20251125083928-1134a582b341/go.mod h1:Lz5SBYWg9+jw+kP0CAyf/b62D5DeUPf6+jE1d0WC4cI=
 kusionstack.io/resourceconsist v0.0.4 h1:wRqLJuNh8O4TT6p0uOklFpHUKiRdRxcAH71Sw/q9LhE=

pkg/controllers/rollout/rollout_controller.go

@@ -677,21 +677,20 @@ func (r *RolloutReconciler) applyOneTimeStrategy(ctx context.Context, obj *rollo
 	var batch *rolloutv1alpha1.RolloutRunBatchStrategy
 
 	// Check if using BatchV2 (V2 strategy scenario)
-	// Note: BatchStrategyV2 does not support Toleration, only V1 BatchStrategy has it
 	if strategy.BatchV2 != nil {
 		batch = &rolloutv1alpha1.RolloutRunBatchStrategy{
 			Batches: constructRolloutRunBatchesV2(strategy.BatchV2, workloads),
 		}
 	} else {
 		// Use original Batch field (V1 scenario)
 		batch = &rolloutv1alpha1.RolloutRunBatchStrategy{
-			Batches:    constructRolloutRunBatches(&strategy.Batch, workloads),
-			Toleration: strategy.Batch.Toleration,
+			Batches: constructRolloutRunBatches(&strategy.Batch, workloads),
 		}
 	}
 
-	if batch.Toleration == nil && run.Spec.Batch != nil {
-		batch.Toleration = run.Spec.Batch.Toleration
+	// Inherit Tolerations from the existing run spec if not changed
+	if run.Spec.Batch != nil {
+		batch.Tolerations = run.Spec.Batch.Tolerations
 	}
 
 	if equality.Semantic.DeepEqual(batch, run.Spec.Batch) {

pkg/controllers/rollout/utils.go

@@ -92,7 +92,6 @@ func constructRolloutRun(obj *rolloutv1alpha1.Rollout, strategy *rolloutv1alpha1
 	// Determine V1 vs V2 path based on BatchV2 presence
 	if strategy.BatchV2 != nil {
 		// V2 path: BatchV2 + optional CanaryV2
-		// Note: BatchStrategyV2 does not support Toleration, only V1 BatchStrategy has it
 		if strategy.CanaryV2 != nil {
 			run.Spec.Canary = constructRolloutRunCanaryV2(strategy.CanaryV2, workloadWrappers)
 		}
@@ -105,8 +104,7 @@ func constructRolloutRun(obj *rolloutv1alpha1.Rollout, strategy *rolloutv1alpha1
 			run.Spec.Canary = constructRolloutRunCanary(strategy.Canary, workloadWrappers)
 		}
 		run.Spec.Batch = &rolloutv1alpha1.RolloutRunBatchStrategy{
-			Toleration: strategy.Batch.Toleration,
-			Batches:    constructRolloutRunBatches(strategy.Batch, workloadWrappers),
+			Batches: constructRolloutRunBatches(strategy.Batch, workloadWrappers),
 		}
 	}
 
@@ -238,6 +236,7 @@ func resolveRolloutTargets(targets []rolloutv1alpha1.RolloutStrategyTargets, wor
 				},
 				Replicas:             t.Replicas,
 				ReplicaSlidingWindow: t.ReplicaSlidingWindow,
+				Toleration:           t.Toleration,
 			}
 			result = append(result, target)
 		}

pkg/controllers/rolloutrun/executor/batch.go

@@ -205,6 +205,8 @@ func (e *batchExecutor) doBatchUpgrading(ctx *ExecutorContext) (bool, time.Durat
 	batchTargetStatuses := make([]rolloutv1alpha1.RolloutWorkloadStatus, 0)
 
 	allWorkloadReady := true
+	allWorkloadsAutoSkippable := true
+
 	for _, item := range currentBatch.Targets {
 		info := ctx.Workloads.Get(item.Cluster, item.Name)
 		if info == nil {
@@ -227,6 +229,11 @@ func (e *batchExecutor) doBatchUpgrading(ctx *ExecutorContext) (bool, time.Durat
 		allWorkloadReady = false
 		logger.V(3).Info("still waiting for target to be ready", "target", item.CrossClusterObjectNameReference, "reason", reason)
 
+		// Check auto-skip toleration for this workload
+		if !e.canAutoSkipTarget(item, info, currentBatchExpectedReplicas, isLastBatch, newStatus) {
+			allWorkloadsAutoSkippable = false
+		}
+
 		expectedReplicas, err := e.calculateExpectedReplicasBySlidingWindow(status, currentBatchExpectedReplicas, item.ReplicaSlidingWindow)
 		if err != nil {
 			return false, retryStop, err
@@ -250,10 +257,64 @@ func (e *batchExecutor) doBatchUpgrading(ctx *ExecutorContext) (bool, time.Durat
 		return true, retryImmediately, nil
 	}
 
+	if allWorkloadsAutoSkippable {
+		logger.Info("auto-skipping batch due to toleration")
+		newStatus.BatchStatus.Records[currentBatchIndex].State = StepSkipped
+		recordRolloutRunTolerations(&newStatus.BatchStatus.Tolerations, rolloutRun.Spec.Batch.Batches, ctx.Workloads, currentBatchIndex)
+		return true, retryImmediately, nil
+	}
+
 	// wait for next reconcile
 	return false, retryDefault, nil
 }
 
+// canAutoSkipTarget checks if the workload target meets the auto-skip toleration conditions.
+// Returns true only when the workload is not ready due to a real deficit (gap > 0) within
+// the toleration threshold and the initial delay has elapsed.
+// Transient states (Generation mismatch, terminating replicas, last-batch overscaling)
+// are NOT auto-skippable because the gap is unreliable until the workload stabilizes.
+func (e *batchExecutor) canAutoSkipTarget(item rolloutv1alpha1.RolloutRunStepTarget, info *workload.Info, currentBatchExpectedReplicas int32, isLastBatch bool, newStatus *rolloutv1alpha1.RolloutRunStatus) bool {
+	if item.Toleration == nil || item.Toleration.FailureThreshold == nil {
+		return false
+	}
+
+	// Not skippable while workload has not been reconciled yet (Generation mismatch).
+	// UpdatedAvailableReplicas may be stale from the previous generation.
+	if info.Generation != info.Status.ObservedGeneration {
+		return false
+	}
+
+	// On last batch, not skippable if observed replicas exceed desired or terminating replicas exist(strict check).
+	if isLastBatch && info.Status.ObservedReplicas > info.Status.DesiredReplicas || info.Status.TerminatingReplicas != 0 {
+		return false
+	}
+
+	// Only evaluate toleration on a real deficit.
+	gap := currentBatchExpectedReplicas - info.Status.UpdatedAvailableReplicas
+	if gap <= 0 {
+		return false
+	}
+
+	if gap > *item.Toleration.FailureThreshold {
+		return false
+	}
+
+	// gap is within threshold, check timeout
+	if item.Toleration.InitialDelaySeconds != nil {
+		currentBatchIndex := newStatus.BatchStatus.CurrentBatchIndex
+		startTime := newStatus.BatchStatus.Records[currentBatchIndex].StartTime
+		if startTime == nil {
+			return false
+		}
+		elapsed := time.Since(startTime.Time)
+		if elapsed < time.Duration(*item.Toleration.InitialDelaySeconds)*time.Second {
+			return false
+		}
+	}
+
+	return true
+}
+
 // calculateExpectedReplicasBySlidingWindow calculate expected replicas by sliding window
 // if window is nil, return currentBatchExpectedReplicas
 // if window is not nil, return min(currentBatchExpectedReplicas, updatedAvailableReplicas + increment)