New mechanism to skip tasks and suites

README.md

@@ -12,6 +12,7 @@ Spread
 [Blacklisting and whitelisting](#blacklisting)  
 [Preparing and restoring](#preparing)  
 [Functions](#functions)
+[Skipping](#skipping)  
 [Rebooting](#rebooting)  
 [Timeouts](#timeouts)  
 [Fast iterations with reuse](#reuse)  
@@ -417,6 +418,36 @@ A few helper functions are available for scripts to use:
  * _FATAL_ - Similar to ERROR, but prevents retries. Specific to [adhoc backend](#adhoc).
  * _ADDRESS_ - Set allocated system address. Specific to [adhoc backend](#adhoc).
 
+<a name="skipping"/>
+
+## Skipping
+
+A skip condition determines whether a task or suite should be skipped before it begins execution.
+
+Skip conditions can only be defined at the task or suite level. They are evaluated before the preparation phase. If a condition evaluates to true, then:
+
+ * Task: the task is completely skipped — including the prepare, execute, and restore phases.
+
+ * Suite: the suite is not prepared or restored, and all tasks inside it are skipped as well.
+
+Each skip condition must include:
+
+ * if: a shell expression (or list of expressions) evaluated to decide whether the task/suite should be skipped.
+
+ * reason: a human-readable explanation of why the task or suite was skipped.
+
+Below is an example showing how to define skip conditions for a task:
+
+```
+summary: skip condition example
+skip:
+    - reason: This is the first skip reason
+      if: [ -d /path/to/dir ]
+    - reason: This is the second skip reason
+      if: [ -f /path/to/file ]
+execute: |
+    echo "This is an example"
+```
 
 <a name="rebooting"/>
 

spread/project.go

@@ -316,6 +316,11 @@ func (e *Environment) Replace(oldkey, newkey, value string) {
 	e.vals[newkey] = value
 }
 
+type Skip struct {
+	Reason string `yaml:"reason"`
+	If     string `yaml:"if"`
+}
+
 type Suite struct {
 	Summary  string
 	Systems  []string
@@ -340,6 +345,7 @@ type Suite struct {
 
 	Priority OptionalInt
 	Manual   bool
+	Skip     []Skip
 }
 
 func (s *Suite) String() string { return "suite " + s.Name }
@@ -371,6 +377,7 @@ type Task struct {
 
 	Priority OptionalInt
 	Manual   bool
+	Skip     []Skip
 }
 
 func (t *Task) String() string { return t.Name }
@@ -387,7 +394,8 @@ type Job struct {
 	Environment *Environment
 	Sample      int
 
-	Priority int64
+	Priority   int64
+	SkipReason string
 }
 
 func (job *Job) String() string {
@@ -625,6 +633,13 @@ func Load(path string) (*Project, error) {
 		suite.PrepareEach = strings.TrimSpace(suite.PrepareEach)
 		suite.RestoreEach = strings.TrimSpace(suite.RestoreEach)
 		suite.DebugEach = strings.TrimSpace(suite.DebugEach)
+		for i := range suite.Skip {
+			suite.Skip[i].Reason = strings.TrimSpace(suite.Skip[i].Reason)
+			suite.Skip[i].If = strings.TrimSpace(suite.Skip[i].If)
+			if suite.Skip[i].If == "" || suite.Skip[i].Reason == "" {
+				return nil, fmt.Errorf("%s is missing either the if or reason for the skip", suite)
+			}
+		}
 
 		project.Suites[suite.Name] = suite
 
@@ -677,6 +692,13 @@ func Load(path string) (*Project, error) {
 			task.Prepare = strings.TrimSpace(task.Prepare)
 			task.Restore = strings.TrimSpace(task.Restore)
 			task.Debug = strings.TrimSpace(task.Debug)
+			for _, skip := range task.Skip {
+				skip.Reason = strings.TrimSpace(skip.Reason)
+				skip.If = strings.TrimSpace(skip.If)
+				if skip.If == "" || skip.Reason == "" {
+					return nil, fmt.Errorf("%s is missing either the if or reason for the skip", task)
+				}
+			}
 			if !validTask.MatchString(task.Name) {
 				return nil, fmt.Errorf("invalid task name: %q", task.Name)
 			}

spread/runner.go

@@ -13,9 +13,10 @@ import (
 	"sync"
 	"time"
 
-	"gopkg.in/tomb.v2"
 	"math"
 	"math/rand"
+
+	"gopkg.in/tomb.v2"
 )
 
 type Options struct {
@@ -433,6 +434,8 @@ const (
 	preparing = "preparing"
 	executing = "executing"
 	restoring = "restoring"
+	checking  = "checking"
+	skipping  = "skipping"
 )
 
 func (r *Runner) run(client *Client, job *Job, verb string, context interface{}, script, debug string, abend *bool) bool {
@@ -476,6 +479,9 @@ func (r *Runner) run(client *Client, job *Job, verb string, context interface{},
 	client.SetKillTimeout(job.KillTimeoutFor(context))
 	_, err := client.Trace(script, dir, job.Environment)
 	printft(start, endTime, "")
+	if verb == checking {
+		return err == nil
+	}
 	if err != nil {
 		// Use a different time so it has a different id on Travis, but keep
 		// the original start time so the error message shows the task time.
@@ -542,13 +548,15 @@ func (r *Runner) worker(backend *Backend, system *System, order []int) {
 	var abend bool
 	var badProject bool
 	var badSuite = make(map[*Suite]bool)
+	var skippedSuite = make(map[*Suite]string)
 
 	var insideProject bool
 	var insideBackend bool
 	var insideSuite *Suite
 
 	var job, last *Job
 
+outer:
 	for {
 		r.mu.Lock()
 		if job != nil {
@@ -573,6 +581,11 @@ func (r *Runner) worker(backend *Backend, system *System, order []int) {
 			r.add(&stats.TaskAbort, job)
 			continue
 		}
+		if skippedSuite[job.Suite] != "" {
+			job.SkipReason = skippedSuite[job.Suite]
+			r.add(&stats.TaskSkip, job)
+			continue
+		}
 
 		if insideSuite != nil && insideSuite != job.Suite {
 			if false {
@@ -608,6 +621,19 @@ func (r *Runner) worker(backend *Backend, system *System, order []int) {
 
 		if insideSuite != job.Suite {
 			insideSuite = job.Suite
+
+			// Check if the suite should be skipped
+			for _, skip := range job.Suite.Skip {
+				if r.run(client, job, checking, job.Suite, skip.If, job.Suite.Debug, &abend) {
+					job.SkipReason = skip.Reason
+					r.add(&stats.SuiteSkip, job)
+					r.add(&stats.TaskSkip, job)
+					skippedSuite[job.Suite] = skip.Reason
+					printft(time.Now(), startTime|endTime, "%s %s (%s)...", strings.Title(skipping), job, client.server.Label())
+					continue outer
+				}
+			}
+
 			if !r.options.Restore && !r.run(client, job, preparing, job.Suite, job.Suite.Prepare, job.Suite.Debug, &abend) {
 				r.add(&stats.SuitePrepareError, job)
 				r.add(&stats.TaskAbort, job)
@@ -617,32 +643,46 @@ func (r *Runner) worker(backend *Backend, system *System, order []int) {
 		}
 
 		debug := job.Debug()
-		for repeat := r.options.Repeat; repeat >= 0; repeat-- {
-			if r.options.Restore {
-				// Do not prepare or execute, and don't repeat.
-				repeat = -1
-			} else if !r.options.Restore && !r.run(client, job, preparing, job, job.Prepare(), debug, &abend) {
-				r.add(&stats.TaskPrepareError, job)
-				r.add(&stats.TaskAbort, job)
-				debug = ""
-				repeat = -1
-			} else if !r.options.Restore && r.run(client, job, executing, job, job.Task.Execute, debug, &abend) {
-				r.add(&stats.TaskDone, job)
-			} else if !r.options.Restore {
-				r.add(&stats.TaskError, job)
-				debug = ""
-				repeat = -1
+
+		// Check if the task should be skipped
+		skipRun := false
+		for _, skip := range job.Task.Skip {
+			if r.run(client, job, checking, job, skip.If, debug, &abend) {
+				skipRun = true
+				job.SkipReason = skip.Reason
+				r.add(&stats.TaskSkip, job)
+				printft(time.Now(), startTime|endTime, "%s %s (%s)...", strings.Title(skipping), job, client.server.Label())
+				break
 			}
-			if !abend && !r.options.Restore && repeat <= 0 {
-				if err := r.fetchArtifacts(client, job); err != nil {
-					printf("Cannot fetch artifacts of %s: %v", job, err)
-					r.tomb.Killf("cannot fetch artifacts of %s: %v", job, err)
+		}
+		if !skipRun {
+			for repeat := r.options.Repeat; repeat >= 0; repeat-- {
+				if r.options.Restore {
+					// Do not prepare or execute, and don't repeat.
+					repeat = -1
+				} else if !r.options.Restore && !r.run(client, job, preparing, job, job.Prepare(), debug, &abend) {
+					r.add(&stats.TaskPrepareError, job)
+					r.add(&stats.TaskAbort, job)
+					debug = ""
+					repeat = -1
+				} else if !r.options.Restore && r.run(client, job, executing, job, job.Task.Execute, debug, &abend) {
+					r.add(&stats.TaskDone, job)
+				} else if !r.options.Restore {
+					r.add(&stats.TaskError, job)
+					debug = ""
+					repeat = -1
+				}
+				if !abend && !r.options.Restore && repeat <= 0 {
+					if err := r.fetchArtifacts(client, job); err != nil {
+						printf("Cannot fetch artifacts of %s: %v", job, err)
+						r.tomb.Killf("cannot fetch artifacts of %s: %v", job, err)
+					}
+				}
+				if !abend && !r.run(client, job, restoring, job, job.Restore(), debug, &abend) {
+					r.add(&stats.TaskRestoreError, job)
+					badProject = true
+					repeat = -1
 				}
-			}
-			if !abend && !r.run(client, job, restoring, job, job.Restore(), debug, &abend) {
-				r.add(&stats.TaskRestoreError, job)
-				badProject = true
-				repeat = -1
 			}
 		}
 	}
@@ -1030,8 +1070,10 @@ type stats struct {
 	TaskDone            []*Job
 	TaskError           []*Job
 	TaskAbort           []*Job
+	TaskSkip            []*Job
 	TaskPrepareError    []*Job
 	TaskRestoreError    []*Job
+	SuiteSkip           []*Job
 	SuitePrepareError   []*Job
 	SuiteRestoreError   []*Job
 	BackendPrepareError []*Job
@@ -1063,9 +1105,11 @@ func (s *stats) log() {
 	printf("Successful tasks: %d", len(s.TaskDone))
 	printf("Aborted tasks: %d", len(s.TaskAbort))
 
+	logNames(printf, "Skipped tasks", s.TaskSkip, taskSkipReason)
 	logNames(printf, "Failed tasks", s.TaskError, taskName)
 	logNames(printf, "Failed task prepare", s.TaskPrepareError, taskName)
 	logNames(printf, "Failed task restore", s.TaskRestoreError, taskName)
+	logNames(printf, "Skipped suites", s.SuiteSkip, suiteSkipReason)
 	logNames(printf, "Failed suite prepare", s.SuitePrepareError, suiteName)
 	logNames(printf, "Failed suite restore", s.SuiteRestoreError, suiteName)
 	logNames(printf, "Failed backend prepare", s.BackendPrepareError, backendName)
@@ -1085,6 +1129,14 @@ func taskName(job *Job) string {
 	return job.Task.Name + ":" + job.Variant
 }
 
+func taskSkipReason(job *Job) string {
+	return taskName(job) + " - " + job.SkipReason
+}
+
+func suiteSkipReason(job *Job) string {
+	return job.Suite.Name + " - " + job.SkipReason
+}
+
 func logNames(f func(format string, args ...interface{}), prefix string, jobs []*Job, name func(job *Job) string) {
 	names := make([]string, 0, len(jobs))
 	for _, job := range jobs {

tests/skip/checks/multi-skip/task.yaml

@@ -0,0 +1,20 @@
+summary: Ensure skip condition works.
+
+environment:
+    skip_1/T1: true
+    skip_2/T1: true
+    skip_1/T2: false
+    skip_2/T2: true
+    skip_1/T3: false
+    skip_2/T3: false
+
+skip:
+  - reason: This is the first skip reason
+    if: |
+      [ "$skip_1" = "true" ]
+  - reason: This is the second skip reason
+    if: |
+      [ "$skip_2" = "true" ]
+
+execute: |
+    exit 0

tests/skip/checks/single-skip/task.yaml

@@ -0,0 +1,19 @@
+summary: Ensure skip condition works.
+
+environment:
+    skip/T1: true
+    exec/T1: true
+    skip/T2: false
+    exec/T2: true
+    skip/T3: false
+    exec/T3: false
+    skip/T4: true
+    exec/T4: false
+
+skip:
+  - reason: This is the skip reason
+    if: |
+      [ "$skip" = "true" ]
+
+execute: |
+    [ "$exec" = "true" ]

tests/skip/spread.yaml

@@ -0,0 +1,18 @@
+project: spread
+
+backends:
+    lxd:
+        systems:
+            - ubuntu-22.04
+
+path: /home/test
+
+suites:
+    checks/:
+        summary: Verification tasks.
+        environment: 
+            MY_TEST_VAR: '$(HOST: echo "${MY_TEST_VAR:-}")'
+        skip:
+          - reason: Skip all tests for checks suite
+            if: |
+                [ "$MY_TEST_VAR" = 1 ]