From 2ec0763861c0a5192fb8899f0476c5a7cd1c235e Mon Sep 17 00:00:00 2001 From: Sergio Cazzolato Date: Fri, 30 Jan 2026 15:36:45 -0300 Subject: [PATCH] New mechanism to skip tasks and suites MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A skip condition determines whether a task or suite should be skipped before it begins execution. Skip conditions can only be defined at the task or suite level. They are evaluated before the task preparation phase. If the condition evaluates to true, the entire task or suite is skipped — including the prepare, execute, and restore phases. We considered introducing a SKIP command similar to REBOOT, but this approach implies partially running the prepare phases (e.g., prepare-each at project/backend/suite level). This means the task wouldn’t actually be skipped, and determining what should be restored becomes unclear. Using a skip condition is cleaner and more predictable from a developer’s perspective, and aligns with how skip/conditional execution works in most testing tools across languages. --- README.md | 31 +++++++ spread/project.go | 24 +++++- spread/runner.go | 102 ++++++++++++++++++------ tests/skip/checks/multi-skip/task.yaml | 20 +++++ tests/skip/checks/single-skip/task.yaml | 19 +++++ tests/skip/spread.yaml | 18 +++++ tests/skip/task.yaml | 44 ++++++++++ 7 files changed, 232 insertions(+), 26 deletions(-) create mode 100644 tests/skip/checks/multi-skip/task.yaml create mode 100644 tests/skip/checks/single-skip/task.yaml create mode 100644 tests/skip/spread.yaml create mode 100644 tests/skip/task.yaml diff --git a/README.md b/README.md index 1ad706b1..f63a0f40 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Spread [Blacklisting and whitelisting](#blacklisting) [Preparing and restoring](#preparing) [Functions](#functions) +[Skipping](#skipping) [Rebooting](#rebooting) [Timeouts](#timeouts) [Fast iterations with reuse](#reuse) @@ -417,6 +418,36 @@ A few helper functions are available for scripts to use: * _FATAL_ - Similar to ERROR, but prevents retries. Specific to [adhoc backend](#adhoc). * _ADDRESS_ - Set allocated system address. Specific to [adhoc backend](#adhoc). + + +## Skipping + +A skip condition determines whether a task or suite should be skipped before it begins execution. + +Skip conditions can only be defined at the task or suite level. They are evaluated before the preparation phase. If a condition evaluates to true, then: + + * Task: the task is completely skipped — including the prepare, execute, and restore phases. + + * Suite: the suite is not prepared or restored, and all tasks inside it are skipped as well. + +Each skip condition must include: + + * if: a shell expression (or list of expressions) evaluated to decide whether the task/suite should be skipped. + + * reason: a human-readable explanation of why the task or suite was skipped. + +Below is an example showing how to define skip conditions for a task: + +``` +summary: skip condition example +skip: + - reason: This is the first skip reason + if: [ -d /path/to/dir ] + - reason: This is the second skip reason + if: [ -f /path/to/file ] +execute: | + echo "This is an example" +``` diff --git a/spread/project.go b/spread/project.go index 2c9d01b1..03d9993d 100644 --- a/spread/project.go +++ b/spread/project.go @@ -316,6 +316,11 @@ func (e *Environment) Replace(oldkey, newkey, value string) { e.vals[newkey] = value } +type Skip struct { + Reason string `yaml:"reason"` + If string `yaml:"if"` +} + type Suite struct { Summary string Systems []string @@ -340,6 +345,7 @@ type Suite struct { Priority OptionalInt Manual bool + Skip []Skip } func (s *Suite) String() string { return "suite " + s.Name } @@ -371,6 +377,7 @@ type Task struct { Priority OptionalInt Manual bool + Skip []Skip } func (t *Task) String() string { return t.Name } @@ -387,7 +394,8 @@ type Job struct { Environment *Environment Sample int - Priority int64 + Priority int64 + SkipReason string } func (job *Job) String() string { @@ -625,6 +633,13 @@ func Load(path string) (*Project, error) { suite.PrepareEach = strings.TrimSpace(suite.PrepareEach) suite.RestoreEach = strings.TrimSpace(suite.RestoreEach) suite.DebugEach = strings.TrimSpace(suite.DebugEach) + for i := range suite.Skip { + suite.Skip[i].Reason = strings.TrimSpace(suite.Skip[i].Reason) + suite.Skip[i].If = strings.TrimSpace(suite.Skip[i].If) + if suite.Skip[i].If == "" || suite.Skip[i].Reason == "" { + return nil, fmt.Errorf("%s is missing either the if or reason for the skip", suite) + } + } project.Suites[suite.Name] = suite @@ -677,6 +692,13 @@ func Load(path string) (*Project, error) { task.Prepare = strings.TrimSpace(task.Prepare) task.Restore = strings.TrimSpace(task.Restore) task.Debug = strings.TrimSpace(task.Debug) + for _, skip := range task.Skip { + skip.Reason = strings.TrimSpace(skip.Reason) + skip.If = strings.TrimSpace(skip.If) + if skip.If == "" || skip.Reason == "" { + return nil, fmt.Errorf("%s is missing either the if or reason for the skip", task) + } + } if !validTask.MatchString(task.Name) { return nil, fmt.Errorf("invalid task name: %q", task.Name) } diff --git a/spread/runner.go b/spread/runner.go index 36417447..bf4ea130 100644 --- a/spread/runner.go +++ b/spread/runner.go @@ -13,9 +13,10 @@ import ( "sync" "time" - "gopkg.in/tomb.v2" "math" "math/rand" + + "gopkg.in/tomb.v2" ) type Options struct { @@ -433,6 +434,8 @@ const ( preparing = "preparing" executing = "executing" restoring = "restoring" + checking = "checking" + skipping = "skipping" ) func (r *Runner) run(client *Client, job *Job, verb string, context interface{}, script, debug string, abend *bool) bool { @@ -476,6 +479,9 @@ func (r *Runner) run(client *Client, job *Job, verb string, context interface{}, client.SetKillTimeout(job.KillTimeoutFor(context)) _, err := client.Trace(script, dir, job.Environment) printft(start, endTime, "") + if verb == checking { + return err == nil + } if err != nil { // Use a different time so it has a different id on Travis, but keep // the original start time so the error message shows the task time. @@ -542,6 +548,7 @@ func (r *Runner) worker(backend *Backend, system *System, order []int) { var abend bool var badProject bool var badSuite = make(map[*Suite]bool) + var skippedSuite = make(map[*Suite]string) var insideProject bool var insideBackend bool @@ -549,6 +556,7 @@ func (r *Runner) worker(backend *Backend, system *System, order []int) { var job, last *Job +outer: for { r.mu.Lock() if job != nil { @@ -573,6 +581,11 @@ func (r *Runner) worker(backend *Backend, system *System, order []int) { r.add(&stats.TaskAbort, job) continue } + if skippedSuite[job.Suite] != "" { + job.SkipReason = skippedSuite[job.Suite] + r.add(&stats.TaskSkip, job) + continue + } if insideSuite != nil && insideSuite != job.Suite { if false { @@ -608,6 +621,19 @@ func (r *Runner) worker(backend *Backend, system *System, order []int) { if insideSuite != job.Suite { insideSuite = job.Suite + + // Check if the suite should be skipped + for _, skip := range job.Suite.Skip { + if r.run(client, job, checking, job.Suite, skip.If, job.Suite.Debug, &abend) { + job.SkipReason = skip.Reason + r.add(&stats.SuiteSkip, job) + r.add(&stats.TaskSkip, job) + skippedSuite[job.Suite] = skip.Reason + printft(time.Now(), startTime|endTime, "%s %s (%s)...", strings.Title(skipping), job, client.server.Label()) + continue outer + } + } + if !r.options.Restore && !r.run(client, job, preparing, job.Suite, job.Suite.Prepare, job.Suite.Debug, &abend) { r.add(&stats.SuitePrepareError, job) r.add(&stats.TaskAbort, job) @@ -617,32 +643,46 @@ func (r *Runner) worker(backend *Backend, system *System, order []int) { } debug := job.Debug() - for repeat := r.options.Repeat; repeat >= 0; repeat-- { - if r.options.Restore { - // Do not prepare or execute, and don't repeat. - repeat = -1 - } else if !r.options.Restore && !r.run(client, job, preparing, job, job.Prepare(), debug, &abend) { - r.add(&stats.TaskPrepareError, job) - r.add(&stats.TaskAbort, job) - debug = "" - repeat = -1 - } else if !r.options.Restore && r.run(client, job, executing, job, job.Task.Execute, debug, &abend) { - r.add(&stats.TaskDone, job) - } else if !r.options.Restore { - r.add(&stats.TaskError, job) - debug = "" - repeat = -1 + + // Check if the task should be skipped + skipRun := false + for _, skip := range job.Task.Skip { + if r.run(client, job, checking, job, skip.If, debug, &abend) { + skipRun = true + job.SkipReason = skip.Reason + r.add(&stats.TaskSkip, job) + printft(time.Now(), startTime|endTime, "%s %s (%s)...", strings.Title(skipping), job, client.server.Label()) + break } - if !abend && !r.options.Restore && repeat <= 0 { - if err := r.fetchArtifacts(client, job); err != nil { - printf("Cannot fetch artifacts of %s: %v", job, err) - r.tomb.Killf("cannot fetch artifacts of %s: %v", job, err) + } + if !skipRun { + for repeat := r.options.Repeat; repeat >= 0; repeat-- { + if r.options.Restore { + // Do not prepare or execute, and don't repeat. + repeat = -1 + } else if !r.options.Restore && !r.run(client, job, preparing, job, job.Prepare(), debug, &abend) { + r.add(&stats.TaskPrepareError, job) + r.add(&stats.TaskAbort, job) + debug = "" + repeat = -1 + } else if !r.options.Restore && r.run(client, job, executing, job, job.Task.Execute, debug, &abend) { + r.add(&stats.TaskDone, job) + } else if !r.options.Restore { + r.add(&stats.TaskError, job) + debug = "" + repeat = -1 + } + if !abend && !r.options.Restore && repeat <= 0 { + if err := r.fetchArtifacts(client, job); err != nil { + printf("Cannot fetch artifacts of %s: %v", job, err) + r.tomb.Killf("cannot fetch artifacts of %s: %v", job, err) + } + } + if !abend && !r.run(client, job, restoring, job, job.Restore(), debug, &abend) { + r.add(&stats.TaskRestoreError, job) + badProject = true + repeat = -1 } - } - if !abend && !r.run(client, job, restoring, job, job.Restore(), debug, &abend) { - r.add(&stats.TaskRestoreError, job) - badProject = true - repeat = -1 } } } @@ -1030,8 +1070,10 @@ type stats struct { TaskDone []*Job TaskError []*Job TaskAbort []*Job + TaskSkip []*Job TaskPrepareError []*Job TaskRestoreError []*Job + SuiteSkip []*Job SuitePrepareError []*Job SuiteRestoreError []*Job BackendPrepareError []*Job @@ -1063,9 +1105,11 @@ func (s *stats) log() { printf("Successful tasks: %d", len(s.TaskDone)) printf("Aborted tasks: %d", len(s.TaskAbort)) + logNames(printf, "Skipped tasks", s.TaskSkip, taskSkipReason) logNames(printf, "Failed tasks", s.TaskError, taskName) logNames(printf, "Failed task prepare", s.TaskPrepareError, taskName) logNames(printf, "Failed task restore", s.TaskRestoreError, taskName) + logNames(printf, "Skipped suites", s.SuiteSkip, suiteSkipReason) logNames(printf, "Failed suite prepare", s.SuitePrepareError, suiteName) logNames(printf, "Failed suite restore", s.SuiteRestoreError, suiteName) logNames(printf, "Failed backend prepare", s.BackendPrepareError, backendName) @@ -1085,6 +1129,14 @@ func taskName(job *Job) string { return job.Task.Name + ":" + job.Variant } +func taskSkipReason(job *Job) string { + return taskName(job) + " - " + job.SkipReason +} + +func suiteSkipReason(job *Job) string { + return job.Suite.Name + " - " + job.SkipReason +} + func logNames(f func(format string, args ...interface{}), prefix string, jobs []*Job, name func(job *Job) string) { names := make([]string, 0, len(jobs)) for _, job := range jobs { diff --git a/tests/skip/checks/multi-skip/task.yaml b/tests/skip/checks/multi-skip/task.yaml new file mode 100644 index 00000000..e5485b83 --- /dev/null +++ b/tests/skip/checks/multi-skip/task.yaml @@ -0,0 +1,20 @@ +summary: Ensure skip condition works. + +environment: + skip_1/T1: true + skip_2/T1: true + skip_1/T2: false + skip_2/T2: true + skip_1/T3: false + skip_2/T3: false + +skip: + - reason: This is the first skip reason + if: | + [ "$skip_1" = "true" ] + - reason: This is the second skip reason + if: | + [ "$skip_2" = "true" ] + +execute: | + exit 0 diff --git a/tests/skip/checks/single-skip/task.yaml b/tests/skip/checks/single-skip/task.yaml new file mode 100644 index 00000000..a15af910 --- /dev/null +++ b/tests/skip/checks/single-skip/task.yaml @@ -0,0 +1,19 @@ +summary: Ensure skip condition works. + +environment: + skip/T1: true + exec/T1: true + skip/T2: false + exec/T2: true + skip/T3: false + exec/T3: false + skip/T4: true + exec/T4: false + +skip: + - reason: This is the skip reason + if: | + [ "$skip" = "true" ] + +execute: | + [ "$exec" = "true" ] diff --git a/tests/skip/spread.yaml b/tests/skip/spread.yaml new file mode 100644 index 00000000..2a7f3881 --- /dev/null +++ b/tests/skip/spread.yaml @@ -0,0 +1,18 @@ +project: spread + +backends: + lxd: + systems: + - ubuntu-22.04 + +path: /home/test + +suites: + checks/: + summary: Verification tasks. + environment: + MY_TEST_VAR: '$(HOST: echo "${MY_TEST_VAR:-}")' + skip: + - reason: Skip all tests for checks suite + if: | + [ "$MY_TEST_VAR" = 1 ] diff --git a/tests/skip/task.yaml b/tests/skip/task.yaml new file mode 100644 index 00000000..6e585f2e --- /dev/null +++ b/tests/skip/task.yaml @@ -0,0 +1,44 @@ +summary: Test the skip condition feature + +prepare: | + if [ ! -f .spread-reuse.yaml ]; then + touch /run/spread-reuse.yaml + ln -s /run/spread-reuse.yaml .spread-reuse.yaml + fi + +execute: | + # Check when the skip condition matches with a single skip condition + spread -reuse -resend lxd:ubuntu-22.04:checks/single-skip &> task1.out || true + grep 'Successful tasks: 1' task1.out + grep 'Failed tasks: 1' task1.out + grep 'Skipped tasks: 2' task1.out + test "$(grep -c 'Skipping ' task1.out)" = 2 + grep 'single-skip:T1 - This is the skip reason' task1.out + grep 'single-skip:T4 - This is the skip reason' task1.out + + # Check when the skip condition matches with multiple skip conditions + spread -reuse -resend lxd:ubuntu-22.04:checks/multi-skip &> task2.out || true + grep 'Successful tasks: 1' task2.out + grep 'Skipped tasks: 2' task2.out + test "$(grep -c 'Skipping ' task2.out)" = 2 + grep 'multi-skip:T1 - This is the first skip reason' task2.out + grep 'multi-skip:T2 - This is the second skip reason' task2.out + + # Check when the skip condition is set at suite level + MY_TEST_VAR=1 spread -reuse -resend lxd:ubuntu-22.04:checks/single-skip &> task3.out || true + grep 'Successful tasks: 0' task3.out + grep 'Skipped tasks: 4' task3.out + grep 'Skipped suites: 1' task3.out + test "$(grep -c 'Skipping ' task3.out)" = 1 + grep 'checks/ - Skip all tests for checks suite' task3.out + grep 'single-skip:T1 - Skip all tests for checks suite' task3.out + grep 'single-skip:T2 - Skip all tests for checks suite' task3.out + grep 'single-skip:T3 - Skip all tests for checks suite' task3.out + grep 'single-skip:T4 - Skip all tests for checks suite' task3.out + +debug: | + cat task1.out || true + echo + cat task2.out || true + echo + cat task3.out || true