Add -setup flag

Author: arp242

CHANGELOG.md

@@ -1,3 +1,13 @@
+Unreleased
+----------
+- Add `-setup` flag to run one or more commands before running the test suite;
+  useful for setting up or compiling the decoder program or script in one
+  command. For example:
+
+      toml-test test \
+          -setup='go build ./cmd/toml-test-decoder' -decoder=./toml-test-decoder \
+          -setup='go build ./cmd/toml-test-encoder' -encoder=./toml-test-encoder
+
 v2.2.0 2026-03-30
 -----------------
 This contains several minor bug fixes to the test runner and one additional

cmd/toml-test/script.gotxt

@@ -8,6 +8,11 @@ decoder="{{join .Decoder " "}}"
 {{if .Encoder}}encoder="{{join .Encoder " "}}"{{else}}
 encoder=  # No encoder tests{{end}}
 
+# Setup command(s), one -setup per array entry.
+setup=({{range $s := .Setup}}
+{{"\t"}}-setup {{$s | quote}}{{end}}
+)
+
 # Version of the TOML specification to test.
 toml={{.TOML}}
 
@@ -52,4 +57,4 @@ if ! command -v "$tt" >/dev/null; then
 fi
 
 # Run toml-test
-"$tt" test -toml="$toml" -skip-must-err ${skip[@]} -decoder="$decoder" -encoder="${encoder:-}" "$@"
+"$tt" test -toml="$toml" -skip-must-err ${skip[@]} "${setup[@]}" -decoder="$decoder" -encoder="${encoder:-}" "$@"

cmd/toml-test/test.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -22,13 +23,31 @@ var hlErr = zli.Color256(224).Bg() | zli.Color256(0) | zli.Bold
 //go:embed script.gotxt
 var script []byte
 
+func shellQuote(s string) string {
+	if strings.Contains(s, "'") {
+		s = strings.ReplaceAll(s, `'`, `'"'"'`) // cmd'foo → 'cmd'"'"'foo'
+	}
+	return "'" + s + "'"
+}
+
 var scriptTemplate = template.Must(template.New("").
 	Option("missingkey=error").
-	Funcs(template.FuncMap{"join": strings.Join}).
+	Funcs(template.FuncMap{"join": strings.Join, "quote": shellQuote}).
 	Parse(string(script)))
 
 func cmdTest(f zli.Flags) {
-	runner, verbose, script, asJSON := parseTestFlags(f)
+	runner, verbose, script, asJSON, setup := parseTestFlags(f)
+
+	for _, s := range setup {
+		f := strings.Fields(s)
+		if verbose > 0 {
+			fmt.Printf("SETUP %v\n", s)
+		}
+		out, err := exec.Command(f[0], f[1:]...).CombinedOutput()
+		if err != nil {
+			zli.Fatalf("error running -setup=%q: %s: command output:\n%s", s, err, out)
+		}
+	}
 
 	tests, err := runner.Run()
 	zli.F(err)
@@ -54,12 +73,13 @@ func cmdTest(f zli.Flags) {
 		err := scriptTemplate.Execute(os.Stdout, struct {
 			Decoder       []string
 			Encoder       []string
+			Setup         []string
 			TOML          string
 			Version       string
 			FailedValid   []string
 			FailedEncoder []string
 			FailedInvalid []string
-		}{runner.Decoder.Cmd(), enc, runner.Version, v, failedValid, failedEncoder, failedInvalid})
+		}{runner.Decoder.Cmd(), enc, setup, runner.Version, v, failedValid, failedEncoder, failedInvalid})
 		zli.F(err)
 		return
 	}
@@ -76,10 +96,11 @@ func cmdTest(f zli.Flags) {
 	zli.Exit(0)
 }
 
-func parseTestFlags(f zli.Flags) (tomltest.Runner, int, bool, bool) {
+func parseTestFlags(f zli.Flags) (tomltest.Runner, int, bool, bool, []string) {
 	var (
 		decoder       = f.String("", "decoder")
 		encoder       = f.String("", "encoder")
+		setup         = f.StringList(nil, "setup")
 		tomlVersion   = f.String(tomltest.DefaultVersion, "toml")
 		verbose       = f.IntCounter(0, "v")
 		color         = f.String("always", "color")
@@ -175,7 +196,7 @@ func parseTestFlags(f zli.Flags) (tomltest.Runner, int, bool, bool) {
 		}
 	}
 
-	return runner, verbose.Int(), script.Bool(), asJSON.Bool()
+	return runner, verbose.Int(), script.Bool(), asJSON.Bool(), setup.Strings()
 }
 
 func newEnc() *json.Encoder {

cmd/toml-test/usage.go

@@ -135,6 +135,22 @@ There are three types of tests:
                    specified in the toml-test README. May be omitted if writing
                    TOML isn't supported.
 
+    -setup         Run once before any tests, to setup/compile the decoder.
+                   toml-test exits with an error and won't run any tests if
+                   this exits with non-zero code. Like -decoder and -encoder,
+                   this isn't run through a shell but arguments are split on
+                   whitespace. This flag can be added more than once to run
+                   several commands (which are run in the order they are given
+                   on the commandline).
+
+                   For example:
+
+                      toml-test test \
+                        -setup='go build ./cmd/toml-test-decoder' \
+                        -setup='go build ./cmd/toml-test-encoder' \
+                        -decoder=./toml-test-decoder \
+                        -encoder=./toml-test-encoder
+
     -json          Output report as JSON rather than text.
 
     -script        Print a small bash/zsh script with -skip flag for failing