Conformance redesign

This reworks the conformance tests to verify the various APIs with different sets of data.
The configuration allows various APIs and types of data to be disabled.
The configuration will default to the tests appropriate for a specific version of the spec.
However, new types of data may still be pushed when testing older versions of the spec.

The tests are also designed to be run with `go build` rather than `go test`.
This will allow the conformance tests themselves to be tested in the future.

Attempts to run the tests with `go test` and the previous configuration variables should still work, but support for this is deprecated and warning will be included in the output to update.

The results now include a new results.yaml including the configuration and the result of each API and type of data tested.
Processing this on the oci-conformance repo requires a separate PR there.

This redesign was necessary to allow the conformance tests to scale with the different types of data we are seeing.
This includes data with a subject, artifacts, and new digest algorithms.

Signed-off-by: Brandon Mitchell <git@bmitch.net>

Author: sudo-bmitch

.github/workflows/build-pr.yml

@@ -4,14 +4,17 @@ on:
   push:
     branches:
       - main
+  pull_request:
+    branches:
+      - main
 
 jobs:
   run:
     runs-on: ubuntu-latest
 
     strategy:
       matrix:
-        go: ['1.21', '1.22']
+        go: ['stable', 'oldstable']
 
     steps:
       - name: checkout source code
@@ -47,5 +50,6 @@ jobs:
         with:
           name: oci-conformance-results-${{ matrix.go }}
           path: |
+            ./results.yaml
             ./report.html
             ./junit.xml

.github/workflows/build.yml

@@ -4,23 +4,27 @@ on:
   push:
     branches:
       - main
+  pull_request:
+    branches:
+      - main
 
 jobs:
   run:
     runs-on: ubuntu-latest
     steps:
       - name: checkout source code
         uses: actions/checkout@v4
-      - name: Start a test registry (zot)
+      - name: Start a test registry
+        id: setup
         run: |
           set -x
           make registry-ci
+          echo "port=$(docker port oci-conformance-olareg 5000| head -1 | cut -f2 -d:)" >>$GITHUB_OUTPUT
       - name: Run OCI distribution-spec conformance
         env:
-          OCI_ROOT_URL: http://localhost:5000
-          OCI_NAMESPACE: myorg/myrepo
-          OCI_TEST_PULL: 1
-          OCI_TEST_PUSH: 1
-          OCI_TEST_CONTENT_DISCOVERY: 1
-          OCI_TEST_CONTENT_MANAGEMENT: 1
+          OCI_REGISTRY: "localhost:${{ steps.setup.outputs.port }}"
+          OCI_TLS: "disabled"
+          OCI_REPO1: "myorg/myrepo"
+          OCI_REPO2: "myorg/myrepo2"
+          OCI_RESULTS_DIR: "."
         uses: ./

.github/workflows/conformance-action-pr.yml

@@ -26,10 +26,6 @@ jobs:
           fi
           VERSION=$(echo "${VERSION}" | sed -r 's#/+#-#g')
           TAGS="${IMAGE}:${VERSION}"
-          if [[ $VERSION == "${{ github.event.repository.default_branch }}" ]]; then
-            GITSHA="$(git rev-parse --short HEAD)"
-            TAGS="${TAGS},${IMAGE}:${GITSHA}"
-          fi
           if [[ $VERSION =~ ^v[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
             MINOR=${VERSION%.*}
             MAJOR=${MINOR%.*}

.github/workflows/conformance-action.yml

@@ -6,3 +6,4 @@ tags
 go.work
 junit.xml
 report.html
+results.yaml

.github/workflows/release.yml

@@ -23,14 +23,13 @@ ifeq "$(strip $(PANDOC))" ''
 	endif
 endif
 
-GOLANGCILINT_CONTAINER ?= ghcr.io/opencontainers/golangci-lint:v1.52.1@sha256:d3d3d56f9706ebe843c1b06686c385877ba65b33f39507cdbeb22f482adce65a
+GOLANGCILINT_CONTAINER ?= docker.io/golangci/golangci-lint:v2.11.3@sha256:e838e8ab68aaefe83e2408691510867ade9329c0e0b895a3fb35eb93d1c2a4ba
 ifeq "$(strip $(GOLANGCILINT))" ''
 	ifneq "$(strip $(DOCKER))" ''
 		GOLANGCILINT = $(DOCKER) run \
 			--rm \
 			-v $(shell pwd)/:/input:ro \
 			-e GOCACHE=/tmp/.cache \
-			-e GO111MODULE=on \
 			-e GOLANGCI_LINT_CACHE=/tmp/.cache \
 			--entrypoint /bin/bash \
 			-u $(shell id -u) \
@@ -83,35 +82,70 @@ install.tools: .install.gitvalidation
 .install.gitvalidation:
 	go install github.com/vbatts/git-validation@latest
 
-conformance: conformance-test conformance-binary
+conformance: conformance-test conformance-cmd
 
 conformance-test:
 	$(GOLANGCILINT) -c 'cd conformance && golangci-lint run -v'
 
 conformance-binary: $(OUTPUT_DIRNAME)/conformance.test
 
-TEST_REGISTRY_CONTAINER ?= ghcr.io/project-zot/zot-minimal-linux-amd64:v2.1.7@sha256:2114797f00696011f38cc94c72f5773c84b1036562df5034d05ea19075179ad1
-registry-ci:
-	docker rm -f oci-conformance && \
+conformance-cmd: $(OUTPUT_DIRNAME)/conformance
+
+registry-ci: registry-ci-olareg
+
+TEST_REGISTRY_IMAGE_OLAREG ?= ghcr.io/olareg/olareg:edge
+registry-ci-olareg:
+	docker rm -f oci-conformance-olareg && \
+		docker run --rm -d \
+			--name=oci-conformance-olareg \
+			-p 5000 \
+			$(TEST_REGISTRY_IMAGE_OLAREG) serve --store-type mem --api-delete --api-blob-delete --api-sparse-image --api-sparse-index && \
+		sleep 2
+
+TEST_REGISTRY_IMAGE_ZOT ?= ghcr.io/project-zot/zot-minimal-linux-amd64:v2.1.7@sha256:2114797f00696011f38cc94c72f5773c84b1036562df5034d05ea19075179ad1
+registry-ci-zot:
+	docker rm -f oci-conformance-zot && \
 		mkdir -p $(OUTPUT_DIRNAME) && \
 		echo '{"distSpecVersion":"1.1.0-dev","storage":{"rootDirectory":"/tmp/zot","gc":false,"dedupe":false},"http":{"address":"0.0.0.0","port":"5000"},"log":{"level":"debug"}}' > $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json
-		docker run -d \
+		docker run --rm -d \
 			-v $(shell pwd)/$(OUTPUT_DIRNAME)/zot-config.json:/etc/zot/config.json \
-			--name=oci-conformance \
-			-p 5000:5000 \
-			$(TEST_REGISTRY_CONTAINER) && \
+			--name=oci-conformance-zot \
+			-p 5000 \
+			$(TEST_REGISTRY_IMAGE_ZOT) && \
 		sleep 5
 
-conformance-ci:
-	export OCI_ROOT_URL="http://localhost:5000" && \
-		export OCI_NAMESPACE="myorg/myrepo" && \
-		export OCI_TEST_PULL=1 && \
-		export OCI_TEST_PUSH=1 && \
-		export OCI_TEST_CONTENT_DISCOVERY=1 && \
-		export OCI_TEST_CONTENT_MANAGEMENT=1 && \
-		$(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test
+conformance-ci: conformance-ci-olareg
+
+conformance-ci-olareg: $(OUTPUT_DIRNAME)/conformance
+	export OCI_VERSION="dev" && \
+		export OCI_REGISTRY="localhost:$$(docker port oci-conformance-olareg 5000| head -1 | cut -f2 -d:)" && \
+		export OCI_TLS="disabled" && \
+		export OCI_REPO1="myorg/myrepo" && \
+		export OCI_REPO2="myorg/myrepo2" && \
+		export OCI_RESULTS_DIR="." && \
+		export OCI_DATA_SPARSE=true && \
+		$(shell pwd)/$(OUTPUT_DIRNAME)/conformance
+
+conformance-ci-zot: $(OUTPUT_DIRNAME)/conformance
+	export OCI_REGISTRY="localhost:$$(docker port oci-conformance-zot 5000| head -1 | cut -f2 -d:)" && \
+		export OCI_TLS="disabled" && \
+		export OCI_REPO1="myorg/myrepo" && \
+		export OCI_REPO2="myorg/myrepo2" && \
+		export OCI_RESULTS_DIR="." && \
+		$(shell pwd)/$(OUTPUT_DIRNAME)/conformance
+
+clean-ci:
+	docker rm -f oci-conformance-olareg oci-conformance-zot
+
+$(OUTPUT_DIRNAME)/conformance: conformance/*.go conformance/go.mod
+	cd conformance && \
+		CGO_ENABLED=0 go build -o $(shell pwd)/$(OUTPUT_DIRNAME)/conformance \
+			--ldflags="-X github.com/opencontainers/distribution-spec/conformance.Version=$(CONFORMANCE_VERSION)"
 
-$(OUTPUT_DIRNAME)/conformance.test:
+$(OUTPUT_DIRNAME)/conformance.test: conformance/*.go conformance/go.mod
 	cd conformance && \
 		CGO_ENABLED=0 go test -c -o $(shell pwd)/$(OUTPUT_DIRNAME)/conformance.test \
 			--ldflags="-X github.com/opencontainers/distribution-spec/conformance.Version=$(CONFORMANCE_VERSION)"
+
+clean: clean-ci
+	rm -rf header.html junit.xml report.html results.yaml output conformance/results

.gitignore

@@ -14,16 +14,39 @@ runs:
       id: check-conformance
       run: |
         set -x
+        required_env_vars=(
+          "OCI_REGISTRY"
+          "OCI_REPO1"
+        )
+        missing=false
+        for v in ${required_env_vars[@]}; do
+          if [[ "${!v}" == "" ]]; then
+            echo "::error title=Missing variable::Missing required variable ${v}"
+            missing=true
+          fi
+        done
+        if [[ "${missing}" == "false" ]]; then
+          exit 0
+        fi
+        # fallback to previous variables
         required_env_vars=(
           "OCI_ROOT_URL"
           "OCI_NAMESPACE"
         )
+        missing=false
         for v in ${required_env_vars[@]}; do
           if [[ "${!v}" == "" ]]; then
-            echo "Error: the following environment variable is required: ${v}"
-            exit 1
+            echo "::error title=Missing fallback variable::Missing fallback variable ${v}"
+            missing=true
+          else
+            echo "::warning title=Deprecated fallback variable::Variable ${v} has been deprecated"
           fi
         done
+        if [[ "${missing}" == "false" ]]; then
+          exit 0
+        else
+          exit 1
+        fi
 
     - name: Build OCI distribution-spec conformance binary
       shell: bash
@@ -34,14 +57,16 @@ runs:
         # Enter the directory containing the checkout of this action which is surpisingly hard to do (but we did it... #OCI)
         cd "$(dirname $(find $(find ~/work/_actions -name distribution-spec -print -quit) -name Makefile -print -quit))"
         
-        # The .git folder is not present, but the dirname is the requested action ref, so use this as the conformance version
-        conformance_version="$(basename "${PWD}")"
-        echo "conformance-version=${conformance_version}" >> $GITHUB_OUTPUT
+        # The .git folder is not present, but the dirname is the requested action ref, so use this as the commit version
+        commit_version="$(basename "${PWD}")"
 
         # Build the conformance binary
-        CONFORMANCE_VERSION="${conformance_version}" OUTPUT_DIRNAME=bin make conformance-binary
+        CONFORMANCE_VERSION="${commit_version}" OUTPUT_DIRNAME=bin make conformance-cmd
+
+        # The spec version is independent of the conformance git commit
+        echo "oci-spec-version=${OCI_VERSION:-stable}" >> $GITHUB_OUTPUT
 
-        # Add bin to the PATH so we can just run "conformance.test"
+        # Add bin to the PATH so we can just run "conformance"
         echo "${PWD}/bin" >> $GITHUB_PATH
 
     - name: Run OCI distribution-spec conformance binary
@@ -50,7 +75,8 @@ runs:
       run: |
         set -x
         set +e
-        conformance.test
+        OCI_RESULTS_DIR=${OCI_RESULTS_DIR:-.}
+        conformance
         conformance_rc="$?"
         set -e
         if [[ -f report.html ]]; then
@@ -62,9 +88,10 @@ runs:
 
     - name: Upload OCI distribution-spec conformance results as build artifact
       if: always() && steps.run-conformance.outputs.has-report == 'true'
-      uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
       with:
-        name: oci-distribution-spec-conformance-results-${{ steps.build-conformance.outputs.conformance-version }}
+        name: oci-distribution-spec-conformance-results-${{ steps.build-conformance.outputs.oci-spec-version }}
         path: |
+          results.yaml
           report.html
           junit.xml

Makefile

@@ -1,6 +1,5 @@
-vendor/
-junit.xml
-report.html
+conformance
 conformance.test
-tags
-env.sh
+oci-conformance.yaml
+results/**
+vendor/