chore: add visual tests to reduce regressions

.github/workflows/test.yml

@@ -33,6 +33,38 @@ jobs:
       - run: npm run build
       - run: npm run test
 
+  browser:
+    name: Test in Browser
+
+    runs-on: ubuntu-latest
+    container:
+      image: mcr.microsoft.com/playwright:v1.55.1-noble
+      options: --user 1001
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22.x"
+          cache: "npm"
+
+      - name: Install Node dependencies
+        run: npm ci
+
+      - run: npm run build
+      - run: npm run test:visual -w @uwdata/mosaic-spec
+
+      - name: Upload Playwright results
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: playwright-test-results
+          path: |
+            packages/vgplot/spec/test-results/
+            packages/vgplot/spec/test/visual.test.js-snapshots/
+
   python:
     name: Test in Python
 
@@ -64,11 +96,11 @@ jobs:
           uv build
           uv run --group dev mypy
           uv run --group dev pytest --cov-report=term-missing --color=yes --cov=pkg
-      
+
       - name: Build and test widget
         run: |
           cd packages/vgplot/widget
-          
+
           # Mock the presence of the static files which would be generated by `npm run build`.
           # Since we are running pure Python tests, we don't need the actual anywidget dist files.
           mkdir -p mosaic_widget/static && touch mosaic_widget/static/index.js mosaic_widget/static/index.css

package.json

@@ -30,12 +30,14 @@
   },
   "devDependencies": {
     "@eslint/js": "^9.35.0",
+    "@playwright/test": "^1.40.0",
     "ajv": "^8.17.1",
     "ajv-formats": "^3.0.1",
     "eslint": "^9.35.0",
     "jsdom": "^26.1.0",
     "lerna": "^8.2.3",
     "nodemon": "^3.1.10",
+    "playwright": "^1.55.1",
     "rimraf": "^6.0.1",
     "timezone-mock": "^1.3.6",
     "typescript": "^5.9.2",

packages/vgplot/spec/.gitignore

@@ -0,0 +1,3 @@
+test-results
+test/visual.test.js-snapshots/*
+!test/visual.test.js-snapshots/*-linux.png

packages/vgplot/spec/README.md

@@ -57,3 +57,26 @@ const spec = parse(yaml); // parse yaml to JS objects
 // parse specification to internal AST (abstract syntax tree)
 const ast = parseSpec(spec);
 ```
+
+## Developers
+
+To rebuild the playwright snapshots, use docker. On macOS 26+, you can use
+
+```sh
+brew install --cask container
+container system start
+```
+
+Then run this at the Mosaic repo root level.
+
+```sh
+container run --rm -it \
+  -v $(pwd):/workspace \
+  -w /workspace \
+  mcr.microsoft.com/playwright:v1.55.1-noble \
+  bash -c "npm i && npm run test:visual:update -w @uwdata/mosaic-spec"
+```
+
+Run `container system stop` when you are done.
+
+Alternatively, you can remove the snapshots and download them from GitHub. At the end of the `browser` action, the snapshots will be uploaded as an artifact.

packages/vgplot/spec/package.json

@@ -33,6 +33,8 @@
     "preschema": "tsc --build",
     "schema": "ts-json-schema-generator -f tsconfig.json -p src/spec/Spec.ts -t Spec --no-type-check --no-ref-encode --functions hide > dist/mosaic-schema.json",
     "test": "npm run schema && vitest run",
+    "test:visual": "playwright test test/visual.test.js",
+    "test:visual:update": "npm run test:visual -- --update-snapshots",
     "version": "cd ../../.. && npm run docs:schema",
     "prepublishOnly": "npm run test && npm run lint && npm run build"
   },

packages/vgplot/spec/playwright.config.js

@@ -0,0 +1,25 @@
+import { defineConfig, devices } from '@playwright/test';
+
+export default defineConfig({
+  // Run all tests in parallel.
+  fullyParallel: true,
+
+  // Retry on CI only.
+  retries: process.env.CI ? 2 : 0,
+
+  // Opt out of parallel tests on CI.
+  workers: process.env.CI ? 1 : undefined,
+
+  testMatch: '**/visual*.test.js',
+
+  use: {
+    viewport: { height: 1280, width: 720 },
+  },
+
+  webServer: {
+    command: 'npx vite --port 5173 --host --config vite.config.docker.js',
+    port: 5173,
+    reuseExistingServer: true,
+    cwd: '../../../',
+  },
+});

packages/vgplot/spec/test/visual.test.js

@@ -0,0 +1,77 @@
+import { test, expect } from '@playwright/test';
+import { readFile, readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { dirname } from 'node:path';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+async function getSpecFiles() {
+  const specsDir = join(__dirname, '../../../../specs/json');
+  const files = await readdir(specsDir);
+  return files
+    .filter(file => file.endsWith('.json'))
+    .map(file => file.replace('.json', ''))
+    .sort();
+}
+
+const specFiles = await getSpecFiles();
+
+test.describe('Visual regression tests for JSON specs', () => {
+  for (const specName of specFiles) {
+    test(`renders ${specName} spec correctly`, async ({ page }) => {
+      const specPath = join(__dirname, '../../../../specs/json', `${specName}.json`);
+      const specData = JSON.parse(await readFile(specPath, 'utf8'));
+
+      await page.goto('http://localhost:5173/');
+
+      await page.setContent(`
+        <!DOCTYPE html>
+        <html>
+        <head>
+          <meta charset="UTF-8">
+          <style>
+            body { margin: 0; padding: 20px; font-family: system-ui; background: white; }
+            #container { display: inline-block; }
+          </style>
+        </head>
+        <body>
+          <div id="container"></div>
+          <script type="module">
+            import { DuckDBWASMConnector } from '/packages/mosaic/core/src/index.js';
+            import { parseSpec, astToDOM } from '/packages/vgplot/spec/src/index.js';
+            import { createAPIContext } from '/packages/vgplot/vgplot/src/index.js';
+
+            async function renderSpec() {
+              const vg = createAPIContext();
+              const wasm = new DuckDBWASMConnector({ log: false });
+              vg.coordinator().databaseConnector(wasm);
+
+              const specData = ${JSON.stringify(specData)};
+              const ast = parseSpec(specData);
+
+              const { element } = await astToDOM(ast, {
+                api: vg,
+                baseURL: window.location.origin + '/'
+              });
+
+              document.getElementById('container').appendChild(element);
+              document.body.setAttribute('data-render-complete', 'true');
+            }
+
+            renderSpec();
+          </script>
+        </body>
+        </html>
+      `);
+
+      await page.waitForFunction(() =>
+        document.body.hasAttribute('data-render-complete')
+      );
+
+      await page.waitForTimeout(1000);  // give mosaic some time to render
+      await expect(page).toHaveScreenshot(`${specName}.png`, { maxDiffPixelRatio: 0.05 });
+    });
+  }
+});

packages/vgplot/spec/vitest.config.ts

@@ -3,4 +3,7 @@ import viteConfig from '../../../vite.config.js';
 
 export default defineConfig({
   resolve: viteConfig.resolve,
+  test: {
+    exclude: ['test/visual.test.js'], // Exclude visual tests from default run
+  },
 });

vite.config.docker.js

@@ -0,0 +1,22 @@
+import { defineConfig } from 'vite';
+
+export default defineConfig({
+  server: {
+    watch: {
+      ignored: [
+        '**/.venv/**',
+        '**/node_modules/**',
+        '**/dist/**',
+        '**/build/**',
+        '**/.git/**',
+        '**/docs/**',
+        '**/__pycache__/**',
+        '**/coverage/**',
+        '**/test-results/**'
+      ]
+    },
+    fs: {
+      allow: ['..']
+    }
+  }
+});