diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml
index bd3ca55ba0..8aade18738 100644
--- a/.github/workflows/ci-test.yml
+++ b/.github/workflows/ci-test.yml
@@ -24,10 +24,28 @@ jobs:
env:
CI: true
- name: build and test
+ id: test
run: npm test
+ continue-on-error: true
env:
CI: true
+ - name: Generate Visual Test Report
+ if: always()
+ run: node visual-report.js
+ env:
+ CI: true
+ - name: Upload Visual Test Report
+ if: always()
+ uses: actions/upload-artifact@v4
+ with:
+ name: visual-test-report
+ path: test/unit/visual/visual-report.html
+ retention-days: 14
- name: report test coverage
+ if: steps.test.outcome == 'success'
run: bash <(curl -s https://codecov.io/bash) -f coverage/coverage-final.json
env:
CI: true
+ - name: fail job if tests failed
+ if: steps.test.outcome != 'success'
+ run: exit 1
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index 6d6bb7e175..1504e8b5a4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,4 +23,6 @@ yarn.lock
docs/data.json
analyzer/
preview/
-__screenshots__/
\ No newline at end of file
+__screenshots__/
+actual-screenshots/
+visual-report.html
\ No newline at end of file
diff --git a/test/unit/visual/cases/typography.js b/test/unit/visual/cases/typography.js
index 46fbab5777..dff725c48c 100644
--- a/test/unit/visual/cases/typography.js
+++ b/test/unit/visual/cases/typography.js
@@ -385,6 +385,18 @@ visualSuite("Typography", function () {
});
}
);
+ visualTest("intentionally failing test", function (p5, screenshot) {
+ p5.createCanvas(100, 100);
+ p5.background(255);
+ // initially put a red circle for storing in screenshots
+ p5.fill(255, 0, 0); // Red fill
+ p5.noStroke();
+
+ // Then change circle to rect to make it fail
+ p5.rect(30, 30, 40, 40);
+
+ screenshot();
+ });
});
}
});
diff --git a/test/unit/visual/screenshots/Typography/textAlign/2d mode/intentionally failing test/000.png b/test/unit/visual/screenshots/Typography/textAlign/2d mode/intentionally failing test/000.png
new file mode 100644
index 0000000000..57a75785d1
Binary files /dev/null and b/test/unit/visual/screenshots/Typography/textAlign/2d mode/intentionally failing test/000.png differ
diff --git a/test/unit/visual/screenshots/Typography/textAlign/2d mode/intentionally failing test/metadata.json b/test/unit/visual/screenshots/Typography/textAlign/2d mode/intentionally failing test/metadata.json
new file mode 100644
index 0000000000..2d4bfe30da
--- /dev/null
+++ b/test/unit/visual/screenshots/Typography/textAlign/2d mode/intentionally failing test/metadata.json
@@ -0,0 +1,3 @@
+{
+ "numScreenshots": 1
+}
\ No newline at end of file
diff --git a/test/unit/visual/screenshots/Typography/textAlign/webgl mode/intentionally failing test/000.png b/test/unit/visual/screenshots/Typography/textAlign/webgl mode/intentionally failing test/000.png
new file mode 100644
index 0000000000..57a75785d1
Binary files /dev/null and b/test/unit/visual/screenshots/Typography/textAlign/webgl mode/intentionally failing test/000.png differ
diff --git a/test/unit/visual/screenshots/Typography/textAlign/webgl mode/intentionally failing test/metadata.json b/test/unit/visual/screenshots/Typography/textAlign/webgl mode/intentionally failing test/metadata.json
new file mode 100644
index 0000000000..2d4bfe30da
--- /dev/null
+++ b/test/unit/visual/screenshots/Typography/textAlign/webgl mode/intentionally failing test/metadata.json
@@ -0,0 +1,3 @@
+{
+ "numScreenshots": 1
+}
\ No newline at end of file
diff --git a/test/unit/visual/visualTest.js b/test/unit/visual/visualTest.js
index 120ce79565..257643ee00 100644
--- a/test/unit/visual/visualTest.js
+++ b/test/unit/visual/visualTest.js
@@ -441,9 +441,15 @@ export function visualTest(
: [];
for (let i = 0; i < actual.length; i++) {
+ const flatName = name.replace(/\//g, '-');
+ const actualFilename = `../actual-screenshots/${flatName}-${i.toString().padStart(3, '0')}.png`;
if (expected[i]) {
const result = await checkMatch(actual[i], expected[i], myp5);
+ // Always save the actual image before potentially throwing an error
+ writeImageFile(actualFilename, toBase64(actual[i]));
if (!result.ok) {
+ const diffFilename = `../actual-screenshots/${flatName}-${i.toString().padStart(3, '0')}-diff.png`;
+ writeImageFile(diffFilename, toBase64(result.diff));
throw new Error(
`Screenshots do not match! Expected:\n${toBase64(expected[i])}\n\nReceived:\n${toBase64(actual[i])}\n\nDiff:\n${toBase64(result.diff)}\n\n` +
'If this is unexpected, paste these URLs into your browser to inspect them.\n\n' +
@@ -452,6 +458,7 @@ export function visualTest(
}
} else {
writeImageFile(expectedFilenames[i], toBase64(actual[i]));
+ writeImageFile(actualFilename, toBase64(actual[i]));
}
}
});
diff --git a/visual-report.js b/visual-report.js
new file mode 100644
index 0000000000..1989c0b72a
--- /dev/null
+++ b/visual-report.js
@@ -0,0 +1,427 @@
+const fs = require('fs');
+const path = require('path');
+
+async function generateVisualReport() {
+ const expectedDir = path.join(process.cwd(), 'test/unit/visual/screenshots');
+ const actualDir = path.join(process.cwd(), 'test/unit/visual/actual-screenshots');
+ const outputFile = path.join(process.cwd(), 'test/unit/visual/visual-report.html');
+
+ // Make sure the output directory exists
+ const outputDir = path.dirname(outputFile);
+ if (!fs.existsSync(outputDir)) {
+ fs.mkdirSync(outputDir, { recursive: true });
+ }
+
+ // Function to read image file and convert to data URL
+ function imageToDataURL(filePath) {
+ try {
+ const data = fs.readFileSync(filePath);
+ const base64 = data.toString('base64');
+ return `data:image/png;base64,${base64}`;
+ } catch (error) {
+ console.error(`Failed to read image: ${filePath}`, error);
+ return null;
+ }
+ }
+
+ // Create a lookup map for actual screenshots
+ function createActualScreenshotMap() {
+ const actualMap = new Map();
+ if (!fs.existsSync(actualDir)) {
+ console.warn(`Actual screenshots directory does not exist: ${actualDir}`);
+ return actualMap;
+ }
+
+ const files = fs.readdirSync(actualDir);
+ for (const file of files) {
+ if (file.endsWith('.png') && !file.endsWith('-diff.png')) {
+ actualMap.set(file, path.join(actualDir, file));
+ }
+ }
+
+ return actualMap;
+ }
+
+ const actualScreenshotMap = createActualScreenshotMap();
+
+ // Recursively find all test cases
+ function findTestCases(dir, prefix = '') {
+ const testCases = [];
+
+ if (!fs.existsSync(path.join(dir, prefix))) {
+ console.warn(`Directory does not exist: ${path.join(dir, prefix)}`);
+ return testCases;
+ }
+
+ const entries = fs.readdirSync(path.join(dir, prefix), { withFileTypes: true });
+
+ for (const entry of entries) {
+ const fullPath = path.join(prefix, entry.name);
+
+ if (entry.isDirectory()) {
+ // Recursively search subdirectories
+ testCases.push(...findTestCases(dir, fullPath));
+ } else if (entry.name === 'metadata.json') {
+ // Found a test case
+ const metadataPath = path.join(dir, fullPath);
+ let metadata;
+
+ try {
+ metadata = JSON.parse(fs.readFileSync(metadataPath, 'utf8'));
+ } catch (error) {
+ console.error(`Failed to read metadata: ${metadataPath}`, error);
+ continue;
+ }
+
+ const testDir = path.dirname(fullPath);
+
+ const test = {
+ name: testDir,
+ numScreenshots: metadata.numScreenshots || 0,
+ screenshots: []
+ };
+
+ // Create flattened name for lookup
+ const flattenedName = testDir.replace(/\//g, '-');
+
+ // Collect all screenshots for this test
+ for (let i = 0; i < test.numScreenshots; i++) {
+ const screenshotName = i.toString().padStart(3, '0') + '.png';
+ const expectedPath = path.join(dir, testDir, screenshotName);
+
+ // Use flattened name for actual screenshots
+ const actualScreenshotName = `${flattenedName}-${i.toString().padStart(3, '0')}.png`;
+ const actualPath = actualScreenshotMap.get(actualScreenshotName) || null;
+
+ // Use flattened name for diff image
+ const diffScreenshotName = `${flattenedName}-${i.toString().padStart(3, '0')}-diff.png`;
+ const diffPath = path.join(actualDir, diffScreenshotName);
+
+ const hasExpected = fs.existsSync(expectedPath);
+ const hasActual = actualPath && fs.existsSync(actualPath);
+ const hasDiff = fs.existsSync(diffPath);
+
+ const screenshot = {
+ index: i,
+ expectedImage: hasExpected ? imageToDataURL(expectedPath) : null,
+ actualImage: hasActual ? imageToDataURL(actualPath) : null,
+ diffImage: hasDiff ? imageToDataURL(diffPath) : null,
+ passed: hasExpected && hasActual && !hasDiff
+ };
+
+ test.screenshots.push(screenshot);
+ }
+
+ // Don't add tests with no screenshots
+ if (test.screenshots.length > 0) {
+ testCases.push(test);
+ }
+ }
+ }
+
+ return testCases;
+ }
+
+ // Find all test cases from the expected directory
+ const testCases = findTestCases(expectedDir);
+
+ if (testCases.length === 0) {
+ console.warn('No test cases found. Check if the expected directory is correct.');
+ }
+
+ // Count passed/failed tests and screenshots
+ const totalTests = testCases.length;
+ let passedTests = 0;
+ let totalScreenshots = 0;
+ let passedScreenshots = 0;
+
+ for (const test of testCases) {
+ const testPassed = test.screenshots.every(screenshot => screenshot.passed);
+ if (testPassed) passedTests++;
+
+ totalScreenshots += test.screenshots.length;
+ passedScreenshots += test.screenshots.filter(s => s.passed).length;
+ }
+
+ // Generate HTML
+ const html = `
+
+
+
+
+
+ p5.js Visual Test Results
+
+
+
+
+
+
+
Summary
+
+ Total Tests: ${totalTests}
+ Passed Tests: ${passedTests} (${totalTests > 0 ? Math.round(passedTests/totalTests*100) : 0}%)
+ Failed Tests: ${totalTests - passedTests} (${totalTests > 0 ? Math.round((totalTests-passedTests)/totalTests*100) : 0}%)
+ Total Screenshots: ${totalScreenshots}
+ Passed Screenshots: ${passedScreenshots} (${totalScreenshots > 0 ? Math.round(passedScreenshots/totalScreenshots*100) : 0}%)
+ Report Generated: ${new Date().toLocaleString()}
+
+
+ ${testCases.map(test => {
+ const passed = test.screenshots.every(s => s.passed);
+ return `
+
+
+
+ ${test.screenshots.map(screenshot => `
+
+
+
+
+
+ ${screenshot.expectedImage ?
+ `
` :
+ `
No expected image found
`}
+
+
+
+ ${screenshot.actualImage ?
+ `
` :
+ `
No actual image found
`}
+
+ ${screenshot.diffImage ? `
+
+
+
+
+ ` : ''}
+
+
+ `).join('')}
+
+
+ `;
+ }).join('')}
+