three.js/test/e2e/puppeteer.js

import puppeteer from 'puppeteer';
import handler from 'serve-handler';
import http from 'http';
import pixelmatch from 'pixelmatch';
import jimp from 'jimp';
import fs from 'fs';

const port = 1234;
const pixelThreshold = 0.1; // threshold error in one pixel
const maxFailedPixels = 0.05; // total failed pixels

const networkTimeout = 600;
const networkTax = 2000; // additional timeout for resources size
const pageSizeMinTax = 1.0; // in mb, when networkTax = 0
const pageSizeMaxTax = 5.0; // in mb, when networkTax = networkTax
const renderTimeout = 1200;
const maxAttemptId = 3; // progresseve attempts
const progressFunc = n => 1 + n;

const width = 400;
const height = 250;
const viewScale = 2;
const jpgQuality = 95;

const exceptionList = [

	'index',
	'css3d_youtube', // video tag not deterministic enough
	'webaudio_visualizer', // audio can't be analyzed without proper audio hook
	'webgl_effects_ascii', // blink renders text differently in every platform
	'webgl_loader_imagebitmap', // takes too long to load?
	'webgl_loader_texture_lottie', // not sure why this fails
	'webgl_loader_texture_pvrtc', // not supported in CI, useless
	'webgl_morphtargets_face', // To investigate...
	'webgl_nodes_materials_standard', // puppeteer does not support import maps yet
	'webgl_postprocessing_crossfade', // fails for some misterious reason
	'webgl_raymarching_reflect', // exception for Github Actions
	'webgl_renderer_pathtracer', // slow to render
	'webgl_test_memory2', // gives fatal error in puppeteer
	'webgl_tiled_forward', // exception for Github Actions
	'webgl_video_kinect', // video tag not deterministic enough
	'webgl_video_panorama_equirectangular', // video tag not deterministic enough?
	'webgl_worker_offscreencanvas', // in a worker, not robust
	// webxr
	'webxr_ar_lighting'
];

console.green = ( msg ) => console.log( `\x1b[32m${ msg }\x1b[37m` );
console.red = ( msg ) => console.log( `\x1b[31m${ msg }\x1b[37m` );
console.null = () => {};


/* Launch server */

const server = http.createServer( ( req, resp ) => handler( req, resp ) );
server.listen( port, async () => await pup );
server.on( 'SIGINT', () => process.exit( 1 ) );


/* Launch browser */

const pup = puppeteer.launch( {
	headless: ! process.env.VISIBLE,
	args: [
		'--use-gl=swiftshader',
		'--no-sandbox',
		'--enable-surface-synchronization',

		'--enable-unsafe-webgpu',
		'--enable-features=Vulkan',
		'--use-angle=swiftshader',
		'--use-vulkan=swiftshader',
		'--use-webgpu-adapter=swiftshader'
	]
} ).then( async browser => {


	/* Prepare page */

	const page = ( await browser.pages() )[ 0 ];
	await page.setViewport( { width: width * viewScale, height: height * viewScale } );

	const cleanPage = fs.readFileSync( 'test/e2e/clean-page.js', 'utf8' );
	const injection = fs.readFileSync( 'test/e2e/deterministic-injection.js', 'utf8' );
	await page.evaluateOnNewDocument( injection );

	const threeJsBuild = fs.readFileSync( 'build/three.module.js', 'utf8' )
		.replace( /Math\.random\(\) \* 0xffffffff/g, 'Math._random() * 0xffffffff' );
	await page.setRequestInterception( true );

	page.on( 'console', msg => ( msg.text().slice( 0, 8 ) === 'Warning.' ) ? console.null( msg.text() ) : {} );
	page.on( 'request', async ( request ) => {

		if ( request.url() === 'http://localhost:1234/build/three.module.js' ) {

			await request.respond( {
				status: 200,
				contentType: 'application/javascript; charset=utf-8',
				body: threeJsBuild
			} );

		} else {

			await request.continue();

		}

	} );
	page.on( 'response', async ( response ) => {

		try {

			await response.buffer().then( buffer => pageSize += buffer.length );

		} catch ( e ) {

			console.null( `Warning. Wrong request. \n${ e }` );

		}

	} );


	/* Find files */

	const isMakeScreenshot = process.argv[ 2 ] == '--make';
	const isExactList = process.argv.length > ( 2 + isMakeScreenshot );

	const exactList = process.argv.slice( isMakeScreenshot ? 3 : 2 )
		.map( f => f.replace( '.html', '' ) );

	const files = fs.readdirSync( './examples' )
		.filter( s => s.slice( - 5 ) === '.html' )
		.map( s => s.slice( 0, s.length - 5 ) )
		.filter( f => isExactList ? exactList.includes( f ) : ! exceptionList.includes( f ) );


	/* Loop for each file, with CI parallelism */

	let pageSize, file, attemptProgress;
	const failedScreenshots = [];

	let beginId = 0;
	let endId = files.length;

	if ( 'CI' in process.env ) {

		const jobs = 8;

		beginId = Math.floor( parseInt( process.env.CI.slice( 0, 1 ) ) * files.length / jobs );
		endId = Math.floor( ( parseInt( process.env.CI.slice( - 1 ) ) + 1 ) * files.length / jobs );

	}

	for ( let id = beginId; id < endId; ++ id ) {

		/* At least 3 attempts before fail */

		let attemptId = isMakeScreenshot ? 1.5 : 0;

		while ( attemptId < maxAttemptId ) {

			/* Load target page */

			file = files[ id ];
			attemptProgress = progressFunc( attemptId );
			pageSize = 0;

			try {

				await page.goto( `http://localhost:${ port }/examples/${ file }.html`, {
					waitUntil: 'networkidle2',
					timeout: networkTimeout * attemptProgress
				} );

			} catch {

				console.null( 'Warning. Network timeout exceeded...' );

			}

			try {

				/* Render page */

				await page.evaluate( cleanPage );

				await page.evaluate( async ( pageSize, pageSizeMinTax, pageSizeMaxTax, networkTax, renderTimeout, attemptProgress ) => {


					/* Resource timeout */

					const resourcesSize = Math.min( 1, ( pageSize / 1024 / 1024 - pageSizeMinTax ) / pageSizeMaxTax );
					await new Promise( resolve => setTimeout( resolve, networkTax * resourcesSize * attemptProgress ) );


					/* Resolve render promise */

					window._renderStarted = true;

					await new Promise( function ( resolve ) {

						performance._now = performance._now || performance.now;

						const renderStart = performance._now();

						const waitingLoop = setInterval( function () {

							const renderEcceded = ( performance._now() - renderStart > renderTimeout * attemptProgress );
							if ( window._renderFinished || renderEcceded ) {

								if ( renderEcceded ) {

									console.log( 'Warning. Render timeout exceeded...' );

								}

								clearInterval( waitingLoop );
								resolve();

							}

						}, 0 );

					} );

				}, pageSize, pageSizeMinTax, pageSizeMaxTax, networkTax, renderTimeout, attemptProgress );

			} catch ( e ) {

				if ( ++ attemptId === maxAttemptId ) {

					console.red( `Something completely wrong. 'Network timeout' is small for your machine. file: ${ file } \n${ e }` );
					failedScreenshots.push( file );
					continue;

				} else {

					console.log( 'Another attempt..' );
					await new Promise( resolve => setTimeout( resolve, networkTimeout * attemptProgress ) );

				}

			}


			if ( isMakeScreenshot ) {


				/* Make screenshots */

				attemptId = maxAttemptId;
				( await jimp.read( await page.screenshot() ) )
					.scale( 1 / viewScale ).quality( jpgQuality )
					.write( `./examples/screenshots/${ file }.jpg` );

				console.green( `file: ${ file } generated` );


			} else if ( fs.existsSync( `./examples/screenshots/${ file }.jpg` ) ) {


				/* Diff screenshots */

				const actual = ( await jimp.read( await page.screenshot() ) ).scale( 1 / viewScale ).quality( jpgQuality ).bitmap;
				const expected = ( await jimp.read( fs.readFileSync( `./examples/screenshots/${ file }.jpg` ) ) ).bitmap;
				const diff = actual;

				let numFailedPixels;

				try {

					numFailedPixels = pixelmatch( expected.data, actual.data, diff.data, actual.width, actual.height, {
						threshold: pixelThreshold,
						alpha: 0.2,
						diffMask: process.env.FORCE_COLOR === '0',
						diffColor: process.env.FORCE_COLOR === '0' ? [ 255, 255, 255 ] : [ 255, 0, 0 ]
					} );

				} catch {

					attemptId = maxAttemptId;
					console.red( `Something completely wrong. Image sizes does not match in file: ${ file }` );
					failedScreenshots.push( file );
					continue;

				}

				numFailedPixels /= actual.width * actual.height;

				/* Print results */
				const percFailedPixels = 100 * numFailedPixels;
				if ( numFailedPixels < maxFailedPixels ) {

					attemptId = maxAttemptId;
					console.green( `diff: ${ percFailedPixels.toFixed( 1 ) }%, file: ${ file }` );

				} else {

					if ( ++ attemptId === maxAttemptId ) {

						console.red( `ERROR! Diff wrong in ${ percFailedPixels.toFixed( 1 ) }% of pixels in file: ${ file }` );
						failedScreenshots.push( file );
						continue;

					} else {

						console.log( 'Another attempt...' );

					}

				}

			} else {

				attemptId = maxAttemptId;
				console.log( `Warning! Screenshot not exists: ${ file }` );
				continue;

			}

		}

	}


	/* Finish */

	if ( failedScreenshots.length ) {

		if ( failedScreenshots.length > 1 ) {

			console.red( 'List of failed screenshots: ' + failedScreenshots.join( ' ' ) );

		} else {

			console.red( `If you sure that all is right, try to run \`npm run make-screenshot ${ failedScreenshots[ 0 ] }\`` );

		}

		console.red( `TEST FAILED! ${ failedScreenshots.length } from ${ endId - beginId } screenshots not pass.` );

	} else if ( ! isMakeScreenshot ) {

		console.green( `TEST PASSED! ${ endId - beginId } screenshots correctly rendered.` );

	}

	setTimeout( () => {

		server.close();
		browser.close();
		process.exit( failedScreenshots.length );

	}, 300 );

} );