puppeteer.js 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628
  1. import chalk from 'chalk';
  2. import puppeteer from 'puppeteer-core';
  3. import { install, computeExecutablePath/*, resolveBuildId*/, detectBrowserPlatform } from '@puppeteer/browsers';
  4. import express from 'express';
  5. import path from 'path';
  6. import pixelmatch from 'pixelmatch';
  7. import jimp from 'jimp';
  8. import * as fs from 'fs/promises';
  9. class PromiseQueue {
  10. constructor( func, ...args ) {
  11. this.func = func.bind( this, ...args );
  12. this.promises = [];
  13. }
  14. add( ...args ) {
  15. const promise = this.func( ...args );
  16. this.promises.push( promise );
  17. promise.then( () => this.promises.splice( this.promises.indexOf( promise ), 1 ) );
  18. }
  19. async waitForAll() {
  20. while ( this.promises.length > 0 ) {
  21. await Promise.all( this.promises );
  22. }
  23. }
  24. }
  25. /* CONFIG VARIABLES START */
  26. const idleTime = 9; // 9 seconds - for how long there should be no network requests
  27. const parseTime = 6; // 6 seconds per megabyte
  28. const exceptionList = [
  29. // video tag isn't deterministic enough?
  30. 'css3d_youtube',
  31. 'webgl_materials_video',
  32. 'webgl_video_kinect',
  33. 'webgl_video_panorama_equirectangular',
  34. 'webaudio_visualizer', // audio can't be analyzed without proper audio hook
  35. // WebXR also isn't determinstic enough?
  36. 'webxr_ar_lighting',
  37. 'webxr_vr_sandbox',
  38. 'webxr_vr_video',
  39. 'webxr_xr_ballshooter',
  40. 'webgl_worker_offscreencanvas', // in a worker, not robust
  41. // Windows-Linux text rendering differences
  42. // TODO: Fix these by e.g. disabling text rendering altogether -- this can also fix a bunch of 0.1%-0.2% examples
  43. 'css3d_periodictable',
  44. 'misc_controls_pointerlock',
  45. 'misc_uv_tests',
  46. 'webgl_camera_logarithmicdepthbuffer',
  47. 'webgl_effects_ascii',
  48. 'webgl_geometry_extrude_shapes',
  49. 'webgl_interactive_lines',
  50. 'webgl_loader_collada_kinematics',
  51. 'webgl_loader_ldraw',
  52. 'webgl_loader_pdb',
  53. 'webgl_modifier_simplifier',
  54. 'webgl_multiple_canvases_circle',
  55. 'webgl_multiple_elements_text',
  56. // Unknown
  57. // TODO: most of these can be fixed just by increasing idleTime and parseTime
  58. 'webgl_animation_skinning_blending',
  59. 'webgl_buffergeometry_glbufferattribute',
  60. 'webgl_clipping_advanced',
  61. 'webgl_lensflares',
  62. 'webgl_lights_spotlights',
  63. 'webgl_loader_imagebitmap',
  64. 'webgl_loader_texture_lottie',
  65. 'webgl_loader_texture_pvrtc',
  66. 'webgl_materials_blending',
  67. 'webgl_mirror',
  68. 'webgl_morphtargets_face',
  69. 'webgl_nodes_materials_standard',
  70. 'webgl_postprocessing_crossfade',
  71. 'webgl_postprocessing_dof2',
  72. 'webgl_raymarching_reflect',
  73. 'webgl_renderer_pathtracer',
  74. 'webgl_shadowmap',
  75. 'webgl_shadowmap_progressive',
  76. 'webgl_test_memory2',
  77. 'webgl_tiled_forward',
  78. // TODO: implement determinism for setTimeout and setInterval
  79. // could it fix some examples from above?
  80. 'physics_rapier_instancing',
  81. // Awaiting for WebGPU support
  82. 'webgpu_audio_processing',
  83. 'webgpu_backdrop',
  84. 'webgpu_backdrop_area',
  85. 'webgpu_clearcoat',
  86. 'webgpu_compute',
  87. 'webgpu_cubemap_adjustments',
  88. 'webgpu_cubemap_dynamic',
  89. 'webgpu_cubemap_mix',
  90. 'webgpu_depth_texture',
  91. 'webgpu_equirectangular',
  92. 'webgpu_instance_mesh',
  93. 'webgpu_instance_uniform',
  94. 'webgpu_lights_custom',
  95. 'webgpu_lights_ies_spotlight',
  96. 'webgpu_lights_phong',
  97. 'webgpu_lights_selective',
  98. 'webgpu_loader_gltf',
  99. 'webgpu_loader_gltf_compressed',
  100. 'webgpu_loader_gltf_sheen',
  101. 'webgpu_materials',
  102. 'webgpu_materials_video',
  103. 'webgpu_morphtargets',
  104. 'webgpu_particles',
  105. 'webgpu_rtt',
  106. 'webgpu_sandbox',
  107. 'webgpu_shadowmap',
  108. 'webgpu_skinning',
  109. 'webgpu_skinning_instancing',
  110. 'webgpu_skinning_points',
  111. 'webgpu_sprites',
  112. 'webgpu_tsl_editor',
  113. 'webgpu_video_panorama'
  114. ];
  115. /* CONFIG VARIABLES END */
  116. const chromiumChannel = 'stable'; // stable -- beta -- dev -- canary -- latest
  117. const installedBrowsersDir = 'test/e2e/chromium';
  118. const port = 1234;
  119. const pixelThreshold = 0.1; // threshold error in one pixel
  120. const maxDifferentPixels = 0.3; // at most 0.3% different pixels
  121. const networkTimeout = 5; // 5 minutes, set to 0 to disable
  122. const renderTimeout = 5; // 5 seconds, set to 0 to disable
  123. const numAttempts = 2; // perform 2 attempts before failing
  124. const numPages = 8; // use 8 browser pages
  125. const numCIJobs = 4; // GitHub Actions run the script in 4 threads
  126. const width = 400;
  127. const height = 250;
  128. const viewScale = 2;
  129. const jpgQuality = 95;
  130. console.red = msg => console.log( chalk.red( msg ) );
  131. console.yellow = msg => console.log( chalk.yellow( msg ) );
  132. console.green = msg => console.log( chalk.green( msg ) );
  133. let browser, platform;
  134. /* Launch server */
  135. const app = express();
  136. app.use( express.static( path.resolve() ) );
  137. const server = app.listen( port, main );
  138. process.on( 'SIGINT', () => close() );
  139. async function main() {
  140. /* Create output directory */
  141. try { await fs.rm( 'test/e2e/output-screenshots', { recursive: true, force: true } ); } catch {}
  142. try { await fs.mkdir( 'test/e2e/output-screenshots' ); } catch {}
  143. /* Find files */
  144. const isMakeScreenshot = process.argv[ 2 ] === '--make';
  145. const exactList = process.argv.slice( isMakeScreenshot ? 3 : 2 )
  146. .map( f => f.replace( '.html', '' ) );
  147. const isExactList = exactList.length !== 0;
  148. let files = ( await fs.readdir( 'examples' ) )
  149. .filter( s => s.slice( - 5 ) === '.html' && s !== 'index.html' )
  150. .map( s => s.slice( 0, s.length - 5 ) )
  151. .filter( f => isExactList ? exactList.includes( f ) : ! exceptionList.includes( f ) );
  152. if ( isExactList ) {
  153. for ( const file of exactList ) {
  154. if ( ! files.includes( file ) ) {
  155. console.log( `Warning! Unrecognised example name: ${ file }` );
  156. }
  157. }
  158. }
  159. /* CI parallelism */
  160. if ( 'CI' in process.env ) {
  161. const CI = parseInt( process.env.CI );
  162. files = files.slice(
  163. Math.floor( CI * files.length / numCIJobs ),
  164. Math.floor( ( CI + 1 ) * files.length / numCIJobs )
  165. );
  166. }
  167. /* Download browser */
  168. const executablePath = await downloadLatestChromium();
  169. /* Launch browser */
  170. const flags = [ '--hide-scrollbars', '--enable-gpu' ];
  171. // flags.push( '--enable-unsafe-webgpu', '--enable-features=Vulkan', '--use-gl=swiftshader', '--use-angle=swiftshader', '--use-vulkan=swiftshader', '--use-webgpu-adapter=swiftshader' );
  172. // if ( process.platform === 'linux' ) flags.push( '--enable-features=Vulkan,UseSkiaRenderer', '--use-vulkan=native', '--disable-vulkan-surface', '--disable-features=VaapiVideoDecoder', '--ignore-gpu-blocklist', '--use-angle=vulkan' );
  173. const viewport = { width: width * viewScale, height: height * viewScale };
  174. browser = await puppeteer.launch( {
  175. executablePath,
  176. headless: process.env.VISIBLE ? false : 'new',
  177. args: flags,
  178. defaultViewport: viewport,
  179. handleSIGINT: false,
  180. protocolTimeout: 0
  181. } );
  182. // this line is intended to stop the script if the browser (in headful mode) is closed by user (while debugging)
  183. // browser.on( 'targetdestroyed', target => ( target.type() === 'other' ) ? close() : null );
  184. // for some reason it randomly stops the script after about ~30 screenshots processed
  185. /* Prepare injections */
  186. const cleanPage = await fs.readFile( 'test/e2e/clean-page.js', 'utf8' );
  187. const injection = await fs.readFile( 'test/e2e/deterministic-injection.js', 'utf8' );
  188. const build = ( await fs.readFile( 'build/three.module.js', 'utf8' ) ).replace( /Math\.random\(\) \* 0xffffffff/g, 'Math._random() * 0xffffffff' );
  189. /* Prepare pages */
  190. const errorMessagesCache = [];
  191. const pages = await browser.pages();
  192. while ( pages.length < numPages && pages.length < files.length ) pages.push( await browser.newPage() );
  193. for ( const page of pages ) await preparePage( page, injection, build, errorMessagesCache );
  194. /* Loop for each file */
  195. const failedScreenshots = [];
  196. const queue = new PromiseQueue( makeAttempt, pages, failedScreenshots, cleanPage, isMakeScreenshot );
  197. for ( const file of files ) queue.add( file );
  198. await queue.waitForAll();
  199. /* Finish */
  200. failedScreenshots.sort();
  201. const list = failedScreenshots.join( ' ' );
  202. if ( isMakeScreenshot && failedScreenshots.length ) {
  203. console.red( 'List of failed screenshots: ' + list );
  204. console.red( `If you are sure that everything is correct, try to run "npm run make-screenshot ${ list }". If this does not help, try increasing idleTime and parseTime variables in /test/e2e/puppeteer.js file. If this also does not help, add remaining screenshots to the exception list.` );
  205. console.red( `${ failedScreenshots.length } from ${ files.length } screenshots have not generated succesfully.` );
  206. } else if ( isMakeScreenshot && ! failedScreenshots.length ) {
  207. console.green( `${ files.length } screenshots succesfully generated.` );
  208. } else if ( failedScreenshots.length ) {
  209. console.red( 'List of failed screenshots: ' + list );
  210. console.red( `If you are sure that everything is correct, try to run "npm run make-screenshot ${ list }". If this does not help, try increasing idleTime and parseTime variables in /test/e2e/puppeteer.js file. If this also does not help, add remaining screenshots to the exception list.` );
  211. console.red( `TEST FAILED! ${ failedScreenshots.length } from ${ files.length } screenshots have not rendered correctly.` );
  212. } else {
  213. console.green( `TEST PASSED! ${ files.length } screenshots rendered correctly.` );
  214. }
  215. setTimeout( close, 300, failedScreenshots.length );
  216. }
  217. async function downloadLatestChromium() {
  218. platform = detectBrowserPlatform();
  219. const revision = '1108766'; //await resolveBuildId( 'chromium', platform, chromiumChannel );
  220. // the Chromium snapshots server doesn't work properly currently so fix the revision
  221. const options = { browser: 'chromium', buildId: revision, cacheDir: path.resolve( installedBrowsersDir ) };
  222. console.log( `Using Chromium r${ revision }, ${ chromiumChannel } channel on ${ platform }` );
  223. console.log( 'Downloading...' );
  224. await install( options );
  225. console.log( 'Downloaded.' );
  226. return computeExecutablePath( options );
  227. }
  228. async function preparePage( page, injection, build, errorMessages ) {
  229. /* let page.file, page.pageSize, page.error */
  230. await page.evaluateOnNewDocument( injection );
  231. await page.setRequestInterception( true );
  232. page.on( 'console', async msg => {
  233. const type = msg.type();
  234. if ( type !== 'warning' && type !== 'error' ) {
  235. return;
  236. }
  237. const file = page.file;
  238. if ( file === undefined ) {
  239. return;
  240. }
  241. const args = await Promise.all( msg.args().map( async arg => {
  242. try {
  243. return await arg.executionContext().evaluate( arg => arg instanceof Error ? arg.message : arg, arg );
  244. } catch ( e ) { // Execution context might have been already destroyed
  245. return arg;
  246. }
  247. } ) );
  248. let text = args.join( ' ' ); // https://github.com/puppeteer/puppeteer/issues/3397#issuecomment-434970058
  249. text = text.trim();
  250. if ( text === '' ) return;
  251. text = file + ': ' + text.replace( /\[\.WebGL-(.+?)\] /g, '' );
  252. if ( text === `${ file }: JSHandle@error` ) {
  253. text = `${ file }: Unknown error`;
  254. }
  255. if ( text.includes( 'Unable to access the camera/webcam' ) ) {
  256. return;
  257. }
  258. if ( errorMessages.includes( text ) ) {
  259. return;
  260. }
  261. errorMessages.push( text );
  262. if ( type === 'warning' ) {
  263. console.yellow( text );
  264. } else {
  265. page.error = text;
  266. }
  267. } );
  268. page.on( 'response', async ( response ) => {
  269. try {
  270. if ( response.status === 200 ) {
  271. await response.buffer().then( buffer => page.pageSize += buffer.length );
  272. }
  273. } catch {}
  274. } );
  275. page.on( 'request', async ( request ) => {
  276. if ( request.url() === `http://localhost:${ port }/build/three.module.js` ) {
  277. await request.respond( {
  278. status: 200,
  279. contentType: 'application/javascript; charset=utf-8',
  280. body: build
  281. } );
  282. } else {
  283. await request.continue();
  284. }
  285. } );
  286. }
  287. async function makeAttempt( pages, failedScreenshots, cleanPage, isMakeScreenshot, file, attemptID = 0 ) {
  288. const page = await new Promise( ( resolve, reject ) => {
  289. const interval = setInterval( () => {
  290. for ( const page of pages ) {
  291. if ( page.file === undefined ) {
  292. page.file = file; // acquire lock
  293. clearInterval( interval );
  294. resolve( page );
  295. break;
  296. }
  297. }
  298. }, 100 );
  299. } );
  300. try {
  301. page.pageSize = 0;
  302. page.error = undefined;
  303. /* Load target page */
  304. try {
  305. await page.goto( `http://localhost:${ port }/examples/${ file }.html`, {
  306. waitUntil: 'networkidle0',
  307. timeout: networkTimeout * 60000
  308. } );
  309. } catch ( e ) {
  310. throw new Error( `Error happened while loading file ${ file }: ${ e }` );
  311. }
  312. try {
  313. /* Render page */
  314. await page.evaluate( cleanPage );
  315. await page.waitForNetworkIdle( {
  316. timeout: networkTimeout * 60000,
  317. idleTime: idleTime * 1000
  318. } );
  319. await page.evaluate( async ( renderTimeout, parseTime ) => {
  320. await new Promise( resolve => setTimeout( resolve, parseTime ) );
  321. /* Resolve render promise */
  322. window._renderStarted = true;
  323. await new Promise( function ( resolve, reject ) {
  324. const renderStart = performance._now();
  325. const waitingLoop = setInterval( function () {
  326. const renderTimeoutExceeded = ( renderTimeout > 0 ) && ( performance._now() - renderStart > 1000 * renderTimeout );
  327. if ( renderTimeoutExceeded ) {
  328. clearInterval( waitingLoop );
  329. reject( 'Render timeout exceeded' );
  330. } else if ( window._renderFinished ) {
  331. clearInterval( waitingLoop );
  332. resolve();
  333. }
  334. }, 10 );
  335. } );
  336. }, renderTimeout, page.pageSize / 1024 / 1024 * parseTime * 1000 );
  337. } catch ( e ) {
  338. if ( ! e.message.includes( 'Render timeout exceeded' ) ) {
  339. throw new Error( `Error happened while rendering file ${ file }: ${ e }` );
  340. } /* else { // This can mean that the example doesn't use requestAnimationFrame loop
  341. console.yellow( `Render timeout exceeded in file ${ file }` );
  342. } */ // TODO: fix this
  343. }
  344. const screenshot = ( await jimp.read( await page.screenshot() ) ).scale( 1 / viewScale ).quality( jpgQuality );
  345. if ( page.error !== undefined ) throw new Error( page.error );
  346. if ( isMakeScreenshot ) {
  347. /* Make screenshots */
  348. await screenshot.writeAsync( `examples/screenshots/${ file }.jpg` );
  349. console.green( `Screenshot generated for file ${ file }` );
  350. } else {
  351. /* Diff screenshots */
  352. let expected;
  353. try {
  354. expected = ( await jimp.read( `examples/screenshots/${ file }.jpg` ) ).quality( jpgQuality );
  355. } catch {
  356. await screenshot.writeAsync( `test/e2e/output-screenshots/${ platform }-${ file }-actual.jpg` );
  357. throw new Error( `Screenshot does not exist: ${ file }` );
  358. }
  359. const actual = screenshot.bitmap;
  360. const diff = screenshot.clone();
  361. let numDifferentPixels;
  362. try {
  363. numDifferentPixels = pixelmatch( expected.bitmap.data, actual.data, diff.bitmap.data, actual.width, actual.height, {
  364. threshold: pixelThreshold,
  365. alpha: 0.2
  366. } );
  367. } catch {
  368. await screenshot.writeAsync( `test/e2e/output-screenshots/${ platform }-${ file }-actual.jpg` );
  369. await expected.writeAsync( `test/e2e/output-screenshots/${ platform }-${ file }-expected.jpg` );
  370. throw new Error( `Image sizes does not match in file: ${ file }` );
  371. }
  372. /* Print results */
  373. const differentPixels = numDifferentPixels / ( actual.width * actual.height ) * 100;
  374. if ( differentPixels < maxDifferentPixels ) {
  375. console.green( `Diff ${ differentPixels.toFixed( 1 ) }% in file: ${ file }` );
  376. } else {
  377. await screenshot.writeAsync( `test/e2e/output-screenshots/${ platform }-${ file }-actual.jpg` );
  378. await expected.writeAsync( `test/e2e/output-screenshots/${ platform }-${ file }-expected.jpg` );
  379. await diff.writeAsync( `test/e2e/output-screenshots/${ platform }-${ file }-diff.jpg` );
  380. throw new Error( `Diff wrong in ${ differentPixels.toFixed( 1 ) }% of pixels in file: ${ file }` );
  381. }
  382. }
  383. } catch ( e ) {
  384. if ( attemptID === numAttempts - 1 ) {
  385. console.red( e );
  386. failedScreenshots.push( file );
  387. } else {
  388. console.yellow( `${ e }, another attempt...` );
  389. this.add( file, attemptID + 1 );
  390. }
  391. }
  392. page.file = undefined; // release lock
  393. }
  394. function close( exitCode = 1 ) {
  395. console.log( 'Closing...' );
  396. if ( browser !== undefined ) browser.close();
  397. server.close();
  398. process.exit( exitCode );
  399. }