puppeteer.js 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609
  1. import chalk from 'chalk';
  2. import puppeteer from 'puppeteer';
  3. import express from 'express';
  4. import path from 'path';
  5. import pixelmatch from 'pixelmatch';
  6. import jimp from 'jimp';
  7. import * as fs from 'fs/promises';
  8. class PromiseQueue {
  9. constructor( func, ...args ) {
  10. this.func = func.bind( this, ...args );
  11. this.promises = [];
  12. }
  13. add( ...args ) {
  14. const promise = this.func( ...args );
  15. this.promises.push( promise );
  16. promise.then( () => this.promises.splice( this.promises.indexOf( promise ), 1 ) );
  17. }
  18. async waitForAll() {
  19. while ( this.promises.length > 0 ) {
  20. await Promise.all( this.promises );
  21. }
  22. }
  23. }
  24. /* CONFIG VARIABLES START */
  25. const idleTime = 9; // 9 seconds - for how long there should be no network requests
  26. const parseTime = 6; // 6 seconds per megabyte
  27. const exceptionList = [
  28. // video tag isn't deterministic enough?
  29. 'css3d_youtube',
  30. 'webgl_materials_video',
  31. 'webgl_video_kinect',
  32. 'webgl_video_panorama_equirectangular',
  33. 'webaudio_visualizer', // audio can't be analyzed without proper audio hook
  34. // WebXR also isn't determinstic enough?
  35. 'webxr_ar_lighting',
  36. 'webxr_vr_sandbox',
  37. 'webxr_vr_video',
  38. 'webxr_xr_ballshooter',
  39. 'webgl_worker_offscreencanvas', // in a worker, not robust
  40. // Windows-Linux text rendering differences
  41. // TODO: Fix these by e.g. disabling text rendering altogether -- this can also fix a bunch of 0.1%-0.2% examples
  42. 'css3d_periodictable',
  43. 'misc_controls_pointerlock',
  44. 'misc_uv_tests',
  45. 'webgl_camera_logarithmicdepthbuffer',
  46. 'webgl_effects_ascii',
  47. 'webgl_geometry_extrude_shapes',
  48. 'webgl_interactive_lines',
  49. 'webgl_loader_collada_kinematics',
  50. 'webgl_loader_ldraw',
  51. 'webgl_loader_pdb',
  52. 'webgl_modifier_simplifier',
  53. 'webgl_multiple_canvases_circle',
  54. 'webgl_multiple_elements_text',
  55. // Unknown
  56. // TODO: most of these can be fixed just by increasing idleTime and parseTime
  57. 'webgl_animation_skinning_blending',
  58. 'webgl_buffergeometry_glbufferattribute',
  59. 'webgl_clipping_advanced',
  60. 'webgl_lensflares',
  61. 'webgl_lights_spotlights',
  62. 'webgl_loader_imagebitmap',
  63. 'webgl_loader_texture_ktx',
  64. 'webgl_loader_texture_lottie',
  65. 'webgl_loader_texture_pvrtc',
  66. 'webgl_materials_alphahash',
  67. 'webgl_materials_blending',
  68. 'webgl_mirror',
  69. 'webgl_morphtargets_face',
  70. 'webgl_nodes_loader_materialx',
  71. 'webgl_nodes_materials_standard',
  72. 'webgl_nodes_materialx_noise',
  73. 'webgl_postprocessing_crossfade',
  74. 'webgl_postprocessing_dof2',
  75. 'webgl_raymarching_reflect',
  76. 'webgl_renderer_pathtracer',
  77. 'webgl_shadowmap',
  78. 'webgl_shadowmap_progressive',
  79. 'webgl_test_memory2',
  80. 'webgl_tiled_forward',
  81. 'webgl2_volume_instancing',
  82. 'webgl2_multisampled_renderbuffers',
  83. 'webgl_points_dynamic',
  84. // TODO: implement determinism for setTimeout and setInterval
  85. // could it fix some examples from above?
  86. 'physics_rapier_instancing',
  87. // Awaiting for WebGL backend support
  88. 'webgpu_clearcoat',
  89. 'webgpu_compute_audio',
  90. 'webgpu_compute_particles',
  91. 'webgpu_compute_particles_rain',
  92. 'webgpu_compute_particles_snow',
  93. 'webgpu_compute_points',
  94. 'webgpu_compute_texture',
  95. 'webgpu_compute_texture_pingpong',
  96. 'webgpu_cubemap_dynamic',
  97. 'webgpu_loader_gltf',
  98. 'webgpu_loader_gltf_compressed',
  99. 'webgpu_loader_gltf_iridescence',
  100. 'webgpu_loader_gltf_sheen',
  101. 'webgpu_materials',
  102. 'webgpu_portal',
  103. 'webgpu_sandbox',
  104. 'webgpu_sprites',
  105. 'webgpu_video_panorama',
  106. 'webgpu_postprocessing_afterimage',
  107. 'webgpu_multisampled_renderbuffers',
  108. // WebGPURenderer: Unknown problem
  109. 'webgpu_backdrop_water',
  110. 'webgpu_camera_logarithmicdepthbuffer',
  111. 'webgpu_loader_materialx',
  112. 'webgpu_materials_video',
  113. 'webgpu_materialx_noise',
  114. 'webgpu_morphtargets_face',
  115. 'webgpu_occlusion',
  116. 'webgpu_particles',
  117. 'webgpu_shadertoy',
  118. 'webgpu_shadowmap',
  119. 'webgpu_tsl_editor',
  120. 'webgpu_tsl_transpiler'
  121. ];
  122. /* CONFIG VARIABLES END */
  123. const port = 1234;
  124. const pixelThreshold = 0.1; // threshold error in one pixel
  125. const maxDifferentPixels = 0.3; // at most 0.3% different pixels
  126. const networkTimeout = 5; // 5 minutes, set to 0 to disable
  127. const renderTimeout = 5; // 5 seconds, set to 0 to disable
  128. const numAttempts = 2; // perform 2 attempts before failing
  129. const numPages = 8; // use 8 browser pages
  130. const numCIJobs = 4; // GitHub Actions run the script in 4 threads
  131. const width = 400;
  132. const height = 250;
  133. const viewScale = 2;
  134. const jpgQuality = 95;
  135. console.red = msg => console.log( chalk.red( msg ) );
  136. console.yellow = msg => console.log( chalk.yellow( msg ) );
  137. console.green = msg => console.log( chalk.green( msg ) );
  138. let browser;
  139. /* Launch server */
  140. const app = express();
  141. app.use( express.static( path.resolve() ) );
  142. const server = app.listen( port, main );
  143. process.on( 'SIGINT', () => close() );
  144. async function main() {
  145. /* Create output directory */
  146. try { await fs.rm( 'test/e2e/output-screenshots', { recursive: true, force: true } ); } catch {}
  147. try { await fs.mkdir( 'test/e2e/output-screenshots' ); } catch {}
  148. /* Find files */
  149. const isMakeScreenshot = process.argv[ 2 ] === '--make';
  150. const exactList = process.argv.slice( isMakeScreenshot ? 3 : 2 )
  151. .map( f => f.replace( '.html', '' ) );
  152. const isExactList = exactList.length !== 0;
  153. let files = ( await fs.readdir( 'examples' ) )
  154. .filter( s => s.slice( - 5 ) === '.html' && s !== 'index.html' )
  155. .map( s => s.slice( 0, s.length - 5 ) )
  156. .filter( f => isExactList ? exactList.includes( f ) : ! exceptionList.includes( f ) );
  157. if ( isExactList ) {
  158. for ( const file of exactList ) {
  159. if ( ! files.includes( file ) ) {
  160. console.log( `Warning! Unrecognised example name: ${ file }` );
  161. }
  162. }
  163. }
  164. /* CI parallelism */
  165. if ( 'CI' in process.env ) {
  166. const CI = parseInt( process.env.CI );
  167. files = files.slice(
  168. Math.floor( CI * files.length / numCIJobs ),
  169. Math.floor( ( CI + 1 ) * files.length / numCIJobs )
  170. );
  171. }
  172. /* Launch browser */
  173. const flags = [ '--hide-scrollbars', '--enable-gpu' ];
  174. // flags.push( '--enable-unsafe-webgpu', '--enable-features=Vulkan', '--use-gl=swiftshader', '--use-angle=swiftshader', '--use-vulkan=swiftshader', '--use-webgpu-adapter=swiftshader' );
  175. // if ( process.platform === 'linux' ) flags.push( '--enable-features=Vulkan,UseSkiaRenderer', '--use-vulkan=native', '--disable-vulkan-surface', '--disable-features=VaapiVideoDecoder', '--ignore-gpu-blocklist', '--use-angle=vulkan' );
  176. const viewport = { width: width * viewScale, height: height * viewScale };
  177. browser = await puppeteer.launch( {
  178. headless: process.env.VISIBLE ? false : 'new',
  179. args: flags,
  180. defaultViewport: viewport,
  181. handleSIGINT: false,
  182. protocolTimeout: 0
  183. } );
  184. // this line is intended to stop the script if the browser (in headful mode) is closed by user (while debugging)
  185. // browser.on( 'targetdestroyed', target => ( target.type() === 'other' ) ? close() : null );
  186. // for some reason it randomly stops the script after about ~30 screenshots processed
  187. /* Prepare injections */
  188. const cleanPage = await fs.readFile( 'test/e2e/clean-page.js', 'utf8' );
  189. const injection = await fs.readFile( 'test/e2e/deterministic-injection.js', 'utf8' );
  190. const build = ( await fs.readFile( 'build/three.module.js', 'utf8' ) ).replace( /Math\.random\(\) \* 0xffffffff/g, 'Math._random() * 0xffffffff' );
  191. /* Prepare pages */
  192. const errorMessagesCache = [];
  193. const pages = await browser.pages();
  194. while ( pages.length < numPages && pages.length < files.length ) pages.push( await browser.newPage() );
  195. for ( const page of pages ) await preparePage( page, injection, build, errorMessagesCache );
  196. /* Loop for each file */
  197. const failedScreenshots = [];
  198. const queue = new PromiseQueue( makeAttempt, pages, failedScreenshots, cleanPage, isMakeScreenshot );
  199. for ( const file of files ) queue.add( file );
  200. await queue.waitForAll();
  201. /* Finish */
  202. failedScreenshots.sort();
  203. const list = failedScreenshots.join( ' ' );
  204. if ( isMakeScreenshot && failedScreenshots.length ) {
  205. console.red( 'List of failed screenshots: ' + list );
  206. console.red( `If you are sure that everything is correct, try to run "npm run make-screenshot ${ list }". If this does not help, try increasing idleTime and parseTime variables in /test/e2e/puppeteer.js file. If this also does not help, add remaining screenshots to the exception list.` );
  207. console.red( `${ failedScreenshots.length } from ${ files.length } screenshots have not generated succesfully.` );
  208. } else if ( isMakeScreenshot && ! failedScreenshots.length ) {
  209. console.green( `${ files.length } screenshots succesfully generated.` );
  210. } else if ( failedScreenshots.length ) {
  211. console.red( 'List of failed screenshots: ' + list );
  212. console.red( `If you are sure that everything is correct, try to run "npm run make-screenshot ${ list }". If this does not help, try increasing idleTime and parseTime variables in /test/e2e/puppeteer.js file. If this also does not help, add remaining screenshots to the exception list.` );
  213. console.red( `TEST FAILED! ${ failedScreenshots.length } from ${ files.length } screenshots have not rendered correctly.` );
  214. } else {
  215. console.green( `TEST PASSED! ${ files.length } screenshots rendered correctly.` );
  216. }
  217. setTimeout( close, 300, failedScreenshots.length );
  218. }
  219. async function preparePage( page, injection, build, errorMessages ) {
  220. /* let page.file, page.pageSize, page.error */
  221. await page.evaluateOnNewDocument( injection );
  222. await page.setRequestInterception( true );
  223. page.on( 'console', async msg => {
  224. const type = msg.type();
  225. if ( type !== 'warning' && type !== 'error' ) {
  226. return;
  227. }
  228. const file = page.file;
  229. if ( file === undefined ) {
  230. return;
  231. }
  232. const args = await Promise.all( msg.args().map( async arg => {
  233. try {
  234. return await arg.executionContext().evaluate( arg => arg instanceof Error ? arg.message : arg, arg );
  235. } catch ( e ) { // Execution context might have been already destroyed
  236. return arg;
  237. }
  238. } ) );
  239. let text = args.join( ' ' ); // https://github.com/puppeteer/puppeteer/issues/3397#issuecomment-434970058
  240. text = text.trim();
  241. if ( text === '' ) return;
  242. text = file + ': ' + text.replace( /\[\.WebGL-(.+?)\] /g, '' );
  243. if ( text === `${ file }: JSHandle@error` ) {
  244. text = `${ file }: Unknown error`;
  245. }
  246. if ( text.includes( 'Unable to access the camera/webcam' ) ) {
  247. return;
  248. }
  249. if ( errorMessages.includes( text ) ) {
  250. return;
  251. }
  252. errorMessages.push( text );
  253. if ( type === 'warning' ) {
  254. console.yellow( text );
  255. } else {
  256. page.error = text;
  257. }
  258. } );
  259. page.on( 'response', async ( response ) => {
  260. try {
  261. if ( response.status === 200 ) {
  262. await response.buffer().then( buffer => page.pageSize += buffer.length );
  263. }
  264. } catch {}
  265. } );
  266. page.on( 'request', async ( request ) => {
  267. if ( request.url() === `http://localhost:${ port }/build/three.module.js` ) {
  268. await request.respond( {
  269. status: 200,
  270. contentType: 'application/javascript; charset=utf-8',
  271. body: build
  272. } );
  273. } else {
  274. await request.continue();
  275. }
  276. } );
  277. }
  278. async function makeAttempt( pages, failedScreenshots, cleanPage, isMakeScreenshot, file, attemptID = 0 ) {
  279. const page = await new Promise( ( resolve, reject ) => {
  280. const interval = setInterval( () => {
  281. for ( const page of pages ) {
  282. if ( page.file === undefined ) {
  283. page.file = file; // acquire lock
  284. clearInterval( interval );
  285. resolve( page );
  286. break;
  287. }
  288. }
  289. }, 100 );
  290. } );
  291. try {
  292. page.pageSize = 0;
  293. page.error = undefined;
  294. /* Load target page */
  295. try {
  296. await page.goto( `http://localhost:${ port }/examples/${ file }.html`, {
  297. waitUntil: 'networkidle0',
  298. timeout: networkTimeout * 60000
  299. } );
  300. } catch ( e ) {
  301. throw new Error( `Error happened while loading file ${ file }: ${ e }` );
  302. }
  303. try {
  304. /* Render page */
  305. await page.evaluate( cleanPage );
  306. await page.waitForNetworkIdle( {
  307. timeout: networkTimeout * 60000,
  308. idleTime: idleTime * 1000
  309. } );
  310. await page.evaluate( async ( renderTimeout, parseTime ) => {
  311. await new Promise( resolve => setTimeout( resolve, parseTime ) );
  312. /* Resolve render promise */
  313. window._renderStarted = true;
  314. await new Promise( function ( resolve, reject ) {
  315. const renderStart = performance._now();
  316. const waitingLoop = setInterval( function () {
  317. const renderTimeoutExceeded = ( renderTimeout > 0 ) && ( performance._now() - renderStart > 1000 * renderTimeout );
  318. if ( renderTimeoutExceeded ) {
  319. clearInterval( waitingLoop );
  320. reject( 'Render timeout exceeded' );
  321. } else if ( window._renderFinished ) {
  322. clearInterval( waitingLoop );
  323. resolve();
  324. }
  325. }, 10 );
  326. } );
  327. }, renderTimeout, page.pageSize / 1024 / 1024 * parseTime * 1000 );
  328. } catch ( e ) {
  329. if ( e.includes && e.includes( 'Render timeout exceeded' ) === false ) {
  330. throw new Error( `Error happened while rendering file ${ file }: ${ e }` );
  331. } /* else { // This can mean that the example doesn't use requestAnimationFrame loop
  332. console.yellow( `Render timeout exceeded in file ${ file }` );
  333. } */ // TODO: fix this
  334. }
  335. const screenshot = ( await jimp.read( await page.screenshot() ) ).scale( 1 / viewScale ).quality( jpgQuality );
  336. if ( page.error !== undefined ) throw new Error( page.error );
  337. if ( isMakeScreenshot ) {
  338. /* Make screenshots */
  339. await screenshot.writeAsync( `examples/screenshots/${ file }.jpg` );
  340. console.green( `Screenshot generated for file ${ file }` );
  341. } else {
  342. /* Diff screenshots */
  343. let expected;
  344. try {
  345. expected = ( await jimp.read( `examples/screenshots/${ file }.jpg` ) ).quality( jpgQuality );
  346. } catch {
  347. await screenshot.writeAsync( `test/e2e/output-screenshots/${ file }-actual.jpg` );
  348. throw new Error( `Screenshot does not exist: ${ file }` );
  349. }
  350. const actual = screenshot.bitmap;
  351. const diff = screenshot.clone();
  352. let numDifferentPixels;
  353. try {
  354. numDifferentPixels = pixelmatch( expected.bitmap.data, actual.data, diff.bitmap.data, actual.width, actual.height, {
  355. threshold: pixelThreshold,
  356. alpha: 0.2
  357. } );
  358. } catch {
  359. await screenshot.writeAsync( `test/e2e/output-screenshots/${ file }-actual.jpg` );
  360. await expected.writeAsync( `test/e2e/output-screenshots/${ file }-expected.jpg` );
  361. throw new Error( `Image sizes does not match in file: ${ file }` );
  362. }
  363. /* Print results */
  364. const differentPixels = numDifferentPixels / ( actual.width * actual.height ) * 100;
  365. if ( differentPixels < maxDifferentPixels ) {
  366. console.green( `Diff ${ differentPixels.toFixed( 1 ) }% in file: ${ file }` );
  367. } else {
  368. await screenshot.writeAsync( `test/e2e/output-screenshots/${ file }-actual.jpg` );
  369. await expected.writeAsync( `test/e2e/output-screenshots/${ file }-expected.jpg` );
  370. await diff.writeAsync( `test/e2e/output-screenshots/${ file }-diff.jpg` );
  371. throw new Error( `Diff wrong in ${ differentPixels.toFixed( 1 ) }% of pixels in file: ${ file }` );
  372. }
  373. }
  374. } catch ( e ) {
  375. if ( attemptID === numAttempts - 1 ) {
  376. console.red( e );
  377. failedScreenshots.push( file );
  378. } else {
  379. console.yellow( `${ e }, another attempt...` );
  380. this.add( file, attemptID + 1 );
  381. }
  382. }
  383. page.file = undefined; // release lock
  384. }
  385. function close( exitCode = 1 ) {
  386. console.log( 'Closing...' );
  387. browser.close();
  388. server.close();
  389. process.exit( exitCode );
  390. }