Tests: refactor

.travis.yml

@@ -7,21 +7,21 @@ jobs:
   include:
 
     - &lint-n-unit
-      stage: lint & unit
+      stage: Lint & Unit
       name: lint
       script: npm run test-lint
     - <<: *lint-n-unit
       name: unit
       script: npm run test-unit
 
-    - &diff
-      stage: diff
-      name: diff
-      script: npm run test-diff
+    - &e2e
+      stage: E2E
+      name: e2e
+      script: npm run test-e2e
       env: FORCE_COLOR=0 CI=0
-    - <<: *diff
+    - <<: *e2e
       env: FORCE_COLOR=0 CI=1
-    - <<: *diff
+    - <<: *e2e
       env: FORCE_COLOR=0 CI=2
-    - <<: *diff
+    - <<: *e2e
       env: FORCE_COLOR=0 CI=3

package.json

@@ -56,8 +56,8 @@
     "test-lint": "eslint src --ext js --ext ts && tsc -p utils/build/tsconfig.lint.json",
     "test-lint-examples": "eslint examples/jsm --ext js --ext ts --ignore-pattern libs && tsc -p utils/build/tsconfig-examples.lint.json",
     "test-unit": "npm run build-test && qunit -r failonlyreporter test/unit/three.source.unit.js",
-    "test-diff": "node --expose-gc test/diff/puppeteer.js",
-    "make-screenshot": "cross-env MAKE=true npm run test-diff"
+    "test-e2e": "node --expose-gc test/e2e/puppeteer.js",
+    "make-screenshot": "cross-env MAKE=true npm run test-e2e"
   },
   "keywords": [
     "three",

test/diff/README.md → test/e2e/README.md

@@ -1,6 +1,8 @@
-# Three.js automatic regression testing with CI
+# Three.js end-to-end testing
 
-You probably shouldn't run this tests on PC because right now it's not optimized for local usage and you can get different results on different GPUs. Goal is to make quick automated testing inside CI and keep screenshot pack updated for it.
+You probably shouldn't run this tests on PC because it's not optimized for local usage and
+you can get different results on different GPUs. Goal is to make quick automated testing
+inside CI and keep screenshot pack updated for it.
 
 ### Local usage
 ```shell
@@ -8,13 +10,20 @@ You probably shouldn't run this tests on PC because right now it's not optimized
 npm run make-screenshot <example1_name> ... <exampleN_name>
 
 # check examples
-npm run test-diff <example1_name> ... <exampleN_name>
+npm run test-e2e <example1_name> ... <exampleN_name>
 
 # check all examples in browser
-npx cross-env VISIBLE=ture npm run test-diff
+npx cross-env VISIBLE=ture npm run test-e2e
 ```
 
 ### How it works
+- ci configs with parallelism
+- deterministic random/timer/rAF/video for screenshots
+- increased robustness with hided text, datgui, different flags and timeouts.
+- pipeline: turn off rAF -> 'networkidle0' -> networkTax -> turn on rAF -> render promise
+- added 3 progressive attempts for robustness
+
+### Development progress
 
 |           Travis                        |               Attempts               |
 |-----------------------------------------|--------------------------------------|
@@ -24,17 +33,15 @@ npx cross-env VISIBLE=ture npm run test-diff
 | 4=0+0+2+2 failed, time=5:13             | with network tax and other settings  |
 | 4=0+0+2+2 failed, time=3:26             | with progressive attempts            |
 
-- ci configs with parallelism
-- deterministic random/timer/rAF/video for screenshots
-- increased robustness with hided text, datgui, different flags and timeouts.
-- pipeline: turn off rAF -> 'networkidle0' -> networkTax -> turn on rAF -> render promise
-- added 3 progressive attempts for robustness
-
 ### Status
-97% examples are covered with tests. Random robusness in CI >93%. Robustness on different machines ~97%. For example in Windows webgl_effects_ascii example always fails or on integrated GPU you will have additional artifacts: webgl_materials_texture_anisotropy, webgl_postprocessing_procedural, webgl_shaders_tonemapping.
+97% examples are covered with tests. Random robusness in CI >93%. Robustness on different machines ~97%.
+For example on integrated GPU you can have additional artifacts: webgl_materials_texture_anisotropy,
+webgl_postprocessing_procedural, webgl_shaders_tonemapping.
 
-### Probably wrong screenshots
-webgl2_multisampled_renderbuffers, webgl_simple_gi, webgl_postprocessing_dof2, webgl_loader_texture_pvrtc
+### Wrong screenshots but ok for CI
+webgl_loader_bvh, webgl_simple_gi, webgl_postprocessing_dof2, webgl_loader_texture_pvrtc, webgl_physics_volume
 
 ### Contribution
-You can help to simplify puppeteer script by suggesting example with [HeadlessExperimental.beginFrame](https://chromedevtools.github.io/devtools-protocol/tot/HeadlessExperimental) CDP API.
+Proper determinism for video/audio is welcome. You can cover more examples: `dof2` can be rendered with 2 rAF,
+`offsceencanvas`/`webgl_test_memory2` with additional puppeteer flags, `webgl_simple_gi` with
+[beginFrame](https://chromedevtools.github.io/devtools-protocol/tot/HeadlessExperimental) CDP API.

test/e2e/clean-page.js

@@ -0,0 +1,45 @@
+/**
+ * @author munrocket / https://github.com/munrocket
+ */
+
+( function () {
+
+
+  /* Start VR pages */
+
+  let button = document.getElementById( 'startButton' );
+  if ( button ) {
+
+    button.click();
+
+  }
+
+
+  /* Remove dat.gui and fonts */
+
+  let style = document.createElement( 'style' );
+  style.type = 'text/css';
+  style.innerHTML = `body { font size: 0 !important; }
+      #info, button, input, body > div.dg.ac, body > div.lbl { display: none !important; }`;
+  let head = document.getElementsByTagName( 'head' );
+  if ( head.length > 0 ) {
+
+    head[ 0 ].appendChild( style );
+
+  }
+
+
+  /* Remove stats.js */
+
+  let canvas = document.getElementsByTagName( 'canvas' );
+  for ( let i = 0; i < canvas.length; ++ i ) {
+
+    if ( canvas[ i ].height === 48 ) {
+
+      canvas[ i ].style.display = 'none';
+
+    }
+
+  }
+
+}() );

test/diff/deterministic-injection.js → test/e2e/deterministic-injection.js

@@ -17,11 +17,8 @@
 
 	/* Deterministic timer */
 
-	const now = function () {
-
-		return frameId * 16;
-
-	};
+	let frameId = 0;
+	const now = () => frameId * 16;
 	window.Date.now = now;
 	window.Date.prototype.getTime = now;
 	window.performance.wow = performance.now;
@@ -30,7 +27,6 @@
 
 	/* Deterministic RAF */
 
-	let frameId = 0;
 	window.chromeMaxFrameId = 1;
 	window.chromeRenderStarted = false;
 	window.chromeRenderFinished = false;

test/diff/puppeteer.js → test/e2e/puppeteer.js

@@ -86,7 +86,8 @@ const pup = puppeteer.launch( {
 	const page = ( await browser.pages() )[ 0 ];
 	await page.setViewport( { width: 800, height: 600 } );
 
-	const injection = fs.readFileSync( 'test/diff/deterministic-injection.js', 'utf8' );
+	const cleanPage = fs.readFileSync( 'test/e2e/clean-page.js', 'utf8' );
+	const injection = fs.readFileSync( 'test/e2e/deterministic-injection.js', 'utf8' );
 	await page.evaluateOnNewDocument( injection );
 
 	page.on( 'console', msg => ( msg.text().slice( 0, 8 ) === 'Warning.' ) ? console.null( msg.text() ) : {} );
@@ -157,39 +158,15 @@ const pup = puppeteer.launch( {
 
 			try {
 
-				await page.evaluate( async ( pageSize, pageSizeMinTax, pageSizeMaxTax, networkTax, renderTimeout, attemptProgress ) => {
-
-
-					/* Prepare page */
-
-					let button = document.getElementById( 'startButton' );
-					if ( button ) {
-
-						button.click();
-
-					}
 
-					let style = document.createElement( 'style' );
-					style.type = 'text/css';
-					style.innerHTML = `body { font size: 0 !important; }
-							#info, button, input, body > div.dg.ac, body > div.lbl { display: none !important; }`;
-					let head = document.getElementsByTagName( 'head' );
-					if ( head.length > 0 ) {
+				/* Render page */
 
-						head[ 0 ].appendChild( style );
+				await page.evaluate( cleanPage );
 
-					}
-
-					let canvas = document.getElementsByTagName( 'canvas' );
-					for ( let i = 0; i < canvas.length; ++ i ) {
-
-						if ( canvas[ i ].height === 48 ) {
-
-							canvas[ i ].style.display = 'none';
+				await page.evaluate( async ( pageSize, pageSizeMinTax, pageSizeMaxTax, networkTax, renderTimeout, attemptProgress ) => {
 
-						}
 
-					}
+					/* Resource timeout */
 
 					let resourcesSize = Math.min( 1, ( pageSize / 1024 / 1024 - pageSizeMinTax ) / pageSizeMaxTax );
 					await new Promise( resolve => setTimeout( resolve, networkTax * resourcesSize * attemptProgress ) );
@@ -208,7 +185,7 @@ const pup = puppeteer.launch( {
 						let renderStart = performance.wow();
 						let waitingLoop = setInterval( function () {
 
-							let renderEcceded = ( performance.wow() - renderStart > renderTimeout * window.chromeMaxFrameId * attemptProgress );
+							let renderEcceded = ( performance.wow() - renderStart > renderTimeout * attemptProgress );
 							if ( window.chromeRenderFinished || renderEcceded ) {
 
 								if ( renderEcceded ) {