Improve WebXR Layers example. (#22144)

* Add 4 eye charts to demonstrate the effects of mipmaps on text.
* Add a GUI to demonstrate interacting with layers.
* Improve info text and add more comments.
* Add a preview screenshot.

Tested:
  * Oculus Quest 2 shows layers properly.
  * Chrome on Windows shows error message.

Co-authored-by: Sameer Padala <[email protected]>

examples/webxr_vr_layers.html

@@ -11,12 +11,21 @@
 		<div id="info">
 			<a href="https://threejs.org" target="_blank" rel="noopener">three.js</a> media and projection layers<br/>
 			(Oculus Browser 16.1+)
+
+			<p> This example demonstrates the use of <a href="https://www.w3.org/TR/webxrlayers-1/">WebXR Layers</a> to render high quality text and video.
+				  For static content such as text, using layers increases the usable resolution of the content by avoiding the extra resampling pass that occurs during normal VR rendering.
+				  For dynamic content such as video, using layers also improves performance by only copying data when new frames are available.	 </p>
+			<br />
+			<p><i>See the comments in the code for more information.</i></p>
+
 		</div>
 
 		<script type="module">
 
 			import * as THREE from '../build/three.module.js';
-			import { OrbitControls } from './jsm/controls/OrbitControls.js';
+			import { GUI } from './jsm/libs/dat.gui.module.js';
+			import { HTMLMesh } from './jsm/interactive/HTMLMesh.js';
+			import { InteractiveGroup } from './jsm/interactive/InteractiveGroup.js';
 			import { VRButton } from './jsm/webxr/VRButton.js';
 			import { XRControllerModelFactory } from './jsm/webxr/XRControllerModelFactory.js';
 			import { XRHandModelFactory } from './jsm/webxr/XRHandModelFactory.js';
@@ -26,14 +35,37 @@
 			let hand1, hand2;
 			let controller1, controller2;
 			let controllerGrip1, controllerGrip2;
-
-			let controls;
 			let video;
+
+			// Four eye charts are rendered to demonstrate the differences in text quality. The two
+			// charts on the bottom are rendered to the eye buffer while the two charts on the top are
+			// rendered into XRQuadLayers, and the latter pair are substantially more legible.
+			//
+			// The two charts on the left are rendered without mipmaps and have aliasing artifacts while
+			// the two charts on the right are with mipmaps an don't twinkle but are blurrier. To
+			// maximize text legibility, it's important to choose a texture size optimized for the
+			// distance of the text. (This example intentionally uses incorrectly large textures to
+			// demonstrate this issue.) If the optimal text size can't be determined beforehand, then
+			// mipmaps are required to avoid aliasing.
+			//
+			// The background of the scene is an equirectangular layer. It uses an XRMediaBinding to
+			// render the contents of a video element into the scene. This example uses a low resolution
+			// video to avoid large files, but using media layers allows video that is higher resolution than normal rendering.
+
 			let snellenTexture;
-			let quadLayer;
+			let quadLayerPlain;
+			let quadLayerMips;
+			let guiLayer;
+			let guiMesh;
+			let errorMesh;
+
+			// Set via GUI.
+			const parameters = {
+				eyeChartDistanceFt: 20,
+			};
 
-			// Data shared between the THREE.Mesh on the left side and WebXR Layer on the right side for
-			// the eye chart. See https://en.wikipedia.org/wiki/Snellen_chart for details about the math.
+			// Data shared between the THREE.Meshes on the bottom and WebXR Layers on the top for the eye
+			// charts. See https://en.wikipedia.org/wiki/Snellen_chart for details about the math.
 			//
 			// The image was designed so that each 2x2px block on the 20/20 line subtends 1 minute of
 			// arc. That is
@@ -45,19 +77,29 @@
 			// legible when using layers. Without layers, you lose ~sqrt(2) in resolution due to the
 			// extra resampling.
 			const snellenConfig = {
-				// The height & width of the WebXR layer needs to match the given image size.
+				// The texture is a power of two so that mipmaps can be generated.
+				textureSizePx: 512,
+				// This is the valid part of the image.
 				widthPx: 320,
 				heightPx: 450,
 
-				x: 0, // +/- widthMeters/2
+				x: 0,
 				y: 1.5,
-
 				z: - 6.1, // 20/20 vision @ 20ft = 6.1m
 
+				// This is the size of mesh and the visible part of the quad layer.
 				widthMeters: .268, // 320px image * (142mm/160px scale factor)
 				heightMeters: .382 // 450px image * (142mm/160px scale factor)
 			};
 
+			snellenConfig.cropX = snellenConfig.widthPx / snellenConfig.textureSizePx;
+			snellenConfig.cropY = snellenConfig.heightPx / snellenConfig.textureSizePx;
+
+			// The quad layer is a [-1, 1] quad but only a part of it has image data. Scale the layer so
+			// that the part with image data is the same size as the mesh.
+			snellenConfig.quadWidth = .5 * snellenConfig.widthMeters / snellenConfig.cropX;
+			snellenConfig.quadHeight = .5 * snellenConfig.heightMeters / snellenConfig.cropY;
+
 			init();
 			animate();
 
@@ -71,9 +113,6 @@
 				camera = new THREE.PerspectiveCamera( 50, window.innerWidth / window.innerHeight, 0.1, 10 );
 				camera.position.set( 0, 1.6, 3 );
 
-				controls = new OrbitControls( camera, container );
-				controls.target.set( 0, 1.6, 0 );
-				controls.update();
 
 				scene.add( new THREE.HemisphereLight( 0x808080, 0x606060 ) );
 
@@ -91,8 +130,8 @@
 
 				renderer = new THREE.WebGLRenderer( { antialias: false, alpha: true } );
 				renderer.setPixelRatio( window.devicePixelRatio );
-				renderer.setClearAlpha(1);
-				renderer.setClearColor(new THREE.Color(0), 0);
+				renderer.setClearAlpha( 1 );
+				renderer.setClearColor( new THREE.Color( 0 ), 0 );
 				renderer.setSize( window.innerWidth, window.innerHeight );
 				renderer.outputEncoding = THREE.sRGBEncoding;
 				renderer.shadowMap.enabled = true;
@@ -136,30 +175,129 @@
 
 				const geometry = new THREE.BufferGeometry().setFromPoints( [ new THREE.Vector3( 0, 0, 0 ), new THREE.Vector3( 0, 0, - 1 ) ] );
 
-				const line = new THREE.Line( geometry );
+				const line = new THREE.Line( geometry, new THREE.LineBasicMaterial( { color: 0x5555ff } ) );
 				line.name = 'line';
-				line.scale.z = 5;
+				line.scale.z = 10;
 
 				controller1.add( line.clone() );
 				controller2.add( line.clone() );
 
-				// Eye chart
+				// Eye charts
+				const eyeCharts = new THREE.Group();
+				eyeCharts.position.z = snellenConfig.z;
+				scene.add( eyeCharts );
+
 				snellenTexture = new THREE.TextureLoader().load( "textures/snellen.png" );
-				const snellenMesh = new THREE.Mesh(
+				snellenTexture.repeat.x = snellenConfig.cropX;
+				snellenTexture.repeat.y = snellenConfig.cropY;
+				snellenTexture.generateMipmaps = false;
+				snellenTexture.minFilter = THREE.LinearFilter;
+				const snellenMeshPlain = new THREE.Mesh(
 					new THREE.PlaneGeometry( snellenConfig.widthMeters, snellenConfig.heightMeters ),
 					new THREE.MeshBasicMaterial( { map: snellenTexture } ) );
-				snellenMesh.position.x = snellenConfig.x - snellenConfig.widthMeters / 2;
-				snellenMesh.position.y = snellenConfig.y;
-				snellenMesh.position.z = snellenConfig.z;
-				scene.add( snellenMesh );
+				snellenMeshPlain.position.x = snellenConfig.x - snellenConfig.widthMeters;
+				snellenMeshPlain.position.y = snellenConfig.y - snellenConfig.heightMeters;
+				eyeCharts.add( snellenMeshPlain );
 
-				//
+				snellenTexture = new THREE.TextureLoader().load( "textures/snellen.png" );
+				snellenTexture.repeat.x = snellenConfig.cropX;
+				snellenTexture.repeat.y = snellenConfig.cropY;
+				const snellenMeshMipMap = new THREE.Mesh(
+					new THREE.PlaneGeometry( snellenConfig.widthMeters, snellenConfig.heightMeters ),
+					new THREE.MeshBasicMaterial( { map: snellenTexture } ) );
+				snellenMeshMipMap.position.x = snellenConfig.x + snellenConfig.widthMeters;
+				snellenMeshMipMap.position.y = snellenConfig.y - snellenConfig.heightMeters;
+				eyeCharts.add( snellenMeshMipMap );
+
+				// The layers don't participate depth testing between each other. Since the projection
+				// layer is rendered last, any 3D object will incorrecly overlap layers. To avoid this,
+				// invisible quads can be placed into the scene to participate in depth testing when the
+				// projection layer is rendered.
+				const dummyMeshLeft = new THREE.Mesh(
+					new THREE.PlaneGeometry( snellenConfig.widthMeters, snellenConfig.heightMeters ),
+					new THREE.MeshBasicMaterial( { opacity: 0 } ) );
+				dummyMeshLeft.position.x = snellenConfig.x - snellenConfig.widthMeters;
+				dummyMeshLeft.position.y = snellenConfig.y + snellenConfig.heightMeters;
+				eyeCharts.add( dummyMeshLeft );
+
+				const dummyMeshRight = dummyMeshLeft.clone( true );
+				dummyMeshRight.position.x = snellenConfig.x + snellenConfig.widthMeters;
+				eyeCharts.add( dummyMeshRight );
+
+				// The GUI is rendered into an invisible HTMLMesh and the backing canvas's data is copied
+				// into a layer as required. Hit testing and interaction is done using standard HTMLMesh
+				// behavior, but since the layer is in the same place as the invisible mesh, the user
+				// thinks they're directly interacting with the layer.
+				const gui = new GUI( { width: 300 } );
+				gui.add( parameters, 'eyeChartDistanceFt', 1.0, 20.0 ).onChange( onChange );
+				gui.domElement.style.visibility = 'hidden';
+
+				function onChange() {
+
+					eyeCharts.position.z = - parameters.eyeChartDistanceFt * 0.3048;
+
+					if ( quadLayerPlain ) {
+
+						quadLayerPlain.transform = new XRRigidTransform( {
+							x: snellenConfig.x - snellenConfig.widthMeters,
+							y: snellenConfig.y + snellenConfig.heightMeters,
+							z: eyeCharts.position.z
+						} );
+
+					}
+
+					if ( quadLayerMips ) {
+
+						quadLayerMips.transform = new XRRigidTransform( {
+							x: snellenConfig.x + snellenConfig.widthMeters,
+							y: snellenConfig.y + snellenConfig.heightMeters,
+							z: eyeCharts.position.z
+						} );
+
+					}
+
+					guiLayer.needsUpdate = true;
+
+				}
+
+				const group = new InteractiveGroup( renderer, camera );
+				scene.add( group );
+
+				guiMesh = new HTMLMesh( gui.domElement );
+				guiMesh.position.x = 1.0;
+				guiMesh.position.y = 1.5;
+				guiMesh.position.z = - 1.0;
+				guiMesh.rotation.y = - Math.PI / 4;
+				guiMesh.scale.setScalar( 2 );
+				guiMesh.material.opacity = 0;
+				group.add( guiMesh );
+
+				// Error message if layer initialization fails.
+				const errorCanvas = document.createElement( 'canvas' );
+				errorCanvas.width = 400;
+				errorCanvas.height = 40;
+				const errorContext = errorCanvas.getContext( '2d' );
+				errorContext.fillStyle = '#FF0000';
+				errorContext.fillRect( 0, 0, errorCanvas.width, errorCanvas.height );
+				errorContext.fillStyle = '#000000';
+				errorContext.font = '28px sans-serif';
+				errorContext.fillText( 'ERROR: Layers not initialized!', 10, 30 );
+
+				errorMesh = new THREE.Mesh(
+					new THREE.PlaneGeometry( 1, .1 ),
+					new THREE.MeshBasicMaterial( { map: new THREE.CanvasTexture( errorCanvas ) } )
+				);
+				errorMesh.position.z = - 1;
+				errorMesh.position.y = 1.5;
+				errorMesh.visible = false;
+				scene.add( errorMesh );
 
 				window.addEventListener( 'resize', onWindowResize, false );
 
-				video = document.createElement('video');
+				video = document.createElement( 'video' );
 				video.loop = true;
 				video.src = 'textures/MaryOculus.webm';
+
 			}
 
 			function onWindowResize() {
@@ -185,42 +323,117 @@
 				const gl = renderer.getContext();
 
 				// Init layers once in immersive mode and video is ready.
+				if ( session && session.renderState.layers === undefined ) {
+
+					errorMesh.visible = true;
+
+				}
+
 				if ( session && session.renderState.layers !== undefined && session.hasMediaLayer === undefined && video.readyState >= 2 ) {
 
 					session.hasMediaLayer = true;
-					session.requestReferenceSpace( 'local' ).then( ( refSpace ) => {
+					session.requestReferenceSpace( 'local-floor' ).then( ( refSpace ) => {
 
-						// Create Quad layer for Snellen chart.
+						// Create Quad layers for Snellen chart.
 						const glBinding = new XRWebGLBinding( session, gl );
-						quadLayer = glBinding.createQuadLayer( {
-							width: snellenConfig.widthMeters / 2,
-							height: snellenConfig.heightMeters / 2,
-							viewPixelWidth: snellenConfig.widthPx,
-							viewPixelHeight: snellenConfig.heightPx,
+						const quadLayerConfig = {
+							width: snellenConfig.quadWidth,
+							height: snellenConfig.quadHeight,
+							viewPixelWidth: snellenConfig.textureSizePx,
+							viewPixelHeight: snellenConfig.textureSizePx,
+							isStatic: true,
 							space: refSpace, layout: "mono",
-							transform: new XRRigidTransform( { x: snellenConfig.x + snellenConfig.widthMeters / 2, y: snellenConfig.y, z: snellenConfig.z } )
+							transform: new XRRigidTransform( {
+								x: snellenConfig.x - snellenConfig.widthMeters,
+								y: snellenConfig.y + snellenConfig.heightMeters,
+								z: snellenConfig.z
+							} )
+
+						};
+
+						quadLayerPlain = glBinding.createQuadLayer( quadLayerConfig );
+
+						quadLayerConfig.mipLevels = 3;
+						quadLayerConfig.transform = new XRRigidTransform(	{
+							x: snellenConfig.x + snellenConfig.widthMeters,
+							y: snellenConfig.y + snellenConfig.heightMeters,
+							z: snellenConfig.z
+						} );
+						quadLayerMips = glBinding.createQuadLayer( quadLayerConfig );
+
+						// Create GUI layer.
+						guiLayer = glBinding.createQuadLayer( {
+							width: guiMesh.geometry.parameters.width,
+							height: guiMesh.geometry.parameters.height,
+							viewPixelWidth: guiMesh.material.map.image.width,
+							viewPixelHeight: guiMesh.material.map.image.height,
+							space: refSpace,
+							transform: new XRRigidTransform( guiMesh.position, guiMesh.quaternion )
 						} );
 
 						// Create background EQR video layer.
 						const mediaBinding = new XRMediaBinding( session );
-						const equirectLayer = mediaBinding.createEquirectLayer( video, { space: refSpace, layout: "stereo-left-right" } );
-
-						session.updateRenderState( { layers: [ equirectLayer, quadLayer, session.renderState.layers[ 0 ] ] } );
+						const equirectLayer = mediaBinding.createEquirectLayer(
+							video,
+							{
+								space: refSpace,
+								layout: "stereo-left-right",
+								// Rotate by 45 deg to avoid stereo conflict with the 3D geometry.
+								transform: new XRRigidTransform(
+									{},
+									{ x: 0, y: .28, z: 0, w: .96 } )
+							}
+						);
+
+						errorMesh.visible = false;
+						session.updateRenderState( { layers: [ equirectLayer, quadLayerPlain, quadLayerMips, guiLayer, session.renderState.layers[ 0 ] ] } );
 						video.play();
 
 					} );
 
 				}
 
-				// Copy image to canvas as required.
+				// Copy image to layers as required.
 				// needsRedraw is set on creation or if the underlying GL resources of a layer are lost.
-				if ( quadLayer && quadLayer.needsRedraw ) {
+				if ( quadLayerPlain && quadLayerPlain.needsRedraw ) {
 
 					const glBinding = new XRWebGLBinding( session, gl );
-					const glayer = glBinding.getSubImage( quadLayer, frame );
+					const glayer = glBinding.getSubImage( quadLayerPlain, frame );
 					renderer.state.bindTexture( gl.TEXTURE_2D, glayer.colorTexture );
 					gl.pixelStorei( gl.UNPACK_FLIP_Y_WEBGL, true );
-					gl.texSubImage2D( gl.TEXTURE_2D, 0, 0, 0, gl.RGBA, gl.UNSIGNED_BYTE, snellenTexture.image );
+					gl.texSubImage2D( gl.TEXTURE_2D, 0,
+									( snellenConfig.textureSizePx - snellenConfig.widthPx ) / 2,
+									( snellenConfig.textureSizePx - snellenConfig.heightPx ) / 2,
+									snellenConfig.widthPx, snellenConfig.heightPx,
+									gl.RGBA, gl.UNSIGNED_BYTE, snellenTexture.image );
+
+				}
+
+				// Same as above but also gl.generateMipmap.
+				if ( quadLayerMips && quadLayerMips.needsRedraw ) {
+
+					const glBinding = new XRWebGLBinding( session, gl );
+					const glayer = glBinding.getSubImage( quadLayerMips, frame );
+					renderer.state.bindTexture( gl.TEXTURE_2D, glayer.colorTexture );
+					gl.pixelStorei( gl.UNPACK_FLIP_Y_WEBGL, true );
+					gl.texSubImage2D( gl.TEXTURE_2D, 0,
+									( snellenConfig.textureSizePx - snellenConfig.widthPx ) / 2,
+									( snellenConfig.textureSizePx - snellenConfig.heightPx ) / 2,
+									snellenConfig.widthPx, snellenConfig.heightPx,
+									gl.RGBA, gl.UNSIGNED_BYTE, snellenTexture.image );
+					gl.generateMipmap( gl.TEXTURE_2D );
+
+				}
+
+				// Same as above, but guiLayer.needsUpdate is set when the user interacts with the GUI.
+				if ( guiLayer && ( guiLayer.needsRedraw || guiLayer.needsUpdate ) ) {
+
+					const glBinding = new XRWebGLBinding( session, gl );
+					const glayer = glBinding.getSubImage( guiLayer, frame );
+					renderer.state.bindTexture( gl.TEXTURE_2D, glayer.colorTexture );
+					gl.pixelStorei( gl.UNPACK_FLIP_Y_WEBGL, true );
+					const canvas = guiMesh.material.map.image;
+					gl.texSubImage2D( gl.TEXTURE_2D, 0, 0, 0, canvas.width, canvas.height, gl.RGBA, gl.UNSIGNED_BYTE, canvas );
 
 				}
 
@@ -228,6 +441,7 @@
 
 			}
 
+
 		</script>
 	</body>
 </html>