WebGPURenderer: Added VideoTexture support (#25530)

* WebGPURenderer: Added VideoTexture support

* Examples: Added webgpu_materials_video

* Test if WebGPU is available

* cleanup

* Node: Fix update .getUpdateType()

* Fix updates

* WebGPUBindings: Fix resource.

* WebGPUNodeBuilder: Improve error message.

examples/files.json

@@ -341,6 +341,7 @@
 		"webgpu_lights_selective",
 		"webgpu_loader_gltf",
 		"webgpu_materials",
+		"webgpu_materials_video",
 		"webgpu_nodes_playground",
 		"webgpu_particles",
 		"webgpu_rtt",

examples/jsm/nodes/accessors/TextureNode.js

@@ -106,7 +106,11 @@ class TextureNode extends UniformNode {
 
 				let snippet = null;
 
-				if ( levelNode && levelNode.isNode === true ) {
+				if ( texture.isVideoTexture === true ) {
+
+					snippet = builder.getVideoTexture( textureProperty, uvSnippet );
+
+				} else if ( levelNode && levelNode.isNode === true ) {
 
 					const levelSnippet = levelNode.build( builder, 'float' );
 

examples/jsm/nodes/core/Node.js

@@ -78,7 +78,7 @@ class Node {
 
 	}
 
-	getUpdateType( /*builder*/ ) {
+	getUpdateType() {
 
 		return this.updateType;
 
@@ -424,4 +424,4 @@ export function createNodeFromType( type ) {
 
 	}
 
-};
+}

examples/jsm/nodes/core/NodeBuilder.js

@@ -87,7 +87,7 @@ class NodeBuilder {
 
 		if ( this.nodes.indexOf( node ) === - 1 ) {
 
-			const updateType = node.getUpdateType( this );
+			const updateType = node.getUpdateType();
 
 			if ( updateType !== NodeUpdateType.NONE ) {
 

examples/jsm/nodes/core/NodeFrame.js

@@ -22,7 +22,9 @@ class NodeFrame {
 
 	updateNode( node ) {
 
-		if ( node.updateType === NodeUpdateType.FRAME ) {
+		const updateType = node.getUpdateType();
+
+		if ( updateType === NodeUpdateType.FRAME ) {
 
 			if ( this.updateMap.get( node ) !== this.frameId ) {
 
@@ -32,7 +34,7 @@ class NodeFrame {
 
 			}
 
-		} else if ( node.updateType === NodeUpdateType.OBJECT ) {
+		} else if ( updateType === NodeUpdateType.OBJECT ) {
 
 			node.update( this );
 

examples/jsm/renderers/webgpu/WebGPUBindings.js

@@ -144,6 +144,7 @@ class WebGPUBindings {
 				const texture = binding.getTexture();
 
 				const needsTextureRefresh = textures.updateTexture( texture );
+
 				const textureGPU = textures.getTextureGPU( texture );
 
 				if ( textureGPU !== undefined && binding.textureGPU !== textureGPU || needsTextureRefresh === true ) {
@@ -227,6 +228,10 @@ class WebGPUBindings {
 
 						binding.textureGPU = this.textures.getDefaultCubeTexture();
 
+					} else if ( binding.texture.isVideoTexture ) {
+
+						binding.textureGPU = this.textures.getVideoDefaultTexture();
+
 					} else {
 
 						binding.textureGPU = this.textures.getDefaultTexture();
@@ -235,7 +240,9 @@ class WebGPUBindings {
 
 				}
 
-				entries.push( { binding: bindingPoint, resource: binding.textureGPU.createView( { dimension: binding.dimension } ) } );
+				const resource = binding.textureGPU instanceof GPUTexture ? binding.textureGPU.createView( { dimension: binding.dimension } ) : binding.textureGPU;
+
+				entries.push( { binding: bindingPoint, resource } );
 
 			}
 

examples/jsm/renderers/webgpu/WebGPUTextures.js

@@ -1,5 +1,5 @@
 import { GPUTextureFormat, GPUAddressMode, GPUFilterMode, GPUTextureDimension } from './constants.js';
-import { CubeTexture, Texture, NearestFilter, NearestMipmapNearestFilter, NearestMipmapLinearFilter, LinearFilter, RepeatWrapping, MirroredRepeatWrapping,
+import { VideoTexture, CubeTexture, Texture, NearestFilter, NearestMipmapNearestFilter, NearestMipmapLinearFilter, LinearFilter, RepeatWrapping, MirroredRepeatWrapping,
 	RGBAFormat, RedFormat, RGFormat, RGBA_S3TC_DXT1_Format, RGBA_S3TC_DXT3_Format, RGBA_S3TC_DXT5_Format, UnsignedByteType, FloatType, HalfFloatType, sRGBEncoding
 } from 'three';
 import WebGPUTextureUtils from './WebGPUTextureUtils.js';
@@ -13,6 +13,7 @@ class WebGPUTextures {
 		this.info = info;
 
 		this.defaultTexture = null;
+		this.defaultVideoTexture = null;
 		this.defaultCubeTexture = null;
 		this.defaultSampler = null;
 
@@ -51,6 +52,26 @@ class WebGPUTextures {
 
 	}
 
+	getVideoDefaultTexture() {
+
+		if ( this.defaultVideoTexture === null ) {
+
+			const video = document.getElementById( 'video' );
+
+			const texture = new VideoTexture( video );
+			texture.minFilter = NearestFilter;
+			texture.magFilter = NearestFilter;
+
+			this._uploadVideoTexture( texture );
+
+			this.defaultVideoTexture = this.getTextureGPU( texture );
+
+		}
+
+		return this.defaultVideoTexture;
+
+	}
+
 	getDefaultCubeTexture() {
 
 		if ( this.defaultCubeTexture === null ) {
@@ -122,7 +143,15 @@ class WebGPUTextures {
 
 				//
 
-				needsUpdate = this._uploadTexture( texture );
+				if ( texture.isVideoTexture ) {
+
+					needsUpdate = this._uploadVideoTexture( texture );
+
+				} else {
+
+					needsUpdate = this._uploadTexture( texture );
+
+				}
 
 			}
 
@@ -296,6 +325,23 @@ class WebGPUTextures {
 
 	}
 
+	_uploadVideoTexture( texture ) {
+
+		const device = this.device;
+
+		const textureProperties = this.properties.get( texture );
+
+		const textureGPU = device.importExternalTexture( {
+			source: texture.source.data
+		} );
+
+		textureProperties.textureGPU = textureGPU;
+		//textureProperties.version = texture.version; // @TODO: Force update for now, study a better solution soon using native VideoTexture.update() to fix warns
+
+		return true;
+
+	}
+
 	_uploadTexture( texture ) {
 
 		let needsUpdate = false;
@@ -341,7 +387,6 @@ class WebGPUTextures {
 		if ( textureGPU === undefined ) {
 
 			textureGPU = device.createTexture( textureGPUDescriptor );
-			textureProperties.textureGPU = textureGPU;
 
 			needsUpdate = true;
 
@@ -367,24 +412,23 @@ class WebGPUTextures {
 
 			}
 
-		} else {
+		} else if ( image !== null ) {
 
-			if ( image !== null ) {
+			// assume HTMLImageElement, HTMLCanvasElement or ImageBitmap
 
-				// assume HTMLImageElement, HTMLCanvasElement or ImageBitmap
+			this._getImageBitmap( image, texture ).then( imageBitmap => {
 
-				this._getImageBitmap( image, texture ).then( imageBitmap => {
+				this._copyExternalImageToTexture( imageBitmap, textureGPU );
 
-					this._copyExternalImageToTexture( imageBitmap, textureGPU );
+				if ( needsMipmaps === true ) this._generateMipmaps( textureGPU, textureGPUDescriptor );
 
-					if ( needsMipmaps === true ) this._generateMipmaps( textureGPU, textureGPUDescriptor );
-
-				} );
-
-			}
+			} );
 
 		}
 
+		//
+
+		textureProperties.textureGPU = textureGPU;
 		textureProperties.version = texture.version;
 
 		return needsUpdate;

examples/jsm/renderers/webgpu/nodes/WebGPUNodeBuilder.js

@@ -187,6 +187,20 @@ class WebGPUNodeBuilder extends NodeBuilder {
 
 	}
 
+	getVideoTexture( textureProperty, uvSnippet, shaderStage = this.shaderStage ) {
+
+		if ( shaderStage === 'fragment' ) {
+
+			return `textureSampleBaseClampToEdge( ${textureProperty}, ${textureProperty}_sampler, vec2<f32>( ${uvSnippet}.x, 1.0 - ${uvSnippet}.y ) )`;
+
+		} else {
+
+			console.error( `WebGPURenderer: THREE.VideoTexture does not support ${ shaderStage } shader.` );
+
+		}
+
+	}
+
 	getPropertyName( node, shaderStage = this.shaderStage ) {
 
 		if ( node.isNodeVarying === true && node.needsInterpolation === true ) {
@@ -537,7 +551,17 @@ class WebGPUNodeBuilder extends NodeBuilder {
 
 				}
 
-				bindingSnippets.push( `@group( 0 ) @binding( ${index ++} ) var ${uniform.name} : texture_2d<f32>;` );
+				const texture = uniform.node.value;
+
+				if ( texture.isVideoTexture === true ) {
+
+					bindingSnippets.push( `@group( 0 ) @binding( ${index ++} ) var ${uniform.name} : texture_external;` );
+
+				} else {
+
+					bindingSnippets.push( `@group( 0 ) @binding( ${index ++} ) var ${uniform.name} : texture_2d<f32>;` );
+
+				}
 
 			} else if ( uniform.type === 'cubeTexture' ) {
 

examples/webgpu_materials_video.html

@@ -0,0 +1,273 @@
+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<title>three.js webgpu - materials - video</title>
+		<meta charset="utf-8">
+		<meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
+		<link type="text/css" rel="stylesheet" href="main.css">
+	</head>
+	<body>
+
+		<div id="overlay">
+			<button id="startButton">Play</button>
+		</div>
+		<div id="container"></div>
+
+		<div id="info">
+			<a href="https://threejs.org" target="_blank" rel="noopener">three.js</a> - webgpu video demo<br/>
+			playing <a href="http://durian.blender.org/" target="_blank" rel="noopener">sintel</a> trailer
+		</div>
+
+		<video id="video" loop crossOrigin="anonymous" playsinline style="display:none">
+			<source src="textures/sintel.ogv" type='video/ogg; codecs="theora, vorbis"'>
+			<source src="textures/sintel.mp4" type='video/mp4; codecs="avc1.42E01E, mp4a.40.2"'>
+		</video>
+
+		<!-- Import maps polyfill -->
+		<!-- Remove this when import maps will be widely supported -->
+		<script async src="https://unpkg.com/[email protected]/dist/es-module-shims.js"></script>
+
+		<script type="importmap">
+			{
+				"imports": {
+					"three": "../build/three.module.js",
+					"three/addons/": "./jsm/",
+					"three/nodes": "./jsm/nodes/Nodes.js"
+				}
+			}
+		</script>
+
+		<script type="module">
+
+			import * as THREE from 'three';
+
+			import WebGPU from 'three/addons/capabilities/WebGPU.js';
+			import WebGPURenderer from 'three/addons/renderers/webgpu/WebGPURenderer.js';
+
+			let container;
+
+			let camera, scene, renderer;
+
+			let video, texture, material, mesh;
+
+			let mouseX = 0;
+			let mouseY = 0;
+
+			let windowHalfX = window.innerWidth / 2;
+			let windowHalfY = window.innerHeight / 2;
+
+			let cube_count;
+
+			const meshes = [],
+				materials = [],
+
+				xgrid = 20,
+				ygrid = 10;
+
+			const startButton = document.getElementById( 'startButton' );
+			startButton.addEventListener( 'click', function () {
+
+				init();
+
+			} );
+
+			function init() {
+
+				if ( WebGPU.isAvailable() === false ) {
+
+					document.body.appendChild( WebGPU.getErrorMessage() );
+
+					throw new Error( 'No WebGPU support' );
+
+				}
+
+				const overlay = document.getElementById( 'overlay' );
+				overlay.remove();
+
+				container = document.createElement( 'div' );
+				document.body.appendChild( container );
+
+				camera = new THREE.PerspectiveCamera( 40, window.innerWidth / window.innerHeight, 1, 10000 );
+				camera.position.z = 500;
+
+				scene = new THREE.Scene();
+
+				const light = new THREE.DirectionalLight( 0xffffff, 7 );
+				light.position.set( 0.5, 1, 1 ).normalize();
+				scene.add( light );
+
+				renderer = new WebGPURenderer();
+				renderer.setPixelRatio( window.devicePixelRatio );
+				renderer.setSize( window.innerWidth, window.innerHeight );
+				renderer.setAnimationLoop( render );
+				container.appendChild( renderer.domElement );
+
+				video = document.getElementById( 'video' );
+				video.play();
+				video.addEventListener( 'play', function () {
+
+					this.currentTime = 3;
+
+				} );
+
+				texture = new THREE.VideoTexture( video );
+
+				//
+
+				let i, j, ox, oy, geometry;
+
+				const ux = 1 / xgrid;
+				const uy = 1 / ygrid;
+
+				const xsize = 480 / xgrid;
+				const ysize = 204 / ygrid;
+
+				const parameters = { color: 0xffffff, map: texture };
+
+				cube_count = 0;
+
+				for ( i = 0; i < xgrid; i ++ ) {
+
+					for ( j = 0; j < ygrid; j ++ ) {
+
+						ox = i;
+						oy = j;
+
+						geometry = new THREE.BoxGeometry( xsize, ysize, xsize );
+
+						change_uvs( geometry, ux, uy, ox, oy );
+
+						materials[ cube_count ] = new THREE.MeshPhongMaterial( parameters );
+
+						material = materials[ cube_count ];
+
+						material.hue = i / xgrid;
+						material.saturation = 1 - j / ygrid;
+
+						material.color.setHSL( material.hue, material.saturation, 0.5 );
+
+						mesh = new THREE.Mesh( geometry, material );
+
+						mesh.position.x = ( i - xgrid / 2 ) * xsize;
+						mesh.position.y = ( j - ygrid / 2 ) * ysize;
+						mesh.position.z = 0;
+
+						mesh.scale.x = mesh.scale.y = mesh.scale.z = 1;
+
+						scene.add( mesh );
+
+						mesh.dx = 0.001 * ( 0.5 - Math.random() );
+						mesh.dy = 0.001 * ( 0.5 - Math.random() );
+
+						meshes[ cube_count ] = mesh;
+
+						cube_count += 1;
+
+					}
+
+				}
+
+				document.addEventListener( 'mousemove', onDocumentMouseMove );
+
+				//
+
+				window.addEventListener( 'resize', onWindowResize );
+
+			}
+
+			function onWindowResize() {
+
+				windowHalfX = window.innerWidth / 2;
+				windowHalfY = window.innerHeight / 2;
+
+				camera.aspect = window.innerWidth / window.innerHeight;
+				camera.updateProjectionMatrix();
+
+				renderer.setSize( window.innerWidth, window.innerHeight );
+
+			}
+
+			function change_uvs( geometry, unitx, unity, offsetx, offsety ) {
+
+				const uvs = geometry.attributes.uv.array;
+
+				for ( let i = 0; i < uvs.length; i += 2 ) {
+
+					uvs[ i ] = ( uvs[ i ] + offsetx ) * unitx;
+					uvs[ i + 1 ] = ( uvs[ i + 1 ] + offsety ) * unity;
+
+				}
+
+			}
+
+
+			function onDocumentMouseMove( event ) {
+
+				mouseX = ( event.clientX - windowHalfX );
+				mouseY = ( event.clientY - windowHalfY ) * 0.3;
+
+			}
+
+			//
+
+			let h, counter = 1;
+
+			function render() {
+
+				const time = Date.now() * 0.00005;
+
+				camera.position.x += ( mouseX - camera.position.x ) * 0.05;
+				camera.position.y += ( - mouseY - camera.position.y ) * 0.05;
+
+				camera.lookAt( scene.position );
+
+				for ( let i = 0; i < cube_count; i ++ ) {
+
+					material = materials[ i ];
+
+					h = ( 360 * ( material.hue + time ) % 360 ) / 360;
+					material.color.setHSL( h, material.saturation, 0.5 );
+
+				}
+
+				if ( counter % 1000 > 200 ) {
+
+					for ( let i = 0; i < cube_count; i ++ ) {
+
+						mesh = meshes[ i ];
+
+						mesh.rotation.x += 10 * mesh.dx;
+						mesh.rotation.y += 10 * mesh.dy;
+
+						mesh.position.x -= 150 * mesh.dx;
+						mesh.position.y += 150 * mesh.dy;
+						mesh.position.z += 300 * mesh.dx;
+
+					}
+
+				}
+
+				if ( counter % 1000 === 0 ) {
+
+					for ( let i = 0; i < cube_count; i ++ ) {
+
+						mesh = meshes[ i ];
+
+						mesh.dx *= - 1;
+						mesh.dy *= - 1;
+
+					}
+
+				}
+
+				counter ++;
+
+				renderer.render( scene, camera );
+
+			}
+
+
+		</script>
+
+	</body>
+</html>