Browse Source

Nodes: Add PixelationNode (#28802)

* sketched out draft of pixelation pass.

* Have normal and depth edges working

* Pixel size modifier

* playing with render targets

* fix renderTarget issue

* auto-mrt version of pixelation

* cleanup

* remove any outside tests, logs, and changes

* screenshot and cleanup

* more cleanup

* differentiate lighting from webgl version and modify to remove lighting errors

* final lighting adjustment

* Revert lighting

* bring back directionToColor

* fix screenshot

* filtering fix

* fix normalView and add new screenshot

* normalzie uvNodeNormal

* remove unused directionToColor import, floor widtth and height of resolution, change uniform naming to match other uniforms, fix comment spacing in updateBefore

* replace single expression tslFn function

* update lowerResolutionMaterial, remove unnecessary const color assignment

* revert to pixelationPass approach

* fix lint issue, ignore puppeteer test for now

* Update webgpu_postprocessing_pixel.html

---------

Co-authored-by: Michael Herzog <[email protected]>
Christian Helgeson 1 year ago
parent
commit
83bffe6901

+ 1 - 0
examples/files.json

@@ -375,6 +375,7 @@
 		"webgpu_postprocessing_anamorphic",
 		"webgpu_postprocessing_ao",
 		"webgpu_postprocessing_dof",
+		"webgpu_postprocessing_pixel",
 		"webgpu_postprocessing_fxaa",
 		"webgpu_postprocessing_sobel",
 		"webgpu_postprocessing",

BIN
examples/screenshots/webgpu_postprocessing_pixel.jpg


+ 277 - 0
examples/webgpu_postprocessing_pixel.html

@@ -0,0 +1,277 @@
+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<title>three.js webgpu - postprocessing pixel</title>
+		<meta charset="utf-8">
+		<meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
+		<link type="text/css" rel="stylesheet" href="main.css">
+	</head>
+
+<body>
+	<div id="info">
+		<a href="https://threejs.org" target="_blank" rel="noopener">three.js</a> - Node based pixelation pass with optional single pixel outlines by
+		<a href="https://github.com/KodyJKing" target="_blank" rel="noopener">Kody King</a><br /><br />
+	</div>
+
+	<div id="container"></div>
+
+	<script type="importmap">
+		{
+			"imports": {
+				"three": "../build/three.webgpu.js",
+				"three/tsl": "../build/three.webgpu.js",
+				"three/addons/": "./jsm/"
+			}
+		}
+	</script>
+	
+
+	<script type="module">
+
+		import * as THREE from 'three';
+
+		import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
+		import { GUI } from 'three/addons/libs/lil-gui.module.min.js';
+
+		import { uniform, pixelationPass } from 'three/tsl';
+
+		let camera, scene, renderer, postProcessing, crystalMesh, clock;
+		let gui, effectController;
+
+		init();
+
+		function init() {
+
+			const aspectRatio = window.innerWidth / window.innerHeight;
+
+			camera = new THREE.OrthographicCamera( - aspectRatio, aspectRatio, 1, - 1, 0.1, 10 );
+			camera.position.y = 2 * Math.tan( Math.PI / 6 );
+			camera.position.z = 2;
+
+			scene = new THREE.Scene();
+			scene.background = new THREE.Color( 0x151729 );
+
+			clock = new THREE.Clock();
+
+			// textures
+
+			const loader = new THREE.TextureLoader();
+			const texChecker = pixelTexture( loader.load( 'textures/checker.png' ) );
+			const texChecker2 = pixelTexture( loader.load( 'textures/checker.png' ) );
+			texChecker.repeat.set( 3, 3 );
+			texChecker2.repeat.set( 1.5, 1.5 );
+
+			// meshes
+
+			const boxMaterial = new THREE.MeshPhongMaterial( { map: texChecker2 } );
+
+			function addBox( boxSideLength, x, z, rotation ) {
+
+				const mesh = new THREE.Mesh( new THREE.BoxGeometry( boxSideLength, boxSideLength, boxSideLength ), boxMaterial );
+				mesh.castShadow = true;
+				//mesh.receiveShadow = true;
+				mesh.rotation.y = rotation;
+				mesh.position.y = boxSideLength / 2;
+				mesh.position.set( x, boxSideLength / 2 + .0001, z );
+				scene.add( mesh );
+				return mesh;
+
+			}
+
+			addBox( .4, 0, 0, Math.PI / 4 );
+			addBox( .5, - .5, - .5, Math.PI / 4 );
+
+			const planeSideLength = 2;
+			const planeMesh = new THREE.Mesh(
+				new THREE.PlaneGeometry( planeSideLength, planeSideLength ),
+				new THREE.MeshPhongMaterial( { map: texChecker } )
+			);
+			planeMesh.receiveShadow = true;
+			planeMesh.rotation.x = - Math.PI / 2;
+			scene.add( planeMesh );
+
+			const radius = .2;
+			const geometry = new THREE.IcosahedronGeometry( radius );
+			crystalMesh = new THREE.Mesh(
+				geometry,
+				new THREE.MeshPhongMaterial( {
+					color: 0x68b7e9,
+					emissive: 0x4f7e8b,
+					shininess: 10,
+					specular: 0xffffff
+				} )
+			);
+			//crystalMesh.receiveShadow = true;
+			crystalMesh.castShadow = true;
+			scene.add( crystalMesh );
+
+			// lights
+
+			scene.add( new THREE.AmbientLight( 0x757f8e, 3 ) );
+
+			const directionalLight = new THREE.DirectionalLight( 0xfffecd, 1.5 );
+			directionalLight.position.set( 100, 100, 100 );
+			directionalLight.castShadow = true;
+			directionalLight.shadow.mapSize.set( 2048, 2048 );
+			scene.add( directionalLight );
+
+			const spotLight = new THREE.SpotLight( 0xffc100, 10, 10, Math.PI / 16, .02, 2 );
+			spotLight.position.set( 2, 2, 0 );
+			const target = spotLight.target;
+			scene.add( target );
+			target.position.set( 0, 0, 0 );
+			spotLight.castShadow = true;
+			scene.add( spotLight );
+
+			renderer = new THREE.WebGPURenderer( { antialias: false } );
+			renderer.shadowMap.enabled = true;
+			//renderer.setPixelRatio( window.devicePixelRatio );
+			renderer.setSize( window.innerWidth, window.innerHeight );
+			renderer.setAnimationLoop( animate );
+			document.body.appendChild( renderer.domElement );
+
+			effectController = {
+				pixelSize: uniform( 6 ),
+				normalEdgeStrength: uniform( 0.3 ),
+				depthEdgeStrength: uniform( 0.4 ),
+				pixelAlignedPanning: true
+			};
+
+			postProcessing = new THREE.PostProcessing( renderer );
+			const scenePass = pixelationPass( scene, camera, effectController.pixelSize, effectController.normalEdgeStrength, effectController.depthEdgeStrength );
+			postProcessing.outputNode = scenePass;
+
+			window.addEventListener( 'resize', onWindowResize );
+
+			const controls = new OrbitControls( camera, renderer.domElement );
+			controls.maxZoom = 2;
+
+			// gui
+
+			gui = new GUI();
+			gui.add( effectController.pixelSize, 'value', 1, 16, 1 ).name( 'Pixel Size' );
+			gui.add( effectController.normalEdgeStrength, 'value', 0, 2, 0.05 ).name( 'Normal Edge Strength' );
+			gui.add( effectController.depthEdgeStrength, 'value', 0, 1, 0.05 ).name( 'Depth Edge Strength' );
+			gui.add( effectController, 'pixelAlignedPanning' );
+
+		}
+
+		function onWindowResize() {
+
+			const aspectRatio = window.innerWidth / window.innerHeight;
+			camera.left = - aspectRatio;
+			camera.right = aspectRatio;
+			camera.updateProjectionMatrix();
+
+			renderer.setSize( window.innerWidth, window.innerHeight );
+
+		}
+
+		function animate() {
+
+			const t = clock.getElapsedTime();
+
+			crystalMesh.material.emissiveIntensity = Math.sin( t * 3 ) * .5 + .5;
+			crystalMesh.position.y = .7 + Math.sin( t * 2 ) * .05;
+			crystalMesh.rotation.y = stopGoEased( t, 2, 4 ) * 2 * Math.PI;
+
+			const rendererSize = renderer.getSize( new THREE.Vector2() );
+			const aspectRatio = rendererSize.x / rendererSize.y;
+
+			if ( effectController.pixelAlignedPanning ) {
+
+				const pixelSize = effectController.pixelSize.value;
+
+				pixelAlignFrustum( camera, aspectRatio, Math.floor( rendererSize.x / pixelSize ),
+					Math.floor( rendererSize.y / pixelSize ) );
+
+			} else if ( camera.left != - aspectRatio || camera.top != 1.0 ) {
+
+				// Reset the Camera Frustum if it has been modified
+				camera.left = - aspectRatio;
+				camera.right = aspectRatio;
+				camera.top = 1.0;
+				camera.bottom = - 1.0;
+				camera.updateProjectionMatrix();
+
+			}
+
+			postProcessing.render();
+
+		}
+
+		// Helper functions
+
+		function pixelTexture( texture ) {
+
+			texture.minFilter = THREE.NearestFilter;
+			texture.magFilter = THREE.NearestFilter;
+			texture.generateMipmaps = false;
+			texture.wrapS = THREE.RepeatWrapping;
+			texture.wrapT = THREE.RepeatWrapping;
+			texture.colorSpace = THREE.SRGBColorSpace;
+			return texture;
+
+		}
+
+		function easeInOutCubic( x ) {
+
+			return x ** 2 * 3 - x ** 3 * 2;
+
+		}
+
+		function linearStep( x, edge0, edge1 ) {
+
+			const w = edge1 - edge0;
+			const m = 1 / w;
+			const y0 = - m * edge0;
+			return THREE.MathUtils.clamp( y0 + m * x, 0, 1 );
+
+		}
+
+		function stopGoEased( x, downtime, period ) {
+
+			const cycle = ( x / period ) | 0;
+			const tween = x - cycle * period;
+			const linStep = easeInOutCubic( linearStep( tween, downtime, period ) );
+			return cycle + linStep;
+
+		}
+
+		function pixelAlignFrustum( camera, aspectRatio, pixelsPerScreenWidth, pixelsPerScreenHeight ) {
+
+			// 0. Get Pixel Grid Units
+			const worldScreenWidth = ( ( camera.right - camera.left ) / camera.zoom );
+			const worldScreenHeight = ( ( camera.top - camera.bottom ) / camera.zoom );
+			const pixelWidth = worldScreenWidth / pixelsPerScreenWidth;
+			const pixelHeight = worldScreenHeight / pixelsPerScreenHeight;
+
+			// 1. Project the current camera position along its local rotation bases
+			const camPos = new THREE.Vector3(); camera.getWorldPosition( camPos );
+			const camRot = new THREE.Quaternion(); camera.getWorldQuaternion( camRot );
+			const camRight = new THREE.Vector3( 1.0, 0.0, 0.0 ).applyQuaternion( camRot );
+			const camUp = new THREE.Vector3( 0.0, 1.0, 0.0 ).applyQuaternion( camRot );
+			const camPosRight = camPos.dot( camRight );
+			const camPosUp = camPos.dot( camUp );
+
+			// 2. Find how far along its position is along these bases in pixel units
+			const camPosRightPx = camPosRight / pixelWidth;
+			const camPosUpPx = camPosUp / pixelHeight;
+
+			// 3. Find the fractional pixel units and convert to world units
+			const fractX = camPosRightPx - Math.round( camPosRightPx );
+			const fractY = camPosUpPx - Math.round( camPosUpPx );
+
+			// 4. Add fractional world units to the left/right top/bottom to align with the pixel grid
+			camera.left = - aspectRatio - ( fractX * pixelWidth );
+			camera.right = aspectRatio - ( fractX * pixelWidth );
+			camera.top = 1.0 - ( fractY * pixelHeight );
+			camera.bottom = - 1.0 - ( fractY * pixelHeight );
+			camera.updateProjectionMatrix();
+
+		}
+
+	</script>
+</body>
+
+</html>

+ 1 - 0
src/nodes/Nodes.js

@@ -137,6 +137,7 @@ export { default as Lut3DNode, lut3D } from './display/Lut3DNode.js';
 export { default as GTAONode, ao } from './display/GTAONode.js';
 export { default as FXAANode, fxaa } from './display/FXAANode.js';
 export { default as RenderOutputNode, renderOutput } from './display/RenderOutputNode.js';
+export { default as PixelationPassNode, pixelationPass } from './display/PixelationPassNode.js';
 
 export { default as PassNode, pass, passTexture, depthPass } from './display/PassNode.js';
 

+ 201 - 0
src/nodes/display/PixelationPassNode.js

@@ -0,0 +1,201 @@
+import TempNode from '../core/TempNode.js';
+import { uv } from '../accessors/UVNode.js';
+import { addNodeElement, tslFn, nodeObject, vec2, vec3, float, If } from '../shadernode/ShaderNode.js';
+import { NodeUpdateType } from '../core/constants.js';
+import { uniform } from '../core/UniformNode.js';
+import { dot, clamp, smoothstep, sign, step, floor } from '../math/MathNode.js';
+import { Vector4 } from '../../math/Vector4.js';
+import { output, property } from '../core/PropertyNode.js';
+import PassNode from './PassNode.js';
+import { mrt } from '../core/MRTNode.js';
+import { normalView } from '../accessors/NormalNode.js';
+import { NearestFilter } from '../../constants.js';
+
+class PixelationNode extends TempNode {
+
+	constructor( textureNode, depthNode, normalNode, pixelSize, normalEdgeStrength, depthEdgeStrength ) {
+
+		super();
+
+		// Input textures
+
+		this.textureNode = textureNode;
+		this.depthNode = depthNode;
+		this.normalNode = normalNode;
+
+		// Input uniforms
+
+		this.pixelSize = pixelSize;
+		this.normalEdgeStrength = normalEdgeStrength;
+		this.depthEdgeStrength = depthEdgeStrength;
+
+		// Private uniforms
+
+		this._resolution = uniform( new Vector4() );
+
+		this.updateBeforeType = NodeUpdateType.RENDER;
+
+	}
+
+	updateBefore() {
+
+		const map = this.textureNode.value;
+
+		const width = map.image.width;
+		const height = map.image.height;
+
+		this._resolution.value.set( width, height, 1 / width, 1 / height );
+
+	}
+
+	setup() {
+
+		const { textureNode, depthNode, normalNode } = this;
+
+		const uvNodeTexture = textureNode.uvNode || uv();
+		const uvNodeDepth = depthNode.uvNode || uv();
+		const uvNodeNormal = normalNode.uvNode || uv();
+
+		const sampleTexture = () => textureNode.uv( uvNodeTexture );
+
+		const sampleDepth = ( x, y ) => depthNode.uv( uvNodeDepth.add( vec2( x, y ).mul( this._resolution.zw ) ) ).r;
+
+		const sampleNormal = ( x, y ) => normalNode.uv( uvNodeNormal.add( vec2( x, y ).mul( this._resolution.zw ) ) ).rgb.normalize();
+
+		const depthEdgeIndicator = ( depth ) => {
+
+			const diff = property( 'float', 'diff' );
+			diff.addAssign( clamp( sampleDepth( 1, 0 ).sub( depth ) ) );
+			diff.addAssign( clamp( sampleDepth( - 1, 0 ).sub( depth ) ) );
+			diff.addAssign( clamp( sampleDepth( 0, 1 ).sub( depth ) ) );
+			diff.addAssign( clamp( sampleDepth( 0, - 1 ).sub( depth ) ) );
+
+			return floor( smoothstep( 0.01, 0.02, diff ).mul( 2 ) ).div( 2 );
+
+		};
+
+		const neighborNormalEdgeIndicator = ( x, y, depth, normal ) => {
+
+			const depthDiff = sampleDepth( x, y ).sub( depth );
+			const neighborNormal = sampleNormal( x, y );
+
+			// Edge pixels should yield to faces who's normals are closer to the bias normal.
+
+			const normalEdgeBias = vec3( 1, 1, 1 ); // This should probably be a parameter.
+			const normalDiff = dot( normal.sub( neighborNormal ), normalEdgeBias );
+			const normalIndicator = clamp( smoothstep( - 0.01, 0.01, normalDiff ), 0.0, 1.0 );
+
+			// Only the shallower pixel should detect the normal edge.
+
+			const depthIndicator = clamp( sign( depthDiff.mul( .25 ).add( .0025 ) ), 0.0, 1.0 );
+
+			return float( 1.0 ).sub( dot( normal, neighborNormal ) ).mul( depthIndicator ).mul( normalIndicator );
+
+		};
+
+		const normalEdgeIndicator = ( depth, normal ) => {
+
+			const indicator = property( 'float', 'indicator' );
+
+			indicator.addAssign( neighborNormalEdgeIndicator( 0, - 1, depth, normal ) );
+			indicator.addAssign( neighborNormalEdgeIndicator( 0, 1, depth, normal ) );
+			indicator.addAssign( neighborNormalEdgeIndicator( - 1, 0, depth, normal ) );
+			indicator.addAssign( neighborNormalEdgeIndicator( 1, 0, depth, normal ) );
+
+			return step( 0.1, indicator );
+
+		};
+
+		const pixelation = tslFn( () => {
+
+			const texel = sampleTexture();
+
+			const depth = property( 'float', 'depth' );
+			const normal = property( 'vec3', 'normal' );
+
+			If( this.depthEdgeStrength.greaterThan( 0.0 ).or( this.normalEdgeStrength.greaterThan( 0.0 ) ), () => {
+
+				depth.assign( sampleDepth( 0, 0 ) );
+				normal.assign( sampleNormal( 0, 0 ) );
+
+			} );
+
+			const dei = property( 'float', 'dei' );
+
+			If( this.depthEdgeStrength.greaterThan( 0.0 ), () => {
+
+				dei.assign( depthEdgeIndicator( depth ) );
+
+			} );
+
+			const nei = property( 'float', 'nei' );
+
+			If( this.normalEdgeStrength.greaterThan( 0.0 ), () => {
+
+				nei.assign( normalEdgeIndicator( depth, normal ) );
+
+			} );
+
+			const strength = dei.greaterThan( 0 ).cond( float( 1.0 ).sub( dei.mul( this.depthEdgeStrength ) ), nei.mul( this.normalEdgeStrength ).add( 1 ) );
+
+			return texel.mul( strength );
+
+		} );
+
+		const outputNode = pixelation();
+
+		return outputNode;
+
+	}
+
+}
+
+const pixelation = ( node, depthNode, normalNode, pixelSize = 6, normalEdgeStrength = 0.3, depthEdgeStrength = 0.4 ) => nodeObject( new PixelationNode( nodeObject( node ).toTexture(), nodeObject( depthNode ).toTexture(), nodeObject( normalNode ).toTexture(), nodeObject( pixelSize ), nodeObject( normalEdgeStrength ), nodeObject( depthEdgeStrength ) ) );
+
+addNodeElement( 'pixelation', pixelation );
+
+class PixelationPassNode extends PassNode {
+
+	constructor( scene, camera, pixelSize = 6, normalEdgeStrength = 0.3, depthEdgeStrength = 0.4 ) {
+
+		super( 'color', scene, camera, { minFilter: NearestFilter, magFilter: NearestFilter } );
+
+		this.pixelSize = pixelSize;
+		this.normalEdgeStrength = normalEdgeStrength;
+		this.depthEdgeStrength = depthEdgeStrength;
+
+		this.isPixelationPassNode = true;
+
+		this._mrt = mrt( {
+			output: output,
+			normal: normalView
+		} );
+
+	}
+
+	setSize( width, height ) {
+
+		const pixelSize = this.pixelSize.value ? this.pixelSize.value : this.pixelSize;
+
+		const adjustedWidth = Math.floor( width / pixelSize );
+		const adjustedHeight = Math.floor( height / pixelSize );
+
+		super.setSize( adjustedWidth, adjustedHeight );
+
+	}
+
+	setup() {
+
+		const color = super.getTextureNode( 'output' );
+		const depth = super.getTextureNode( 'depth' );
+		const normal = super.getTextureNode( 'normal' );
+
+		return pixelation( color, depth, normal, this.pixelSize, this.normalEdgeStrength, this.depthEdgeStrength );
+
+	}
+
+}
+
+export const pixelationPass = ( scene, camera, pixelSize, normalEdgeStrength, depthEdgeStrength ) => nodeObject( new PixelationPassNode( scene, camera, pixelSize, normalEdgeStrength, depthEdgeStrength ) );
+
+export default PixelationPassNode;

+ 1 - 0
test/e2e/puppeteer.js

@@ -126,6 +126,7 @@ const exceptionList = [
 	// WebGPURenderer: Unknown problem
 	'webgpu_postprocessing_afterimage',
 	'webgpu_postprocessing_3dlut',
+	"webgpu_postprocessing_pixel",
 	'webgpu_postprocessing_ao',
 	'webgpu_backdrop_water',
 	'webgpu_camera_logarithmicdepthbuffer',