Browse Source

Merge pull request #19922 from fernandojsg/handtracking

Add experimental WebXR Handtracking API support
Mr.doob 5 years ago
parent
commit
1168ec4bd5

+ 1 - 1
examples/jsm/webxr/VRButton.js

@@ -71,7 +71,7 @@ var VRButton = {
 					// ('local' is always available for immersive sessions and doesn't need to
 					// be requested separately.)
 
-					var sessionInit = { optionalFeatures: [ 'local-floor', 'bounded-floor' ] };
+					var sessionInit = { optionalFeatures: [ 'local-floor', 'bounded-floor', 'hand-tracking' ] };
 					navigator.xr.requestSession( 'immersive-vr', sessionInit ).then( onSessionStarted );
 
 				} else {

+ 124 - 0
examples/jsm/webxr/WebXRHandController.js

@@ -0,0 +1,124 @@
+import {
+	Object3D,
+	SphereBufferGeometry,
+	MeshStandardMaterial,
+	Mesh
+} from "../../../build/three.module.js";
+
+function XRHandModel( controller ) {
+
+	Object3D.call( this );
+
+	this.controller = controller;
+	this.envMap = null;
+
+	if ( window.XRHand ) {
+
+		var geometry = new SphereBufferGeometry( 1, 10, 10 );
+		var jointMaterial = new MeshStandardMaterial( { color: 0x000000, roughness: 0.2, metalness: 0.8 } );
+		var tipMaterial = new MeshStandardMaterial( { color: 0x333333, roughness: 0.2, metalness: 0.8 } );
+
+		const tipIndexes = [
+			XRHand.THUMB_PHALANX_TIP,
+			XRHand.INDEX_PHALANX_TIP,
+			XRHand.MIDDLE_PHALANX_TIP,
+			XRHand.RING_PHALANX_TIP,
+			XRHand.LITTLE_PHALANX_TIP
+		];
+		for ( let i = 0; i <= XRHand.LITTLE_PHALANX_TIP; i ++ ) {
+
+			var cube = new Mesh( geometry, tipIndexes.indexOf( i ) !== - 1 ? tipMaterial : jointMaterial );
+			cube.castShadow = true;
+			this.add( cube );
+
+		}
+
+	}
+
+}
+
+XRHandModel.prototype = Object.assign( Object.create( Object3D.prototype ), {
+
+	constructor: XRHandModel,
+
+	updateMatrixWorld: function ( force ) {
+
+		Object3D.prototype.updateMatrixWorld.call( this, force );
+
+		this.updateMesh();
+
+	},
+
+	updateMesh: function () {
+
+		const defaultRadius = 0.008;
+
+		// XR Joints
+		const XRJoints = this.controller.joints;
+		for ( var i = 0; i < this.children.length; i ++ ) {
+
+			const jointMesh = this.children[ i ];
+			const XRJoint = XRJoints[ i ];
+
+			if ( XRJoint.visible ) {
+
+				jointMesh.position.copy( XRJoint.position );
+				jointMesh.quaternion.copy( XRJoint.quaternion );
+				jointMesh.scale.setScalar( XRJoint.jointRadius || defaultRadius );
+
+			}
+
+			jointMesh.visible = XRJoint.visible;
+
+		}
+
+	}
+} );
+
+
+var XRHandModelFactory = ( function () {
+
+	function XRHandModelFactory() {}
+
+	XRHandModelFactory.prototype = {
+
+		constructor: XRHandModelFactory,
+
+		createHandModel: function ( controller ) {
+
+			const handModel = new XRHandModel( controller );
+			let scene = null;
+
+			controller.addEventListener( 'connected', ( event ) => {
+
+				const xrInputSource = event.data;
+				console.log( "Connected!", xrInputSource );
+
+				if ( xrInputSource.hand ) {
+
+					handModel.xrInputSource = xrInputSource;
+
+				}
+
+			} );
+
+			controller.addEventListener( 'disconnected', () => {
+
+				handModel.motionController = null;
+				handModel.remove( scene );
+				scene = null;
+
+			} );
+
+			return handModel;
+
+		}
+
+	};
+
+	return XRHandModelFactory;
+
+} )();
+
+
+export { XRHandModelFactory };

+ 284 - 0
examples/webxr_vr_handtracking.html

@@ -0,0 +1,284 @@
+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<title>three.js vr - dragging</title>
+		<meta charset="utf-8">
+		<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
+		<link type="text/css" rel="stylesheet" href="main.css">
+	</head>
+	<body>
+
+		<div id="info">
+			<a href="https://threejs.org" target="_blank" rel="noopener">three.js</a> vr - hand and controller support
+		</div>
+
+		<script type="module">
+
+			import * as THREE from '../build/three.module.js';
+			import { OrbitControls } from './jsm/controls/OrbitControls.js';
+			import { VRButton } from './jsm/webxr/VRButton.js';
+			import { XRControllerModelFactory } from './jsm/webxr/XRControllerModelFactory.js';
+			import { XRHandModelFactory } from './jsm/webxr/WebXRHandController.js';
+
+			var container;
+			var camera, scene, renderer;
+			var hand1, hand2;
+			var controller1, controller2;
+			var controllerGrip1, controllerGrip2;
+
+			var controls;
+
+			var grabbing = false;
+			var scaling = {
+				active: false,
+				initialDistance: 0,
+				object: null,
+				initialScale: 1
+			};
+
+			var spheres = [];
+
+			init();
+			animate();
+
+			function init() {
+
+				container = document.createElement( 'div' );
+				document.body.appendChild( container );
+
+				scene = new THREE.Scene();
+				scene.background = new THREE.Color( 0x808080 );
+
+				camera = new THREE.PerspectiveCamera( 50, window.innerWidth / window.innerHeight, 0.1, 10 );
+				camera.position.set( 0, 1.6, 3 );
+
+				controls = new OrbitControls( camera, container );
+				controls.target.set( 0, 1.6, 0 );
+				controls.update();
+
+				var geometry = new THREE.PlaneBufferGeometry( 4, 4 );
+				var material = new THREE.MeshStandardMaterial( {
+					color: 0xeeeeee,
+					roughness: 1.0,
+					metalness: 0.0
+				} );
+				var floor = new THREE.Mesh( geometry, material );
+				floor.rotation.x = - Math.PI / 2;
+				floor.receiveShadow = true;
+				scene.add( floor );
+
+				scene.add( new THREE.HemisphereLight( 0x808080, 0x606060 ) );
+
+				var light = new THREE.DirectionalLight( 0xffffff );
+				light.position.set( 0, 6, 0 );
+				light.castShadow = true;
+				light.shadow.camera.top = 2;
+				light.shadow.camera.bottom = - 2;
+				light.shadow.camera.right = 2;
+				light.shadow.camera.left = - 2;
+				light.shadow.mapSize.set( 4096, 4096 );
+				scene.add( light );
+
+				//
+
+				renderer = new THREE.WebGLRenderer( { antialias: true } );
+				renderer.setPixelRatio( window.devicePixelRatio );
+				renderer.setSize( window.innerWidth, window.innerHeight );
+				renderer.outputEncoding = THREE.sRGBEncoding;
+				renderer.shadowMap.enabled = true;
+				renderer.xr.enabled = true;
+
+				container.appendChild( renderer.domElement );
+
+				document.body.appendChild( VRButton.createButton( renderer ) );
+
+				// controllers
+
+				controller1 = renderer.xr.getController( 0 );
+				scene.add( controller1 );
+
+				controller2 = renderer.xr.getController( 1 );
+				scene.add( controller2 );
+
+				var controllerModelFactory = new XRControllerModelFactory();
+				var handModelFactory = new XRHandModelFactory();
+
+				// Hand 1
+				controllerGrip1 = renderer.xr.getControllerGrip( 0 );
+				controllerGrip1.add( controllerModelFactory.createControllerModel( controllerGrip1 ) );
+				scene.add( controllerGrip1 );
+
+				hand1 = renderer.xr.getHand( 0 );
+				hand1.addEventListener( 'pinchstart', onPinchStartLeft );
+				hand1.addEventListener( 'pinchend', () => {
+
+					scaling.active = false;
+
+				} );
+				hand1.add( handModelFactory.createHandModel( hand1 ) );
+
+				scene.add( hand1 );
+
+				// Hand 2
+				controllerGrip2 = renderer.xr.getControllerGrip( 1 );
+				controllerGrip2.add( controllerModelFactory.createControllerModel( controllerGrip2 ) );
+				scene.add( controllerGrip2 );
+
+				hand2 = renderer.xr.getHand( 1 );
+				hand2.addEventListener( 'pinchstart', onPinchStartRight );
+				hand2.addEventListener( 'pinchend', onPinchEndRight );
+				hand2.add( handModelFactory.createHandModel( hand2 ) );
+				scene.add( hand2 );
+
+				//
+
+				var geometry = new THREE.BufferGeometry().setFromPoints( [ new THREE.Vector3( 0, 0, 0 ), new THREE.Vector3( 0, 0, - 1 ) ] );
+
+				var line = new THREE.Line( geometry );
+				line.name = 'line';
+				line.scale.z = 5;
+
+				controller1.add( line.clone() );
+				controller2.add( line.clone() );
+
+				//
+
+				window.addEventListener( 'resize', onWindowResize, false );
+
+			}
+
+			function onWindowResize() {
+
+				camera.aspect = window.innerWidth / window.innerHeight;
+				camera.updateProjectionMatrix();
+
+				renderer.setSize( window.innerWidth, window.innerHeight );
+
+			}
+
+			const SphereRadius = 0.05;
+			function onPinchStartLeft( event ) {
+
+				var controller = event.target;
+
+				if ( grabbing ) {
+
+					const indexTip = controller.joints[ XRHand.INDEX_PHALANX_TIP ];
+					const sphere = collideObject( indexTip );
+
+					if ( sphere ) {
+
+						const sphere2 = hand2.userData.selected;
+						console.log( "sphere1", sphere, "sphere2", sphere2 );
+						if ( sphere === sphere2 ) {
+
+							scaling.active = true;
+							scaling.object = sphere;
+							scaling.initialScale = sphere.scale.x;
+							scaling.initialDistance = indexTip.position.distanceTo( hand2.joints[ XRHand.INDEX_PHALANX_TIP ].position );
+							return;
+
+						}
+
+					}
+
+				}
+
+				var geometry = new THREE.BoxBufferGeometry( SphereRadius, SphereRadius, SphereRadius );
+				var material = new THREE.MeshStandardMaterial( {
+					color: Math.random() * 0xffffff,
+					roughness: 1.0,
+					metalness: 0.0
+				} );
+				var spawn = new THREE.Mesh( geometry, material );
+				spawn.geometry.computeBoundingSphere();
+
+				const indexTip = controller.joints[ XRHand.INDEX_PHALANX_TIP ];
+				spawn.position.copy( indexTip.position );
+				spawn.quaternion.copy( indexTip.quaternion );
+
+				spheres.push( spawn );
+
+				scene.add( spawn );
+			}
+
+			function collideObject( indexTip ) {
+
+				for ( var i = 0; i < spheres.length; i ++ ) {
+
+					const sphere = spheres[ i ];
+					const distance = indexTip.getWorldPosition().distanceTo( sphere.getWorldPosition() );
+
+					if ( distance < sphere.geometry.boundingSphere.radius * sphere.scale.x ) {
+
+						return sphere;
+
+					}
+
+				}
+
+				return null;
+
+			}
+
+			function onPinchStartRight( event ) {
+
+				var controller = event.target;
+				const indexTip = controller.joints[ XRHand.INDEX_PHALANX_TIP ];
+				const object = collideObject( indexTip );
+				if ( object ) {
+
+					grabbing = true;
+					indexTip.attach( object );
+					controller.userData.selected = object;
+					console.log( "Selected", object );
+
+				}
+
+			}
+
+			function onPinchEndRight( event ) {
+
+				var controller = event.target;
+
+				if ( controller.userData.selected !== undefined ) {
+
+					var object = controller.userData.selected;
+					object.material.emissive.b = 0;
+					scene.attach( object );
+
+					controller.userData.selected = undefined;
+					grabbing = false;
+
+				}
+
+				scaling.active = false;
+
+			}
+
+			//
+
+			function animate() {
+
+				renderer.setAnimationLoop( render );
+
+			}
+
+			function render() {
+
+				if ( scaling.active ) {
+
+					const indexTip1Pos = hand1.joints[ XRHand.INDEX_PHALANX_TIP ].position;
+					const indexTip2Pos = hand2.joints[ XRHand.INDEX_PHALANX_TIP ].position;
+					const distance = indexTip1Pos.distanceTo( indexTip2Pos );
+					const newScale = scaling.initialScale + distance / scaling.initialDistance - 1;
+					scaling.object.scale.setScalar( newScale );
+				}
+
+				renderer.render( scene, camera );
+
+			}
+
+		</script>
+	</body>
+</html>

+ 156 - 0
examples/webxr_vr_handtracking_simple.html

@@ -0,0 +1,156 @@
+<!DOCTYPE html>
+<html lang="en">
+	<head>
+		<title>three.js vr - dragging</title>
+		<meta charset="utf-8">
+		<meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
+		<link type="text/css" rel="stylesheet" href="main.css">
+	</head>
+	<body>
+
+		<div id="info">
+			<a href="https://threejs.org" target="_blank" rel="noopener">three.js</a> vr - hand and controller support
+		</div>
+
+		<script type="module">
+
+			import * as THREE from '../build/three.module.js';
+			import { OrbitControls } from './jsm/controls/OrbitControls.js';
+			import { VRButton } from './jsm/webxr/VRButton.js';
+			import { XRControllerModelFactory } from './jsm/webxr/XRControllerModelFactory.js';
+			import { XRHandModelFactory } from './jsm/webxr/WebXRHandController.js';
+
+			var container;
+			var camera, scene, renderer;
+			var hand1, hand2;
+			var controller1, controller2;
+			var controllerGrip1, controllerGrip2;
+
+			var controls;
+
+			init();
+			animate();
+
+			function init() {
+
+				container = document.createElement( 'div' );
+				document.body.appendChild( container );
+
+				scene = new THREE.Scene();
+				scene.background = new THREE.Color( 0x808080 );
+
+				camera = new THREE.PerspectiveCamera( 50, window.innerWidth / window.innerHeight, 0.1, 10 );
+				camera.position.set( 0, 1.6, 3 );
+
+				controls = new OrbitControls( camera, container );
+				controls.target.set( 0, 1.6, 0 );
+				controls.update();
+
+				var geometry = new THREE.PlaneBufferGeometry( 4, 4 );
+				var material = new THREE.MeshStandardMaterial( {
+					color: 0xeeeeee,
+					roughness: 1.0,
+					metalness: 0.0
+				} );
+				var floor = new THREE.Mesh( geometry, material );
+				floor.rotation.x = - Math.PI / 2;
+				floor.receiveShadow = true;
+				scene.add( floor );
+
+				scene.add( new THREE.HemisphereLight( 0x808080, 0x606060 ) );
+
+				var light = new THREE.DirectionalLight( 0xffffff );
+				light.position.set( 0, 6, 0 );
+				light.castShadow = true;
+				light.shadow.camera.top = 2;
+				light.shadow.camera.bottom = - 2;
+				light.shadow.camera.right = 2;
+				light.shadow.camera.left = - 2;
+				light.shadow.mapSize.set( 4096, 4096 );
+				scene.add( light );
+
+				//
+
+				renderer = new THREE.WebGLRenderer( { antialias: true } );
+				renderer.setPixelRatio( window.devicePixelRatio );
+				renderer.setSize( window.innerWidth, window.innerHeight );
+				renderer.outputEncoding = THREE.sRGBEncoding;
+				renderer.shadowMap.enabled = true;
+				renderer.xr.enabled = true;
+
+				container.appendChild( renderer.domElement );
+
+				document.body.appendChild( VRButton.createButton( renderer ) );
+
+				// controllers
+
+				controller1 = renderer.xr.getController( 0 );
+				scene.add( controller1 );
+
+				controller2 = renderer.xr.getController( 1 );
+				scene.add( controller2 );
+
+				var controllerModelFactory = new XRControllerModelFactory();
+				var handModelFactory = new XRHandModelFactory();
+
+				// Hand 1
+				controllerGrip1 = renderer.xr.getControllerGrip( 0 );
+				controllerGrip1.add( controllerModelFactory.createControllerModel( controllerGrip1 ) );
+				scene.add( controllerGrip1 );
+
+				hand1 = renderer.xr.getHand( 0 );
+				hand1.add( handModelFactory.createHandModel( hand1 ) );
+
+				scene.add( hand1 );
+
+				// Hand 2
+				controllerGrip2 = renderer.xr.getControllerGrip( 1 );
+				controllerGrip2.add( controllerModelFactory.createControllerModel( controllerGrip2 ) );
+				scene.add( controllerGrip2 );
+
+				hand2 = renderer.xr.getHand( 1 );
+				hand2.add( handModelFactory.createHandModel( hand2 ) );
+				scene.add( hand2 );
+
+				//
+
+				var geometry = new THREE.BufferGeometry().setFromPoints( [ new THREE.Vector3( 0, 0, 0 ), new THREE.Vector3( 0, 0, - 1 ) ] );
+
+				var line = new THREE.Line( geometry );
+				line.name = 'line';
+				line.scale.z = 5;
+
+				controller1.add( line.clone() );
+				controller2.add( line.clone() );
+
+				//
+
+				window.addEventListener( 'resize', onWindowResize, false );
+
+			}
+
+			function onWindowResize() {
+
+				camera.aspect = window.innerWidth / window.innerHeight;
+				camera.updateProjectionMatrix();
+
+				renderer.setSize( window.innerWidth, window.innerHeight );
+
+			}
+			//
+
+			function animate() {
+
+				renderer.setAnimationLoop( render );
+
+			}
+
+			function render() {
+
+				renderer.render( scene, camera );
+
+			}
+
+		</script>
+	</body>
+</html>

+ 121 - 11
src/renderers/webxr/WebXRController.js

@@ -8,6 +8,7 @@ function WebXRController() {
 
 	this._targetRay = null;
 	this._grip = null;
+	this._hand = null;
 
 }
 
@@ -15,6 +16,39 @@ Object.assign( WebXRController.prototype, {
 
 	constructor: WebXRController,
 
+	getHandSpace: function () {
+
+		if ( this._hand === null ) {
+
+			this._hand = new Group();
+			this._hand.matrixAutoUpdate = false;
+			this._hand.visible = false;
+
+			this._hand.joints = [];
+			this._hand.inputState = { pinching: false };
+
+			if ( window.XRHand ) {
+
+				for ( let i = 0; i <= XRHand.LITTLE_PHALANX_TIP; i ++ ) {
+
+					// The transform of this joint will be updated with the joint pose on each frame
+					let joint = new Group();
+					joint.matrixAutoUpdate = false;
+					joint.visible = false;
+					this._hand.joints.push( joint );
+					// ??
+					this._hand.add( joint );
+
+				}
+
+			}
+
+		}
+
+		return this._hand;
+
+	},
+
 	getTargetRaySpace: function () {
 
 		if ( this._targetRay === null ) {
@@ -57,6 +91,12 @@ Object.assign( WebXRController.prototype, {
 
 		}
 
+		if ( this._hand !== null ) {
+
+			this._hand.dispatchEvent( event );
+
+		}
+
 		return this;
 
 	},
@@ -77,6 +117,12 @@ Object.assign( WebXRController.prototype, {
 
 		}
 
+		if ( this._hand !== null ) {
+
+			this._hand.visible = false;
+
+		}
+
 		return this;
 
 	},
@@ -85,33 +131,91 @@ Object.assign( WebXRController.prototype, {
 
 		let inputPose = null;
 		let gripPose = null;
+		let handPose = null;
 
 		const targetRay = this._targetRay;
 		const grip = this._grip;
+		const hand = this._hand;
 
 		if ( inputSource ) {
 
-			if ( targetRay !== null ) {
+			if ( inputSource.hand ) {
+
+				handPose = true;
+				for ( let i = 0; i <= XRHand.LITTLE_PHALANX_TIP; i ++ ) {
 
-				inputPose = frame.getPose( inputSource.targetRaySpace, referenceSpace );
+					if ( inputSource.hand[ i ] ) {
 
-				if ( inputPose !== null ) {
+						// Update the joints groups with the XRJoint poses
+						let jointPose = frame.getJointPose( inputSource.hand[ i ], referenceSpace );
+						const joint = hand.joints[ i ];
 
-					targetRay.matrix.fromArray( inputPose.transform.matrix );
-					targetRay.matrix.decompose( targetRay.position, targetRay.rotation, targetRay.scale );
+						if ( jointPose !== null ) {
+
+							joint.matrix.fromArray( jointPose.transform.matrix );
+							joint.matrix.decompose( joint.position, joint.rotation, joint.scale );
+							joint.jointRadius = jointPose.radius;
+
+						}
+
+						joint.visible = jointPose !== null;
+
+						// Custom events
+
+						// Check pinch
+						const indexTip = hand.joints[ XRHand.INDEX_PHALANX_TIP ];
+						const thumbTip = hand.joints[ XRHand.THUMB_PHALANX_TIP ];
+						const distance = indexTip.position.distanceTo( thumbTip.position );
+
+						const distanceToPinch = 0.02;
+						const threshold = 0.005;
+						if ( hand.inputState.pinching && distance > distanceToPinch + threshold ) {
+
+							hand.inputState.pinching = false;
+							this.dispatchEvent( {
+								type: "pinchend",
+								target: this
+							} );
+
+						} else if ( ! hand.inputState.pinching && distance <= distanceToPinch - threshold ) {
+
+							hand.inputState.pinching = true;
+							this.dispatchEvent( {
+								type: "pinchstart",
+								target: this
+							} );
+
+						}
+
+					}
 
 				}
 
-			}
+			} else {
 
-			if ( grip !== null && inputSource.gripSpace ) {
+				if ( targetRay !== null ) {
 
-				gripPose = frame.getPose( inputSource.gripSpace, referenceSpace );
+					inputPose = frame.getPose( inputSource.targetRaySpace, referenceSpace );
 
-				if ( gripPose !== null ) {
+					if ( inputPose !== null ) {
 
-					grip.matrix.fromArray( gripPose.transform.matrix );
-					grip.matrix.decompose( grip.position, grip.rotation, grip.scale );
+						targetRay.matrix.fromArray( inputPose.transform.matrix );
+						targetRay.matrix.decompose( targetRay.position, targetRay.rotation, targetRay.scale );
+
+					}
+
+				}
+
+				if ( grip !== null && inputSource.gripSpace ) {
+
+					gripPose = frame.getPose( inputSource.gripSpace, referenceSpace );
+
+					if ( gripPose !== null ) {
+
+						grip.matrix.fromArray( gripPose.transform.matrix );
+						grip.matrix.decompose( grip.position, grip.rotation, grip.scale );
+
+					}
 
 				}
 
@@ -131,6 +235,12 @@ Object.assign( WebXRController.prototype, {
 
 		}
 
+		if ( hand !== null ) {
+
+			hand.visible = ( handPose !== null );
+
+		}
+
 		return this;
 
 	}

+ 17 - 0
src/renderers/webxr/WebXRManager.js

@@ -81,6 +81,21 @@ function WebXRManager( renderer, gl ) {
 
 	};
 
+	this.getHand = function ( index ) {
+
+		let controller = controllers[ index ];
+
+		if ( controller === undefined ) {
+
+			controller = new WebXRController();
+			controllers[ index ] = controller;
+
+		}
+
+		return controller.getHandSpace();
+
+	};
+
 	//
 
 	function onSessionEvent( event ) {
@@ -215,6 +230,8 @@ function WebXRManager( renderer, gl ) {
 
 		const inputSources = session.inputSources;
 
+		console.log("Update sources", inputSources);
+
 		// Assign inputSources to available controllers
 
 		for ( let i = 0; i < controllers.length; i ++ ) {