three.js/examples/jsm/loaders/UltraHDRLoader.js

import {
	ClampToEdgeWrapping,
	DataTexture,
	DataUtils,
	FileLoader,
	HalfFloatType,
	LinearFilter,
	LinearMipMapLinearFilter,
	LinearSRGBColorSpace,
	Loader,
	RGBAFormat,
	UVMapping,
} from 'three';

// UltraHDR Image Format - https://developer.android.com/media/platform/hdr-image-format
// Originally proposed as gainmap-js using a WASM dependency - https://github.com/MONOGRID/gainmap-js

/**
 *
 * Short format brief:
 *
 *  [JPEG headers]
 *  [XMP metadata describing the MPF container and *both* SDR and gainmap images]
 *  [Optional metadata] [EXIF] [ICC Profile]
 *  [SDR image]
 *  [XMP metadata describing only the gainmap image]
 *  [Gainmap image]
 *
 * Each section is separated by a 0xFFXX byte followed by a descriptor byte (0xFFE0, 0xFFE1, 0xFFE2.)
 * Binary image storages are prefixed with a unique 0xFFD8 16-bit descriptor.
 */

/**
 * Current feature set:
 * - JPEG headers (required)
 * - XMP metadata (required)
 *  - XMP validation (not implemented)
 * - EXIF profile (not implemented)
 * - ICC profile (not implemented)
 * - Binary storage for SDR & HDR images (required)
 * - Gainmap metadata (required)
 * - Non-JPEG image formats (not implemented)
 * - Primary image as an HDR image (not implemented)
 */

/* Calculating this SRGB powers is extremely slow for 4K images and can be sufficiently precalculated for a 3-4x speed boost */
const SRGB_TO_LINEAR = Array( 1024 )
	.fill( 0 )
	.map( ( _, value ) =>
		Math.pow( ( value / 255 ) * 0.9478672986 + 0.0521327014, 2.4 )
	);

class UltraHDRLoader extends Loader {

	constructor( manager ) {

		super( manager );

		this.type = HalfFloatType;

	}

	setDataType( value ) {

		this.type = value;

		return this;

	}

	parse( buffer, onLoad ) {

		const xmpMetadata = {
			version: null,
			baseRenditionIsHDR: null,
			gainMapMin: null,
			gainMapMax: null,
			gamma: null,
			offsetSDR: null,
			offsetHDR: null,
			hdrCapacityMin: null,
			hdrCapacityMax: null,
		};
		const textDecoder = new TextDecoder();

		const data = new DataView( buffer );
		const dataAsString = textDecoder.decode( data );

		/* Minimal sufficient validation - https://developer.android.com/media/platform/hdr-image-format#signal_of_the_format */
		if ( ! dataAsString.includes( 'hdrgm:Version="1.0"' ) ) {

			throw new Error( 'THREE.UltraHDRLoader: Not a valid UltraHDR image' );

		}

		let byteOffset = 0;
		const sections = [];

		while ( byteOffset < data.byteLength ) {

			const byte = data.getUint8( byteOffset );

			if ( byte === 0xff ) {

				const leadingByte = data.getUint8( byteOffset + 1 );

				if (
					[
						/* Valid section headers */
						0xd8, // SOI
						0xe0, // APP0
						0xe1, // APP1
						0xe2, // APP2
					].includes( leadingByte )
				) {

					sections.push( {
						sectionType: leadingByte,
						section: [ byte, leadingByte ],
						sectionOffset: byteOffset + 2,
					} );

					byteOffset += 2;

				} else {

					sections[ sections.length - 1 ].section.push( byte, leadingByte );

					byteOffset += 2;

				}

			} else {

				sections[ sections.length - 1 ].section.push( byte );

				byteOffset ++;

			}

		}

		let primaryImage, gainmapImage;

		for ( let i = 0; i < sections.length; i ++ ) {

			const { sectionType, section, sectionOffset } = sections[ i ];

			if ( sectionType === 0xe0 ) {

				/* JPEG Header - no useful information */

			} else if ( sectionType === 0xe1 ) {

				/* XMP Metadata */

				this._parseXMPMetadata(
					textDecoder.decode( new Uint8Array( section ) ),
					xmpMetadata
				);

			} else if ( sectionType === 0xe2 ) {

				/* Data Sections - MPF / EXIF / ICC Profile */

				const sectionData = new DataView(
					new Uint8Array( section.slice( 2 ) ).buffer
				);
				const sectionHeader = sectionData.getUint32( 2, false );

				if ( sectionHeader === 0x4d504600 ) {

					/* MPF Section */

					/* Section contains a list of static bytes and ends with offsets indicating location of SDR and gainmap images */
					/* First bytes after header indicate little / big endian ordering (0x49492A00 - LE / 0x4D4D002A - BE) */
					/*
            ... 60 bytes indicating tags, versions, etc. ...

            bytes | bits | description

            4       32     primary image size
            4       32     primary image offset
            2       16     0x0000
            2       16     0x0000

            4       32     0x00000000
            4       32     gainmap image size
            4       32     gainmap image offset
            2       16     0x0000
            2       16     0x0000
          */

					const mpfLittleEndian = sectionData.getUint32( 6 ) === 0x49492a00;
					const mpfBytesOffset = 60;

					/* SDR size includes the metadata length, SDR offset is always 0 */

					const primaryImageSize = sectionData.getUint32(
						mpfBytesOffset,
						mpfLittleEndian
					);
					const primaryImageOffset = sectionData.getUint32(
						mpfBytesOffset + 4,
						mpfLittleEndian
					);

					/* Gainmap size is an absolute value starting from its offset, gainmap offset needs 6 bytes padding to take into account 0x00 bytes at the end of XMP */
					const gainmapImageSize = sectionData.getUint32(
						mpfBytesOffset + 16,
						mpfLittleEndian
					);
					const gainmapImageOffset =
						sectionData.getUint32( mpfBytesOffset + 20, mpfLittleEndian ) +
						sectionOffset +
						6;

					primaryImage = new Uint8Array(
						data.buffer,
						primaryImageOffset,
						primaryImageSize
					);

					gainmapImage = new Uint8Array(
						data.buffer,
						gainmapImageOffset,
						gainmapImageSize
					);

				}

			}

		}

		if ( primaryImage && gainmapImage ) {

			this._applyGainmapToSDR(
				xmpMetadata,
				primaryImage,
				gainmapImage,
				( hdrBuffer, width, height ) => {

					onLoad( {
						width,
						height,
						data: hdrBuffer,
						format: RGBAFormat,
						type: this.type,
					} );

				},
				( error ) => {

					throw new Error( error );

				}
			);

		} else {

			throw new Error( 'THREE.UltraHDRLoader: Could not parse UltraHDR images' );

		}

	}

	load( url, onLoad, onError ) {

		const texture = new DataTexture(
			this.type === HalfFloatType ? new Uint16Array() : new Float32Array(),
			0,
			0,
			RGBAFormat,
			this.type,
			UVMapping,
			ClampToEdgeWrapping,
			ClampToEdgeWrapping,
			LinearFilter,
			LinearMipMapLinearFilter,
			1,
			LinearSRGBColorSpace
		);
		texture.generateMipmaps = true;
		texture.flipY = true;

		const loader = new FileLoader( this.manager );
		loader.setResponseType( 'arraybuffer' );
		loader.setRequestHeader( this.requestHeader );
		loader.setPath( this.path );
		loader.setWithCredentials( this.withCredentials );
		loader.load( url, ( buffer ) => {

			try {

				this.parse(
					buffer,
					( texData ) => {

						texture.image = {
							data: texData.data,
							width: texData.width,
							height: texData.height,
						};
						texture.needsUpdate = true;

						if ( onLoad ) onLoad( texture, texData );

					},
					onError
				);

			} catch ( error ) {

				if ( onError ) onError( error );

				console.error( error );

			}

		} );

		return texture;

	}

	_parseXMPMetadata( xmpDataString, xmpMetadata ) {

		const domParser = new DOMParser();

		const xmpXml = domParser.parseFromString(
			xmpDataString.substring(
				xmpDataString.indexOf( '<' ),
				xmpDataString.lastIndexOf( '>' ) + 1
			),
			'text/xml'
		);

		/* Determine if given XMP metadata is the primary GContainer descriptor or a gainmap descriptor */
		const [ hasHDRContainerDescriptor ] = xmpXml.getElementsByTagName(
			'Container:Directory'
		);

		if ( hasHDRContainerDescriptor ) {

			/* There's not much useful information in the container descriptor besides memory-validation */

		} else {

			/* Gainmap descriptor - defaults from https://developer.android.com/media/platform/hdr-image-format#HDR_gain_map_metadata */

			const [ gainmapNode ] = xmpXml.getElementsByTagName( 'rdf:Description' );

			xmpMetadata.version = gainmapNode.getAttribute( 'hdrgm:Version' ) || '1.0';
			xmpMetadata.baseRenditionIsHDR =
				gainmapNode.getAttribute( 'hdrgm:BaseRenditionIsHDR' ) === 'True';
			xmpMetadata.gainMapMin = parseFloat(
				gainmapNode.getAttribute( 'hdrgm:GainMapMin' ) || 0.0
			);
			xmpMetadata.gainMapMax = parseFloat(
				gainmapNode.getAttribute( 'hdrgm:GainMapMax' ) || 1.0
			);
			xmpMetadata.gamma = parseFloat(
				gainmapNode.getAttribute( 'hdrgm:Gamma' ) || 1.0
			);
			xmpMetadata.offsetSDR = parseFloat(
				gainmapNode.getAttribute( 'hdrgm:OffsetSDR' ) / ( 1 / 64 )
			);
			xmpMetadata.offsetHDR = parseFloat(
				gainmapNode.getAttribute( 'hdrgm:OffsetHDR' ) / ( 1 / 64 )
			);
			xmpMetadata.hdrCapacityMin = parseFloat(
				gainmapNode.getAttribute( 'hdrgm:HDRCapacityMin' ) || 0.0
			);
			xmpMetadata.hdrCapacityMax = parseFloat(
				gainmapNode.getAttribute( 'hdrgm:HDRCapacityMax' ) || 1.0
			);

		}

	}

	_srgbToLinear( value ) {

		if ( value / 255 < 0.04045 ) {

			return ( value / 255 ) * 0.0773993808;

		}

		if ( value < 1024 ) {

			return SRGB_TO_LINEAR[ ~ ~ value ];

		}

		return Math.pow( ( value / 255 ) * 0.9478672986 + 0.0521327014, 2.4 );

	}

	_applyGainmapToSDR(
		xmpMetadata,
		sdrBuffer,
		gainmapBuffer,
		onSuccess,
		onError
	) {

		const getImageDataFromBuffer = ( buffer ) =>
			new Promise( ( resolve, reject ) => {

				const imageLoader = document.createElement( 'img' );

				imageLoader.onload = () => {

					const image = {
						width: imageLoader.naturalWidth,
						height: imageLoader.naturalHeight,
						source: imageLoader,
					};

					URL.revokeObjectURL( imageLoader.src );

					resolve( image );

				};

				imageLoader.onerror = () => {

					URL.revokeObjectURL( imageLoader.src );

					reject();

				};

				imageLoader.src = URL.createObjectURL(
					new Blob( [ buffer ], { type: 'image/jpeg' } )
				);

			} );

		Promise.all( [
			getImageDataFromBuffer( sdrBuffer ),
			getImageDataFromBuffer( gainmapBuffer ),
		] )
			.then( ( [ sdrImage, gainmapImage ] ) => {

				const sdrImageAspect = sdrImage.width / sdrImage.height;
				const gainmapImageAspect = gainmapImage.width / gainmapImage.height;

				if ( sdrImageAspect !== gainmapImageAspect ) {

					onError(
						'THREE.UltraHDRLoader Error: Aspect ratio mismatch between SDR and Gainmap images'
					);

					return;

				}

				const canvas = document.createElement( 'canvas' );
				const ctx = canvas.getContext( '2d', {
					willReadFrequently: true,
					colorSpace: 'srgb',
				} );

				canvas.width = sdrImage.width;
				canvas.height = sdrImage.height;

				/* Use out-of-the-box interpolation of Canvas API to scale gainmap to fit the SDR resolution */
				ctx.drawImage(
					gainmapImage.source,
					0,
					0,
					gainmapImage.width,
					gainmapImage.height,
					0,
					0,
					sdrImage.width,
					sdrImage.height
				);
				const gainmapImageData = ctx.getImageData(
					0,
					0,
					sdrImage.width,
					sdrImage.height,
					{ colorSpace: 'srgb' }
				);

				ctx.drawImage( sdrImage.source, 0, 0 );
				const sdrImageData = ctx.getImageData(
					0,
					0,
					sdrImage.width,
					sdrImage.height,
					{ colorSpace: 'srgb' }
				);

				/* HDR Recovery formula - https://developer.android.com/media/platform/hdr-image-format#use_the_gain_map_to_create_adapted_HDR_rendition */
				let hdrBuffer;

				if ( this.type === HalfFloatType ) {

					hdrBuffer = new Uint16Array( sdrImageData.data.length ).fill( 23544 );

				} else {

					hdrBuffer = new Float32Array( sdrImageData.data.length ).fill( 255 );

				}

				const maxDisplayBoost = Math.sqrt(
					Math.pow(
						/* 1.8 instead of 2 near-perfectly rectifies approximations introduced by precalculated SRGB_TO_LINEAR values */
						1.8,
						xmpMetadata.hdrCapacityMax
					)
				);
				const unclampedWeightFactor =
					( Math.log2( maxDisplayBoost ) - xmpMetadata.hdrCapacityMin ) /
					( xmpMetadata.hdrCapacityMax - xmpMetadata.hdrCapacityMin );
				const weightFactor = Math.min(
					Math.max( unclampedWeightFactor, 0.0 ),
					1.0
				);
				const useGammaOne = xmpMetadata.gamma === 1.0;

				for (
					let pixelIndex = 0;
					pixelIndex < sdrImageData.data.length;
					pixelIndex += 4
				) {

					const x = ( pixelIndex / 4 ) % sdrImage.width;
					const y = Math.floor( pixelIndex / 4 / sdrImage.width );

					for ( let index = pixelIndex; index < pixelIndex + 3; index ++ ) {

						const sdrValue = sdrImageData.data[ index ];

						const gainmapIndex = ( y * sdrImage.width + x ) * 4;
						const gainmapValue = gainmapImageData.data[ gainmapIndex ] / 255.0;

						/* Gamma is 1.0 by default */
						const logRecovery = useGammaOne
							? gainmapValue
							: Math.pow( gainmapValue, 1.0 / xmpMetadata.gamma );

						const logBoost =
							xmpMetadata.gainMapMin * ( 1.0 - logRecovery ) +
							xmpMetadata.gainMapMax * logRecovery;

						const hdrValue =
						( sdrValue + xmpMetadata.offsetSDR ) *
						( logBoost * weightFactor === 0.0
							? 1.0
							: Math.pow( 2, logBoost * weightFactor ) ) -
					xmpMetadata.offsetHDR;

						const linearHDRValue = Math.min(
							Math.max(
								this._srgbToLinear( hdrValue ),
								0
							),
							65504
						);

						hdrBuffer[ index ] =
							this.type === HalfFloatType
								? DataUtils.toHalfFloat( linearHDRValue )
								: linearHDRValue;

					}

				}

				onSuccess( hdrBuffer, sdrImage.width, sdrImage.height );

			} )
			.catch( () => {

				throw new Error(
					'THREE.UltraHDRLoader Error: Could not parse UltraHDR images'
				);

			} );

	}

}

export { UltraHDRLoader };