From e541fe38c7f8c739aaf1a0626fbc9fd2be641e6b Mon Sep 17 00:00:00 2001 From: Philipp Mildenberger Date: Thu, 21 May 2026 20:22:00 +0200 Subject: [PATCH] Render the gaussian splats relative to the camera --- docs/docs/packed-splats.md | 2 +- src/OldSparkRenderer.ts | 3 ++ src/SparkRenderer.ts | 59 ++++++++++++++++++++++++++------------ src/SplatAccumulator.ts | 9 +++--- src/SplatGenerator.ts | 1 + src/SplatMesh.ts | 37 ++++++++++++++++-------- 6 files changed, 75 insertions(+), 36 deletions(-) diff --git a/docs/docs/packed-splats.md b/docs/docs/packed-splats.md index 1c93034f..8bd5f521 100644 --- a/docs/docs/packed-splats.md +++ b/docs/docs/packed-splats.md @@ -127,7 +127,7 @@ Opacity is encoded on a linear scale where 0..255 maps to 0..1. ### Splat center encoding -The center x/y/z components are encoded as float16, which provides 10 bits of mantissa, or approximately 1K steps (0.1%) of resolution between each successive power of 0 from the origin, with a range of up to 32K in distance. If most of the splats are positioned relative to the origin this provides enough positional resolution. Splats that are transformed far from the origin, however (for example when bringing multiple `SplatMesh`es together in a scene that are far apart) may lose precision when mapped to the space of `SparkRenderer`. For scenes where the user camera may move far from the origin, you may want to tie the `SparkRenderer` origin to your camera by adding it as a child of the camera. +The center x/y/z components are encoded as float16, which provides 10 bits of mantissa, or approximately 1K steps (0.1%) of resolution between each successive power of 0 from the origin, with a range of up to 32K in distance. If most of the splats are positioned relative to the origin this provides enough positional resolution. Splats that are transformed far from the origin, however (for example when bringing multiple `SplatMesh`es together in a scene that are far apart) may lose precision when mapped to the space of `SparkRenderer`. ### Splat scales encoding diff --git a/src/OldSparkRenderer.ts b/src/OldSparkRenderer.ts index 2a3224c6..81316895 100644 --- a/src/OldSparkRenderer.ts +++ b/src/OldSparkRenderer.ts @@ -230,6 +230,7 @@ export class OldSparkRenderer extends THREE.Mesh { // List of cameras used for the current viewpoint (for WebXR) private defaultCameras: THREE.Matrix4[] = []; private lastStochastic: boolean | null = null; + private readonly renderOrigin = new THREE.Vector3(); // Should be set to the defaultView, but can be temporarily changed to another // viewpoint using prepareViewpoint() for rendering from a different viewpoint. @@ -700,6 +701,7 @@ export class OldSparkRenderer extends THREE.Mesh { originToWorld = this.active.toWorld; } viewToWorld = viewToWorld ?? originToWorld.clone(); + this.renderOrigin.setFromMatrixPosition(originToWorld); const time = this.time ?? this.clock.getElapsedTime(); const deltaTime = time - (this.lastUpdateTime ?? time); @@ -723,6 +725,7 @@ export class OldSparkRenderer extends THREE.Mesh { time, deltaTime, viewToWorld, + renderOrigin: this.renderOrigin, globalEdits, }); } diff --git a/src/SparkRenderer.ts b/src/SparkRenderer.ts index 21d6f6e1..69a60a45 100644 --- a/src/SparkRenderer.ts +++ b/src/SparkRenderer.ts @@ -379,6 +379,14 @@ export class SparkRenderer extends THREE.Mesh { sortTimeoutId = -1; sortedCenter = new THREE.Vector3().setScalar(Number.NEGATIVE_INFINITY); sortedDir = new THREE.Vector3().setScalar(0); + private readonly accumToCamera = new THREE.Matrix4(); + private readonly accumToCameraScale = new THREE.Vector3(); + private readonly renderCameraToWorld = new THREE.Matrix4(); + private readonly renderCameraOrigin = new THREE.Vector3(); + private readonly lodCameraToWorld = new THREE.Matrix4(); + private readonly lodCameraScale = new THREE.Vector3(); + private readonly lodViewToObject = new THREE.Matrix4(); + private readonly lodMeshOrigin = new THREE.Vector3(); readback32 = new Uint32Array(0); enableLod: boolean; @@ -756,17 +764,20 @@ export class SparkRenderer extends THREE.Mesh { const geometry = this.geometry as SplatGeometry; geometry.instanceCount = spark.activeSplats; - const accumToWorld = new THREE.Matrix4(); - if (!this.display.extSplats) { - accumToWorld.makeTranslation(spark.display.viewOrigin); - } - const cameraToWorld = camera.matrixWorld.clone(); - const worldToCamera = cameraToWorld.invert(); - const accumToCamera = worldToCamera.multiply(accumToWorld); + const renderCameraToWorld = this.renderCameraToWorld + .copy(camera.matrixWorld) + .setPosition( + this.renderCameraOrigin + .setFromMatrixPosition(camera.matrixWorld) + .sub(spark.display.renderOrigin), + ); + const accumToCamera = this.accumToCamera + .copy(renderCameraToWorld) + .invert(); accumToCamera.decompose( this.uniforms.renderToViewPos.value, this.uniforms.renderToViewQuat.value, - new THREE.Vector3(), + this.accumToCameraScale, ); this.uniforms.renderToViewBasis.value.setFromMatrix4(accumToCamera); @@ -997,7 +1008,7 @@ export class SparkRenderer extends THREE.Mesh { const current = this.current; - this.sortedCenter.copy(current.viewOrigin); + this.sortedCenter.copy(current.renderOrigin); this.sortedDir.copy(current.viewDirection); const { numSplats, maxSplats } = current; @@ -1159,8 +1170,9 @@ export class SparkRenderer extends THREE.Mesh { const viewQuat = new THREE.Quaternion(); this.current.viewToWorld.decompose(viewPos, viewQuat, new THREE.Vector3()); + const lodPos = this.lodPosOverride ?? this.current.renderOrigin; if (this.lodPosOverride) { - viewPos.copy(this.lodPosOverride); + viewPos.copy(this.lodPosOverride).sub(this.current.renderOrigin); } if (this.lodQuatOverride) { viewQuat.copy(this.lodQuatOverride).normalize(); @@ -1174,7 +1186,7 @@ export class SparkRenderer extends THREE.Mesh { this.lodDirty = true; } - const distance = viewPos.distanceTo(this.lastLod.pos); + const distance = lodPos.distanceTo(this.lastLod.pos); const distanceRamp = Math.max(0.0, 1.0 - distance / 1.0); const dot = viewQuat.dot(this.lastLod.quat); const quatRamp = Math.max(0.0, 1.0 - (1.0 - dot) / 0.01); @@ -1306,12 +1318,12 @@ export class SparkRenderer extends THREE.Mesh { if (this.lastLod) { const deltaTime = Math.max(1, now - this.lastLod.timestamp); deltaPred - .copy(viewPos) + .copy(lodPos) .sub(this.lastLod.pos) .multiplyScalar(this.lastTraverseTime / deltaTime); } this.lastLod = { - pos: viewPos, + pos: lodPos.clone(), quat: viewQuat, pixelScaleLimit, maxSplats, @@ -1324,8 +1336,10 @@ export class SparkRenderer extends THREE.Mesh { deltaPred, lodMeshes, maxSplats, + this.lastLod.pos, viewPos, viewQuat, + this.current.renderOrigin, pixelScaleLimit, ); this.currentLod = this.lastLod; @@ -1365,25 +1379,32 @@ export class SparkRenderer extends THREE.Mesh { deltaPred: THREE.Vector3, lodMeshes: SplatMesh[], maxSplats: number, + lodPos: THREE.Vector3, viewPos: THREE.Vector3, viewQuat: THREE.Quaternion, + renderOrigin: THREE.Vector3, pixelScaleLimit: number, ) { // Commented out because it makes LoDing less stable // viewPos.add(deltaPred); const uuidToMesh: Map = new Map(); - const cameraToWorld = new THREE.Matrix4().compose( + const cameraToWorld = this.lodCameraToWorld.compose( viewPos, viewQuat, - new THREE.Vector3().setScalar(1), + this.lodCameraScale.setScalar(1), ); const instances = lodMeshes.reduce( (instances, mesh) => { uuidToMesh.set(mesh.uuid, mesh); - const viewToObject = mesh.matrixWorld - .clone() + const viewToObject = this.lodViewToObject + .copy(mesh.matrixWorld) + .setPosition( + this.lodMeshOrigin + .setFromMatrixPosition(mesh.matrixWorld) + .sub(renderOrigin), + ) .invert() .multiply(cameraToWorld); @@ -1407,7 +1428,7 @@ export class SparkRenderer extends THREE.Mesh { instanceId: mesh.uuid, lodId: record.lodId, rootPage: record.rootPage, - viewToObjectCols: viewToObject.elements, + viewToObjectCols: viewToObject.elements.slice(), lodScale: mesh.lodScale, behindFoveate: mesh.behindFoveate ?? this.behindFoveate, coneFov0: mesh.coneFov0 ?? this.coneFov0, @@ -1472,7 +1493,7 @@ export class SparkRenderer extends THREE.Mesh { const meshPosition = mesh.getWorldPosition(new THREE.Vector3()); return { splats: mesh.paged, - distance: meshPosition.distanceTo(viewPos), + distance: meshPosition.distanceTo(lodPos), }; }) .filter((result) => result !== null); diff --git a/src/SplatAccumulator.ts b/src/SplatAccumulator.ts index cfdcdaab..efbefdbd 100644 --- a/src/SplatAccumulator.ts +++ b/src/SplatAccumulator.ts @@ -59,7 +59,7 @@ export class SplatAccumulator { time = 0; deltaTime = 0; viewToWorld = new THREE.Matrix4(); - viewOrigin = new THREE.Vector3(); + renderOrigin = new THREE.Vector3(); viewDirection = new THREE.Vector3(); static viewCenterUniform = new DynoVec3({ value: new THREE.Vector3() }); static viewDirUniform = new DynoVec3({ value: new THREE.Vector3() }); @@ -460,10 +460,10 @@ export class SplatAccumulator { { numSplats: number; texture: THREE.DataTexture } >; }) { - this.viewToWorld.copy(camera.matrixWorld); - camera.getWorldPosition(this.viewOrigin); + camera.getWorldPosition(this.renderOrigin); + this.viewToWorld.copy(camera.matrixWorld).setPosition(0, 0, 0); camera.getWorldDirection(this.viewDirection); - SplatAccumulator.viewCenterUniform.value.copy(this.viewOrigin); + SplatAccumulator.viewCenterUniform.value.set(0, 0, 0); SplatAccumulator.viewDirUniform.value.copy(this.viewDirection); SplatAccumulator.sortRadialUniform.value = sortRadial; @@ -502,6 +502,7 @@ export class SplatAccumulator { time: this.time, deltaTime: this.deltaTime, viewToWorld: this.viewToWorld, + renderOrigin: this.renderOrigin, camera, renderSize, globalEdits, diff --git a/src/SplatGenerator.ts b/src/SplatGenerator.ts index de778c7e..3c47938d 100644 --- a/src/SplatGenerator.ts +++ b/src/SplatGenerator.ts @@ -256,6 +256,7 @@ export interface FrameUpdateContext { time: number; deltaTime: number; viewToWorld: THREE.Matrix4; + renderOrigin: THREE.Vector3; camera?: THREE.Camera; renderSize?: THREE.Vector2; globalEdits: SplatEdit[]; diff --git a/src/SplatMesh.ts b/src/SplatMesh.ts index 1e288e6b..172217c7 100644 --- a/src/SplatMesh.ts +++ b/src/SplatMesh.ts @@ -308,6 +308,11 @@ export class SplatMesh extends SplatGenerator { showLodPage?: number; showLodPageDyno = new DynoInt({ value: 0 }); + private readonly objectToRender = new THREE.Matrix4(); + private readonly objectToRenderOffset = new THREE.Vector3(); + private readonly worldToObject = new THREE.Matrix4(); + private readonly worldToView = new THREE.Matrix4(); + private readonly viewToObjectMatrix = new THREE.Matrix4(); constructor(options: SplatMeshOptions = {}) { super({ @@ -833,6 +838,7 @@ export class SplatMesh extends SplatGenerator { time, deltaTime, viewToWorld, + renderOrigin, camera, renderSize, globalEdits, @@ -870,8 +876,16 @@ export class SplatMesh extends SplatGenerator { this.generatorDirty = true; } + this.updateMatrixWorld(); + const objectToWorld = this.objectToRender.copy(this.matrixWorld); + objectToWorld.setPosition( + this.objectToRenderOffset + .setFromMatrixPosition(this.matrixWorld) + .sub(renderOrigin), + ); + if (!this.covSplats) { - if (this.context.transform.update(this)) { + if (this.context.transform.updateFromMatrix(objectToWorld)) { updated = true; } @@ -881,7 +895,7 @@ export class SplatMesh extends SplatGenerator { ) { updated = true; } - const worldToView = viewToWorld.clone().invert(); + const worldToView = this.worldToView.copy(viewToWorld).invert(); if ( this.context.worldToView.updateFromMatrix(worldToView) && this.enableWorldToView @@ -889,13 +903,10 @@ export class SplatMesh extends SplatGenerator { updated = true; } - const objectToWorld = new THREE.Matrix4().compose( - this.context.transform.translate.value, - this.context.transform.rotate.value, - new THREE.Vector3().setScalar(this.context.transform.scale.value), + const viewToObjectMatrix = this.viewToObjectMatrix.multiplyMatrices( + this.worldToObject.copy(objectToWorld).invert(), + viewToWorld, ); - const worldToObject = objectToWorld.invert(); - const viewToObjectMatrix = worldToObject.multiply(viewToWorld); if ( this.context.viewToObject.updateFromMatrix(viewToObjectMatrix) && (this.enableViewToObject || this.context.splats.hasRgbDir()) @@ -904,7 +915,7 @@ export class SplatMesh extends SplatGenerator { updated = true; } } else { - if (this.context.covTransform.update(this)) { + if (this.context.covTransform.updateFromMatrix(objectToWorld)) { updated = true; } @@ -914,7 +925,7 @@ export class SplatMesh extends SplatGenerator { ) { updated = true; } - const worldToView = viewToWorld.clone().invert(); + const worldToView = this.worldToView.copy(viewToWorld).invert(); if ( this.context.covWorldToView.updateFromMatrix(worldToView) && this.enableWorldToView @@ -922,8 +933,10 @@ export class SplatMesh extends SplatGenerator { updated = true; } - const worldToObject = this.matrixWorld.clone().invert(); - const viewToObjectMatrix = worldToObject.multiply(viewToWorld); + const viewToObjectMatrix = this.viewToObjectMatrix.multiplyMatrices( + this.worldToObject.copy(objectToWorld).invert(), + viewToWorld, + ); if ( this.context.covViewToObject.updateFromMatrix(viewToObjectMatrix) && (this.enableViewToObject || this.context.splats.hasRgbDir())