Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion docs/docs/packed-splats.md
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ Opacity is encoded on a linear scale where 0..255 maps to 0..1.

### Splat center encoding

The center x/y/z components are encoded as float16, which provides 10 bits of mantissa, or approximately 1K steps (0.1%) of resolution between each successive power of 0 from the origin, with a range of up to 32K in distance. If most of the splats are positioned relative to the origin this provides enough positional resolution. Splats that are transformed far from the origin, however (for example when bringing multiple `SplatMesh`es together in a scene that are far apart) may lose precision when mapped to the space of `SparkRenderer`. For scenes where the user camera may move far from the origin, you may want to tie the `SparkRenderer` origin to your camera by adding it as a child of the camera.
The center x/y/z components are encoded as float16, which provides 10 bits of mantissa, or approximately 1K steps (0.1%) of resolution between each successive power of 0 from the origin, with a range of up to 32K in distance. If most of the splats are positioned relative to the origin this provides enough positional resolution. Splats that are transformed far from the origin, however (for example when bringing multiple `SplatMesh`es together in a scene that are far apart) may lose precision when mapped to the space of `SparkRenderer`.

### Splat scales encoding

Expand Down
3 changes: 3 additions & 0 deletions src/OldSparkRenderer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,7 @@ export class OldSparkRenderer extends THREE.Mesh {
// List of cameras used for the current viewpoint (for WebXR)
private defaultCameras: THREE.Matrix4[] = [];
private lastStochastic: boolean | null = null;
private readonly renderOrigin = new THREE.Vector3();

// Should be set to the defaultView, but can be temporarily changed to another
// viewpoint using prepareViewpoint() for rendering from a different viewpoint.
Expand Down Expand Up @@ -700,6 +701,7 @@ export class OldSparkRenderer extends THREE.Mesh {
originToWorld = this.active.toWorld;
}
viewToWorld = viewToWorld ?? originToWorld.clone();
this.renderOrigin.setFromMatrixPosition(originToWorld);

const time = this.time ?? this.clock.getElapsedTime();
const deltaTime = time - (this.lastUpdateTime ?? time);
Expand All @@ -723,6 +725,7 @@ export class OldSparkRenderer extends THREE.Mesh {
time,
deltaTime,
viewToWorld,
renderOrigin: this.renderOrigin,
globalEdits,
});
}
Expand Down
59 changes: 40 additions & 19 deletions src/SparkRenderer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -379,6 +379,14 @@ export class SparkRenderer extends THREE.Mesh {
sortTimeoutId = -1;
sortedCenter = new THREE.Vector3().setScalar(Number.NEGATIVE_INFINITY);
sortedDir = new THREE.Vector3().setScalar(0);
private readonly accumToCamera = new THREE.Matrix4();
private readonly accumToCameraScale = new THREE.Vector3();
private readonly renderCameraToWorld = new THREE.Matrix4();
private readonly renderCameraOrigin = new THREE.Vector3();
private readonly lodCameraToWorld = new THREE.Matrix4();
private readonly lodCameraScale = new THREE.Vector3();
private readonly lodViewToObject = new THREE.Matrix4();
private readonly lodMeshOrigin = new THREE.Vector3();
readback32 = new Uint32Array(0);

enableLod: boolean;
Expand Down Expand Up @@ -756,17 +764,20 @@ export class SparkRenderer extends THREE.Mesh {
const geometry = this.geometry as SplatGeometry;
geometry.instanceCount = spark.activeSplats;

const accumToWorld = new THREE.Matrix4();
if (!this.display.extSplats) {
accumToWorld.makeTranslation(spark.display.viewOrigin);
}
const cameraToWorld = camera.matrixWorld.clone();
const worldToCamera = cameraToWorld.invert();
const accumToCamera = worldToCamera.multiply(accumToWorld);
const renderCameraToWorld = this.renderCameraToWorld
.copy(camera.matrixWorld)
.setPosition(
this.renderCameraOrigin
.setFromMatrixPosition(camera.matrixWorld)
.sub(spark.display.renderOrigin),
);
const accumToCamera = this.accumToCamera
.copy(renderCameraToWorld)
.invert();
accumToCamera.decompose(
this.uniforms.renderToViewPos.value,
this.uniforms.renderToViewQuat.value,
new THREE.Vector3(),
this.accumToCameraScale,
);
this.uniforms.renderToViewBasis.value.setFromMatrix4(accumToCamera);

Expand Down Expand Up @@ -997,7 +1008,7 @@ export class SparkRenderer extends THREE.Mesh {

const current = this.current;

this.sortedCenter.copy(current.viewOrigin);
this.sortedCenter.copy(current.renderOrigin);
this.sortedDir.copy(current.viewDirection);

const { numSplats, maxSplats } = current;
Expand Down Expand Up @@ -1159,8 +1170,9 @@ export class SparkRenderer extends THREE.Mesh {
const viewQuat = new THREE.Quaternion();
this.current.viewToWorld.decompose(viewPos, viewQuat, new THREE.Vector3());

const lodPos = this.lodPosOverride ?? this.current.renderOrigin;
if (this.lodPosOverride) {
viewPos.copy(this.lodPosOverride);
viewPos.copy(this.lodPosOverride).sub(this.current.renderOrigin);
}
if (this.lodQuatOverride) {
viewQuat.copy(this.lodQuatOverride).normalize();
Expand All @@ -1174,7 +1186,7 @@ export class SparkRenderer extends THREE.Mesh {
this.lodDirty = true;
}

const distance = viewPos.distanceTo(this.lastLod.pos);
const distance = lodPos.distanceTo(this.lastLod.pos);
const distanceRamp = Math.max(0.0, 1.0 - distance / 1.0);
const dot = viewQuat.dot(this.lastLod.quat);
const quatRamp = Math.max(0.0, 1.0 - (1.0 - dot) / 0.01);
Expand Down Expand Up @@ -1306,12 +1318,12 @@ export class SparkRenderer extends THREE.Mesh {
if (this.lastLod) {
const deltaTime = Math.max(1, now - this.lastLod.timestamp);
deltaPred
.copy(viewPos)
.copy(lodPos)
.sub(this.lastLod.pos)
.multiplyScalar(this.lastTraverseTime / deltaTime);
}
this.lastLod = {
pos: viewPos,
pos: lodPos.clone(),
quat: viewQuat,
pixelScaleLimit,
maxSplats,
Expand All @@ -1324,8 +1336,10 @@ export class SparkRenderer extends THREE.Mesh {
deltaPred,
lodMeshes,
maxSplats,
this.lastLod.pos,
viewPos,
viewQuat,
this.current.renderOrigin,
pixelScaleLimit,
);
this.currentLod = this.lastLod;
Expand Down Expand Up @@ -1365,25 +1379,32 @@ export class SparkRenderer extends THREE.Mesh {
deltaPred: THREE.Vector3,
lodMeshes: SplatMesh[],
maxSplats: number,
lodPos: THREE.Vector3,
viewPos: THREE.Vector3,
viewQuat: THREE.Quaternion,
renderOrigin: THREE.Vector3,
pixelScaleLimit: number,
) {
// Commented out because it makes LoDing less stable
// viewPos.add(deltaPred);

const uuidToMesh: Map<string, SplatMesh> = new Map();
const cameraToWorld = new THREE.Matrix4().compose(
const cameraToWorld = this.lodCameraToWorld.compose(
viewPos,
viewQuat,
new THREE.Vector3().setScalar(1),
this.lodCameraScale.setScalar(1),
);

const instances = lodMeshes.reduce(
(instances, mesh) => {
uuidToMesh.set(mesh.uuid, mesh);
const viewToObject = mesh.matrixWorld
.clone()
const viewToObject = this.lodViewToObject
.copy(mesh.matrixWorld)
.setPosition(
this.lodMeshOrigin
.setFromMatrixPosition(mesh.matrixWorld)
.sub(renderOrigin),
)
.invert()
.multiply(cameraToWorld);

Expand All @@ -1407,7 +1428,7 @@ export class SparkRenderer extends THREE.Mesh {
instanceId: mesh.uuid,
lodId: record.lodId,
rootPage: record.rootPage,
viewToObjectCols: viewToObject.elements,
viewToObjectCols: viewToObject.elements.slice(),
lodScale: mesh.lodScale,
behindFoveate: mesh.behindFoveate ?? this.behindFoveate,
coneFov0: mesh.coneFov0 ?? this.coneFov0,
Expand Down Expand Up @@ -1472,7 +1493,7 @@ export class SparkRenderer extends THREE.Mesh {
const meshPosition = mesh.getWorldPosition(new THREE.Vector3());
return {
splats: mesh.paged,
distance: meshPosition.distanceTo(viewPos),
distance: meshPosition.distanceTo(lodPos),
};
})
.filter((result) => result !== null);
Expand Down
9 changes: 5 additions & 4 deletions src/SplatAccumulator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ export class SplatAccumulator {
time = 0;
deltaTime = 0;
viewToWorld = new THREE.Matrix4();
viewOrigin = new THREE.Vector3();
renderOrigin = new THREE.Vector3();
viewDirection = new THREE.Vector3();
static viewCenterUniform = new DynoVec3({ value: new THREE.Vector3() });
static viewDirUniform = new DynoVec3({ value: new THREE.Vector3() });
Expand Down Expand Up @@ -460,10 +460,10 @@ export class SplatAccumulator {
{ numSplats: number; texture: THREE.DataTexture }
>;
}) {
this.viewToWorld.copy(camera.matrixWorld);
camera.getWorldPosition(this.viewOrigin);
camera.getWorldPosition(this.renderOrigin);
this.viewToWorld.copy(camera.matrixWorld).setPosition(0, 0, 0);
camera.getWorldDirection(this.viewDirection);
SplatAccumulator.viewCenterUniform.value.copy(this.viewOrigin);
SplatAccumulator.viewCenterUniform.value.set(0, 0, 0);
SplatAccumulator.viewDirUniform.value.copy(this.viewDirection);
SplatAccumulator.sortRadialUniform.value = sortRadial;

Expand Down Expand Up @@ -502,6 +502,7 @@ export class SplatAccumulator {
time: this.time,
deltaTime: this.deltaTime,
viewToWorld: this.viewToWorld,
renderOrigin: this.renderOrigin,
camera,
renderSize,
globalEdits,
Expand Down
1 change: 1 addition & 0 deletions src/SplatGenerator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -256,6 +256,7 @@ export interface FrameUpdateContext {
time: number;
deltaTime: number;
viewToWorld: THREE.Matrix4;
renderOrigin: THREE.Vector3;
camera?: THREE.Camera;
renderSize?: THREE.Vector2;
globalEdits: SplatEdit[];
Expand Down
37 changes: 25 additions & 12 deletions src/SplatMesh.ts
Original file line number Diff line number Diff line change
Expand Up @@ -308,6 +308,11 @@ export class SplatMesh extends SplatGenerator {

showLodPage?: number;
showLodPageDyno = new DynoInt({ value: 0 });
private readonly objectToRender = new THREE.Matrix4();
private readonly objectToRenderOffset = new THREE.Vector3();
private readonly worldToObject = new THREE.Matrix4();
private readonly worldToView = new THREE.Matrix4();
private readonly viewToObjectMatrix = new THREE.Matrix4();

constructor(options: SplatMeshOptions = {}) {
super({
Expand Down Expand Up @@ -833,6 +838,7 @@ export class SplatMesh extends SplatGenerator {
time,
deltaTime,
viewToWorld,
renderOrigin,
camera,
renderSize,
globalEdits,
Expand Down Expand Up @@ -870,8 +876,16 @@ export class SplatMesh extends SplatGenerator {
this.generatorDirty = true;
}

this.updateMatrixWorld();
const objectToWorld = this.objectToRender.copy(this.matrixWorld);
objectToWorld.setPosition(
this.objectToRenderOffset
.setFromMatrixPosition(this.matrixWorld)
.sub(renderOrigin),
);

if (!this.covSplats) {
if (this.context.transform.update(this)) {
if (this.context.transform.updateFromMatrix(objectToWorld)) {
updated = true;
}

Expand All @@ -881,21 +895,18 @@ export class SplatMesh extends SplatGenerator {
) {
updated = true;
}
const worldToView = viewToWorld.clone().invert();
const worldToView = this.worldToView.copy(viewToWorld).invert();
if (
this.context.worldToView.updateFromMatrix(worldToView) &&
this.enableWorldToView
) {
updated = true;
}

const objectToWorld = new THREE.Matrix4().compose(
this.context.transform.translate.value,
this.context.transform.rotate.value,
new THREE.Vector3().setScalar(this.context.transform.scale.value),
const viewToObjectMatrix = this.viewToObjectMatrix.multiplyMatrices(
this.worldToObject.copy(objectToWorld).invert(),
viewToWorld,
);
const worldToObject = objectToWorld.invert();
const viewToObjectMatrix = worldToObject.multiply(viewToWorld);
if (
this.context.viewToObject.updateFromMatrix(viewToObjectMatrix) &&
(this.enableViewToObject || this.context.splats.hasRgbDir())
Expand All @@ -904,7 +915,7 @@ export class SplatMesh extends SplatGenerator {
updated = true;
}
} else {
if (this.context.covTransform.update(this)) {
if (this.context.covTransform.updateFromMatrix(objectToWorld)) {
updated = true;
}

Expand All @@ -914,16 +925,18 @@ export class SplatMesh extends SplatGenerator {
) {
updated = true;
}
const worldToView = viewToWorld.clone().invert();
const worldToView = this.worldToView.copy(viewToWorld).invert();
if (
this.context.covWorldToView.updateFromMatrix(worldToView) &&
this.enableWorldToView
) {
updated = true;
}

const worldToObject = this.matrixWorld.clone().invert();
const viewToObjectMatrix = worldToObject.multiply(viewToWorld);
const viewToObjectMatrix = this.viewToObjectMatrix.multiplyMatrices(
this.worldToObject.copy(objectToWorld).invert(),
viewToWorld,
);
if (
this.context.covViewToObject.updateFromMatrix(viewToObjectMatrix) &&
(this.enableViewToObject || this.context.splats.hasRgbDir())
Expand Down