diff --git a/examples/src/examples/gaussian-splatting/relighting.controls.jsx b/examples/src/examples/gaussian-splatting/relighting.controls.jsx new file mode 100644 index 00000000000..1fa084fcc2c --- /dev/null +++ b/examples/src/examples/gaussian-splatting/relighting.controls.jsx @@ -0,0 +1,211 @@ +import { + BindingTwoWay, + ColorPicker, + LabelGroup, + BooleanInput, + Panel, + SelectInput, + SliderInput, + Label +} from '@playcanvas/pcui/react'; + +/** + * @import { Observer } from '@playcanvas/observer' + * @import { ReactElement } from 'react' + */ + +/** + * @param {{ observer: Observer }} props - The control panel props. + * @returns {ReactElement} The control panel. + */ +export function Controls({ observer }) { + return ( + <> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ); +} diff --git a/examples/src/examples/gaussian-splatting/relighting.example.mjs b/examples/src/examples/gaussian-splatting/relighting.example.mjs new file mode 100644 index 00000000000..c6cdd7e0739 --- /dev/null +++ b/examples/src/examples/gaussian-splatting/relighting.example.mjs @@ -0,0 +1,774 @@ +// @config +// +// Demonstrates relighting of a Gaussian Splat scene using a matching simplified mesh +// with positions and normals, loaded from a draco compressed glb file. The proxy mesh is lit by +// lights and rendered into an offscreen texture (lit color in RGB, mesh coverage mask in A), +// which is then used to relight the splats. +// +// @flag NO_MINISTATS +// +// @credit +// title: Roman Parish +// author: Andrii Shramko +// source: https://www.linkedin.com/in/andrii-shramko/ +// +// @credit +// title: HDRI Environments +// author: Poly Haven +// source: https://polyhaven.com +// license: CC0 + +import * as pc from 'playcanvas'; +import { CameraControls } from 'playcanvas/scripts/esm/camera-controls.mjs'; +import { GsplatRelighting } from 'playcanvas/scripts/esm/gsplat/gsplat-relighting.mjs'; + +import { data, deviceType, win } from 'examples/context'; + +// allow overriding scene url and orientation via hash query params, e.g. +// #/gaussian-splatting/relighting?url=https://example.com/scene/lod-meta.json&orientation=90 +const hashQuery = (win.location.hash || window.location.hash || '').split('?')[1] || ''; +const hashParams = new URLSearchParams(hashQuery); +const paramUrl = hashParams.get('url'); +const paramOrientation = hashParams.get('orientation'); + +const canvas = /** @type {HTMLCanvasElement} */ (document.getElementById('application-canvas')); +window.focus(); + +// Set up and load draco module, as the mesh glb we load is draco compressed +pc.WasmModule.setConfig('DracoDecoderModule', { + glueUrl: './assets/wasm/draco/draco.wasm.js', + wasmUrl: './assets/wasm/draco/draco.wasm.wasm', + fallbackUrl: './assets/wasm/draco/draco.js' +}); + +await new Promise((resolve) => { + pc.WasmModule.getInstance('DracoDecoderModule', () => resolve(true)); +}); + +const gfxOptions = { + deviceTypes: [deviceType], + + // disable antialiasing as gaussian splats do not benefit from it and it's expensive + antialias: false +}; + +const device = await pc.createGraphicsDevice(canvas, gfxOptions); + +const createOptions = new pc.AppOptions(); +createOptions.graphicsDevice = device; +createOptions.mouse = new pc.Mouse(document.body); +createOptions.touch = new pc.TouchDevice(document.body); +createOptions.keyboard = new pc.Keyboard(document.body); + +createOptions.componentSystems = [ + pc.RenderComponentSystem, + pc.CameraComponentSystem, + pc.LightComponentSystem, + pc.ScriptComponentSystem, + pc.GSplatComponentSystem +]; +createOptions.resourceHandlers = [pc.TextureHandler, pc.ContainerHandler, pc.ScriptHandler, pc.GSplatHandler]; + +const app = new pc.AppBase(canvas); +app.init(createOptions); + +// Set the canvas to fill the window and automatically change resolution to be the same as the canvas size +app.setCanvasFillMode(pc.FILLMODE_FILL_WINDOW); +app.setCanvasResolution(pc.RESOLUTION_AUTO); + +// High Res toggle (false by default): when false, use half native DPR; when true, use min(DPR, 2) +data.set('highRes', !!data.get('highRes')); +const applyResolution = () => { + const dpr = window.devicePixelRatio || 1; + // auto: treat DPR >= 2 as high-DPI (drops to half); High Res forces native capped at 2 + device.maxPixelRatio = data.get('highRes') ? Math.min(dpr, 2) : (dpr >= 2 ? dpr * 0.5 : dpr); +}; +applyResolution(); +const applyAndResize = () => { + applyResolution(); app.resizeCanvas(); +}; +data.on('highRes:set', applyAndResize); + +// Ensure DPR and canvas are updated when window changes size +window.addEventListener('resize', applyAndResize); +app.on('destroy', () => { + window.removeEventListener('resize', applyAndResize); +}); + +// Roman-Parish configuration +// original dataset: https://www.youtube.com/watch?v=3RtY_cLK13k +const config = { + name: 'Roman-Parish', + url: 'https://code.playcanvas.com/examples_data/example_roman_parish_02/lod-meta.json', + lodUpdateDistance: 0.5, + lodUnderfillLimit: 5, + cameraPosition: [10.3, 2, -10], + eulerAngles: [-90, 0, 0], + moveSpeed: 4, + moveFastSpeed: 15, + enableOrbit: false, + enablePan: false, + focusPoint: [12, 3, 0] +}; + +// HDRI environment presets (Poly Haven, infinite projection) +/** @type {Record} */ +const ENV_PRESETS = { + 'none': null, + 'rosendal': 'https://dl.polyhaven.org/file/ph-assets/HDRIs/hdr/2k/rosendal_park_sunset_puresky_2k.hdr', + 'industrial-sunset': 'https://dl.polyhaven.org/file/ph-assets/HDRIs/hdr/2k/industrial_sunset_puresky_2k.hdr', + 'partly-cloudy': 'https://dl.polyhaven.org/file/ph-assets/HDRIs/hdr/2k/kloofendal_48d_partly_cloudy_puresky_2k.hdr', + 'moonlit': 'https://dl.polyhaven.org/file/ph-assets/HDRIs/hdr/2k/qwantani_moon_noon_puresky_2k.hdr', + 'sunflowers': 'https://dl.polyhaven.org/file/ph-assets/HDRIs/hdr/2k/sunflowers_puresky_2k.hdr', + 'table-mountain': 'https://dl.polyhaven.org/file/ph-assets/HDRIs/hdr/2k/table_mountain_2_puresky_2k.hdr', + 'cloud-layers': 'https://dl.polyhaven.org/file/ph-assets/HDRIs/hdr/2k/cloud_layers_2k.hdr', + 'night': 'https://dl.polyhaven.org/file/ph-assets/HDRIs/hdr/2k/qwantani_night_puresky_2k.hdr' +}; + +// LOD preset definitions +/** @type {Record} */ +const LOD_PRESETS = { + 'desktop-max': { + range: [0, 5], + lodBaseDistance: 7, + lodMultiplier: 3 + }, + 'desktop': { + range: [1, 5], + lodBaseDistance: 5, + lodMultiplier: 4 + }, + 'mobile-max': { + range: [2, 5], + lodBaseDistance: 5, + lodMultiplier: 2 + }, + 'mobile': { + range: [3, 5], + lodBaseDistance: 2, + lodMultiplier: 2 + } +}; + +const assets = { + church: new pc.Asset('gsplat', 'gsplat', { url: config.url }), + + // draco compressed mesh matching the splat scene, with positions and normals + mesh: new pc.Asset('mesh', 'container', { url: 'https://code.playcanvas.com/examples_data/example_roman_parish_02/roman-parish-mesh.glb' }), + + envatlas: new pc.Asset( + 'env-atlas', + 'texture', + { url: './assets/cubemaps/table-mountain-env-atlas.png' }, + { type: pc.TEXTURETYPE_RGBP, mipmaps: false } + ) +}; + +const assetListLoader = new pc.AssetListLoader(Object.values(assets), app.assets); +assetListLoader.load(async () => { + app.start(); + + const miniStats = new pc.MiniStats(app, pc.MiniStats.getDefaultOptions(['gsplats', 'gsplatsCopy'])); // eslint-disable-line no-unused-vars + + // enable rotation-based LOD updates and behind-camera penalty + app.scene.gsplat.lodUpdateAngle = 90; + app.scene.gsplat.lodBehindPenalty = 3; + app.scene.gsplat.radialSorting = true; + + data.on('radialSorting:set', () => { + app.scene.gsplat.radialSorting = !!data.get('radialSorting'); + }); + + app.scene.gsplat.lodUpdateDistance = config.lodUpdateDistance; + app.scene.gsplat.lodUnderfillLimit = config.lodUnderfillLimit; + + data.on('renderer:set', () => { + app.scene.gsplat.renderer = data.get('renderer'); + const current = app.scene.gsplat.currentRenderer; + if (current !== data.get('renderer')) { + setTimeout(() => data.set('renderer', current), 0); + } + }); + data.on('minPixelSize:set', () => { + app.scene.gsplat.minPixelSize = data.get('minPixelSize'); + }); + data.on('alphaClipForward:set', () => { + app.scene.gsplat.alphaClipForward = data.get('alphaClipForward'); + }); + data.on('minContribution:set', () => { + app.scene.gsplat.minContribution = data.get('minContribution'); + }); + data.on('debug:set', () => { + app.scene.gsplat.debug = data.get('debug'); + }); + data.on('compact:set', () => { + app.scene.gsplat.dataFormat = data.get('compact') ? pc.GSPLATDATA_COMPACT : pc.GSPLATDATA_LARGE; + }); + + const MAX_PERSPECTIVE_FOV = 140; + + // initialize UI settings (must be after observer registration) + const initialSettings = { + fisheye: 0, + cameraFov: 75, + toneMapping: pc.TONEMAP_LINEAR, + exposure: 0.3, + minPixelSize: 2, + alphaClipForward: 1 / 255, + minContribution: 3, + radialSorting: true, + renderer: pc.GSPLAT_RENDERER_AUTO, + culling: device.isWebGPU, + compact: true, + debug: pc.GSPLAT_DEBUG_NONE, + lodPreset: pc.platform.mobile ? 'mobile' : 'desktop', + splatBudget: pc.platform.mobile ? 1 : 4, + environment: 'rosendal', + fogDensity: 0, + url: paramUrl || '', + orientation: paramOrientation ? parseFloat(paramOrientation) : 270, + blend: 0.5, + brightness: 1, + textureScale: 1, + debugRt: false, + lightIntensity: 3, + lightColor: [1, 1, 1], + shadows: true, + omniRadius: 12.9, + omniIntensity: 3, + omniColor: [1, 1, 1], + omniShadows: true, + envRotation: 20, + skyExposure: 1 + }; + Object.entries(initialSettings).forEach(([key, value]) => data.set(key, value)); + + const gsplatSystem = /** @type {any} */ (app.systems.gsplat); + + // Create a camera with fly controls + const camera = new pc.Entity('camera'); + camera.addComponent('camera', { + clearColor: new pc.Color(1, 1, 1), + fov: 75, + toneMapping: pc.TONEMAP_LINEAR + }); + + const [camX, camY, camZ] = /** @type {[number, number, number]} */ (config.cameraPosition); + const [focusX, focusY, focusZ] = /** @type {[number, number, number]} */ (config.focusPoint || [0, 0.6, 0]); + const focusPoint = new pc.Vec3(focusX, focusY, focusZ); + + camera.setLocalPosition(camX, camY, camZ); + app.root.addChild(camera); + + camera.addComponent('script'); + const cc = /** @type { CameraControls} */ ((/** @type {any} */ (camera.script)).create(CameraControls)); + Object.assign(cc, { + sceneSize: 500, + moveSpeed: /** @type {number} */ (config.moveSpeed), + moveFastSpeed: /** @type {number} */ (config.moveFastSpeed), + enableOrbit: false, + enablePan: false, + focusPoint: focusPoint + }); + + // Relighting renderer: renders the relighting layer (proxy mesh) from a camera matching the + // main camera into an RGBA16F texture - lit mesh color in RGB, mesh coverage mask in A + const relighting = /** @type {GsplatRelighting} */ ( + (/** @type {any} */ (camera.script)).create(GsplatRelighting, { + properties: { + textureScale: data.get('textureScale'), + blend: data.get('blend'), + brightness: data.get('brightness'), + background: data.get('skyExposure') + } + }) + ); + const relightLayer = /** @type {pc.Layer} */ (relighting.layer); + + data.on('textureScale:set', () => { + relighting.textureScale = data.get('textureScale'); + }); + data.on('blend:set', () => { + relighting.blend = data.get('blend'); + }); + data.on('brightness:set', () => { + relighting.brightness = data.get('brightness'); + }); + + // Directional light with PCSS soft shadows, lighting the proxy mesh on the relighting layer + const light = new pc.Entity('light'); + light.addComponent('light', { + type: 'directional', + color: new pc.Color(1, 1, 1), + intensity: data.get('lightIntensity'), + layers: [relightLayer.id], + castShadows: !!data.get('shadows'), + shadowType: pc.SHADOW_PCSS_32F, + shadowResolution: 4096, + shadowDistance: 150, + shadowBias: 0.3, + normalOffsetBias: 0.2, + penumbraSize: 0.015, + penumbraFalloff: 4, + shadowSamples: 16, + shadowBlockerSamples: 16 + }); + light.setLocalPosition(0, 1, 0); + light.setEulerAngles(-20, 30, 0); + light.enabled = data.get('lightIntensity') > 0; + app.root.addChild(light); + + const gizmoLayer = pc.Gizmo.createLayer(app); + + // rotation gizmo to orient the directional light; disable camera controls while dragging it + const lightGizmo = new pc.RotateGizmo(camera.camera, gizmoLayer); + lightGizmo.size = 0.5; + lightGizmo.attach(light); + lightGizmo.on('pointer:down', (/** @type {number} */ _x, /** @type {number} */ _y, /** @type {pc.MeshInstance} */ meshInstance) => { + if (meshInstance) cc.enabled = false; + }); + lightGizmo.on('pointer:up', () => { + cc.enabled = true; + }); + + data.on('lightIntensity:set', () => { + const intensity = data.get('lightIntensity'); + light.light.intensity = intensity; + + // disable the light completely when intensity is 0, including its gizmo + light.enabled = intensity > 0; + if (intensity > 0) { + lightGizmo.attach(light); + } else { + lightGizmo.detach(); + } + }); + + data.on('lightColor:set', () => { + const c = data.get('lightColor'); + light.light.color = new pc.Color(c[0], c[1], c[2]); + }); + + data.on('shadows:set', () => { + light.light.castShadows = !!data.get('shadows'); + }); + + // Omni lights with translate gizmos, used to interactively tune light placements - their + // settings are logged to the console once a second + const OMNI_LIGHTS = [ + { position: [12.02, 4.45, 9.04], radius: 12.9, intensity: 3.67, color: [1.00, 1.00, 1.00] }, + { position: [8.19, 4.67, 11.76], radius: 12.9, intensity: 3.67, color: [0.97, 0.91, 0.72] }, + { position: [4.22, 4.80, 13.69], radius: 12.9, intensity: 3.67, color: [0.97, 0.72, 0.72] }, + { position: [4.22, 4.80, 13.69], radius: 12.9, intensity: 3.67, color: [0.97, 0.72, 0.72] }, + { position: [-6.58, 0.52, -7.01], radius: 12.9, intensity: 3.67, color: [0.97, 0.72, 0.72] } + ]; + + /** @type {pc.Entity[]} */ + const omniLights = []; + /** @type {pc.TranslateGizmo[]} */ + const omniGizmos = []; + + // Color / radius edits apply to all lights until one is moved using its gizmo - from then on + // they apply only to the light last grabbed (-1 means all) + let selectedOmni = -1; + + OMNI_LIGHTS.forEach((def, index) => { + const entity = new pc.Entity(`omni-light-${index}`); + entity.addComponent('light', { + type: 'omni', + color: new pc.Color(def.color[0], def.color[1], def.color[2]), + intensity: def.intensity * data.get('omniIntensity'), + range: def.radius, + layers: [relightLayer.id], + castShadows: !!data.get('omniShadows'), + shadowType: pc.SHADOW_PCF3_32F, + shadowResolution: 1024, + shadowBias: 0.2, + normalOffsetBias: 0.05, + + // the lights are static unless moved by their gizmo, so render their shadows once + shadowUpdateMode: pc.SHADOWUPDATE_THISFRAME + }); + entity.setLocalPosition(def.position[0], def.position[1], def.position[2]); + app.root.addChild(entity); + omniLights.push(entity); + + // translate gizmo to position the light; disable camera controls while dragging it + const gizmo = new pc.TranslateGizmo(camera.camera, gizmoLayer); + gizmo.size = 0.5; + + // double the size of the plane-movement squares + gizmo.axisPlaneSize *= 2; + gizmo.attach(entity); + gizmo.on('pointer:down', (/** @type {number} */ _x, /** @type {number} */ _y, /** @type {pc.MeshInstance} */ meshInstance) => { + if (meshInstance) { + cc.enabled = false; + + // select this light for color / radius edits and sync the UI to its values + selectedOmni = index; + const lightComponent = /** @type {pc.LightComponent} */ (entity.light); + data.set('omniColor', [lightComponent.color.r, lightComponent.color.g, lightComponent.color.b]); + data.set('omniRadius', lightComponent.range); + + // update shadows every frame while the light is being moved + lightComponent.shadowUpdateMode = pc.SHADOWUPDATE_REALTIME; + } + }); + gizmo.on('pointer:up', () => { + cc.enabled = true; + + // render the shadows once more, then stop updating them + entity.light.shadowUpdateMode = pc.SHADOWUPDATE_THISFRAME; + }); + omniGizmos.push(gizmo); + }); + + // UI intensity is a multiplier on the per-light intensities; when zero, the lights are + // disabled and their gizmos hidden + const applyOmniIntensity = () => { + const multiplier = data.get('omniIntensity'); + omniLights.forEach((entity, index) => { + entity.light.intensity = OMNI_LIGHTS[index].intensity * multiplier; + entity.enabled = multiplier > 0; + if (multiplier > 0) { + // refresh the static shadows after re-enabling + entity.light.shadowUpdateMode = pc.SHADOWUPDATE_THISFRAME; + } + }); + omniGizmos.forEach((gizmo, index) => { + if (multiplier > 0) { + gizmo.attach(omniLights[index]); + } else { + gizmo.detach(); + } + }); + }; + data.on('omniIntensity:set', applyOmniIntensity); + + data.on('omniRadius:set', () => { + const radius = data.get('omniRadius'); + omniLights.forEach((entity, index) => { + if (selectedOmni === -1 || selectedOmni === index) { + entity.light.range = radius; + + // radius affects the shadow projection, refresh the static shadows + entity.light.shadowUpdateMode = pc.SHADOWUPDATE_THISFRAME; + } + }); + }); + data.on('omniColor:set', () => { + const c = data.get('omniColor'); + omniLights.forEach((entity, index) => { + if (selectedOmni === -1 || selectedOmni === index) { + entity.light.color = new pc.Color(c[0], c[1], c[2]); + } + }); + }); + data.on('omniShadows:set', () => { + const castShadows = !!data.get('omniShadows'); + omniLights.forEach((entity) => { + entity.light.castShadows = castShadows; + if (castShadows) { + // render the newly enabled shadows once + entity.light.shadowUpdateMode = pc.SHADOWUPDATE_THISFRAME; + } + }); + }); + + // Rotation of the image based lighting around the Y axis + const envRotationQuat = new pc.Quat(); + const applyEnvRotation = () => { + app.scene.skyboxRotation = envRotationQuat.setFromEulerAngles(0, data.get('envRotation'), 0); + }; + applyEnvRotation(); + data.on('envRotation:set', applyEnvRotation); + + // Intensity of the image based lighting. The same value is used as the background multiplier + // of the relighting effect, so the splat based sky follows the environment exposure. + data.on('skyExposure:set', () => { + app.scene.skyboxIntensity = data.get('skyExposure'); + relighting.background = data.get('skyExposure'); + }); + + // Image based lighting for the proxy mesh. This needs to be the scene environment (not a + // per-material env atlas), as skyboxIntensity and skyboxRotation only apply to materials + // using the scene environment. + app.scene.envAtlas = assets.envatlas.resource; + + // Instantiate the draco compressed proxy mesh matching the splat scene. It renders only to + // the relighting layer, with a lit gray material configured to write a coverage mask to alpha. + const meshEntity = assets.mesh.resource.instantiateRenderEntity(); + const meshMaterial = new pc.StandardMaterial(); + meshMaterial.diffuse = new pc.Color(0.5, 0.5, 0.5); + meshMaterial.update(); + relighting.configureMaterial(meshMaterial); + meshEntity.findComponents('render').forEach((/** @type {pc.RenderComponent} */ render) => { + render.layers = [relightLayer.id]; + render.meshInstances.forEach((meshInstance) => { + meshInstance.material = meshMaterial; + }); + }); + + // wrap in a parent so the same orientation as the splat can be applied + const meshParent = new pc.Entity('mesh-parent'); + meshParent.addChild(meshEntity); + meshParent.setLocalEulerAngles(data.get('orientation'), 0, 0); + app.root.addChild(meshParent); + + // CameraFrame for HDR linear rendering (created lazily on first enable) + /** @type {pc.CameraFrame|null} */ + let cameraFrame = null; + + const applyToneMapping = () => { + const tm = data.get('toneMapping'); + if (cameraFrame?.enabled) { + cameraFrame.rendering.toneMapping = tm; + cameraFrame.update(); + } else { + camera.camera.toneMapping = tm; + } + }; + + data.set('cameraFrame', false); + data.on('cameraFrame:set', () => { + if (data.get('cameraFrame')) { + if (!cameraFrame) { + cameraFrame = new pc.CameraFrame(app, camera.camera); + cameraFrame.rendering.toneMapping = data.get('toneMapping'); + } + cameraFrame.enabled = true; + cameraFrame.update(); + } else if (cameraFrame) { + cameraFrame.destroy(); + cameraFrame = null; + } + applyToneMapping(); + }); + + const applyFov = () => { + const fov = data.get('cameraFov'); + camera.camera.fov = (data.get('fisheye') === 0) ? Math.min(fov, MAX_PERSPECTIVE_FOV) : fov; + }; + data.on('cameraFov:set', applyFov); + data.on('fisheye:set', () => { + const fisheye = data.get('fisheye'); + app.scene.gsplat.fisheye = fisheye; + app.scene.sky.fisheye = fisheye; + applyFov(); + }); + data.on('toneMapping:set', applyToneMapping); + data.on('exposure:set', () => { + app.scene.exposure = data.get('exposure'); + }); + app.scene.exposure = data.get('exposure'); + + data.on('fogDensity:set', () => { + const density = data.get('fogDensity'); + if (density > 0) { + app.scene.fog.type = pc.FOG_EXP; + app.scene.fog.density = density; + app.scene.fog.color.copy(camera.camera.clearColor); + } else { + app.scene.fog.type = pc.FOG_NONE; + } + }); + + // HDRI environment loading + /** @type {Map} */ + const hdriCache = new Map(); + + const applyEnvironment = async (/** @type {string} */ name) => { + const url = ENV_PRESETS[name]; + if (!url) { + app.scene.skybox = null; + // restore the default environment used to light the proxy mesh + app.scene.envAtlas = assets.envatlas.resource; + return; + } + + if (!hdriCache.has(url)) { + const asset = new pc.Asset('hdri', 'texture', { url: url }, { mipmaps: false }); + await new Promise((resolve, reject) => { + asset.on('load', resolve); + asset.on('error', (/** @type {string} */ err) => { + console.error('Failed to load HDRI:', err); + reject(err); + }); + app.assets.add(asset); + app.assets.load(asset); + }); + + const source = asset.resource; + const skybox = pc.EnvLighting.generateSkyboxCubemap(source); + const lighting = pc.EnvLighting.generateLightingSource(source); + const envAtlas = pc.EnvLighting.generateAtlas(lighting); + lighting.destroy(); + hdriCache.set(url, { skybox, envAtlas }); + } + + const cached = /** @type {{ skybox: pc.Texture, envAtlas: pc.Texture }} */ (hdriCache.get(url)); + app.scene.skybox = cached.skybox; + app.scene.envAtlas = cached.envAtlas; + app.scene.sky.type = pc.SKYTYPE_INFINITE; + }; + + data.on('environment:set', () => { + applyEnvironment(data.get('environment')).catch((err) => { + console.warn('Environment load failed:', err); + }); + }); + + // apply the initial environment + applyEnvironment(data.get('environment')).catch((err) => { + console.warn('Environment load failed:', err); + }); + + // Gsplat loading state + /** @type {pc.Entity|null} */ + let gsplatEntity = null; + /** @type {any} */ + let gsplatGs = null; + /** @type {pc.Asset|null} */ + let customAsset = null; + + const applyPreset = () => { + const preset = data.get('lodPreset'); + const presetData = LOD_PRESETS[preset] || LOD_PRESETS.desktop; + app.scene.gsplat.lodRangeMin = presetData.range[0]; + app.scene.gsplat.lodRangeMax = presetData.range[1]; + if (gsplatGs) { + gsplatGs.lodBaseDistance = presetData.lodBaseDistance; + gsplatGs.lodMultiplier = presetData.lodMultiplier; + } + data.set('lodBaseDistance', presetData.lodBaseDistance); + data.set('lodMultiplier', presetData.lodMultiplier); + }; + + const loadGsplat = async (/** @type {string|null} */ url) => { + if (gsplatEntity) { + gsplatEntity.destroy(); + gsplatEntity = null; + gsplatGs = null; + } + + if (customAsset) { + app.assets.remove(customAsset); + customAsset.unload(); + customAsset = null; + } + + /** @type {pc.Asset} */ + let asset; + if (url) { + asset = new pc.Asset('gsplat', 'gsplat', { url: url }); + app.assets.add(asset); + await new Promise((resolve, reject) => { + asset.on('load', resolve); + asset.on('error', (/** @type {string} */ err) => { + console.error('Failed to load gsplat:', err); + reject(err); + }); + app.assets.load(asset); + }); + customAsset = asset; // eslint-disable-line require-atomic-updates + } else { + asset = assets.church; + } + + gsplatEntity = new pc.Entity(config.name || 'gsplat'); // eslint-disable-line require-atomic-updates + gsplatEntity.addComponent('gsplat', { + asset: asset + }); + gsplatEntity.setLocalPosition(0, 0, 0); + gsplatEntity.setLocalEulerAngles(data.get('orientation'), 0, 0); + gsplatEntity.setLocalScale(1, 1, 1); + app.root.addChild(gsplatEntity); + gsplatGs = /** @type {any} */ (gsplatEntity.gsplat); + + const presetData = LOD_PRESETS[data.get('lodPreset')] || LOD_PRESETS.desktop; + gsplatGs.lodBaseDistance = presetData.lodBaseDistance; + gsplatGs.lodMultiplier = presetData.lodMultiplier; + + // Start with lowest LOD for fast initial display, then stream up + const lodLevels = gsplatGs.resource?.octree?.lodLevels; + if (lodLevels) { + const worstLod = lodLevels - 1; + app.scene.gsplat.lodRangeMin = worstLod; + app.scene.gsplat.lodRangeMax = worstLod; + } + + const onFrameReady = (/** @type {any} */ cam, /** @type {any} */ layer, /** @type {boolean} */ ready, /** @type {number} */ loadingCount) => { + if (ready && loadingCount === 0) { + gsplatSystem.off('frame:ready', onFrameReady); + applyPreset(); + } + }; + gsplatSystem.on('frame:ready', onFrameReady); + }; + + // Initial load — use the observer's current url, which is paramUrl from the + // hash query if set, or the share-URL state value applied during app.start(). + await loadGsplat(data.get('url') || null); + + data.on('lodPreset:set', applyPreset); + + data.on('lodBaseDistance:set', () => { + if (gsplatGs) gsplatGs.lodBaseDistance = data.get('lodBaseDistance'); + }); + data.on('lodMultiplier:set', () => { + if (gsplatGs) gsplatGs.lodMultiplier = data.get('lodMultiplier'); + }); + + const applySplatBudget = () => { + const millions = data.get('splatBudget'); + app.scene.gsplat.splatBudget = Math.round(millions * 1000000); + }; + + applySplatBudget(); + data.on('splatBudget:set', applySplatBudget); + + data.on('orientation:set', () => { + if (gsplatEntity) { + gsplatEntity.setLocalEulerAngles(data.get('orientation'), 0, 0); + } + meshParent.setLocalEulerAngles(data.get('orientation'), 0, 0); + }); + + data.on('url:set', () => { + const url = data.get('url'); + loadGsplat(url || null).catch((err) => { + console.warn('Loading failed, reverting to default:', err); + loadGsplat(null); + }); + }); + + let logTexturesRequested = false; + data.on('logTextures', () => { + logTexturesRequested = true; + }); + + let logBuffersRequested = false; + data.on('logBuffers', () => { + logBuffersRequested = true; + }); + + app.on('update', () => { + + // debug display of the relighting texture + if (data.get('debugRt') && relighting.texture) { + // @ts-ignore engine-tsd + app.drawTexture(0.6, -0.6, 0.7, 0.7, relighting.texture); + } + + // log textures for one frame if requested + pc.Tracing.set(pc.TRACEID_TEXTURES, logTexturesRequested); + logTexturesRequested = false; + + pc.Tracing.set(pc.TRACEID_BUFFERS, logBuffersRequested); + logBuffersRequested = false; + + data.set('data.stats.gsplats', app.stats.frame.gsplats.toLocaleString()); + const bb = app.graphicsDevice.backBufferSize; + data.set('data.stats.resolution', `${bb.x} x ${bb.y}`); + }); +}); diff --git a/examples/thumbnails/gaussian-splatting_relighting_large.webp b/examples/thumbnails/gaussian-splatting_relighting_large.webp new file mode 100644 index 00000000000..0c4120d94ee Binary files /dev/null and b/examples/thumbnails/gaussian-splatting_relighting_large.webp differ diff --git a/examples/thumbnails/gaussian-splatting_relighting_small.webp b/examples/thumbnails/gaussian-splatting_relighting_small.webp new file mode 100644 index 00000000000..9e43f3be9db Binary files /dev/null and b/examples/thumbnails/gaussian-splatting_relighting_small.webp differ diff --git a/scripts/esm/gsplat/gsplat-relighting.mjs b/scripts/esm/gsplat/gsplat-relighting.mjs new file mode 100644 index 00000000000..0a7a0afd9bc --- /dev/null +++ b/scripts/esm/gsplat/gsplat-relighting.mjs @@ -0,0 +1,323 @@ +import { + Color, Entity, Layer, RenderTarget, Script, Texture, + ADDRESS_CLAMP_TO_EDGE, FILTER_LINEAR, GAMMA_NONE, PIXELFORMAT_RGBA16F, PIXELFORMAT_SRGBA8, + SHADERLANGUAGE_GLSL, SHADERLANGUAGE_WGSL, TONEMAP_NONE +} from 'playcanvas'; + +/** + * @import { Material } from 'playcanvas' + */ + +// outputPS chunk overrides for the proxy mesh material: keep the lit color in RGB and write 1 to +// A, marking the pixel as covered by the mesh. The render target clears alpha to 0, so the alpha +// channel acts as a coverage mask - splats sampling uncovered pixels are left untinted. +const meshOutputGLSL = /* glsl */` + gl_FragColor.a = 1.0; +`; + +const meshOutputWGSL = /* wgsl */` + output.color = vec4f(output.color.rgb, 1.0); +`; + +// gsplatModifyPS chunk: per-pixel relighting. The relighting texture is screen-aligned with the +// main camera, so each fragment samples it at its own screen position - no matrix, no per-splat +// flicker at screen edges. The brightness uniform compensates for the gray albedo of the proxy +// mesh material (2 for 0.5 gray albedo) and allows the overall lighting to be brightened. +const splatModifyGLSL = /* glsl */` +uniform sampler2D uRelightMap; +uniform vec4 uScreenSize; +uniform float uRelightBlend; +uniform float uRelightBrightness; +uniform float uRelightBackground; + +void modifySplatColor(vec2 gaussianUV, inout vec4 color) { + vec4 lit = textureLod(uRelightMap, gl_FragCoord.xy * uScreenSize.zw, 0.0); + + // the texture alpha is a mesh coverage mask - splats not covered by the mesh (e.g. the sky) + // are modulated by the background multiplier instead of the mesh lighting + vec3 factor = mix(vec3(uRelightBackground), lit.rgb * uRelightBrightness, lit.a); + color.rgb = mix(color.rgb, color.rgb * factor, uRelightBlend); +} +`; + +const splatModifyWGSL = /* wgsl */` +var uRelightMap: texture_2d; +var uRelightMapSampler: sampler; +uniform uScreenSize: vec4f; +uniform uRelightBlend: f32; +uniform uRelightBrightness: f32; +uniform uRelightBackground: f32; + +fn modifySplatColor(gaussianUV: vec2f, color: ptr) { + let lit = textureSampleLevel(uRelightMap, uRelightMapSampler, pcPosition.xy * uniform.uScreenSize.zw, 0.0); + + // the texture alpha is a mesh coverage mask - splats not covered by the mesh (e.g. the sky) + // are modulated by the background multiplier instead of the mesh lighting + let factor = mix(vec3f(uniform.uRelightBackground), lit.rgb * uniform.uRelightBrightness, lit.a); + *color = vec4f(mix((*color).rgb, (*color).rgb * factor, uniform.uRelightBlend), (*color).a); +} +`; + +/** + * Relights a gaussian splat scene using a proxy mesh. A proxy mesh of the splat scene is lit by + * standard lights and rendered into an offscreen texture (lit mesh color in RGB, mesh coverage + * mask in A) by a camera matching the main camera. The texture is screen-aligned with the main + * camera, so each splat fragment samples the mesh lighting at its own screen position and the + * splat color is modulated by it per pixel. + * + * Attach this script to the entity holding the main camera that renders the gsplat scene. Under + * the hood it creates a child entity with a camera matching the main camera, which renders a + * dedicated layer into the texture before the main camera renders. Place the proxy mesh and the + * lights that should light it on that layer ({@link GsplatRelighting#layer}), and apply + * {@link GsplatRelighting#configureMaterial} to the proxy mesh material so it writes the + * coverage mask to alpha. + * + * Note: only gsplat components in unified mode are supported - the splat customization is + * applied to `app.scene.gsplat.material`, and only by the raster gsplat renderers (CPU and GPU + * sort), as the fragment chunk is not used by the compute renderer. + */ +class GsplatRelighting extends Script { + static scriptName = 'gsplatRelighting'; + + /** + * Scale of the relighting texture resolution relative to the back buffer. + * @attribute + * @range [0.1, 1] + */ + textureScale = 1; + + /** + * Priority of the relighting camera. Keep it lower than the main camera priority (default 0) + * so the relighting texture is rendered first each frame. + * @attribute + */ + priority = -1; + + /** + * Name of the layer the proxy mesh and its lights should be placed on. The layer is created + * if it does not exist. + * @attribute + */ + layerName = 'Relighting'; + + /** + * How much the mesh lighting affects the splat colors. 0 leaves splats unchanged, 1 fully + * modulates the splat color by the mesh lighting. + * @attribute + * @range [0, 1] + */ + blend = 1; + + /** + * Brightness of the lighting texture when tinting the splats. The default of 2 compensates + * for the 0.5 gray albedo of the proxy mesh material. + * @attribute + * @range [0, 5] + */ + brightness = 2; + + /** + * Multiplier applied to splats not covered by the proxy mesh (e.g. the sky), allowing them + * to follow the environment exposure. + * @attribute + * @range [0, 5] + */ + background = 1; + + /** @type {Layer|null} */ + _layer = null; + + /** @type {boolean} */ + _ownsLayer = false; + + /** @type {Entity|null} */ + _rtEntity = null; + + /** @type {Texture|null} */ + _texture = null; + + /** @type {RenderTarget|null} */ + _renderTarget = null; + + /** @type {number} */ + _format = PIXELFORMAT_RGBA16F; + + initialize() { + const camera = this.entity.camera; + if (!camera) { + console.error('GsplatRelighting requires a Camera component on the entity.'); + return; + } + + // HDR format with alpha for the relighting texture, with LDR fallback when not + // renderable / filterable - sRGB to limit banding as the texture stores linear lighting + this._format = this.app.graphicsDevice.getRenderableHdrFormat([PIXELFORMAT_RGBA16F], true) ?? + PIXELFORMAT_SRGBA8; + + // find or create the relighting layer + let layer = this.app.scene.layers.getLayerByName(this.layerName); + if (!layer) { + layer = new Layer({ name: this.layerName }); + this.app.scene.layers.push(layer); + this._ownsLayer = true; + } + this._layer = layer; + + // child entity with a camera matching the host camera, rendering the relighting layer + // into the texture; inherits the host camera world transform automatically + const rtEntity = new Entity('RelightingCamera'); + rtEntity.addComponent('camera', { + layers: [layer.id], + priority: this.priority, + clearColor: new Color(0, 0, 0, 0), + fov: camera.fov, + nearClip: camera.nearClip, + farClip: camera.farClip, + + // keep the texture linear HDR + toneMapping: TONEMAP_NONE, + gammaCorrection: GAMMA_NONE + }); + this.entity.addChild(rtEntity); + this._rtEntity = rtEntity; + + this._updateRenderTarget(); + this._applySplatChunk(); + + this.on('enable', () => { + if (this._rtEntity) this._rtEntity.enabled = true; + this._applySplatChunk(); + }); + + this.on('disable', () => { + if (this._rtEntity) this._rtEntity.enabled = false; + this._removeSplatChunk(); + }); + + this.on('destroy', () => { + this._removeSplatChunk(); + this._destroyRenderTarget(); + this._rtEntity?.destroy(); + this._rtEntity = null; + if (this._ownsLayer && this._layer) { + this.app.scene.layers.remove(this._layer); + } + this._layer = null; + }); + } + + /** + * The layer the proxy mesh and its lights should be placed on. + * @type {Layer|null} + */ + get layer() { + return this._layer; + } + + /** + * The relighting texture: lit mesh color in RGB, mesh coverage mask in A. + * @type {Texture|null} + */ + get texture() { + return this._texture; + } + + /** + * Overrides the output shader chunk of a material to write the mesh coverage mask to the + * alpha channel, as expected by the relighting effect. Apply this to the proxy mesh + * material. + * + * @param {Material} material - The material to configure. + */ + configureMaterial(material) { + material.getShaderChunks(SHADERLANGUAGE_GLSL).set('outputPS', meshOutputGLSL); + material.getShaderChunks(SHADERLANGUAGE_WGSL).set('outputPS', meshOutputWGSL); + material.shaderChunksVersion = '2.8'; + material.update(); + } + + update() { + const camera = this.entity.camera; + const rtCamera = this._rtEntity?.camera; + if (!camera || !rtCamera) return; + + // keep the relighting camera in sync with the host camera + rtCamera.fov = camera.fov; + rtCamera.nearClip = camera.nearClip; + rtCamera.farClip = camera.farClip; + + this._updateRenderTarget(); + + // update the splat customization uniforms on the unified gsplat material + const material = this.app.scene.gsplat?.material; + if (material && this._texture) { + material.setParameter('uRelightMap', this._texture); + material.setParameter('uRelightBlend', this.blend); + material.setParameter('uRelightBrightness', this.brightness); + material.setParameter('uRelightBackground', this.background); + material.update(); + } + } + + _applySplatChunk() { + const material = this.app.scene.gsplat?.material; + if (!material) return; + + const isWebGPU = this.app.graphicsDevice.isWebGPU; + const shaderLanguage = isWebGPU ? SHADERLANGUAGE_WGSL : SHADERLANGUAGE_GLSL; + material.getShaderChunks(shaderLanguage).set('gsplatModifyPS', isWebGPU ? splatModifyWGSL : splatModifyGLSL); + material.update(); + } + + _removeSplatChunk() { + const material = this.app.scene.gsplat?.material; + if (!material) return; + + const shaderLanguage = this.app.graphicsDevice.isWebGPU ? SHADERLANGUAGE_WGSL : SHADERLANGUAGE_GLSL; + material.getShaderChunks(shaderLanguage).delete('gsplatModifyPS'); + material.update(); + } + + _destroyRenderTarget() { + this._renderTarget?.destroy(); + this._renderTarget = null; + this._texture?.destroy(); + this._texture = null; + } + + _updateRenderTarget() { + const device = this.app.graphicsDevice; + const width = Math.max(1, Math.floor(device.width * this.textureScale)); + const height = Math.max(1, Math.floor(device.height * this.textureScale)); + + if (this._texture && this._texture.width === width && this._texture.height === height) { + return; + } + + this._destroyRenderTarget(); + + this._texture = new Texture(device, { + name: 'RelightingTexture', + width: width, + height: height, + format: this._format, + mipmaps: false, + minFilter: FILTER_LINEAR, + magFilter: FILTER_LINEAR, + addressU: ADDRESS_CLAMP_TO_EDGE, + addressV: ADDRESS_CLAMP_TO_EDGE + }); + + this._renderTarget = new RenderTarget({ + name: 'RelightingRT', + colorBuffer: this._texture, + depth: true + }); + + if (this._rtEntity?.camera) { + this._rtEntity.camera.renderTarget = this._renderTarget; + } + } +} + +export { GsplatRelighting }; diff --git a/scripts/esm/gsplat/gsplat-shader-effect.mjs b/scripts/esm/gsplat/gsplat-shader-effect.mjs index 634468d3da7..d13e802b425 100644 --- a/scripts/esm/gsplat/gsplat-shader-effect.mjs +++ b/scripts/esm/gsplat/gsplat-shader-effect.mjs @@ -21,9 +21,9 @@ import { Script } from 'playcanvas'; * When enabled, the shader effect is applied and effectTime starts tracking from 0. * When disabled, the custom shader is removed and materials revert to default rendering. * - * Subclasses must implement: - * - getShaderGLSL(): Return GLSL shader string - * - getShaderWGSL(): Return WGSL shader string + * Subclasses override some of: + * - getShaderGLSL() / getShaderWGSL(): Return the gsplatModifyVS vertex chunk string + * - getFragmentShaderGLSL() / getFragmentShaderWGSL(): Return the gsplatModifyPS fragment chunk string * - updateEffect(effectTime, dt): Update effect each frame * * @abstract @@ -101,7 +101,9 @@ class GsplatShaderEffect extends Script { const device = this.app.graphicsDevice; const shaderLanguage = device?.isWebGPU ? 'wgsl' : 'glsl'; - this.material.getShaderChunks(shaderLanguage).delete('gsplatModifyVS'); + const chunks = this.material.getShaderChunks(shaderLanguage); + chunks.delete('gsplatModifyVS'); + chunks.delete('gsplatModifyPS'); this.material.update(); this.material = null; } @@ -137,9 +139,18 @@ class GsplatShaderEffect extends Script { applyShaderToMaterial(material) { const device = this.app.graphicsDevice; const shaderLanguage = device?.isWebGPU ? 'wgsl' : 'glsl'; - const customShader = shaderLanguage === 'wgsl' ? this.getShaderWGSL() : this.getShaderGLSL(); + const chunks = material.getShaderChunks(shaderLanguage); + + const vertexShader = shaderLanguage === 'wgsl' ? this.getShaderWGSL() : this.getShaderGLSL(); + if (vertexShader) { + chunks.set('gsplatModifyVS', vertexShader); + } + + const fragmentShader = shaderLanguage === 'wgsl' ? this.getFragmentShaderWGSL() : this.getFragmentShaderGLSL(); + if (fragmentShader) { + chunks.set('gsplatModifyPS', fragmentShader); + } - material.getShaderChunks(shaderLanguage).set('gsplatModifyVS', customShader); material.update(); } @@ -184,23 +195,39 @@ class GsplatShaderEffect extends Script { } /** - * Get the GLSL shader string. - * Must be implemented by subclasses. - * @returns {string} GLSL shader code - * @abstract + * Get the GLSL shader string for the gsplatModifyVS vertex chunk, or null to leave the + * default chunk in place. + * @returns {string|null} GLSL shader code */ getShaderGLSL() { - throw new Error(`${this.constructor.name} must implement getShaderGLSL()`); + return null; } /** - * Get the WGSL shader string. - * Must be implemented by subclasses. - * @returns {string} WGSL shader code - * @abstract + * Get the WGSL shader string for the gsplatModifyVS vertex chunk, or null to leave the + * default chunk in place. + * @returns {string|null} WGSL shader code */ getShaderWGSL() { - throw new Error(`${this.constructor.name} must implement getShaderWGSL()`); + return null; + } + + /** + * Get the GLSL shader string for the gsplatModifyPS fragment chunk, or null to leave the + * default chunk in place. + * @returns {string|null} GLSL shader code + */ + getFragmentShaderGLSL() { + return null; + } + + /** + * Get the WGSL shader string for the gsplatModifyPS fragment chunk, or null to leave the + * default chunk in place. + * @returns {string|null} WGSL shader code + */ + getFragmentShaderWGSL() { + return null; } /** diff --git a/src/scene/gsplat-unified/gsplat-hybrid-renderer.js b/src/scene/gsplat-unified/gsplat-hybrid-renderer.js index 76afacdb6bd..a25179bb329 100644 --- a/src/scene/gsplat-unified/gsplat-hybrid-renderer.js +++ b/src/scene/gsplat-unified/gsplat-hybrid-renderer.js @@ -74,6 +74,9 @@ class GSplatHybridRenderer extends GSplatRenderer { /** @type {boolean} */ forceCopyMaterial = true; + /** @type {string} */ + _lastSourceChunksKey = ''; + /** * @param {GraphicsDevice} device - The graphics device. * @param {GraphNode} node - The graph node. @@ -348,6 +351,64 @@ class GSplatHybridRenderer extends GSplatRenderer { this._material.setDefine('GSPLAT_NO_FOG', noFog); this._material.update(); } + + // Copy material settings from params.material if dirty or on first update + if (this.forceCopyMaterial || params.material.dirty) { + this.copyMaterialSettings(params.material); + this.forceCopyMaterial = false; + } + } + + /** + * Copies material settings from a source material to the internal material. + * Preserves internal defines while copying user defines, parameters, and shader chunks. + * This delivers user customizations (e.g. the `gsplatModifyPS` fragment chunk and its + * parameters) set on `app.scene.gsplat.material` to the hybrid render material. Note that + * the `gsplatModifyVS` chunk is handled by the projector compute instead, and even when + * copied here it is not referenced by the hybrid vertex shader. + * + * @param {ShaderMaterial} sourceMaterial - The source material to copy settings from. + * @private + */ + copyMaterialSettings(sourceMaterial) { + // Sync defines via setDefine so _definesDirty tracks real changes. Only delete keys the + // source no longer has (and that aren't internal). Deleting all user defines and re-adding + // them every frame would force _definesDirty true forever and trigger clearVariants on + // every frame. + const keysToDelete = []; + this._material.defines.forEach((value, key) => { + if (!this._internalDefines.has(key) && !sourceMaterial.defines.has(key)) { + keysToDelete.push(key); + } + }); + keysToDelete.forEach(key => this._material.setDefine(key, undefined)); + + // Add/update defines from the source. setDefine is conditional — it only flips + // _definesDirty when the value actually changed, so unchanged entries stay cheap. + sourceMaterial.defines.forEach((value, key) => { + this._material.setDefine(key, value); + }); + + // Copy parameters + const srcParams = sourceMaterial.parameters; + for (const paramName in srcParams) { + if (srcParams.hasOwnProperty(paramName)) { + this._material.setParameter(paramName, srcParams[paramName].data); + } + } + + // Copy shader chunks only when they actually changed on the source (chunks.key is a + // stable content hash), to avoid marking chunks dirty every frame and forcing a + // per-frame clearVariants. + if (sourceMaterial.hasShaderChunks) { + const sourceChunksKey = sourceMaterial.shaderChunks.key; + if (sourceChunksKey !== this._lastSourceChunksKey) { + this._material.shaderChunks.copy(sourceMaterial.shaderChunks); + this._lastSourceChunksKey = sourceChunksKey; + } + } + + this._material.update(); } /** diff --git a/src/scene/shader-lib/glsl/chunks/gsplat/frag/gsplat.js b/src/scene/shader-lib/glsl/chunks/gsplat/frag/gsplat.js index 22549904385..790f7565ba7 100644 --- a/src/scene/shader-lib/glsl/chunks/gsplat/frag/gsplat.js +++ b/src/scene/shader-lib/glsl/chunks/gsplat/frag/gsplat.js @@ -30,6 +30,8 @@ varying mediump vec4 gaussianColor; #include "pickPS" #endif +#include "gsplatModifyPS" + const float EXP4 = exp(-4.0); const float INV_EXP4 = 1.0 / (1.0 - EXP4); @@ -81,7 +83,9 @@ void main(void) { opacityDither(alpha, id * 0.013); #endif - gl_FragColor = vec4(gaussianColor.xyz * alpha, alpha); + vec4 fragColor = vec4(gaussianColor.xyz, alpha); + modifySplatColor(gaussianUV, fragColor); + gl_FragColor = vec4(fragColor.xyz * fragColor.a, fragColor.a); #endif } `; diff --git a/src/scene/shader-lib/glsl/chunks/gsplat/frag/gsplatModify.js b/src/scene/shader-lib/glsl/chunks/gsplat/frag/gsplatModify.js new file mode 100644 index 00000000000..947ff985834 --- /dev/null +++ b/src/scene/shader-lib/glsl/chunks/gsplat/frag/gsplatModify.js @@ -0,0 +1,11 @@ +export default /* glsl */` +// Modify the final splat fragment color in the forward pass. +// Parameters: +// gaussianUV - position of the fragment within the gaussian footprint: (0,0) at the splat +// center, length 1 at the edge where the splat is clipped +// color - rgb: splat color, a: fragment alpha after gaussian falloff, before +// premultiplication +void modifySplatColor(vec2 gaussianUV, inout vec4 color) { + // Example: color.rgb *= 0.5; // darken all splats +} +`; diff --git a/src/scene/shader-lib/glsl/collections/gsplat-chunks-glsl.js b/src/scene/shader-lib/glsl/collections/gsplat-chunks-glsl.js index b6af44a9b7a..c62884db18a 100644 --- a/src/scene/shader-lib/glsl/collections/gsplat-chunks-glsl.js +++ b/src/scene/shader-lib/glsl/collections/gsplat-chunks-glsl.js @@ -4,6 +4,7 @@ import gsplatSplatVS from '../chunks/gsplat/vert/gsplatSplat.js'; import gsplatEvalSHVS from '../chunks/gsplat/vert/gsplatEvalSH.js'; import gsplatHelpersVS from '../chunks/gsplat/vert/gsplatHelpers.js'; import gsplatModifyVS from '../chunks/gsplat/vert/gsplatModify.js'; +import gsplatModifyPS from '../chunks/gsplat/frag/gsplatModify.js'; import gsplatQuatToMat3VS from '../chunks/gsplat/vert/gsplatQuatToMat3.js'; import gsplatStructsVS from '../chunks/gsplat/vert/gsplatStructs.js'; import gsplatCornerVS from '../chunks/gsplat/vert/gsplatCorner.js'; @@ -33,6 +34,7 @@ export const gsplatChunksGLSL = { gsplatEvalSHVS, gsplatHelpersVS, gsplatModifyVS, + gsplatModifyPS, gsplatQuatToMat3VS, gsplatStructsVS, gsplatOutputVS, diff --git a/src/scene/shader-lib/wgsl/chunks/gsplat/frag/gsplat.js b/src/scene/shader-lib/wgsl/chunks/gsplat/frag/gsplat.js index 1bd6b07cdc9..daae22ba9b8 100644 --- a/src/scene/shader-lib/wgsl/chunks/gsplat/frag/gsplat.js +++ b/src/scene/shader-lib/wgsl/chunks/gsplat/frag/gsplat.js @@ -37,6 +37,8 @@ varying gaussianColor: half4; #include "pickPS" #endif +#include "gsplatModifyPS" + @fragment fn fragmentMain(input: FragmentInput) -> FragmentOutput { var output: FragmentOutput; @@ -89,7 +91,9 @@ fn fragmentMain(input: FragmentInput) -> FragmentOutput { opacityDither(f32(alpha), id * 0.013); #endif - output.color = vec4f(vec3f(gaussianColor.xyz * alpha), f32(alpha)); + var fragColor: vec4f = vec4f(vec3f(gaussianColor.xyz), f32(alpha)); + modifySplatColor(vec2f(gaussianUV), &fragColor); + output.color = vec4f(fragColor.xyz * fragColor.a, fragColor.a); #endif return output; diff --git a/src/scene/shader-lib/wgsl/chunks/gsplat/frag/gsplatModify.js b/src/scene/shader-lib/wgsl/chunks/gsplat/frag/gsplatModify.js new file mode 100644 index 00000000000..cf889060aff --- /dev/null +++ b/src/scene/shader-lib/wgsl/chunks/gsplat/frag/gsplatModify.js @@ -0,0 +1,11 @@ +export default /* wgsl */` +// Modify the final splat fragment color in the forward pass. +// Parameters: +// gaussianUV - position of the fragment within the gaussian footprint: (0,0) at the splat +// center, length 1 at the edge where the splat is clipped +// color - rgb: splat color, a: fragment alpha after gaussian falloff, before +// premultiplication +fn modifySplatColor(gaussianUV: vec2f, color: ptr) { + // Example: *color = vec4f((*color).rgb * 0.5, (*color).a); // darken all splats +} +`; diff --git a/src/scene/shader-lib/wgsl/collections/gsplat-chunks-wgsl.js b/src/scene/shader-lib/wgsl/collections/gsplat-chunks-wgsl.js index 4f0b395b642..49ed955908c 100644 --- a/src/scene/shader-lib/wgsl/collections/gsplat-chunks-wgsl.js +++ b/src/scene/shader-lib/wgsl/collections/gsplat-chunks-wgsl.js @@ -4,6 +4,7 @@ import gsplatSplatVS from '../chunks/gsplat/vert/gsplatSplat.js'; import gsplatEvalSHVS from '../chunks/gsplat/vert/gsplatEvalSH.js'; import gsplatHelpersVS from '../chunks/gsplat/vert/gsplatHelpers.js'; import gsplatModifyVS from '../chunks/gsplat/vert/gsplatModify.js'; +import gsplatModifyPS from '../chunks/gsplat/frag/gsplatModify.js'; import gsplatQuatToMat3VS from '../chunks/gsplat/vert/gsplatQuatToMat3.js'; import gsplatStructsVS from '../chunks/gsplat/vert/gsplatStructs.js'; import gsplatCornerVS from '../chunks/gsplat/vert/gsplatCorner.js'; @@ -37,6 +38,7 @@ export const gsplatChunksWGSL = { gsplatEvalSHVS, gsplatHelpersVS, gsplatModifyVS, + gsplatModifyPS, gsplatStructsVS, gsplatQuatToMat3VS, gsplatOutputVS,