Skip to content

Commit 2a4cf33

Browse files
author
Kazi Miftahul
committed
[ST-2164] (+) Separate renderer
- separate range based depth overlay renderer - configurable feather between ranges
1 parent 131f8d4 commit 2a4cf33

6 files changed

Lines changed: 275 additions & 53 deletions

File tree

‎Sources/StructureKit/Metal/STKDepthBandOverlay.metal‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
// StructureKit - A collection of extension utilities for Structure SDK
2+
// Copyright 2022 XRPro, LLC. All rights reserved.
3+
// http://structure.io
4+
//
5+
// Redistribution and use in source and binary forms, with or without
6+
// modification, are permitted provided that the following conditions are met:
7+
//
8+
// * Redistributions of source code must retain the above copyright notice,
9+
// this list of conditions and the following disclaimer.
10+
// * Redistributions in binary form must reproduce the above copyright notice,
11+
// this list of conditions and the following disclaimer in the documentation
12+
// and/or other materials provided with the distribution.
13+
// * Neither the name of XRPro, LLC nor the names of its contributors may be
14+
// used to endorse or promote products derived from this software without
15+
// specific prior written permission.
16+
//
17+
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18+
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19+
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20+
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21+
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
22+
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23+
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24+
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25+
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26+
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27+
// POSSIBILITY OF SUCH DAMAGE.
28+
29+
#include "STKMetalCommon.h"
30+
#include "STKMetalData.h"
31+
32+
#include <metal_stdlib>
33+
34+
using namespace metal;
35+
36+
vertex STKVertexTexOut vertexDepthBandOverlay(
37+
const device STKVertexTex* vertex_array [[buffer(0)]],
38+
const device STKUniformsDepthOverlay& uniforms [[buffer(1)]],
39+
unsigned int vid [[vertex_id]])
40+
{
41+
const STKVertexTex VertexIn = vertex_array[vid];
42+
const float4 texTrans = uniforms.projection * float4(VertexIn.texCoord, 0, 1);
43+
44+
STKVertexTexOut VertexOut;
45+
VertexOut.position = float4(VertexIn.position, 1);
46+
VertexOut.texCoord = float2(texTrans.x, texTrans.y);
47+
return VertexOut;
48+
}
49+
50+
fragment float4 fragmentDepthBandOverlay(
51+
STKVertexTexOut interpolated [[stage_in]],
52+
const device STKUniformsDepthBandOverlay& uniforms [[buffer(0)]],
53+
texture2d<float> texDepth [[texture(0)]],
54+
sampler sampler2D [[sampler(0)]])
55+
{
56+
// depth udefined
57+
const float depth = texDepth.sample(sampler2D, interpolated.texCoord).r;
58+
if (isnan(depth))
59+
return float4(0);
60+
61+
// calculate position of the depth pixel in the world CS using intrinsics
62+
const auto intrinsics = uniforms.cameraIntrinsics;
63+
const float u = interpolated.texCoord.x * intrinsics.width;
64+
const float v = interpolated.texCoord.y * intrinsics.height;
65+
const float depthM = depth / 1000;
66+
const float4 cameraPoint(
67+
depthM * (u - intrinsics.cx) / intrinsics.fx,
68+
depthM * (v - intrinsics.cy) / intrinsics.fy,
69+
depthM,
70+
1);
71+
const float4 worldPoint = uniforms.cameraPose * cameraPoint;
72+
const float4 cubePoint = uniforms.cubeModelInv * worldPoint;
73+
74+
// outside the box
75+
if (cubePoint.x < 0 || cubePoint.x > 1 || cubePoint.y < 0 || cubePoint.y > 1 || cubePoint.z < 0 || cubePoint.z > 1)
76+
return float4(0);
77+
78+
// Use validRangeColor in ideal range, outOfRangeColor elsewhere.
79+
float inAtMin = smoothstep(uniforms.validRangeMinMM - uniforms.feather, uniforms.validRangeMinMM + uniforms.feather, depth);
80+
float inAtMax = 1.0 - smoothstep(uniforms.validRangeMaxMM - uniforms.feather, uniforms.validRangeMaxMM + uniforms.feather, depth);
81+
float t = clamp(inAtMin * inAtMax, 0.0, 1.0);
82+
83+
float4 color = mix(uniforms.outOfRangeColor, uniforms.validRangeColor, t);
84+
color.a = uniforms.alpha;
85+
return color;
86+
}

‎Sources/StructureKit/Metal/STKDepthBandOverlayRenderer.swift‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
// StructureKit - A collection of extension utilities for Structure SDK
2+
// Copyright 2022 XRPro, LLC. All rights reserved.
3+
// http://structure.io
4+
//
5+
// Redistribution and use in source and binary forms, with or without
6+
// modification, are permitted provided that the following conditions are met:
7+
//
8+
// * Redistributions of source code must retain the above copyright notice,
9+
// this list of conditions and the following disclaimer.
10+
// * Redistributions in binary form must reproduce the above copyright notice,
11+
// this list of conditions and the following disclaimer in the documentation
12+
// and/or other materials provided with the distribution.
13+
// * Neither the name of XRPro, LLC nor the names of its contributors may be
14+
// used to endorse or promote products derived from this software without
15+
// specific prior written permission.
16+
//
17+
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18+
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19+
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20+
// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
21+
// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
22+
// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
23+
// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
24+
// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
25+
// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
26+
// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27+
// POSSIBILITY OF SUCH DAMAGE.
28+
29+
import Metal
30+
import MetalKit
31+
import StructureKitCTypes
32+
33+
public class STKDepthBandOverlayRenderer {
34+
private var mtkView: MTKView
35+
private var renderDepthState: MTLRenderPipelineState
36+
private var textureDepth: MTLTexture?
37+
private var vertexBuffer: MTLBuffer
38+
private var samplerState: MTLSamplerState
39+
private var device: MTLDevice
40+
private var intr: STKIntrinsics?
41+
42+
private var validRangeMinMM: Float = 200.0
43+
private var validRangeMaxMM: Float = 400.0
44+
private var validRangeColor: simd_float4 = simd_float4(0,1,0,0.5) // Green
45+
private var outOfRangeColor: simd_float4 = simd_float4(1,0,0,0.5) // Red
46+
private var feather: Float = 40.0 // in mm
47+
48+
public init(view: MTKView, device: MTLDevice) {
49+
mtkView = view
50+
self.device = device
51+
52+
let library = STKMetalLibLoader.load(device: device)
53+
let pipelineDepthDescriptor = MTLRenderPipelineDescriptor()
54+
pipelineDepthDescriptor.sampleCount = 1
55+
pipelineDepthDescriptor.colorAttachments[0].pixelFormat = mtkView.colorPixelFormat
56+
pipelineDepthDescriptor.depthAttachmentPixelFormat = mtkView.depthStencilPixelFormat
57+
pipelineDepthDescriptor.vertexFunction = library.makeFunction(name: "vertexDepthBandOverlay")
58+
pipelineDepthDescriptor.fragmentFunction = library.makeFunction(name: "fragmentDepthBandOverlay")
59+
// blending
60+
pipelineDepthDescriptor.colorAttachments[0].isBlendingEnabled = true
61+
pipelineDepthDescriptor.colorAttachments[0].rgbBlendOperation = .add
62+
pipelineDepthDescriptor.colorAttachments[0].sourceRGBBlendFactor = .sourceAlpha
63+
pipelineDepthDescriptor.colorAttachments[0].destinationRGBBlendFactor = .oneMinusSourceAlpha
64+
65+
renderDepthState = try! device.makeRenderPipelineState(descriptor: pipelineDepthDescriptor)
66+
67+
let vertexData = makeRectangularVertices()
68+
let dataSize = vertexData.count * MemoryLayout.size(ofValue: vertexData[0])
69+
vertexBuffer = device.makeBuffer(bytes: vertexData, length: dataSize, options: [])!
70+
71+
samplerState = makeDefaultSampler(device)
72+
}
73+
74+
public func uploadColorTextureFromDepth(_ depthFrame: STKDepthFrame) {
75+
if let texture = textureDepth {
76+
if texture.width != depthFrame.width || texture.height != depthFrame.height {
77+
textureDepth = nil // invalidate texture
78+
}
79+
}
80+
81+
if textureDepth == nil {
82+
let textureDescriptor = MTLTextureDescriptor.texture2DDescriptor(
83+
pixelFormat: .r32Float, width: Int(depthFrame.width), height: Int(depthFrame.height), mipmapped: false)
84+
textureDescriptor.usage = MTLTextureUsage(
85+
rawValue: MTLTextureUsage.renderTarget.rawValue | MTLTextureUsage.shaderRead.rawValue)
86+
textureDepth = device.makeTexture(descriptor: textureDescriptor)
87+
}
88+
89+
intr = depthFrame.intrinsics()
90+
let depthMap = depthFrame.depthInMillimeters
91+
assert(MemoryLayout<Float32>.size == MemoryLayout<Float>.size)
92+
93+
let bytesPerRow: Int = Int(depthFrame.width) * MemoryLayout<Float32>.stride
94+
let region = MTLRegionMake2D(0, 0, Int(depthFrame.width), Int(depthFrame.height))
95+
textureDepth?.replace(region: region, mipmapLevel: 0, withBytes: depthMap!, bytesPerRow: bytesPerRow)
96+
}
97+
98+
public func renderDepthOverlay(
99+
_ commandEncoder: MTLRenderCommandEncoder,
100+
volumeSizeInMeters: simd_float3,
101+
cameraPosition: float4x4,
102+
textureOrientation: float4x4,
103+
alpha: Float
104+
) {
105+
guard let texture = textureDepth,
106+
let intr = intr
107+
else { return }
108+
109+
commandEncoder.pushDebugGroup("RenderDepthRangeOverlay")
110+
commandEncoder.setRenderPipelineState(renderDepthState)
111+
112+
// rotate and mirror:
113+
// move to [-0.5, 0.5] coordinates
114+
// rotate
115+
// mirror and apply scale to fix the ratio
116+
// move back to [0, 1] coordinates
117+
let projection =
118+
float4x4.makeTranslation(0.5, 0.5, 0) * textureOrientation * float4x4.makeTranslation(-0.5, -0.5, 0)
119+
120+
let intrinsics = STKIntrinsicsMetal(
121+
cx: intr.cx, cy: intr.cy, fx: intr.fx, fy: intr.fy, width: UInt32(texture.width), height: UInt32(texture.height))
122+
123+
var uniforms = STKUniformsDepthBandOverlay(
124+
projection: projection,
125+
cameraPose: cameraPosition,
126+
cameraIntrinsics: intrinsics,
127+
cubeModelInv: float4x4.makeScale(volumeSizeInMeters.x, volumeSizeInMeters.y, volumeSizeInMeters.z).inverse,
128+
alpha: alpha,
129+
validRangeMinMM: validRangeMinMM,
130+
validRangeMaxMM: validRangeMaxMM,
131+
validRangeColor: validRangeColor,
132+
outOfRangeColor: outOfRangeColor,
133+
feather: feather
134+
)
135+
136+
commandEncoder.setVertexBuffer(vertexBuffer, offset: 0, index: 0)
137+
commandEncoder.setVertexBytes(&uniforms, length: MemoryLayout<STKUniformsDepthBandOverlay>.stride, index: 1)
138+
139+
commandEncoder.setFragmentBytes(&uniforms, length: MemoryLayout<STKUniformsDepthBandOverlay>.stride, index: 0)
140+
commandEncoder.setFragmentTexture(texture, index: 0) // [[ texture(0) ]],
141+
commandEncoder.setFragmentSamplerState(samplerState, index: 0) // [[ sampler(0) ]]
142+
commandEncoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 6, instanceCount: 1)
143+
commandEncoder.popDebugGroup()
144+
}
145+
146+
public func configure(
147+
validRangeMinMM: Float, validRangeMaxMM: Float,
148+
validRangeColor: simd_float4 = simd_float4(0,1,0,0.5),
149+
outOfRangeColor: simd_float4 = simd_float4(1,0,0,0.5),
150+
feather: Float = 40.0
151+
) {
152+
self.validRangeMinMM = validRangeMinMM
153+
self.validRangeMaxMM = validRangeMaxMM
154+
self.validRangeColor = validRangeColor
155+
self.outOfRangeColor = outOfRangeColor
156+
self.feather = feather
157+
}
158+
}

‎Sources/StructureKit/Metal/STKDepthOverlay.metal‎

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -84,20 +84,7 @@ fragment float4 fragmentDepthOverlay(
8484
return float4(0);
8585

8686
// calculate the depth color
87-
if (uniforms.mode == 1) {
88-
float4 finalColor = calcDepthColor(depth, float2(uniforms.depthMin, uniforms.depthMax), colors);
89-
finalColor.w = uniforms.alpha;
90-
return finalColor;
91-
} else {
92-
// Use validRangeColor in ideal range, outOfRangeColor elsewhere.
93-
const float feather = 40.0; // Use softness of 40 mm
94-
95-
float inAtMin = smoothstep(uniforms.validRangeMinMM - feather, uniforms.validRangeMinMM + feather, depth);
96-
float inAtMax = 1.0 - smoothstep(uniforms.validRangeMaxMM - feather, uniforms.validRangeMaxMM + feather, depth);
97-
float t = clamp(inAtMin * inAtMax, 0.0, 1.0);
98-
99-
float4 color = mix(uniforms.outOfRangeColor, uniforms.validRangeColor, t);
100-
color.a = uniforms.alpha;
101-
return color;
102-
}
87+
float4 finalColor = calcDepthColor(depth, float2(uniforms.depthMin, uniforms.depthMax), colors);
88+
finalColor.w = uniforms.alpha;
89+
return finalColor;
10390
}

‎Sources/StructureKit/Metal/STKDepthRenderer.swift‎

Lines changed: 1 addition & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,6 @@ import StructureKitCTypes
3535
// 2. the cube
3636
// 3. the depth overlay inside the cube
3737

38-
public enum STKDepthOverlayMode: Int {
39-
case palette = 1
40-
case range = 2
41-
}
42-
4338
public class STKDepthRenderer {
4439
private var mtkView: MTKView
4540
private var renderDepthState: MTLRenderPipelineState
@@ -59,12 +54,6 @@ public class STKDepthRenderer {
5954
private var _vertexCubeBuffer: MTLBuffer!
6055
private var _indexCubeBuffer: MTLBuffer!
6156

62-
private var overlayMode: STKDepthOverlayMode = .palette
63-
private var validRangeMinMM: Float = 0.0
64-
private var validRangeMaxMM: Float = 0.0
65-
private var validRangeColor: simd_float4 = simd_float4(0,1,0,0.5) // Green
66-
private var outOfRangeColor: simd_float4 = simd_float4(1,0,0,0.5) // Red
67-
6857
public init(view: MTKView, device: MTLDevice) {
6958
mtkView = view
7059
self.device = device
@@ -226,12 +215,7 @@ public class STKDepthRenderer {
226215
cubeModelInv: float4x4.makeScale(volumeSizeInMeters.x, volumeSizeInMeters.y, volumeSizeInMeters.z).inverse,
227216
depthMin: minDistM * 1000,
228217
depthMax: maxDistM * 1000,
229-
alpha: alpha,
230-
mode: Int32(overlayMode.rawValue),
231-
validRangeMinMM: validRangeMinMM,
232-
validRangeMaxMM: validRangeMaxMM,
233-
validRangeColor: validRangeColor,
234-
outOfRangeColor: outOfRangeColor
218+
alpha: alpha
235219
)
236220

237221
commandEncoder.setVertexBuffer(vertexBuffer, offset: 0, index: 0)
@@ -315,18 +299,4 @@ public class STKDepthRenderer {
315299
indexBufferOffset: 0)
316300
commandEncoder.popDebugGroup()
317301
}
318-
319-
public func configureDepthOverlay(
320-
_ mode: STKDepthOverlayMode,
321-
validRangeMinMM: Float = 0, validRangeMaxMM: Float = 0,
322-
validRangeColor: simd_float4 = simd_float4(0,1,0,0.5),
323-
outOfRangeColor: simd_float4 = simd_float4(1,0,0,0.5)
324-
) {
325-
self.overlayMode = mode
326-
self.validRangeMinMM = validRangeMinMM
327-
self.validRangeMaxMM = validRangeMaxMM
328-
self.validRangeColor = validRangeColor
329-
self.outOfRangeColor = outOfRangeColor
330-
}
331-
332302
}

‎Sources/StructureKit/Metal/STKMetalData.h‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,12 +71,20 @@ struct STKUniformsDepthOverlay
7171
float depthMin;
7272
float depthMax;
7373
float alpha;
74-
// Variables for colouring depth for distance guide mode
75-
int mode;
74+
};
75+
76+
struct STKUniformsDepthBandOverlay
77+
{
78+
matrix_float4x4 projection;
79+
matrix_float4x4 cameraPose;
80+
struct STKIntrinsicsMetal cameraIntrinsics;
81+
matrix_float4x4 cubeModelInv;
82+
float alpha;
7683
float validRangeMinMM;
7784
float validRangeMaxMM;
7885
vector_float4 validRangeColor;
7986
vector_float4 outOfRangeColor;
87+
float feather;
8088
};
8189

8290
struct STKUniformsLine

0 commit comments

Comments
 (0)