Softer shadows in raytracing and RTAO

code/def_files/data/effects/deferred-f.sdr

@@ -24,7 +24,7 @@ layout(set = 0, binding = 3) uniform samplerCube sEnvmap;
 layout(set = 0, binding = 4) uniform samplerCube sIrrmap;
 #endif
 
-#ifdef RT_SHADOWS
+#if defined(RT_SHADOWS) || defined(RTAO)
 layout(set = 0, binding = 5) uniform accelerationStructureEXT shadow_tlas;
 #endif
 #else
@@ -63,6 +63,11 @@ uniform shadowCascadeParams {
 	int cascade_count;
 	float rtShadowBiasMin;
 	float rtShadowBiasMax;
+	int rtShadowSampleCount;
+	float rtShadowSoftness;
+	int rtaoSampleCount;
+	float rtaoRadius;
+	float rtaoStrength;
 
 	mat4 shadow_mv_matrix;
 	mat4 shadow_proj_matrix[NUM_SHADOW_CASCADES];
@@ -291,6 +296,17 @@ void main()
 
 	if (lightType == LT_AMBIENT) {
 		float ao = position_buffer.w;
+#ifdef RTAO
+		{
+			// Raytraced AO multiplies into the same ao term as the baked AO maps,
+			// so the env-map/IBL ambient below darkens consistently for free.
+			vec3 worldPos = (inv_view_matrix * vec4(position, 1.0)).xyz;
+			vec3 worldNormal = normalize((inv_view_matrix * vec4(normal, 0.0)).xyz);
+			float rtaoBias = computeRtShadowBias(length(position), rtShadowBiasMin, rtShadowBiasMax);
+			ao *= traceAmbientOcclusion(shadow_tlas, worldPos, worldNormal,
+				rtaoRadius, rtaoStrength, rtaoSampleCount, rtaoBias, gl_FragCoord.xy);
+		}
+#endif
 		fragmentColor.rgb = diffuseLightColor * diffColor * ao;
 
 #ifdef ENV_MAP
@@ -315,7 +331,12 @@ void main()
 			vec3 worldLightDir = normalize((inv_view_matrix * vec4(lightDir, 0.0)).xyz);
 
 			float rtShadowBias = computeRtShadowBias(length(position), rtShadowBiasMin, rtShadowBiasMax);
-			attenuation *= traceShadowRay(shadow_tlas, worldPos, worldNormal, worldLightDir, lightDist, rtShadowBias);
+			// Penumbra cone: for directional lights sourceRadius IS the tangent of the
+			// sun's angular radius ($SunAngularSize); for local lights the subtended
+			// half-angle follows from the physical source radius and the light distance.
+			float coneTan = (lightType == LT_DIRECTIONAL) ? sourceRadius : sourceRadius / max(lightDist, 1e-3);
+			attenuation *= traceShadowRayCone(shadow_tlas, worldPos, worldNormal, worldLightDir, lightDist,
+				rtShadowBias, coneTan * rtShadowSoftness, rtShadowSampleCount, gl_FragCoord.xy);
 #else
 			vec4 fragShadowPos = shadow_mv_matrix * inv_view_matrix * vec4(position, 1.0);
 			vec4 fragShadowUV[NUM_SHADOW_CASCADES];

code/def_files/data/effects/main-f.sdr

@@ -101,6 +101,11 @@ uniform shadowCascadeParams {
 	int cascade_count;
 	float rtShadowBiasMin;
 	float rtShadowBiasMax;
+	int rtShadowSampleCount;
+	float rtShadowSoftness;
+	int rtaoSampleCount;
+	float rtaoRadius;
+	float rtaoStrength;
 
 	mat4 shadow_mv_matrix;
 	mat4 shadow_proj_matrix[NUM_SHADOW_CASCADES];
@@ -245,7 +250,10 @@ vec3 CalculateLighting(vec3 normal, vec3 diffuseMaterial, vec3 specularMaterial,
 		if (rtShadowsActive && lights[i].light_type == LT_DIRECTIONAL && shadowedDirectionalCount < MAX_RT_SHADOW_LIGHTS) {
 			vec3 worldSunDir = normalize((invView * vec4(lights[i].position.xyz, 0.0)).xyz);
 			float rtShadowBias = computeRtShadowBias(length(vertIn.position.xyz), rtShadowBiasMin, rtShadowBiasMax);
-			shadow = traceShadowRay(shadow_tlas, worldPos, worldNormal, worldSunDir, RT_SHADOW_MAX_DISTANCE, rtShadowBias);
+			// For directional lights ml_sourceRadius is the tangent of the sun's
+			// angular radius ($SunAngularSize) -- 0 keeps hard shadows.
+			shadow = traceShadowRayCone(shadow_tlas, worldPos, worldNormal, worldSunDir, RT_SHADOW_MAX_DISTANCE,
+				rtShadowBias, lights[i].ml_sourceRadius * rtShadowSoftness, rtShadowSampleCount, gl_FragCoord.xy);
 			++shadowedDirectionalCount;
 		} else {
 			shadow = 1.0;

code/def_files/data/effects/main-v.sdr

@@ -115,6 +115,11 @@ uniform shadowCascadeParams {
 	int cascade_count;
 	float rtShadowBiasMin;
 	float rtShadowBiasMax;
+	int rtShadowSampleCount;
+	float rtShadowSoftness;
+	int rtaoSampleCount;
+	float rtaoRadius;
+	float rtaoStrength;
 
 	mat4 shadow_mv_matrix;
 	mat4 shadow_proj_matrix[NUM_SHADOW_CASCADES];

code/def_files/data/effects/shadow_map-g.sdr

@@ -7,11 +7,19 @@ layout (triangle_strip, max_vertices = 3) out;
 layout(invocations = NUM_SHADOW_CASCADES) in;
 #endif
 
-layout (std140) uniform shadowCascadeParams {
-    int cascade_offset;
-    int cascade_count;
-    float rtShadowBiasMin;
-    float rtShadowBiasMax;
+// OpenGL-only: the Vulkan backend has no geometry-shader stage, and disables the
+// shadow pass outright when it can't write gl_Layer from the vertex shader, so it
+// never requests GEOMETRY_FALLBACK. Hence no Vulkan set/binding qualifier here.
+layout(std140) uniform shadowCascadeParams {
+	int cascade_offset;
+	int cascade_count;
+	float rtShadowBiasMin;
+	float rtShadowBiasMax;
+	int rtShadowSampleCount;
+	float rtShadowSoftness;
+	int rtaoSampleCount;
+	float rtaoRadius;
+	float rtaoStrength;
 
 	mat4 shadow_mv_matrix;
 	mat4 shadow_proj_matrix[NUM_SHADOW_CASCADES];

code/def_files/data/effects/shadow_map-v.sdr

@@ -38,10 +38,15 @@ layout(set = 0, binding = 6, std140)
 layout(std140)
 #endif
 uniform shadowCascadeParams {
-    int cascade_offset;
-    int cascade_count;
-    float rtShadowBiasMin;
-    float rtShadowBiasMax;
+	int cascade_offset;
+	int cascade_count;
+	float rtShadowBiasMin;
+	float rtShadowBiasMax;
+	int rtShadowSampleCount;
+	float rtShadowSoftness;
+	int rtaoSampleCount;
+	float rtaoRadius;
+	float rtaoStrength;
 
 	mat4 shadow_mv_matrix;
 	mat4 shadow_proj_matrix[NUM_SHADOW_CASCADES];

code/def_files/data/effects/shadows.sdr

@@ -79,7 +79,7 @@ float getShadowValue(sampler2DArrayShadow shadow_map, float depth, vec4 shadowUV
 // in deferred-f.sdr/main-f.sdr references it regardless of which shadow method is active.
 const float RT_SHADOW_MAX_DISTANCE = 10000.0;
 
-#if defined(MODEL_SDR_FLAG_RT_SHADOWS) || defined(RT_SHADOWS)
+#if defined(MODEL_SDR_FLAG_RT_SHADOWS) || defined(RT_SHADOWS) || defined(RTAO)
 #extension GL_EXT_ray_query : require
 
 // Ray origin bias grows with distance from the camera, between the rtShadowBiasMin/
@@ -126,6 +126,157 @@ float traceShadowRay(accelerationStructureEXT tlas, vec3 worldPos, vec3 worldNor
 
 	return (rayQueryGetIntersectionTypeEXT(rq, true) == gl_RayQueryCommittedIntersectionNoneEXT) ? 1.0 : 0.0;
 }
+
+// Penumbra/AO sampling: at most this many rays per fragment per light, matching
+// the disc pattern size below. rtShadowSampleCount/rtaoSampleCount
+// (shadowCascadeParams) are clamped against it, so a bad uniform value can't
+// index out of the pattern.
+#define RT_SHADOW_MAX_SAMPLES 16
+
+const vec2 rtPoissonDisc[RT_SHADOW_MAX_SAMPLES] = vec2[](
+	vec2(-0.76275, -0.3432573),
+	vec2(-0.5226235, -0.8277544),
+	vec2(-0.3780261, 0.01528688),
+	vec2(-0.7742821, 0.4245702),
+	vec2(0.04196143, -0.02622231),
+	vec2(-0.2974772, -0.4722782),
+	vec2(-0.516093, 0.71495),
+	vec2(-0.3257416, 0.3910343),
+	vec2(0.2705966, 0.6670476),
+	vec2(0.4918377, 0.1853267),
+	vec2(0.4428544, -0.6251478),
+	vec2(-0.09204347, 0.9267113),
+	vec2(0.391505, -0.2558275),
+	vec2(0.05605913, -0.7570801),
+	vec2(0.81772, -0.02475523),
+	vec2(0.6890262, 0.5191521)
+);
+
+// Orthonormal basis spanning the plane perpendicular to axis (a unit vector).
+void rtBuildOnb(vec3 axis, out vec3 tangent, out vec3 bitangent)
+{
+	vec3 up = (abs(axis.z) < 0.999) ? vec3(0.0, 0.0, 1.0) : vec3(1.0, 0.0, 0.0);
+	tangent = normalize(cross(up, axis));
+	bitangent = cross(axis, tangent);
+}
+
+// Interleaved gradient noise (Jimenez 2014) -> per-fragment [0,1) value used to
+// rotate the shared disc pattern. Callers pass gl_FragCoord.xy as the seed; it is
+// a parameter rather than read here because this file is also compiled in vertex
+// stages, where gl_FragCoord doesn't exist.
+float rtInterleavedGradientNoise(vec2 seed)
+{
+	return fract(52.9829189 * fract(dot(seed, vec2(0.06711056, 0.00583715))));
+}
+
+// Soft (penumbra) variant of traceShadowRay: averages up to sampleCount occlusion
+// rays whose directions are jittered inside a cone of half-angle
+// atan(coneHalfAngleTan) around worldLightDir. Callers derive coneHalfAngleTan
+// from the light's source radius -- for directional lights the light's
+// source_radius IS the tangent of its angular radius (see $SunAngularSize in
+// stars.tbl / starfield.cpp), for local lights it's sourceRadius / lightDist --
+// so penumbras widen with occluder distance like a real area light's would.
+//
+// noiseSeed decorrelates the sample pattern between fragments (see
+// rtInterleavedGradientNoise() for why it's a parameter). There is no temporal
+// accumulation in the engine, so the penumbra must fully resolve in this one
+// pass: the disc pattern is fixed and only its per-fragment rotation is
+// randomized, trading a little visible grain for stable, flicker-free output.
+//
+// sampleCount <= 1 or coneHalfAngleTan <= 0.0 falls back to the single hard ray,
+// bit-identical to traceShadowRay() -- that keeps the pre-penumbra look and cost
+// as the default (mods opt in via source radii, users via the sample option).
+float traceShadowRayCone(accelerationStructureEXT tlas, vec3 worldPos, vec3 worldNormal, vec3 worldLightDir,
+	float tMax, float bias, float coneHalfAngleTan, int sampleCount, vec2 noiseSeed)
+{
+	if (sampleCount <= 1 || coneHalfAngleTan <= 0.0) {
+		return traceShadowRay(tlas, worldPos, worldNormal, worldLightDir, tMax, bias);
+	}
+
+	vec3 origin = worldPos + worldNormal * bias;
+	vec3 direction = normalize(worldLightDir);
+
+	vec3 tangent, bitangent;
+	rtBuildOnb(direction, tangent, bitangent);
+
+	float rotAngle = rtInterleavedGradientNoise(noiseSeed) * 6.2831853;
+	float cosR = cos(rotAngle);
+	float sinR = sin(rotAngle);
+
+	int samples = min(sampleCount, RT_SHADOW_MAX_SAMPLES);
+	float visibility = 0.0;
+	for (int i = 0; i < samples; ++i) {
+		vec2 p = rtPoissonDisc[i];
+		p = vec2(p.x * cosR - p.y * sinR, p.x * sinR + p.y * cosR);
+		vec3 sampleDir = normalize(direction + (tangent * p.x + bitangent * p.y) * coneHalfAngleTan);
+
+		rayQueryEXT rq;
+		rayQueryInitializeEXT(rq, tlas, gl_RayFlagsOpaqueEXT | gl_RayFlagsTerminateOnFirstHitEXT,
+			0xFF, origin, 0.001, sampleDir, tMax);
+		while (rayQueryProceedEXT(rq)) {}
+
+		if (rayQueryGetIntersectionTypeEXT(rq, true) == gl_RayQueryCommittedIntersectionNoneEXT) {
+			visibility += 1.0;
+		}
+	}
+	return visibility / float(samples);
+}
+
+// Raytraced ambient occlusion: averages up to sampleCount occlusion rays of
+// length `radius` over the hemisphere around worldNormal and returns ambient
+// visibility in [0,1] (1 = fully open), shaped by pow(visibility, strength).
+// Mapping the shared disc pattern up onto the hemisphere (z = sqrt(1-|p|^2))
+// makes the samples cosine-weighted, matching the diffuse ambient term being
+// occluded. radius/strength come from the mod's lighting profile ($RTAO Radius /
+// $RTAO Strength); scene scale varies too much (fighters vs. capships) for a
+// built-in constant.
+//
+// Hits accumulate with a (1 - t/radius) falloff so geometry entering the radius
+// occludes gradually instead of popping. That needs the *closest* hit's t, so
+// these rays deliberately omit gl_RayFlagsTerminateOnFirstHitEXT (any-hit t
+// would make the falloff arbitrary); the extra traversal cost is small because
+// tMax = radius keeps the rays short.
+//
+// sampleCount < 1, radius <= 0, or strength <= 0 returns 1.0 (no occlusion) --
+// callers gate on the RTAO shader variant, this just keeps bad uniforms benign.
+float traceAmbientOcclusion(accelerationStructureEXT tlas, vec3 worldPos, vec3 worldNormal,
+	float radius, float strength, int sampleCount, float bias, vec2 noiseSeed)
+{
+	if (sampleCount < 1 || radius <= 0.0 || strength <= 0.0) {
+		return 1.0;
+	}
+
+	vec3 origin = worldPos + worldNormal * bias;
+
+	vec3 tangent, bitangent;
+	rtBuildOnb(worldNormal, tangent, bitangent);
+
+	float rotAngle = rtInterleavedGradientNoise(noiseSeed) * 6.2831853;
+	float cosR = cos(rotAngle);
+	float sinR = sin(rotAngle);
+
+	int samples = min(sampleCount, RT_SHADOW_MAX_SAMPLES);
+	float occlusion = 0.0;
+	for (int i = 0; i < samples; ++i) {
+		vec2 p = rtPoissonDisc[i];
+		p = vec2(p.x * cosR - p.y * sinR, p.x * sinR + p.y * cosR);
+		vec3 sampleDir = normalize(tangent * p.x + bitangent * p.y
+			+ worldNormal * sqrt(max(1.0 - dot(p, p), 0.0)));
+
+		rayQueryEXT rq;
+		rayQueryInitializeEXT(rq, tlas, gl_RayFlagsOpaqueEXT,
+			0xFF, origin, 0.001, sampleDir, radius);
+		while (rayQueryProceedEXT(rq)) {}
+
+		if (rayQueryGetIntersectionTypeEXT(rq, true) != gl_RayQueryCommittedIntersectionNoneEXT) {
+			float hitT = rayQueryGetIntersectionTEXT(rq, true);
+			occlusion += 1.0 - clamp(hitT / radius, 0.0, 1.0);
+		}
+	}
+
+	float visibility = clamp(1.0 - occlusion / float(samples), 0.0, 1.0);
+	return pow(visibility, strength);
+}
 #endif
 
 vec4 transformToShadowMap(mat4 shadow_proj_matrix, int i, vec4 pos)

code/graphics/2d.h

@@ -262,6 +262,7 @@ enum shader_type {
 
 #define SDR_FLAG_ENV_MAP (1 << 0)
 #define SDR_FLAG_DEFERRED_RT_SHADOWS (1 << 1)
+#define SDR_FLAG_DEFERRED_RTAO (1 << 2)
 
 #define SDR_FLAG_SHADOW_FALLBACK (1 << 0)
 

code/graphics/rtao.cpp

@@ -0,0 +1,40 @@
+#include "graphics/rtao.h"
+
+#include "graphics/2d.h"
+#include "options/Option.h"
+
+int Rtao_samples = 0;
+
+// coverity[GLOBAL_INIT_ORDER] -- safe; OptionBuilder::finish() uses Meyers singleton
+auto RtaoQualityOption = options::OptionBuilder<int>("Graphics.RtaoQuality",
+                     std::pair<const char*, int>{"Raytraced Ambient Occlusion", -1},
+                     std::pair<const char*, int>{"Rays traced per pixel for ambient occlusion in the deferred ambient pass; Off keeps baked AO maps only", -1})
+                     .enumerator([]() -> SCP_vector<int> {
+                         if (rtao_supported()) {
+                             return {0, 4, 8, 16};
+                         }
+                         return {0}; // inert default when RT isn't supported
+                     })
+                     .display(options::MapValueDisplay<int>({
+                         {0, {"Off", -1}},
+                         {4, {"Low", -1}},
+                         {8, {"Medium", -1}},
+                         {16, {"High", -1}}
+                     }))
+                     .bind_to(&Rtao_samples)
+                     .flags({options::OptionFlags::ForceMultiValueSelection})
+                     .level(options::ExpertLevel::Advanced)
+                     .category(std::make_pair("Graphics", 1825))
+                     .default_func([]() { return 0; })
+                     .importance(71)
+                     .finish();
+
+bool rtao_supported()
+{
+	return gr_is_capable(gr_capability::CAPABILITY_RAYTRACED_SHADOWS);
+}
+
+bool rtao_enabled()
+{
+	return Rtao_samples > 0 && rtao_supported();
+}

code/graphics/rtao.h

@@ -0,0 +1,25 @@
+#pragma once
+
+// Raytraced ambient occlusion (RTAO): short hemisphere occlusion rays traced via
+// inline ray query against the raytraced-shadow TLAS, inside the deferred ambient
+// light pass (LT_AMBIENT branch of deferred-f.sdr, RTAO shader variant). Composes
+// multiplicatively with baked AO maps through the shared G-buffer `ao` term, so
+// env-map/IBL ambient darkens consistently for free. The AO radius and strength
+// are content-scale-dependent and therefore mod-owned -- see $RTAO Radius: /
+// $RTAO Strength: in lighting_profiles.h.
+
+// Rays traced per pixel for ambient occlusion. 0 (the default) disables RTAO
+// entirely; nonzero values only take effect when rtao_supported(). Clamped
+// in-shader to RT_SHADOW_MAX_SAMPLES (16). See RtaoQualityOption in rtao.cpp.
+extern int Rtao_samples;
+
+// Whether the hardware/renderer can trace AO rays at all -- the same requirement
+// set as raytraced shadows (Vulkan + VK_KHR_acceleration_structure +
+// VK_KHR_ray_query), so this simply forwards to that capability.
+bool rtao_supported();
+
+// Whether the deferred ambient pass should trace AO this frame, i.e. the user has
+// enabled it AND the hardware supports it. This is the single source of truth --
+// gate any RTAO shader-flag, TLAS-build, or uniform code on this, not on
+// Rtao_samples/rtao_supported() separately.
+bool rtao_enabled();

code/graphics/shader_types.cpp

@@ -133,6 +133,8 @@ static ShaderVariantInfo SHADER_VARIANTS[] = {
 
 	{SDR_TYPE_DEFERRED_LIGHTING, false, SDR_FLAG_DEFERRED_RT_SHADOWS, "RT_SHADOWS", {}, "Use raytraced (TLAS ray query) shadows instead of cascaded shadow maps"},
 
+	{SDR_TYPE_DEFERRED_LIGHTING, false, SDR_FLAG_DEFERRED_RTAO, "RTAO", {}, "Trace raytraced ambient occlusion in the ambient light pass"},
+
 	{SDR_TYPE_POST_PROCESS_BLUR, false, SDR_FLAG_BLUR_HORIZONTAL, "PASS_0", {}, "Horizontal blur pass"},
 
 	{SDR_TYPE_POST_PROCESS_BLUR, false, SDR_FLAG_BLUR_VERTICAL, "PASS_1", {}, "Vertical blur pass"},
@@ -208,7 +210,7 @@ bool shader_variant_requires_raytracing(shader_type type, unsigned int flags)
 	case SDR_TYPE_MODEL:
 		return (flags & MODEL_SDR_FLAG_RT_SHADOWS) != 0;
 	case SDR_TYPE_DEFERRED_LIGHTING:
-		return (flags & SDR_FLAG_DEFERRED_RT_SHADOWS) != 0;
+		return (flags & (SDR_FLAG_DEFERRED_RT_SHADOWS | SDR_FLAG_DEFERRED_RTAO)) != 0;
 	default:
 		return false;
 	}