chore: adjust masking (top, bottom)

benmotanoti · doranteseduardo · commit 3a0984d48ddd · 2026-04-01T18:40:53.000-06:00
diff --git a/ViroRenderer/VROMaterial.cpp b/ViroRenderer/VROMaterial.cpp
@@ -445,26 +445,50 @@ void VROMaterial::applySemanticMaskModifier() {
         "uniform highp float semantic_label_mask;",
         // Compute shared UV once. No Y-flip: gl_FragCoord and the transform are both GL-convention.
         "highp vec2 semUV = (ar_semantic_texture_transform * vec4(gl_FragCoord.xy / ar_viewport_size.xy, 0.0, 1.0)).xy;",
-        "highp float label_raw = texture(semantic_texture, semUV).r * 255.0;",
-        "int semLabel = int(floor(label_raw + 0.5));",
-        "bool semMatches = (semLabel >= 1 && semLabel <= 11 && (int(semantic_label_mask) & (1 << semLabel)) != 0);",
+        // Clamp inward by a small margin to skip the 1-2 pixel unlabeled border that ARCore's
+        // neural network outputs at the left/right edges of the landscape image. In portrait mode
+        // those edges map to thin horizontal lines at the top/bottom of the viewport. Without the
+        // clamp those border pixels (label=0) cut the sky effect with a transparent stripe.
+        // 0.005 ≈ half a pixel on a 192-px-tall semantic texture; harmless for center content.
+        "semUV = clamp(semUV, vec2(0.005), vec2(0.995));",
+        "bool semInBounds = (semUV.x >= 0.0 && semUV.x <= 1.0 && semUV.y >= 0.0 && semUV.y <= 1.0);",
+        "if (semInBounds) {",
+        "    highp float label_raw = texture(semantic_texture, semUV).r * 255.0;",
+        "    int semLabel = int(floor(label_raw + 0.5));",
+        "    bool semMatches = (semLabel >= 1 && semLabel <= 11 && (int(semantic_label_mask) & (1 << semLabel)) != 0);",
         // Debug mode (semantic_mask_mode == 2): colorise by label for overlay visualisation.
         // Blue = unlabeled, teal→orange gradient = classified pixels (fully opaque).
-        "if (semantic_mask_mode >= 1.5) {",
-        "    highp float t = label_raw / 11.0;",
-        "    _output_color = vec4(t, float(semLabel > 0) * 0.8, 1.0 - t, 1.0);",
-        "} else {",
+        "    if (semantic_mask_mode >= 1.5 && semantic_mask_mode < 2.5) {",
+        "        highp float t = label_raw / 11.0;",
+        "        _output_color = vec4(t, float(semLabel > 0) * 0.8, 1.0 - t, 1.0);",
+        "    } else {",
         // Confidence value from texture [0,1]. Used as alpha weight for soft edges.
         // On iOS (no platform confidence) the texture is 1x1 white → conf = 1.0 → hard edges.
-        "    highp float conf = texture(semantic_confidence_texture, semUV).r;",
+        "        highp float conf = texture(semantic_confidence_texture, semUV).r;",
         // ShowOnly (mode=0): show matched pixels with weight = conf; hide everything else.
         // Hide (mode=1): hide matched pixels; show others at full opacity.
-        "    if (semantic_mask_mode < 0.5) {",
-        "        _output_color.a *= semMatches ? conf : 0.0;",
-        "    } else {",
-        "        _output_color.a *= semMatches ? (1.0 - conf) : 1.0;",
+        // ShowOnlySky (mode=3): like ShowOnly, but unlabeled pixels use viewport Y as a proxy
+        //   for sky context — upper half of screen shows sphere, lower half hides it. This
+        //   handles the ~10-15% border of the ARCore semantic image where the neural network
+        //   returns label=0 even though the camera sees real sky (or real ground). Regular
+        //   ShowOnly (mode=0) is unchanged so other semantic mask uses are not affected.
+        "        if (semantic_mask_mode < 0.5) {",
+        "            _output_color.a *= semMatches ? conf : 0.0;",
+        "        } else if (semantic_mask_mode > 2.5) {",
+        "            if (semMatches) {",
+        "                _output_color.a *= conf;",
+        "            } else if (semLabel == 0) {",
+        "                highp float normY = gl_FragCoord.y / ar_viewport_size.y;",
+        "                _output_color.a *= (normY > 0.5) ? 1.0 : 0.0;",
+        "            } else {",
+        "                _output_color.a *= 0.0;",
+        "            }",
+        "        } else {",
+        "            _output_color.a *= semMatches ? (1.0 - conf) : 1.0;",
+        "        }",
         "    }",
         "}"
+        // semInBounds == false: no semantic data available for this pixel → pass through unchanged.
     };
 
     _semanticMaskModifier = std::make_shared<VROShaderModifier>(
diff --git a/ViroRenderer/VROPortal.cpp b/ViroRenderer/VROPortal.cpp
@@ -33,6 +33,7 @@
 #include "VROBoundingBox.h"
 #include "VROPortalFrame.h"
 #include "VROShaderModifier.h"
+#include "VROSemantics.h"
 
 // Parameters for sphere backgrounds
 static const float kSphereBackgroundRadius = 1;
@@ -391,6 +392,73 @@ void VROPortal::removeBackground() {
     _background.reset();
 }
 
+#pragma mark - Sky Effect
+
+void VROPortal::setSkyEffectBackground(std::shared_ptr<VROTexture> texture) {
+    passert_thread(__func__);
+
+    // Remove previous sky effect node if present
+    if (_skyEffectNode) {
+        _skyEffectNode->removeFromParentNode();
+        _skyEffectNode = nullptr;
+    }
+
+    // Create sphere geometry (same radius/segments as the normal background sphere)
+    std::shared_ptr<VROGeometry> sphere = VROSphere::createSphere(kSphereBackgroundRadius,
+                                                                   kSphereBackgroundNumSegments,
+                                                                   kSphereBackgroundNumSegments,
+                                                                   false);
+    sphere->setCameraEnclosure(true);
+    sphere->setName("SkyEffectSphere");
+
+    std::shared_ptr<VROMaterial> material = sphere->getMaterials().front();
+    material->setLightingModel(VROLightingModel::Constant);
+    material->getDiffuse().setTexture(texture);
+    material->setWritesToDepthBuffer(false);
+    material->setReadsFromDepthBuffer(true);
+    material->setNeedsToneMapping(false);
+    // Alpha blending so the confidence-weighted mask fades at sky boundaries
+    material->setBlendMode(VROBlendMode::Alpha);
+
+    // Semantic mask: ShowOnlySky (label 1 = Sky).
+    // ShowOnlySky extends ShowOnly with asymmetric unlabeled handling: unlabeled pixels in
+    // the upper screen half show the sphere (sky context), lower half hide it (ground context).
+    // This covers the ~12% top/bottom strips where ARCore returns label=0 at image boundaries.
+    // Set mode and mask BEFORE enabling — setSemanticMaskEnabled(true) calls
+    // applySemanticMaskModifier() which captures the current mask/mode values
+    // into shader uniform lambdas. Calling setSemanticLabelMask() afterwards
+    // would not update the already-captured lambda.
+    material->setSemanticMaskMode(VROSemanticMaskMode::ShowOnlySky);
+    material->setSemanticLabelMask(1u << static_cast<int>(VROSemanticLabel::Sky));
+    material->setSemanticMaskEnabled(true);
+
+    // Depth trick: push to NDC depth=0.9999 so 3D content (depth < 0.9999) renders on top
+    if (!sPortalBackgroundShaderModifier) {
+        std::vector<std::string> modifierCode = {
+            "_vertex.position = _vertex.position.xyww;",
+            "_vertex.position.z = _vertex.position.w * 0.9999;"
+        };
+        sPortalBackgroundShaderModifier = std::make_shared<VROShaderModifier>(VROShaderEntryPoint::Vertex,
+                                                                               modifierCode);
+        sPortalBackgroundShaderModifier->setName("portal_background");
+    }
+    material->addShaderModifier(sPortalBackgroundShaderModifier);
+
+    // Wrap in a node and attach as a child of this portal
+    _skyEffectNode = std::make_shared<VRONode>();
+    _skyEffectNode->setGeometry(sphere);
+    _skyEffectNode->setName("SkyEffectNode");
+    addChildNode(_skyEffectNode);
+}
+
+void VROPortal::removeSkyEffectBackground() {
+    passert_thread(__func__);
+    if (_skyEffectNode) {
+        _skyEffectNode->removeFromParentNode();
+        _skyEffectNode = nullptr;
+    }
+}
+
 #pragma mark - Intersection
 
 bool VROPortal::intersectsLineSegment(VROLineSegment segment) const {
diff --git a/ViroRenderer/VROPortal.h b/ViroRenderer/VROPortal.h
@@ -219,6 +219,16 @@ class VROPortal : public VRONode {
         return _background;
     }
     void removeBackground();
+
+    /*
+     Set a sky-effect background: a sphere textured with the given texture, masked so that
+     it only appears over pixels semantically labeled as Sky. Uses alpha blending via the
+     confidence texture for smooth edges. Unlike setBackgroundSphere(), this sphere is added
+     as a child node (not the portal background geometry) so that it participates in the
+     scene's transparent render pass and the semantic mask shader is properly applied.
+     */
+    void setSkyEffectBackground(std::shared_ptr<VROTexture> texture);
+    void removeSkyEffectBackground();
     
 private:
     
@@ -278,6 +288,12 @@ class VROPortal : public VRONode {
      node content.
      */
     std::shared_ptr<VROGeometry> _background;
+
+    /*
+     Child node holding the sky-effect sphere geometry. Kept as a node so that semantic
+     mask shader modifiers are applied normally via the scene render pass.
+     */
+    std::shared_ptr<VRONode> _skyEffectNode;
     
     /*
      The lighting environment for this portal. Determines the effect of image-based
diff --git a/ViroRenderer/VROSemantics.h b/ViroRenderer/VROSemantics.h
@@ -72,9 +72,13 @@ enum class VROSemanticMode {
  * Controls whether fragments are shown or hidden based on label match.
  */
 enum class VROSemanticMaskMode {
-    ShowOnly = 0,  // Only render fragments where the label matches
-    Hide     = 1,  // Hide fragments where the label matches
-    Debug    = 2   // Color fragments by semantic label for debugging
+    ShowOnly    = 0,  // Only render fragments where the label matches
+    Hide        = 1,  // Hide fragments where the label matches
+    Debug       = 2,  // Color fragments by semantic label for debugging
+    ShowOnlySky = 3   // ShowOnly + unlabeled pixels use viewport Y to decide:
+                      // upper half → show sphere (sky context),
+                      // lower half → hide sphere (ground/foreground context).
+                      // Use this for Viro360Image skyEffect only.
 };
 
 /*
diff --git a/ViroRenderer/capi/PortalScene_JNI.cpp b/ViroRenderer/capi/PortalScene_JNI.cpp
@@ -177,6 +177,32 @@ VRO_METHOD(void, nativeSetBackgroundCubeWithColor)(VRO_ARGS
     });
 }
 
+VRO_METHOD(void, nativeSetSkyEffectTexture)(VRO_ARGS
+                                            VRO_REF(VROPortal) portal_j,
+                                            VRO_REF(VROTexture) texture_j) {
+    std::weak_ptr<VROPortal> portal_w = VRO_REF_GET(VROPortal, portal_j);
+    std::weak_ptr<VROTexture> texture_w = VRO_REF_GET(VROTexture, texture_j);
+
+    VROPlatformDispatchAsyncRenderer([portal_w, texture_w] {
+        std::shared_ptr<VROPortal> portal = portal_w.lock();
+        std::shared_ptr<VROTexture> texture = texture_w.lock();
+        if (portal && texture) {
+            portal->setSkyEffectBackground(texture);
+        }
+    });
+}
+
+VRO_METHOD(void, nativeRemoveSkyEffect)(VRO_ARGS
+                                        VRO_REF(VROPortal) portal_j) {
+    std::weak_ptr<VROPortal> portal_w = VRO_REF_GET(VROPortal, portal_j);
+    VROPlatformDispatchAsyncRenderer([portal_w] {
+        std::shared_ptr<VROPortal> portal = portal_w.lock();
+        if (portal) {
+            portal->removeSkyEffectBackground();
+        }
+    });
+}
+
 VRO_METHOD(void, nativeSetLightingEnvironment)(VRO_ARGS
                                                VRO_REF(VROPortal) portal_j,
                                                VRO_REF(VROTexture) texture_j) {
diff --git a/android/sharedCode/src/main/cpp/arcore/VROARSessionARCore.cpp b/android/sharedCode/src/main/cpp/arcore/VROARSessionARCore.cpp
@@ -2669,11 +2669,12 @@ static void uploadR8Texture(std::shared_ptr<VROTexture> &texture,
                              std::weak_ptr<VRODriver> driverWeak) {
     if (!texture || texture->getWidth() != width || texture->getHeight() != height) {
         auto vData = std::make_shared<VROData>((void *)data, dataLength, VRODataOwnership::Copy);
+        std::vector<std::shared_ptr<VROData>> dataVec{ vData };
+        std::vector<uint32_t> mipSizes;
         texture = std::make_shared<VROTexture>(
             VROTextureType::Texture2D, VROTextureFormat::R8, VROTextureInternalFormat::R8,
             false, VROMipmapMode::None,
-            std::vector<std::shared_ptr<VROData>>{ vData },
-            width, height, std::vector<uint32_t>());
+            dataVec, width, height, mipSizes);
         texture->setMinificationFilter(VROFilterMode::Nearest);
         texture->setMagnificationFilter(VROFilterMode::Nearest);
         texture->setWrapS(VROWrapMode::Clamp);
diff --git a/android/sharedCode/src/main/java/com/viro/core/PortalScene.java b/android/sharedCode/src/main/java/com/viro/core/PortalScene.java
@@ -262,6 +262,27 @@ public Texture getLightingEnvironment() {
         return mLightingEnvironment;
     }
 
+    /**
+     * Set the sky-effect background for this PortalScene. The provided {@link Texture} will
+     * be rendered as a sphere that is only visible over pixels semantically labeled as sky by
+     * ARCore scene semantics. The sphere uses alpha blending via the confidence texture for
+     * smooth sky/non-sky boundary transitions.
+     * <p>
+     * Requires ARCore scene semantics to be enabled on the AR session.
+     *
+     * @param texture The spherical {@link Texture} to display over sky pixels.
+     */
+    public void setSkyEffectTexture(Texture texture) {
+        nativeSetSkyEffectTexture(mNativeRef, texture.mNativeRef);
+    }
+
+    /**
+     * Remove the sky-effect background previously set via {@link #setSkyEffectTexture(Texture)}.
+     */
+    public void removeSkyEffect() {
+        nativeRemoveSkyEffect(mNativeRef);
+    }
+
     private native long nativeCreatePortalScene();
     private native long nativeCreatePortalDelegate();
     private native long nativeDestroyPortalScene(long nativeRef);
@@ -275,6 +296,8 @@ private native void nativeSetBackgroundRotation(long nativeRef, float radiansX,
     private native long nativeAttachDelegate(long nativeRef, long delegateRef);
     private native void nativeSetPassable(long nativeRef, boolean passable);
     private native void nativeSetPortalEntrance(long nativeRef, long portalNativeRef);
+    private native void nativeSetSkyEffectTexture(long nativeRef, long textureRef);
+    private native void nativeRemoveSkyEffect(long nativeRef);
 
     /*
      Invoked from JNI.
diff --git a/ios/ViroKit/VROARSessioniOS.cpp b/ios/ViroKit/VROARSessioniOS.cpp
@@ -2640,11 +2640,12 @@ std::shared_ptr<VROTexture> VROARSessioniOS::getSemanticConfidenceTexture() {
   if (!_defaultConfidenceTexture) {
     static const uint8_t white = 255;
     auto data = std::make_shared<VROData>((void *)&white, 1, VRODataOwnership::Copy);
+    std::vector<std::shared_ptr<VROData>> dataVec{ data };
+    std::vector<uint32_t> mipSizes;
     _defaultConfidenceTexture = std::make_shared<VROTexture>(
         VROTextureType::Texture2D, VROTextureFormat::R8, VROTextureInternalFormat::R8,
         false, VROMipmapMode::None,
-        std::vector<std::shared_ptr<VROData>>{ data },
-        1, 1, std::vector<uint32_t>());
+        dataVec, 1, 1, mipSizes);
     _defaultConfidenceTexture->setMinificationFilter(VROFilterMode::Nearest);
     _defaultConfidenceTexture->setMagnificationFilter(VROFilterMode::Nearest);
     _defaultConfidenceTexture->setWrapS(VROWrapMode::Clamp);
