chore: Make materializer stackless for obj and arr

Author: He-Pin

sjsonnet/src/sjsonnet/Materializer.scala

@@ -2,11 +2,19 @@ package sjsonnet
 
 import sjsonnet.Expr.{FieldName, Member, ObjBody}
 import sjsonnet.Expr.Member.Visibility
-import upickle.core.Visitor
+import upickle.core.{ArrVisitor, ObjVisitor, Visitor}
 
 /**
  * Serializes the given [[Val]] out to the given [[upickle.core.Visitor]], which can transform it
- * into [[ujson.Value]]s or directly serialize it to `String`s
+ * into [[ujson.Value]]s or directly serialize it to `String`s.
+ *
+ * TCO boundary: all [[Val]] values entering materialization — whether from object field evaluation
+ * (`Val.Obj.value`), array element forcing (`Val.Arr.value`), or top-level evaluation — must not
+ * contain unresolved [[TailCall]] sentinels. This invariant is maintained by the evaluator: object
+ * field `invoke` calls `visitExpr` (not `visitExprWithTailCallSupport`), and `Val.Func.apply*`
+ * resolves TailCalls when called with `TailstrictModeDisabled`. A defensive check in
+ * `materializeLeaf` guards against accidental TailCall leakage with a clear internal-error
+ * diagnostic.
  */
 abstract class Materializer {
   def storePos(pos: Position): Unit
@@ -17,64 +25,211 @@ abstract class Materializer {
     apply0(v, new sjsonnet.Renderer()).toString
   }
 
-  def apply0[T](v: Val, visitor: Visitor[T, T])(implicit evaluator: EvalScope): T = try {
+  /**
+   * Materialize a leaf value (non-container) to the given visitor. Callers must ensure that
+   * container values (Obj/Arr) are never passed to this method — they are handled by the iterative
+   * stack-based loop in [[materializeContainer]]. Passing a container will fall through to the
+   * catch-all branch and throw an error.
+   */
+  private def materializeLeaf[T](
+      v: Val,
+      visitor: Visitor[T, T])(implicit evaluator: EvalScope): T = {
     v match {
-      case Val.Str(pos, s) => storePos(pos); visitor.visitString(s, -1)
-      case obj: Val.Obj    =>
-        storePos(obj.pos)
-        obj.triggerAllAsserts(evaluator.settings.brokenAssertionLogic)
-        val objVisitor = visitor.visitObject(obj.visibleKeyNames.length, jsonableKeys = true, -1)
-        val sort = !evaluator.settings.preserveOrder
-        var prevKey: String = null
-        obj.foreachElement(sort, evaluator.emptyMaterializeFileScopePos) { (k, v) =>
-          storePos(v)
-          objVisitor.visitKeyValue(objVisitor.visitKey(-1).visitString(k, -1))
-          objVisitor.visitValue(
-            apply0(v, objVisitor.subVisitor.asInstanceOf[Visitor[T, T]]),
-            -1
-          )
-          if (sort) {
-            if (prevKey != null && Util.compareStringsByCodepoint(k, prevKey) <= 0)
-              Error.fail(
-                s"""Internal error: Unexpected key "$k" after "$prevKey" in sorted object materialization""",
-                v.pos
-              )
-            prevKey = k
-          }
-        }
-        objVisitor.visitEnd(-1)
-      case Val.Num(pos, _) => storePos(pos); visitor.visitFloat64(v.asDouble, -1)
-      case xs: Val.Arr     =>
-        storePos(xs.pos)
-        val arrVisitor = visitor.visitArray(xs.length, -1)
-        var i = 0
-        while (i < xs.length) {
-          val sub = arrVisitor.subVisitor.asInstanceOf[Visitor[T, T]]
-          arrVisitor.visitValue(apply0(xs.value(i), sub), -1)
-          i += 1
-        }
-        arrVisitor.visitEnd(-1)
-      case Val.True(pos)  => storePos(pos); visitor.visitTrue(-1)
-      case Val.False(pos) => storePos(pos); visitor.visitFalse(-1)
-      case Val.Null(pos)  => storePos(pos); visitor.visitNull(-1)
-      case s: Val.Func    =>
+      case Val.Str(pos, s)                  => storePos(pos); visitor.visitString(s, -1)
+      case Val.Num(pos, _)                  => storePos(pos); visitor.visitFloat64(v.asDouble, -1)
+      case Val.True(pos)                    => storePos(pos); visitor.visitTrue(-1)
+      case Val.False(pos)                   => storePos(pos); visitor.visitFalse(-1)
+      case Val.Null(pos)                    => storePos(pos); visitor.visitNull(-1)
+      case mat: Materializer.Materializable => storePos(v.pos); mat.materialize(visitor)
+      case s: Val.Func                      =>
         Error.fail(
           "Couldn't manifest function with params [" + s.params.names.mkString(",") + "]",
           v.pos
         )
-      case mat: Materializer.Materializable => storePos(v.pos); mat.materialize(visitor)
-      case vv: Val                          =>
+      case tc: TailCall =>
+        Error.fail(
+          "Internal error: TailCall sentinel leaked into materialization. " +
+          "This indicates a bug in the TCO protocol — a TailCall was not resolved before " +
+          "reaching the Materializer.",
+          tc.pos
+        )
+      case vv: Val =>
         Error.fail("Unknown value type " + vv.prettyName, vv.pos)
       case null =>
         Error.fail("Unknown value type " + v)
     }
-  } catch {
-    case _: StackOverflowError =>
-      Error.fail("Stackoverflow while materializing, possibly due to recursive value", v.pos)
-    case _: OutOfMemoryError =>
-      Error.fail("Stackoverflow while materializing, possibly due to recursive value", v.pos)
   }
 
+  /**
+   * Iterative materialization using an explicit stack. Replaces the previous recursive
+   * implementation to eliminate StackOverflowError on deeply nested JSON structures.
+   *
+   * The stack holds frames representing in-progress container (object/array) materializations. Each
+   * iteration either:
+   *   1. Processes the next child of the current container, pushing a new frame if the child is
+   *      itself a container.
+   *   2. Completes the current container (visitEnd) and pops the frame, feeding the result back to
+   *      the parent frame via visitValue.
+   *
+   * A depth limit guards against infinite recursion from self-referential values (e.g.
+   * `local L = [L]; L`), throwing a descriptive error instead of OOM.
+   */
+  def apply0[T](v: Val, visitor: Visitor[T, T])(implicit evaluator: EvalScope): T = v match {
+    case obj: Val.Obj => materializeContainer(obj, visitor)
+    case xs: Val.Arr  => materializeContainer(xs, visitor)
+    case _            => materializeLeaf(v, visitor)
+  }
+
+  // Iterative materialization for container values (Obj/Arr). Separated from apply0 to keep
+  // the fast-path method small and inlineable by the JIT.
+  //
+  // Design: the loop processes one child per iteration using pattern matching on the sealed trait
+  // MaterializeFrame. Helper methods (pushObjFrame, pushArrFrame, materializeChild, feedResult)
+  // are kept small so the JIT can unconditionally inline them (< 35 bytes each), giving the same
+  // effect as manual inlining while keeping the main loop compact enough for C2 deep optimization.
+  private def materializeContainer[T](v: Val, visitor: Visitor[T, T])(implicit
+      evaluator: EvalScope): T = {
+    try {
+      val maxDepth = evaluator.settings.maxMaterializeDepth
+      val sort = !evaluator.settings.preserveOrder
+      val brokenAssertionLogic = evaluator.settings.brokenAssertionLogic
+      val emptyPos = evaluator.emptyMaterializeFileScopePos
+      val stack = new java.util.ArrayDeque[Materializer.MaterializeFrame](8)
+
+      // Push the initial container frame
+      v match {
+        case obj: Val.Obj => pushObjFrame(obj, visitor, stack, maxDepth, sort, brokenAssertionLogic)
+        case xs: Val.Arr  => pushArrFrame(xs, visitor, stack, maxDepth)
+        case _            => () // unreachable — apply0 guarantees Obj or Arr
+      }
+
+      while (true) {
+        stack.peekFirst() match {
+          case frame: Materializer.MaterializeObjFrame[T @unchecked] =>
+            val keys = frame.keys
+            val ov = frame.objVisitor
+            if (frame.index < keys.length) {
+              val key = keys(frame.index)
+              val childVal = frame.obj.value(key, emptyPos)
+              storePos(childVal)
+
+              if (frame.sort) {
+                if (
+                  frame.prevKey != null && Util.compareStringsByCodepoint(key, frame.prevKey) <= 0
+                )
+                  Error.fail(
+                    s"""Internal error: Unexpected key "$key" after "${frame.prevKey}" in sorted object materialization""",
+                    childVal.pos
+                  )
+                frame.prevKey = key
+              }
+
+              ov.visitKeyValue(ov.visitKey(-1).visitString(key, -1))
+              frame.index += 1
+
+              val sub = ov.subVisitor.asInstanceOf[Visitor[T, T]]
+              materializeChild(childVal, sub, ov, stack, maxDepth, sort, brokenAssertionLogic)
+            } else {
+              val result = ov.visitEnd(-1)
+              stack.removeFirst()
+              if (stack.isEmpty) return result
+              feedResult(stack.peekFirst(), result)
+            }
+
+          case frame: Materializer.MaterializeArrFrame[T @unchecked] =>
+            val arr = frame.arr
+            val av = frame.arrVisitor
+            if (frame.index < arr.length) {
+              val childVal = arr.value(frame.index)
+              frame.index += 1
+
+              val sub = av.subVisitor.asInstanceOf[Visitor[T, T]]
+              materializeChild(childVal, sub, av, stack, maxDepth, sort, brokenAssertionLogic)
+            } else {
+              val result = av.visitEnd(-1)
+              stack.removeFirst()
+              if (stack.isEmpty) return result
+              feedResult(stack.peekFirst(), result)
+            }
+        }
+      }
+
+      null.asInstanceOf[T] // unreachable — while(true) exits via return
+    } catch {
+      case _: StackOverflowError =>
+        Error.fail("Stackoverflow while materializing, possibly due to recursive value", v.pos)
+      case _: OutOfMemoryError =>
+        Error.fail("Out of memory while materializing, possibly due to recursive value", v.pos)
+    }
+  }
+
+  // Materialize a child value: leaf fast-path avoids a full pattern match for the common case
+  // (strings, numbers, booleans, null). Only containers (Obj/Arr) push a new frame.
+  private def materializeChild[T](
+      childVal: Val,
+      childVisitor: Visitor[T, T],
+      parentVisitor: upickle.core.ObjArrVisitor[T, T],
+      stack: java.util.ArrayDeque[Materializer.MaterializeFrame],
+      maxDepth: Int,
+      sort: Boolean,
+      brokenAssertionLogic: Boolean)(implicit evaluator: EvalScope): Unit = {
+    if (!childVal.isInstanceOf[Val.Obj] && !childVal.isInstanceOf[Val.Arr]) {
+      parentVisitor.visitValue(materializeLeaf(childVal, childVisitor), -1)
+    } else
+      childVal match {
+        case obj: Val.Obj =>
+          pushObjFrame(obj, childVisitor, stack, maxDepth, sort, brokenAssertionLogic)
+        case xs: Val.Arr =>
+          pushArrFrame(xs, childVisitor, stack, maxDepth)
+        case _ => () // unreachable — guarded by isInstanceOf checks above
+      }
+  }
+
+  private def pushObjFrame[T](
+      obj: Val.Obj,
+      visitor: Visitor[T, T],
+      stack: java.util.ArrayDeque[Materializer.MaterializeFrame],
+      maxDepth: Int,
+      sort: Boolean,
+      brokenAssertionLogic: Boolean)(implicit evaluator: EvalScope): Unit = {
+    checkDepth(obj.pos, stack.size, maxDepth)
+    storePos(obj.pos)
+    obj.triggerAllAsserts(brokenAssertionLogic)
+    val keyNames =
+      if (sort) obj.visibleKeyNames.sorted(Util.CodepointStringOrdering)
+      else obj.visibleKeyNames
+    val objVisitor = visitor.visitObject(keyNames.length, jsonableKeys = true, -1)
+    stack.push(new Materializer.MaterializeObjFrame[T](objVisitor, keyNames, obj, sort, 0, null))
+  }
+
+  private def pushArrFrame[T](
+      xs: Val.Arr,
+      visitor: Visitor[T, T],
+      stack: java.util.ArrayDeque[Materializer.MaterializeFrame],
+      maxDepth: Int)(implicit evaluator: EvalScope): Unit = {
+    checkDepth(xs.pos, stack.size, maxDepth)
+    storePos(xs.pos)
+    val arrVisitor = visitor.visitArray(xs.length, -1)
+    stack.push(new Materializer.MaterializeArrFrame[T](arrVisitor, xs, 0))
+  }
+
+  // Feed a completed child result into the parent frame's visitor.
+  private def feedResult[T](parentFrame: Materializer.MaterializeFrame, result: T): Unit =
+    parentFrame match {
+      case f: Materializer.MaterializeObjFrame[T @unchecked] =>
+        f.objVisitor.visitValue(result, -1)
+      case f: Materializer.MaterializeArrFrame[T @unchecked] =>
+        f.arrVisitor.visitValue(result, -1)
+    }
+
+  private def checkDepth(pos: Position, stackSize: Int, maxDepth: Int)(implicit
+      ev: EvalErrorScope): Unit =
+    if (stackSize >= maxDepth)
+      Error.fail(
+        "Stackoverflow while materializing, possibly due to recursive value",
+        pos
+      )
+
   def reverse(pos: Position, v: ujson.Value): Val = v match {
     case ujson.True    => Val.True(pos)
     case ujson.False   => Val.False(pos)
@@ -149,6 +304,26 @@ object Materializer extends Materializer {
   final val emptyStringArray = new Array[String](0)
   final val emptyLazyArray = new Array[Eval](0)
 
+  /** Common parent for stack frames used in iterative materialization. */
+  private[sjsonnet] sealed trait MaterializeFrame
+
+  /** Stack frame for in-progress object materialization. */
+  private[sjsonnet] final class MaterializeObjFrame[T](
+      val objVisitor: ObjVisitor[T, T],
+      val keys: Array[String],
+      val obj: Val.Obj,
+      val sort: Boolean,
+      var index: Int,
+      var prevKey: String)
+      extends MaterializeFrame
+
+  /** Stack frame for in-progress array materialization. */
+  private[sjsonnet] final class MaterializeArrFrame[T](
+      val arrVisitor: ArrVisitor[T, T],
+      val arr: Val.Arr,
+      var index: Int)
+      extends MaterializeFrame
+
   /**
    * Trait for providing custom materialization logic to the Materializer.
    * @since 1.0.0

sjsonnet/src/sjsonnet/Settings.scala

@@ -10,7 +10,8 @@ final case class Settings(
     throwErrorForInvalidSets: Boolean = false,
     useNewEvaluator: Boolean = false,
     maxParserRecursionDepth: Int = 1000,
-    brokenAssertionLogic: Boolean = false
+    brokenAssertionLogic: Boolean = false,
+    maxMaterializeDepth: Int = 1000
 )
 
 object Settings {