Skip to content

Commit 65ebc6b

Browse files
He-PinHe-Pin
committed
chore: Make materializer stackless for obj and arr
1 parent 5dd9b27 commit 65ebc6b

1 file changed

Lines changed: 213 additions & 49 deletions

File tree

‎sjsonnet/src/sjsonnet/Materializer.scala‎

Lines changed: 213 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,19 @@ package sjsonnet
22

33
import sjsonnet.Expr.{FieldName, Member, ObjBody}
44
import sjsonnet.Expr.Member.Visibility
5-
import upickle.core.Visitor
5+
import upickle.core.{ArrVisitor, ObjVisitor, Visitor}
66

77
/**
88
* Serializes the given [[Val]] out to the given [[upickle.core.Visitor]], which can transform it
9-
* into [[ujson.Value]]s or directly serialize it to `String`s
9+
* into [[ujson.Value]]s or directly serialize it to `String`s.
10+
*
11+
* TCO boundary: all [[Val]] values entering materialization — whether from object field evaluation
12+
* (`Val.Obj.value`), array element forcing (`Val.Arr.value`), or top-level evaluation — must not
13+
* contain unresolved [[TailCall]] sentinels. This invariant is maintained by the evaluator: object
14+
* field `invoke` calls `visitExpr` (not `visitExprWithTailCallSupport`), and `Val.Func.apply*`
15+
* resolves TailCalls when called with `TailstrictModeDisabled`. A defensive check in
16+
* `materializeLeaf` guards against accidental TailCall leakage with a clear internal-error
17+
* diagnostic.
1018
*/
1119
abstract class Materializer {
1220
def storePos(pos: Position): Unit
@@ -17,64 +25,200 @@ abstract class Materializer {
1725
apply0(v, new sjsonnet.Renderer()).toString
1826
}
1927

20-
def apply0[T](v: Val, visitor: Visitor[T, T])(implicit evaluator: EvalScope): T = try {
28+
/**
29+
* Materialize a leaf value (non-container) to the given visitor. Returns the visitor result or
30+
* `null` if the value is a container (Obj/Arr) that needs stack-based processing.
31+
*/
32+
private def materializeLeaf[T](
33+
v: Val,
34+
visitor: Visitor[T, T])(implicit evaluator: EvalScope): T = {
2135
v match {
22-
case Val.Str(pos, s) => storePos(pos); visitor.visitString(s, -1)
23-
case obj: Val.Obj =>
24-
storePos(obj.pos)
25-
obj.triggerAllAsserts(evaluator.settings.brokenAssertionLogic)
26-
val objVisitor = visitor.visitObject(obj.visibleKeyNames.length, jsonableKeys = true, -1)
27-
val sort = !evaluator.settings.preserveOrder
28-
var prevKey: String = null
29-
obj.foreachElement(sort, evaluator.emptyMaterializeFileScopePos) { (k, v) =>
30-
storePos(v)
31-
objVisitor.visitKeyValue(objVisitor.visitKey(-1).visitString(k, -1))
32-
objVisitor.visitValue(
33-
apply0(v, objVisitor.subVisitor.asInstanceOf[Visitor[T, T]]),
34-
-1
35-
)
36-
if (sort) {
37-
if (prevKey != null && Util.compareStringsByCodepoint(k, prevKey) <= 0)
38-
Error.fail(
39-
s"""Internal error: Unexpected key "$k" after "$prevKey" in sorted object materialization""",
40-
v.pos
41-
)
42-
prevKey = k
43-
}
44-
}
45-
objVisitor.visitEnd(-1)
46-
case Val.Num(pos, _) => storePos(pos); visitor.visitFloat64(v.asDouble, -1)
47-
case xs: Val.Arr =>
48-
storePos(xs.pos)
49-
val arrVisitor = visitor.visitArray(xs.length, -1)
50-
var i = 0
51-
while (i < xs.length) {
52-
val sub = arrVisitor.subVisitor.asInstanceOf[Visitor[T, T]]
53-
arrVisitor.visitValue(apply0(xs.value(i), sub), -1)
54-
i += 1
55-
}
56-
arrVisitor.visitEnd(-1)
57-
case Val.True(pos) => storePos(pos); visitor.visitTrue(-1)
58-
case Val.False(pos) => storePos(pos); visitor.visitFalse(-1)
59-
case Val.Null(pos) => storePos(pos); visitor.visitNull(-1)
60-
case s: Val.Func =>
36+
case Val.Str(pos, s) => storePos(pos); visitor.visitString(s, -1)
37+
case Val.Num(pos, _) => storePos(pos); visitor.visitFloat64(v.asDouble, -1)
38+
case Val.True(pos) => storePos(pos); visitor.visitTrue(-1)
39+
case Val.False(pos) => storePos(pos); visitor.visitFalse(-1)
40+
case Val.Null(pos) => storePos(pos); visitor.visitNull(-1)
41+
case mat: Materializer.Materializable => storePos(v.pos); mat.materialize(visitor)
42+
case s: Val.Func =>
6143
Error.fail(
6244
"Couldn't manifest function with params [" + s.params.names.mkString(",") + "]",
6345
v.pos
6446
)
65-
case mat: Materializer.Materializable => storePos(v.pos); mat.materialize(visitor)
66-
case vv: Val =>
47+
case tc: TailCall =>
48+
Error.fail(
49+
"Internal error: TailCall sentinel leaked into materialization. " +
50+
"This indicates a bug in the TCO protocol — a TailCall was not resolved before " +
51+
"reaching the Materializer.",
52+
tc.pos
53+
)
54+
case vv: Val =>
6755
Error.fail("Unknown value type " + vv.prettyName, vv.pos)
6856
case null =>
6957
Error.fail("Unknown value type " + v)
7058
}
71-
} catch {
72-
case _: StackOverflowError =>
73-
Error.fail("Stackoverflow while materializing, possibly due to recursive value", v.pos)
74-
case _: OutOfMemoryError =>
75-
Error.fail("Stackoverflow while materializing, possibly due to recursive value", v.pos)
7659
}
7760

61+
/**
62+
* Iterative materialization using an explicit stack. Replaces the previous recursive
63+
* implementation to eliminate StackOverflowError on deeply nested JSON structures.
64+
*
65+
* The stack holds frames representing in-progress container (object/array) materializations. Each
66+
* iteration either:
67+
* 1. Processes the next child of the current container, pushing a new frame if the child is
68+
* itself a container.
69+
* 2. Completes the current container (visitEnd) and pops the frame, feeding the result back to
70+
* the parent frame via visitValue.
71+
*
72+
* A depth limit guards against infinite recursion from self-referential values (e.g.
73+
* `local L = [L]; L`), throwing a descriptive error instead of OOM.
74+
*/
75+
def apply0[T](v: Val, visitor: Visitor[T, T])(implicit evaluator: EvalScope): T = {
76+
import Materializer.{MaterializeObjFrame, MaterializeArrFrame}
77+
78+
// --- fast path for non-container values (avoids stack allocation) ---
79+
v match {
80+
case _: Val.Obj | _: Val.Arr => // fall through to iterative path
81+
case _ => return materializeLeaf(v, visitor)
82+
}
83+
84+
try {
85+
val maxDepth = evaluator.settings.maxMaterializeDepth
86+
val stack = new java.util.ArrayDeque[Materializer.MaterializeFrame](16)
87+
var pendingVal: Val = v
88+
var pendingVisitor: Visitor[T, T] = visitor
89+
var leafResult: T = null.asInstanceOf[T]
90+
var hasLeafResult = false
91+
92+
while (true) {
93+
// Step 1: If we have a pending value to process, handle it
94+
if (pendingVal != null) {
95+
val currentVal = pendingVal
96+
val currentVisitor = pendingVisitor
97+
pendingVal = null
98+
pendingVisitor = null
99+
100+
currentVal match {
101+
case obj: Val.Obj =>
102+
checkDepth(obj.pos, stack.size, maxDepth)
103+
storePos(obj.pos)
104+
obj.triggerAllAsserts(evaluator.settings.brokenAssertionLogic)
105+
val sort = !evaluator.settings.preserveOrder
106+
val keyNames = if (sort) {
107+
obj.visibleKeyNames.sorted(Util.CodepointStringOrdering)
108+
} else {
109+
obj.visibleKeyNames
110+
}
111+
val objVisitor =
112+
currentVisitor.visitObject(keyNames.length, jsonableKeys = true, -1)
113+
stack.push(new MaterializeObjFrame[T](objVisitor, keyNames, obj, sort, 0, null))
114+
115+
case xs: Val.Arr =>
116+
checkDepth(xs.pos, stack.size, maxDepth)
117+
storePos(xs.pos)
118+
val arrVisitor = currentVisitor.visitArray(xs.length, -1)
119+
stack.push(new MaterializeArrFrame[T](arrVisitor, xs, 0))
120+
121+
case _ =>
122+
leafResult = materializeLeaf(currentVal, currentVisitor)
123+
hasLeafResult = true
124+
}
125+
}
126+
127+
// Step 2+3: Process the current top-of-stack frame (feeding any pending leaf result first)
128+
if (stack.isEmpty) {
129+
if (hasLeafResult) return leafResult
130+
Error.fail("Internal error in iterative materializer")
131+
}
132+
133+
stack.peekFirst() match {
134+
case frame: MaterializeObjFrame[T @unchecked] =>
135+
if (hasLeafResult) {
136+
frame.objVisitor.visitValue(leafResult, -1)
137+
hasLeafResult = false
138+
leafResult = null.asInstanceOf[T]
139+
}
140+
if (frame.index < frame.keys.length) {
141+
val key = frame.keys(frame.index)
142+
val value = frame.obj.value(key, evaluator.emptyMaterializeFileScopePos)
143+
storePos(value)
144+
145+
if (frame.sort) {
146+
if (
147+
frame.prevKey != null && Util.compareStringsByCodepoint(key, frame.prevKey) <= 0
148+
)
149+
Error.fail(
150+
s"""Internal error: Unexpected key "$key" after "${frame.prevKey}" in sorted object materialization""",
151+
value.pos
152+
)
153+
frame.prevKey = key
154+
}
155+
156+
frame.objVisitor.visitKeyValue(frame.objVisitor.visitKey(-1).visitString(key, -1))
157+
frame.index += 1
158+
159+
value match {
160+
case _: Val.Obj | _: Val.Arr =>
161+
pendingVal = value
162+
pendingVisitor = frame.objVisitor.subVisitor.asInstanceOf[Visitor[T, T]]
163+
case _ =>
164+
val sub = frame.objVisitor.subVisitor.asInstanceOf[Visitor[T, T]]
165+
val childResult = materializeLeaf(value, sub)
166+
frame.objVisitor.visitValue(childResult, -1)
167+
}
168+
} else {
169+
val result = frame.objVisitor.visitEnd(-1)
170+
stack.pop()
171+
if (stack.isEmpty) return result
172+
leafResult = result
173+
hasLeafResult = true
174+
}
175+
176+
case frame: MaterializeArrFrame[T @unchecked] =>
177+
if (hasLeafResult) {
178+
frame.arrVisitor.visitValue(leafResult, -1)
179+
hasLeafResult = false
180+
leafResult = null.asInstanceOf[T]
181+
}
182+
if (frame.index < frame.arr.length) {
183+
val childVal = frame.arr.value(frame.index)
184+
frame.index += 1
185+
186+
childVal match {
187+
case _: Val.Obj | _: Val.Arr =>
188+
pendingVal = childVal
189+
pendingVisitor = frame.arrVisitor.subVisitor.asInstanceOf[Visitor[T, T]]
190+
case _ =>
191+
val sub = frame.arrVisitor.subVisitor.asInstanceOf[Visitor[T, T]]
192+
val childResult = materializeLeaf(childVal, sub)
193+
frame.arrVisitor.visitValue(childResult, -1)
194+
}
195+
} else {
196+
val result = frame.arrVisitor.visitEnd(-1)
197+
stack.pop()
198+
if (stack.isEmpty) return result
199+
leafResult = result
200+
hasLeafResult = true
201+
}
202+
}
203+
}
204+
205+
null.asInstanceOf[T] // unreachable
206+
} catch {
207+
case _: StackOverflowError =>
208+
Error.fail("Stackoverflow while materializing, possibly due to recursive value", v.pos)
209+
case _: OutOfMemoryError =>
210+
Error.fail("Out of memory while materializing, possibly due to recursive value", v.pos)
211+
}
212+
}
213+
214+
private def checkDepth(pos: Position, stackSize: Int, maxDepth: Int)(implicit
215+
ev: EvalErrorScope): Unit =
216+
if (stackSize >= maxDepth)
217+
Error.fail(
218+
"Stackoverflow while materializing, possibly due to recursive value",
219+
pos
220+
)
221+
78222
def reverse(pos: Position, v: ujson.Value): Val = v match {
79223
case ujson.True => Val.True(pos)
80224
case ujson.False => Val.False(pos)
@@ -149,6 +293,26 @@ object Materializer extends Materializer {
149293
final val emptyStringArray = new Array[String](0)
150294
final val emptyLazyArray = new Array[Eval](0)
151295

296+
/** Common parent for stack frames used in iterative materialization. */
297+
private[sjsonnet] sealed trait MaterializeFrame
298+
299+
/** Stack frame for in-progress object materialization. */
300+
private[sjsonnet] final class MaterializeObjFrame[T](
301+
val objVisitor: ObjVisitor[T, T],
302+
val keys: Array[String],
303+
val obj: Val.Obj,
304+
val sort: Boolean,
305+
var index: Int,
306+
var prevKey: String)
307+
extends MaterializeFrame
308+
309+
/** Stack frame for in-progress array materialization. */
310+
private[sjsonnet] final class MaterializeArrFrame[T](
311+
val arrVisitor: ArrVisitor[T, T],
312+
val arr: Val.Arr,
313+
var index: Int)
314+
extends MaterializeFrame
315+
152316
/**
153317
* Trait for providing custom materialization logic to the Materializer.
154318
* @since 1.0.0

0 commit comments

Comments
 (0)