Skip to content

Commit b2690e6

Browse files
JoshRosenJoshRosen
authored
Fix multiple object comprehension bugs (#358)
This PR fixes multiple bugs in object comprehensions (fixes #357, fixes #331, plus another bug). ## Mis-handling of `self` and `super` in object comprehension locals when inherited Object comprehensions may define local variables and those variables may reference `self` or `super`. Consider the reproduction reported in #357: ```jsonnet local lib = { foo():: { local global = self, [iterParam]: global.base { foo: iterParam } for iterParam in ["foo"] }, }; { base:: {} } + lib.foo() ``` This is supposed to output ```json { "foo": { "foo": "foo" } } ``` but sjsonnet outputs ``` sjsonnet.Error: Field does not exist: base at [Select base].(:7:26) ``` This bug occurs because of how bindings were managed: there's some tricky circular reference and variable rebinding logic (which I believe was added to handle references _between_ locals) and that was rebinding the `self` reference to the comprehension result itself: that is wrong because the `self` reference may change if an object is inherited or extended. While digging into this, I also discovered a related bug where `super` wasn't being properly re-bound in object comprehension locals or values: ```jsonnet local lib = { foo():: { local sx = super.x, [k]: sx + 1 for k in ["x"] }, }; { x: 2 } + lib.foo() ``` was failing with ``` java.lang.Exception: sjsonnet.Error: Attempt to use `super` when there is no super class at [SelectSuper x].(:4:21) at [ValidId sx].(:5:10) at [BinaryOp +].(:5:13) ``` ## Silent dropping of fields if key is not a string The object comprehension ```jsonnet {[k]: k for k in [1]} ``` fails in regular jsonnet with an error ``` RUNTIME ERROR: field must be string, got: number <cmdline>:1:1-22 ``` but in sjsonnet it was silently returning `{}`. This regression was [introduced](https://github.com/databricks/sjsonnet/pull/226/files#diff-7e786f55f42d493f85bfb37e5181c1c942ea039b21a307680e4e6913a8084aa2L628-R631) in #226 by adding a default `case _ =>` (presumably to fix a compiler or IDE warning). ## Support function definition via object comprehensions This fixes #331 by adding parser support for defining functions using method syntax, e.g. ```jsonnet { [a](x): x * 2 for a in ["f1", "f2", "f3"] } ``` ## Fixes This PR fixes the above bugs by changing logic within `Evaluator.visitObjComp`. It is best reviewed commit-by-commit or using a whitespace-free diff.
1 parent 84ac320 commit b2690e6

5 files changed

Lines changed: 154 additions & 61 deletions

File tree

‎sjsonnet/src/sjsonnet/Evaluator.scala‎

Lines changed: 36 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -680,15 +680,15 @@ class Evaluator(
680680
visitExpr(k) match {
681681
case Val.Str(_, k1) => k1
682682
case Val.Null(_) => null
683-
case x =>
684-
Error.fail(
685-
s"Field name must be string or null, not ${x.prettyName}",
686-
pos
687-
)
683+
case x => fieldNameTypeError(x, pos)
688684
}
689685
}
690686
}
691687

688+
private def fieldNameTypeError(fieldName: Val, pos: Position): Nothing = {
689+
Error.fail(s"Field name must be string or null, not ${fieldName.prettyName}", pos)
690+
}
691+
692692
def visitMethod(rhs: Expr, params: Params, outerPos: Position)(implicit
693693
scope: ValScope): Val.Func =
694694
new Val.Func(outerPos, scope, params) {
@@ -697,20 +697,16 @@ class Evaluator(
697697
override def evalDefault(expr: Expr, vs: ValScope, es: EvalScope) = visitExpr(expr)(vs)
698698
}
699699

700-
def visitBindings(
701-
bindings: Array[Bind],
702-
scope: (Val.Obj, Val.Obj) => ValScope): Array[(Val.Obj, Val.Obj) => Lazy] = {
703-
val arrF = new Array[(Val.Obj, Val.Obj) => Lazy](bindings.length)
700+
def visitBindings(bindings: Array[Bind], scope: => ValScope): Array[Lazy] = {
701+
val arrF = new Array[Lazy](bindings.length)
704702
var i = 0
705703
while (i < bindings.length) {
706704
val b = bindings(i)
707705
arrF(i) = b.args match {
708706
case null =>
709-
(self: Val.Obj, sup: Val.Obj) =>
710-
new LazyWithComputeFunc(() => visitExpr(b.rhs)(scope(self, sup)))
707+
new LazyWithComputeFunc(() => visitExpr(b.rhs)(scope))
711708
case argSpec =>
712-
(self: Val.Obj, sup: Val.Obj) =>
713-
new LazyWithComputeFunc(() => visitMethod(b.rhs, argSpec, b.pos)(scope(self, sup)))
709+
new LazyWithComputeFunc(() => visitMethod(b.rhs, argSpec, b.pos)(scope))
714710
}
715711
i += 1
716712
}
@@ -841,42 +837,37 @@ class Evaluator(
841837
def visitObjComp(e: ObjBody.ObjComp, sup: Val.Obj)(implicit scope: ValScope): Val.Obj = {
842838
val binds = e.preLocals ++ e.postLocals
843839
val compScope: ValScope = scope // .clearSuper
844-
845-
lazy val newSelf: Val.Obj = {
846-
val builder = new java.util.LinkedHashMap[String, Val.Obj.Member]
847-
for (s <- visitComp(e.first :: e.rest, Array(compScope))) {
848-
lazy val newScope: ValScope = s.extend(newBindings, newSelf, null)
849-
850-
lazy val newBindings = visitBindings(binds, (self, sup) => newScope)
851-
852-
visitExpr(e.key)(s) match {
853-
case Val.Str(_, k) =>
854-
val prev_length = builder.size()
855-
builder.put(
856-
k,
857-
new Val.Obj.Member(e.plus, Visibility.Normal) {
858-
def invoke(self: Val.Obj, sup: Val.Obj, fs: FileScope, ev: EvalScope): Val =
859-
visitExpr(e.value)(
860-
s.extend(newBindings, self, null)
861-
)
840+
val builder = new java.util.LinkedHashMap[String, Val.Obj.Member]
841+
for (s <- visitComp(e.first :: e.rest, Array(compScope))) {
842+
visitExpr(e.key)(s) match {
843+
case Val.Str(_, k) =>
844+
val prev_length = builder.size()
845+
builder.put(
846+
k,
847+
new Val.Obj.Member(e.plus, Visibility.Normal) {
848+
def invoke(self: Val.Obj, sup: Val.Obj, fs: FileScope, ev: EvalScope): Val = {
849+
// There is a circular dependency between `newScope` and `newBindings` because
850+
// bindings may refer to other bindings (e.g. chains of locals that build on
851+
// each other):
852+
lazy val newScope: ValScope = s.extend(newBindings, self, sup)
853+
lazy val newBindings = visitBindings(binds, newScope)
854+
visitExpr(e.value)(newScope)
862855
}
863-
)
864-
if (prev_length == builder.size() && settings.noDuplicateKeysInComprehension) {
865-
Error.fail(s"Duplicate key ${k} in evaluated object comprehension.", e.pos);
866856
}
867-
case Val.Null(_) => // do nothing
868-
case _ =>
869-
}
870-
}
871-
val valueCache = if (sup == null) {
872-
Val.Obj.getEmptyValueCacheForObjWithoutSuper(builder.size())
873-
} else {
874-
new java.util.HashMap[Any, Val]()
857+
)
858+
if (prev_length == builder.size() && settings.noDuplicateKeysInComprehension) {
859+
Error.fail(s"Duplicate key ${k} in evaluated object comprehension.", e.pos);
860+
}
861+
case Val.Null(_) => // do nothing
862+
case x => fieldNameTypeError(x, e.pos)
875863
}
876-
new Val.Obj(e.pos, builder, false, null, sup, valueCache)
877864
}
878-
879-
newSelf
865+
val valueCache = if (sup == null) {
866+
Val.Obj.getEmptyValueCacheForObjWithoutSuper(builder.size())
867+
} else {
868+
new java.util.HashMap[Any, Val]()
869+
}
870+
new Val.Obj(e.pos, builder, false, null, sup, valueCache)
880871
}
881872

882873
@tailrec

‎sjsonnet/src/sjsonnet/Parser.scala‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -430,8 +430,20 @@ class Parser(
430430
val preLocals = exprs
431431
.takeWhile(_.isInstanceOf[Expr.Bind])
432432
.map(_.asInstanceOf[Expr.Bind])
433-
val Expr.Member.Field(offset, Expr.FieldName.Dyn(lhs), plus, args, Visibility.Normal, rhs) =
433+
val Expr.Member.Field(
434+
offset,
435+
Expr.FieldName.Dyn(lhs),
436+
plus,
437+
args,
438+
Visibility.Normal,
439+
rhsBody
440+
) =
434441
exprs(preLocals.length): @unchecked
442+
val rhs = if (args == null) {
443+
rhsBody
444+
} else {
445+
Expr.Function(offset, args, rhsBody)
446+
}
435447
val postLocals = exprs
436448
.drop(preLocals.length + 1)
437449
.takeWhile(_.isInstanceOf[Expr.Bind])

‎sjsonnet/src/sjsonnet/ValScope.scala‎

Lines changed: 3 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -18,23 +18,11 @@ final class ValScope private (val bindings: Array[Lazy]) extends AnyVal {
1818

1919
def length: Int = bindings.length
2020

21-
def extend(
22-
newBindingsF: Array[(Val.Obj, Val.Obj) => Lazy] = null,
23-
newSelf: Val.Obj,
24-
newSuper: Val.Obj): ValScope = {
25-
val by = if (newBindingsF == null) 2 else newBindingsF.length + 2
26-
val b = Arrays.copyOf(bindings, bindings.length + by)
21+
def extend(newBindings: Array[Lazy], newSelf: Val.Obj, newSuper: Val.Obj): ValScope = {
22+
val b = Arrays.copyOf(bindings, bindings.length + newBindings.length + 2)
2723
b(bindings.length) = newSelf
2824
b(bindings.length + 1) = newSuper
29-
if (newBindingsF != null) {
30-
var i = 0
31-
var j = bindings.length + 2
32-
while (i < newBindingsF.length) {
33-
b(j) = newBindingsF(i).apply(newSelf, newSuper)
34-
i += 1
35-
j += 1
36-
}
37-
}
25+
System.arraycopy(newBindings, 0, b, bindings.length + 2, newBindings.length)
3826
new ValScope(b)
3927
}
4028

‎sjsonnet/test/src-jvm/sjsonnet/ErrorTestsJvmOnly.scala‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,17 @@ object ErrorTestsJvmOnly extends TestSuite {
2222
}
2323

2424
val tests: Tests = Tests {
25+
// Hack: this test suite may flakily fail if this suite runs prior to other error tests:
26+
// if classloading or linking happens inside the StackOverflowError handling then that
27+
// may will trigger a secondary StackOverflowError and cause the test to fail.
28+
// As a temporary solution, we redundantly run one of the other error tests first.
29+
// A better long term solution would be to change how we handle StackOverflowError
30+
// to avoid this failure mode, but for now we add this hack to avoid CI flakiness:
31+
test("02") - check(
32+
"""sjsonnet.Error: Foo.
33+
| at [Error].(sjsonnet/test/resources/test_suite/error.02.jsonnet:17:1)
34+
|""".stripMargin
35+
)
2536
test("array_recursive_manifest") - check(
2637
"""sjsonnet.Error: Stackoverflow while materializing, possibly due to recursive value
2738
| at .(sjsonnet/test/resources/test_suite/error.array_recursive_manifest.jsonnet:17:12)

‎sjsonnet/test/src/sjsonnet/EvaluatorTests.scala‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,97 @@ object EvaluatorTests extends TestSuite {
145145
"""{local y = $["2"], [x]: if x == "1" then y else 0, for x in ["1", "2"]}["1"]""",
146146
useNewEvaluator = useNewEvaluator
147147
) ==> ujson.Num(0)
148+
// References between locals in an object comprehension:
149+
eval(
150+
"""{local a = 1, local b = a + 1, [k]: b + 1 for k in ["x"]}""",
151+
useNewEvaluator = useNewEvaluator
152+
) ==> ujson.Obj("x" -> ujson.Num(3))
153+
// Locals which reference variables from the comprehension:
154+
eval(
155+
"""{local x2 = k*2, [std.toString(k)]: x2 for k in [1]}""",
156+
useNewEvaluator = useNewEvaluator
157+
) ==> ujson.Obj("1" -> ujson.Num(2))
158+
// Regression test for https://github.com/databricks/sjsonnet/issues/357
159+
// self references in object comprehension locals are properly rebound during inheritance:
160+
eval(
161+
"""
162+
|local lib = {
163+
| foo()::
164+
| {
165+
| local global = self,
166+
|
167+
| [iterParam]: global.base {
168+
| foo: iterParam
169+
| }
170+
| for iterParam in ["foo"]
171+
| },
172+
|};
173+
|
174+
|{
175+
| base:: {}
176+
|}
177+
|+ lib.foo()
178+
|""".stripMargin,
179+
useNewEvaluator = useNewEvaluator
180+
) ==> ujson.Obj("foo" -> ujson.Obj("foo" -> "foo"))
181+
// Regression test for a related bug involving local references to `super`:
182+
eval(
183+
"""
184+
|local lib = {
185+
| foo():: {
186+
| local sx = super.x,
187+
| [k]: sx + 1
188+
| for k in ["x"]
189+
| },
190+
|};
191+
|
192+
|{ x: 2 }
193+
|+ lib.foo()
194+
|""".stripMargin,
195+
useNewEvaluator = useNewEvaluator
196+
) ==> ujson.Obj("x" -> ujson.Num(3))
197+
// Yet another related bug involving super references _not_ in locals:
198+
eval(
199+
"""
200+
|local lib = {
201+
| foo():: {
202+
| [k]: super.x + 1
203+
| for k in ["x"]
204+
| },
205+
|};
206+
|
207+
|{ x: 2 }
208+
|+ lib.foo()
209+
|""".stripMargin,
210+
useNewEvaluator = useNewEvaluator
211+
) ==> ujson.Obj("x" -> ujson.Num(3))
212+
// Regression test for a bug in handling of non-string field names:
213+
evalErr("{[k]: k for k in [1]}", useNewEvaluator = useNewEvaluator) ==>
214+
"""sjsonnet.Error: Field name must be string or null, not number
215+
|at .(:1:2)""".stripMargin
216+
// Basic function support:
217+
eval(
218+
"""
219+
|local funcs = {
220+
| [a](x): x * 2
221+
| for a in ["f1", "f2", "f3"]
222+
|};
223+
|funcs.f1(10)
224+
|""".stripMargin,
225+
useNewEvaluator = useNewEvaluator
226+
) ==> ujson.Num(20)
227+
// Functions which use locals from the comprehension:
228+
eval(
229+
"""
230+
|local funcs = {
231+
| local y = b,
232+
| [a](x): x * y
233+
| for a in ["f1", "f2", "f3"] for b in [2]
234+
|};
235+
|funcs.f1(10)
236+
|""".stripMargin,
237+
useNewEvaluator = useNewEvaluator
238+
) ==> ujson.Num(20)
148239
}
149240
test("super") {
150241
test("implicit") {

0 commit comments

Comments
 (0)