pkl
pkl copied to clipboard
apple
Forcing values with `toMap` in dependency loop situations can emit null values from `VmObject.iterateAlreadyForcedMemberValues`
Reproduced in Pkl 0.29.0.
I don't really have a reproducer for this, as the failure happens in an internal codebase which I am not allowed to share and I haven't been able to create an independent reproduction that exhibits the error. This is an approximation of how the code looks (but does not reproduce the error):
// --- meta.pkl ---
amends "PackageMetadata.pkl"
name = "package"
packages {
["package"] {
meta = (import("meta.pkl")) {}
definition = (import("definition.pkl")) {}
}
}
// --- definition.pkl ---
extends "PackageDefinition.pkl"
import "DockerContainerTask.pkl"
import "DockerNetworkTask.pkl"
import "meta.pkl"
tasks {
["Docker"] {
local docker = this
["network"] = new DockerNetworkTask {
name = "network"
}
["backend"] = new DockerContainerTask {
name = "backend"
image = "backend"
networks { docker["network"].asNetwork(meta.self) }
}
["other"] = new DockerContainerTask {
name = "other"
image = "other"
}
}
}
// --- PackageMetadata.pkl ---
module PackageMetadata
import "PackageDefinition.pkl"
import "PackageMetadata.pkl"
name: String
packages: Mapping<String, Package>
hidden fixed self = packages[name]
class Package {
meta: PackageMetadata
definition: PackageDefinition
function absoluteName(providerName: String, taskName: String): String =
let (provider = definition.tasks.getOrNull(providerName))
if (provider == null) throw("Package '\(this.meta.name)' does not support provider '\(providerName)'")
else
// v Error happens here, in `filter`
let (task = provider.toMap().values.filter((t) -> t.name == taskName).firstOrNull)
if (task == null) throw("Package '\(meta.name)' does not contain task '\(taskName)' in provider '\(providerName)'")
else "\(meta.name)-\(taskName)"
}
// --- PackageDefinition.pkl ---
open module PackageDefinition
import "Task.pkl"
tasks: Mapping<String, Mapping<String, Task>>
// --- Task.pkl ---
abstract module Task
name: String
// --- DockerContainerTask.pkl ---
module DockerContainerTask
extends "Task.pkl"
import "DockerNetwork.pkl"
image: String
networks: Listing<DockerNetwork> = new { }
// --- DockerNetworkTask.pkl ---
module DockerNetworkTask
extends "Task.pkl"
import "PackageMetadata.pkl"
import "DockerNetwork.pkl"
driver: String = "bridge"
/// Return this task as a `DockerNetwork`. [package] must be the package this
/// task is defined in.
function asNetwork(package: PackageMetadata.Package) = new DockerNetwork {
kind = "External"
name = package.absoluteName("Docker", outer.name)
}
// --- DockerNetwork.pkl ---
module DockerNetwork
typealias NetworkKind = "Internal" | "External"
name: String
kind: NetworkKind = "Internal"
When I evaluate definition.pkl in the internal codebase, I get the following Pkl error (stack trace redacted/modified to fit the code example):
org.pkl.core.PklBugException: An unexpected error has occurred. Would you mind filing a bug report?
Cmd+Double-click the link below to open an issue.
Please copy and paste the entire error output into the issue's description.
https://github.com/apple/pkl/issues/new
java.lang.NullPointerException
–– Pkl Error ––
None (cause has no message)
52 | let (task = provider.toMap().values.filter((t) -> t.name == taskName).firstOrNull)
^^^^^^^^^^^^^^^^^^^^^^^^^
at PackageMetadata#Package.absoluteName.<function#3> (PackageMetadata.pkl)
52 | let (task = provider.toMap().values.filter((t) -> t.name == taskName).firstOrNull)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at PackageMetadata#Package.absoluteName.<function#1> (PackageMetadata.pkl)
49 | let (provider = definition.tasks.getOrNull(providerName))
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at PackageMetadata#Package.absoluteName (PackageMetadata.pkl)
31 | name = Package.absoluteName("Docker", outer.name)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
at DockerNetworkTask#asNetwork.name (DockerNetworkTask.pkl)
Pkl 0.30.0-dev+dbcd11da (Linux 6.15.3-cachyos, native)
java.lang.NullPointerException
at org.pkl.core.ast.expression.member.ReadPropertyNodeGen.executeGeneric(ReadPropertyNodeGen.java:78)
at org.pkl.core.ast.expression.binary.EqualNodeGen.executeGeneric_generic5(EqualNodeGen.java:205)
at org.pkl.core.ast.expression.binary.EqualNodeGen.executeGeneric(EqualNodeGen.java:93)
at org.pkl.core.ast.member.FunctionNode.executeImpl(FunctionNode.java:116)
at org.pkl.core.ast.PklRootNode.execute(PklRootNode.java:46)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.executeRootNode(OptimizedCallTarget.java:776)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.profiledPERoot(OptimizedCallTarget.java:700)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.callBoundary(OptimizedCallTarget.java:624)
at org.graalvm.truffle.runtime.svm/com.oracle.svm.truffle.api.SubstrateOptimizedCallTarget.invokeCallBoundary(SubstrateOptimizedCallTarget.java:124)
at org.graalvm.truffle.runtime.svm/com.oracle.svm.truffle.api.SubstrateOptimizedCallTargetInstalledCode.doInvoke(SubstrateOptimizedCallTargetInstalledCode.java:232)
at org.graalvm.truffle.runtime.svm/com.oracle.svm.truffle.api.SubstrateOptimizedCallTarget.doInvoke(SubstrateOptimizedCallTarget.java:106)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.callDirect(OptimizedCallTarget.java:556)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedDirectCallNode.call(OptimizedDirectCallNode.java:94)
at org.pkl.core.ast.lambda.ApplyVmFunction1Node.evalDirect(ApplyVmFunction1Node.java:80)
at org.pkl.core.ast.lambda.ApplyVmFunction1NodeGen.execute(ApplyVmFunction1NodeGen.java:71)
at org.pkl.core.ast.lambda.ApplyVmFunction1Node.executeBoolean(ApplyVmFunction1Node.java:41)
at org.pkl.core.stdlib.base.ListNodes$filter.eval(ListNodes.java:430)
at org.pkl.core.stdlib.base.ListNodesFactory$filterNodeGen.executeAndSpecialize(ListNodesFactory.java:3297)
at org.pkl.core.stdlib.base.ListNodesFactory$filterNodeGen.executeGeneric(ListNodesFactory.java:3286)
at org.pkl.core.ast.member.FunctionNode.executeImpl(FunctionNode.java:116)
at org.pkl.core.ast.PklRootNode.execute(PklRootNode.java:46)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.executeRootNode(OptimizedCallTarget.java:776)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.profiledPERoot(OptimizedCallTarget.java:700)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.callBoundary(OptimizedCallTarget.java:624)
at org.graalvm.truffle.runtime.svm/com.oracle.svm.truffle.api.SubstrateOptimizedCallTarget.invokeCallBoundary(SubstrateOptimizedCallTarget.java:124)
at org.graalvm.truffle.runtime.svm/com.oracle.svm.truffle.api.SubstrateOptimizedCallTargetInstalledCode.doInvoke(SubstrateOptimizedCallTargetInstalledCode.java:232)
at org.graalvm.truffle.runtime.svm/com.oracle.svm.truffle.api.SubstrateOptimizedCallTarget.doInvoke(SubstrateOptimizedCallTarget.java:106)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.callDirect(OptimizedCallTarget.java:556)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedDirectCallNode.call(OptimizedDirectCallNode.java:94)
at org.pkl.core.ast.expression.member.InvokeMethodVirtualNode.evalCached(InvokeMethodVirtualNode.java:134)
at org.pkl.core.ast.expression.member.InvokeMethodVirtualNodeGen.executeAndSpecialize(InvokeMethodVirtualNodeGen.java:272)
at org.pkl.core.ast.expression.member.InvokeMethodVirtualNodeGen.executeGeneric(InvokeMethodVirtualNodeGen.java:193)
at org.pkl.core.ast.expression.member.ReadPropertyNodeGen.executeGeneric(ReadPropertyNodeGen.java:73)
at org.pkl.core.ast.expression.binary.LetExprNode.executeGeneric(LetExprNode.java:71)
at org.pkl.core.ast.expression.ternary.IfElseNode.executeGeneric(IfElseNode.java:49)
at org.pkl.core.ast.member.FunctionNode.executeImpl(FunctionNode.java:116)
at org.pkl.core.ast.PklRootNode.execute(PklRootNode.java:46)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.executeRootNode(OptimizedCallTarget.java:776)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.profiledPERoot(OptimizedCallTarget.java:700)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.callBoundary(OptimizedCallTarget.java:624)
at org.graalvm.truffle.runtime.svm/com.oracle.svm.truffle.api.SubstrateOptimizedCallTarget.invokeCallBoundary(SubstrateOptimizedCallTarget.java:124)
at org.graalvm.truffle.runtime.svm/com.oracle.svm.truffle.api.SubstrateOptimizedCallTargetInstalledCode.doInvoke(SubstrateOptimizedCallTargetInstalledCode.java:232)
at org.graalvm.truffle.runtime.svm/com.oracle.svm.truffle.api.SubstrateOptimizedCallTarget.doInvoke(SubstrateOptimizedCallTarget.java:106)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.callDirect(OptimizedCallTarget.java:556)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedDirectCallNode.call(OptimizedDirectCallNode.java:94)
at org.pkl.core.ast.expression.binary.LetExprNode.executeGeneric(LetExprNode.java:73)
at org.pkl.core.ast.member.FunctionNode.executeImpl(FunctionNode.java:116)
at org.pkl.core.ast.PklRootNode.execute(PklRootNode.java:46)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.executeRootNode(OptimizedCallTarget.java:776)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.profiledPERoot(OptimizedCallTarget.java:700)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.callBoundary(OptimizedCallTarget.java:624)
at org.graalvm.truffle.runtime.svm/com.oracle.svm.truffle.api.SubstrateOptimizedCallTarget.invokeCallBoundary(SubstrateOptimizedCallTarget.java:124)
at org.graalvm.truffle.runtime.svm/com.oracle.svm.truffle.api.SubstrateOptimizedCallTargetInstalledCode.doInvoke(SubstrateOptimizedCallTargetInstalledCode.java:232)
at org.graalvm.truffle.runtime.svm/com.oracle.svm.truffle.api.SubstrateOptimizedCallTarget.doInvoke(SubstrateOptimizedCallTarget.java:106)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.callDirect(OptimizedCallTarget.java:556)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedDirectCallNode.call(OptimizedDirectCallNode.java:94)
at org.pkl.core.ast.expression.member.InvokeMethodVirtualNode.evalCached(InvokeMethodVirtualNode.java:134)
at org.pkl.core.ast.expression.member.InvokeMethodVirtualNodeGen.executeAndSpecialize(InvokeMethodVirtualNodeGen.java:272)
at org.pkl.core.ast.expression.member.InvokeMethodVirtualNodeGen.executeGeneric(InvokeMethodVirtualNodeGen.java:193)
at org.pkl.core.ast.member.TypeCheckedPropertyNode.evalTypedObjectCached(TypeCheckedPropertyNode.java:54)
at org.pkl.core.ast.member.TypeCheckedPropertyNodeGen.executeAndSpecialize(TypeCheckedPropertyNodeGen.java:136)
at org.pkl.core.ast.member.TypeCheckedPropertyNodeGen.executeImpl(TypeCheckedPropertyNodeGen.java:101)
at org.pkl.core.ast.PklRootNode.execute(PklRootNode.java:46)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.executeRootNode(OptimizedCallTarget.java:776)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.profiledPERoot(OptimizedCallTarget.java:700)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.callBoundary(OptimizedCallTarget.java:624)
at org.graalvm.truffle.runtime.svm/com.oracle.svm.truffle.api.SubstrateOptimizedCallTarget.invokeCallBoundary(SubstrateOptimizedCallTarget.java:124)
at org.graalvm.truffle.runtime.svm/com.oracle.svm.truffle.api.SubstrateOptimizedCallTargetInstalledCode.doInvoke(SubstrateOptimizedCallTargetInstalledCode.java:232)
at org.graalvm.truffle.runtime.svm/com.oracle.svm.truffle.api.SubstrateOptimizedCallTarget.doInvoke(SubstrateOptimizedCallTarget.java:106)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.callDirect(OptimizedCallTarget.java:556)
at org.graalvm.truffle.runtime/com.oracle.truffle.runtime.OptimizedCallTarget.call(OptimizedCallTarget.java:502)
at org.graalvm.truffle/com.oracle.truffle.api.nodes.IndirectCallNode$1.call(IndirectCallNode.java:96)
at org.pkl.core.runtime.VmUtils.doReadMember(VmUtils.java:312)
at org.pkl.core.runtime.VmUtils.doReadMember(VmUtils.java:214)
at org.pkl.core.runtime.VmObject.force(VmObject.java:177)
at org.pkl.core.runtime.VmObject.force(VmObject.java:197)
at org.pkl.core.runtime.VmValue.force(VmValue.java:67)
at org.pkl.core.runtime.VmObject.force(VmObject.java:185)
at org.pkl.core.runtime.VmObject.force(VmObject.java:197)
at org.pkl.core.runtime.VmValue.force(VmValue.java:67)
at org.pkl.core.runtime.VmObject.force(VmObject.java:185)
at org.pkl.core.runtime.VmObject.force(VmObject.java:197)
at org.pkl.core.runtime.VmValue.force(VmValue.java:67)
at org.pkl.core.runtime.VmObject.force(VmObject.java:185)
at org.pkl.core.runtime.VmObject.force(VmObject.java:197)
at org.pkl.core.runtime.VmValue.force(VmValue.java:67)
at org.pkl.core.runtime.VmObject.force(VmObject.java:185)
at org.pkl.core.runtime.VmObject.force(VmObject.java:197)
at org.pkl.core.runtime.VmValue.force(VmValue.java:67)
at org.pkl.core.runtime.VmObject.force(VmObject.java:185)
at org.pkl.core.runtime.VmObject.force(VmObject.java:197)
at org.pkl.core.runtime.VmValue.force(VmValue.java:67)
at org.pkl.server.BinaryEvaluator.evaluate$lambda$2(BinaryEvaluator.kt:63)
at org.pkl.core.EvaluatorImpl.lambda$doEvaluate$15(EvaluatorImpl.java:368)
at org.pkl.core.EvaluatorImpl.doEvaluate(EvaluatorImpl.java:316)
at org.pkl.core.EvaluatorImpl.doEvaluate(EvaluatorImpl.java:364)
at org.pkl.server.BinaryEvaluator.evaluate(BinaryEvaluator.kt:59)
at org.pkl.server.Server.handleEvaluate$lambda$2(Server.kt:128)
at [email protected]/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1144)
at [email protected]/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:642)
at [email protected]/java.lang.Thread.runWith(Thread.java:1596)
at [email protected]/java.lang.Thread.run(Thread.java:1583)
at org.graalvm.nativeimage.builder/com.oracle.svm.core.thread.PlatformThreads.threadStartRoutine(PlatformThreads.java:896)
at org.graalvm.nativeimage.builder/com.oracle.svm.core.thread.PlatformThreads.threadStartRoutine(PlatformThreads.java:872)
I dug through the VM code a bit, and it seems at this particular line null is returned:
https://github.com/apple/pkl/blob/3f2f0c3a2bb8e33622754c79aafed5544a38f10a/pkl-core/src/main/java/org/pkl/core/runtime/VmObject.java#L130-L132
Which then results in a Java null which is untrace()able. Indeed, if I enable assertions in jpkl, the assert below it triggers.
I'm not sure what exactly is causing forcing to fail like this, but presumably Pkl should either outright reject the toMap because of the dependency loop (definition -> DockerNetworkTask -> meta -> definition) or allow partial forcing.
(FWIW, substituting Mapping.toMap().values.filter with Mapping.fold() works as expected, so it's toMap's forcing in particular that's messing up somewhere.)
Thanks for the bug report. It's a little hard to figure out the root cause without a reproducer, so, definitely let us know if you find one.
