From 0ff37d87a02af328e7430e3570dbbc9f5dfc0362 Mon Sep 17 00:00:00 2001 From: Taha Alamine Boukhalf Date: Sun, 24 Jul 2022 12:06:17 +0100 Subject: [PATCH 01/43] Collect SDK, Regex unittest reports --- regex/mx.regex/mx_regex.py | 5 +++-- sdk/mx.sdk/mx_sdk.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/regex/mx.regex/mx_regex.py b/regex/mx.regex/mx_regex.py index e7eb5a21125..526375c4add 100644 --- a/regex/mx.regex/mx_regex.py +++ b/regex/mx.regex/mx_regex.py @@ -48,9 +48,10 @@ def _tregex_tests_gate_runner(args, tasks): - with Task('UnitTests', tasks, tags=['default', 'all']) as t: + with Task('UnitTests', tasks, tags=['default', 'all'], report=True) as t: if t: - unittest(['--enable-timing', '--very-verbose', 'com.oracle.truffle.regex']) + tags = {'task' : t.title} + unittest(['--enable-timing', '--very-verbose', 'com.oracle.truffle.regex'], test_report_tags=tags) mx_sdk_vm.register_graalvm_component(mx_sdk_vm.GraalVmLanguage( diff --git a/sdk/mx.sdk/mx_sdk.py b/sdk/mx.sdk/mx_sdk.py index e408ada03c1..6dd8d9ea6ca 100644 --- a/sdk/mx.sdk/mx_sdk.py +++ b/sdk/mx.sdk/mx_sdk.py @@ -63,8 +63,10 @@ def _sdk_gate_runner(args, tasks): - with Task('SDK UnitTests', tasks, tags=['test']) as t: - if t: unittest(['--suite', 'sdk', '--enable-timing', '--verbose', '--fail-fast']) + with Task('SDK UnitTests', tasks, tags=['test'], report=True) as t: + if t: + tags = {'task' : t.title} + unittest(['--suite', 'sdk', '--enable-timing', '--verbose', '--fail-fast'], test_report_tags=tags) with Task('Check Copyrights', tasks) as t: if t: if mx.command_function('checkcopyrights')(['--primary', '--', '--projects', 'src']) != 0: From 2dcc422f6e96833544edf6e95c6b17db6edf599b Mon Sep 17 00:00:00 2001 From: Taha Alamine Boukhalf Date: Tue, 21 Feb 2023 17:09:44 +0100 Subject: [PATCH 02/43] tmp --- regex/mx.regex/mx_regex.py | 3 +-- sdk/mx.sdk/mx_sdk.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/regex/mx.regex/mx_regex.py b/regex/mx.regex/mx_regex.py index 526375c4add..15bfb96c3db 100644 --- a/regex/mx.regex/mx_regex.py +++ b/regex/mx.regex/mx_regex.py @@ -50,8 +50,7 @@ def _tregex_tests_gate_runner(args, tasks): with Task('UnitTests', tasks, tags=['default', 'all'], report=True) as t: if t: - tags = {'task' : t.title} - unittest(['--enable-timing', '--very-verbose', 'com.oracle.truffle.regex'], test_report_tags=tags) + unittest(['--enable-timing', '--very-verbose', 'com.oracle.truffle.regex'], test_report_tags={'task': t.title}) mx_sdk_vm.register_graalvm_component(mx_sdk_vm.GraalVmLanguage( diff --git a/sdk/mx.sdk/mx_sdk.py b/sdk/mx.sdk/mx_sdk.py index 6dd8d9ea6ca..6589ddd0bc1 100644 --- a/sdk/mx.sdk/mx_sdk.py +++ b/sdk/mx.sdk/mx_sdk.py @@ -65,8 +65,7 @@ def _sdk_gate_runner(args, tasks): with Task('SDK UnitTests', tasks, tags=['test'], report=True) as t: if t: - tags = {'task' : t.title} - unittest(['--suite', 'sdk', '--enable-timing', '--verbose', '--fail-fast'], test_report_tags=tags) + unittest(['--suite', 'sdk', '--enable-timing', '--verbose', '--fail-fast'], test_report_tags={'task': t.title}) with Task('Check Copyrights', tasks) as t: if t: if mx.command_function('checkcopyrights')(['--primary', '--', '--projects', 'src']) != 0: From 1d8e4f1b5a47bd7ce6822fa8dd2aa784f0297314 Mon Sep 17 00:00:00 2001 From: Andreas Woess Date: Thu, 13 Apr 2023 20:29:03 +0200 Subject: [PATCH 03/43] [GR-45616] Update ASM dependency to version 9.5. --- compiler/mx.compiler/suite.py | 30 +++++++++++++++--------------- espresso/mx.espresso/suite.py | 4 ++-- truffle/mx.truffle/suite.py | 10 +++++----- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/compiler/mx.compiler/suite.py b/compiler/mx.compiler/suite.py index c7ca3e1ba40..d1c2e0685d1 100644 --- a/compiler/mx.compiler/suite.py +++ b/compiler/mx.compiler/suite.py @@ -122,33 +122,33 @@ "urls" : ["https://lafo.ssw.uni-linz.ac.at/pub/graal-external-deps/batik-all-1.7.jar"], }, - "ASM_9.1" : { - "digest" : "sha512:0a586544f3053ec8425d252b6f7e3e6772f010eb81d75020b4fd4759a561a4534dab4f805ffd18130594d1abbeb1ad7116b9d3a1e2e643d427e12bb866655954", + "ASM_9.5" : { + "digest" : "sha512:9e65f2983783725bae196ca939b45246958731246df1c495089c8ea5ce646de77c4e01a5a9ba10642016bb3258e1727e9ebcece6e74d9e3c32f528025d76b955", "maven" : { "groupId" : "org.ow2.asm", "artifactId" : "asm", - "version" : "9.1", + "version" : "9.5", }, }, - "ASM_TREE_9.1" : { - "digest" : "sha512:2e7c7e2453b4453db83aa5f13a7a9ec9fa7896d3b13670b171f6e8f186f3ec2f382a985c69018a510ea1b14a2e986f00e1bd3dd6e77a59a28f82b7fbe738916d", + "ASM_TREE_9.5" : { + "digest" : "sha512:816de8f84c216a7bd97b2458bde64a4b99a039b7b59fbd1ef52edf8bf869edabb93967736fe0c61e8eb3e1520e0cefe69ba59cda12df30f9f85db75fb6c064f3", "maven" : { "groupId" : "org.ow2.asm", "artifactId" : "asm-tree", - "version" : "9.1", + "version" : "9.5", }, - "dependencies" : ["ASM_9.1"], + "dependencies" : ["ASM_9.5"], }, - "ASM_UTIL_9.1" : { - "digest" : "sha512:2182c016c5547cd9e904a4a6d803c45a2c481533e1ffb5b0e18109b40a3d12e106654bbf0673da28ce9ac46cae3b7cfc016dfec68adf5d444917188c70f8b534", + "ASM_UTIL_9.5" : { + "digest" : "sha512:f68284d8f8fd029f3f428112225b2035ed3a4216cf3b34e0aacc83c32a6d44ab5e5d128b60a13ef768e3396041a62cf63f7fd3445dc5a05ce0ae03a2b2ed3080", "maven" : { "groupId" : "org.ow2.asm", "artifactId" : "asm-util", - "version" : "9.1", + "version" : "9.5", }, - "dependencies" : ["ASM_9.1"], + "dependencies" : ["ASM_9.5"], }, "HSDIS" : { @@ -1113,8 +1113,8 @@ "dependencies" : [ "org.graalvm.compiler.core.test", "org.graalvm.compiler.replacements.amd64", - "ASM_TREE_9.1", - "ASM_UTIL_9.1", + "ASM_TREE_9.5", + "ASM_UTIL_9.5", ], "requires" : [ "jdk.unsupported", @@ -1508,8 +1508,8 @@ "org.graalvm.compiler.graph.test", "org.graalvm.compiler.printer", "JAVA_ALLOCATION_INSTRUMENTER", - "ASM_TREE_9.1", - "ASM_UTIL_9.1", + "ASM_TREE_9.5", + "ASM_UTIL_9.5", ], "requires" : [ "jdk.unsupported", diff --git a/espresso/mx.espresso/suite.py b/espresso/mx.espresso/suite.py index 1572c50a28a..a87d4c2a518 100644 --- a/espresso/mx.espresso/suite.py +++ b/espresso/mx.espresso/suite.py @@ -119,7 +119,7 @@ "dependencies": [ "truffle:TRUFFLE_API", "truffle:TRUFFLE_NFI", - "truffle:TRUFFLE_ASM_9.1", + "truffle:TRUFFLE_ASM_9.5", "com.oracle.truffle.espresso.jdwp", ], "requires": [ @@ -323,7 +323,7 @@ "tools:TRUFFLE_PROFILER", ], "exclude": [ - "truffle:TRUFFLE_ASM_9.1", + "truffle:TRUFFLE_ASM_9.5", ], "javaProperties": { "org.graalvm.language.java.home": "", diff --git a/truffle/mx.truffle/suite.py b/truffle/mx.truffle/suite.py index f9b3886ac81..f95589458c5 100644 --- a/truffle/mx.truffle/suite.py +++ b/truffle/mx.truffle/suite.py @@ -112,9 +112,9 @@ "license": ["MIT"], }, - "TRUFFLE_ASM_9.1" : { - "digest" : "sha512:5aa4dbb2886173e17b357c66bc926a75662df559091f007a64000e777fb2bf25f3ca08c40efb8b8120e1e8fd85ca542c76d777f80da6c530db19a3430e4a2cd1", - "urls": ["https://lafo.ssw.uni-linz.ac.at/pub/graal-external-deps/com.oracle.truffle.api.impl.asm-9.1.0.jar"], + "TRUFFLE_ASM_9.5" : { + "digest" : "sha512:7a49aaa0c4b513ca54ce684a74a3848ba4caf486320125f08cb8872720dc1e789538729f45c46d6ccf1b1ea54f7c3770dc9682d13a3f1813a348168ee5c40b82", + "urls": ["https://lafo.ssw.uni-linz.ac.at/pub/graal-external-deps/com.oracle.truffle.api.impl.asm-9.5.0.jar"], }, "ICU4J" : { @@ -251,7 +251,7 @@ "dependencies" : [ "sdk:GRAAL_SDK", "com.oracle.truffle.api.exception", - "truffle:TRUFFLE_ASM_9.1", + "truffle:TRUFFLE_ASM_9.5", ], "requires" : [ "java.sql", @@ -561,7 +561,7 @@ "sourceDirs" : ["src"], "dependencies" : [ "com.oracle.truffle.api", - "truffle:TRUFFLE_ASM_9.1", + "truffle:TRUFFLE_ASM_9.5", ], "requires" : [ "jdk.unsupported", # sun.misc.Unsafe From 027409cc8fbae8b468fd0b462ef2538064730a33 Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Mon, 6 Feb 2023 13:59:13 +0100 Subject: [PATCH 04/43] Add input accessor method to BindToRegisterNode --- .../org/graalvm/compiler/nodes/debug/BindToRegisterNode.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/debug/BindToRegisterNode.java b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/debug/BindToRegisterNode.java index 014505ab041..e840a374dbc 100644 --- a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/debug/BindToRegisterNode.java +++ b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/debug/BindToRegisterNode.java @@ -47,6 +47,10 @@ public BindToRegisterNode(ValueNode value) { this.value = value; } + public ValueNode getValue() { + return value; + } + @Override public void generate(NodeLIRBuilderTool gen) { gen.getLIRGeneratorTool().append(new StandardOp.BindToRegisterOp(gen.getLIRGeneratorTool().asAllocatable(gen.operand(value)))); From 21dbe1167e978cff32741ae5a1f3c35ccb15d6ed Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Thu, 9 Feb 2023 14:39:15 +0100 Subject: [PATCH 05/43] Add special node class and compiler directive to inject switch case probability --- .../SwitchCaseProbabilityDirectiveTest.java | 99 ++++++++++++++ .../api/directives/GraalDirectives.java | 32 +++++ .../extended/SwitchCaseProbabilityNode.java | 128 ++++++++++++++++++ .../compiler/nodes/extended/SwitchNode.java | 5 + .../StandardGraphBuilderPlugins.java | 8 ++ 5 files changed, 272 insertions(+) create mode 100644 compiler/src/org.graalvm.compiler.api.directives.test/src/org/graalvm/compiler/api/directives/test/SwitchCaseProbabilityDirectiveTest.java create mode 100644 compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchCaseProbabilityNode.java diff --git a/compiler/src/org.graalvm.compiler.api.directives.test/src/org/graalvm/compiler/api/directives/test/SwitchCaseProbabilityDirectiveTest.java b/compiler/src/org.graalvm.compiler.api.directives.test/src/org/graalvm/compiler/api/directives/test/SwitchCaseProbabilityDirectiveTest.java new file mode 100644 index 00000000000..e1071b1eed2 --- /dev/null +++ b/compiler/src/org.graalvm.compiler.api.directives.test/src/org/graalvm/compiler/api/directives/test/SwitchCaseProbabilityDirectiveTest.java @@ -0,0 +1,99 @@ +/* + * Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package org.graalvm.compiler.api.directives.test; + +import org.graalvm.compiler.api.directives.GraalDirectives; +import org.graalvm.compiler.core.test.GraalCompilerTest; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.graph.iterators.NodeIterable; +import org.graalvm.compiler.nodes.StructuredGraph; +import org.graalvm.compiler.nodes.extended.IntegerSwitchNode; +import org.graalvm.compiler.nodes.extended.SwitchCaseProbabilityNode; +import org.junit.Assert; +import org.junit.Test; + +import jdk.vm.ci.meta.ResolvedJavaMethod; + +public class SwitchCaseProbabilityDirectiveTest extends GraalCompilerTest { + /** + * Called before a test is compiled. + */ + @Override + protected void before(ResolvedJavaMethod method) { + // don't let -Xcomp pollute profile + method.reprofile(); + } + + public static int switchProbabilitySnippet1(int x) { + switch (x) { + case 0: + GraalDirectives.injectSwitchCaseProbability(0.5); + return 1; + case 1: + GraalDirectives.injectSwitchCaseProbability(0.3); + return 3; + case 2: + GraalDirectives.injectSwitchCaseProbability(0.1); + return x * 3; + default: + GraalDirectives.injectSwitchCaseProbability(0.1); + return x + 5; + } + } + + @Test + public void testSwitch() { + test("switchProbabilitySnippet1", 1); + } + + @Override + protected void checkLowTierGraph(StructuredGraph graph) { + NodeIterable switchProbabilityInjectionNodes = graph.getNodes().filter(SwitchCaseProbabilityNode.class); + Assert.assertEquals("SwitchCaseProbabilityNode count", 0, switchProbabilityInjectionNodes.count()); + + NodeIterable switchNodes = graph.getNodes().filter(IntegerSwitchNode.class); + Assert.assertEquals("IntegerSwitchNode count", 1, switchNodes.count()); + IntegerSwitchNode switchNode = switchNodes.first(); + for (int i = 0; i < switchNode.keyCount(); ++i) { + double expectedProbability = Double.NaN; + switch (switchNode.intKeyAt(i)) { + case 0: + expectedProbability = 0.5; + break; + case 1: + expectedProbability = 0.3; + break; + case 2: + expectedProbability = 0.1; + break; + default: + GraalError.shouldNotReachHereUnexpectedValue(switchNode.intKeyAt(i)); + } + Assert.assertEquals(expectedProbability, switchNode.keyProbability(i), 0.001); + } + Assert.assertEquals(0.1, switchNode.defaultProbability(), 0.001); + } +} diff --git a/compiler/src/org.graalvm.compiler.api.directives/src/org/graalvm/compiler/api/directives/GraalDirectives.java b/compiler/src/org.graalvm.compiler.api.directives/src/org/graalvm/compiler/api/directives/GraalDirectives.java index 5183627972a..d97150407cd 100644 --- a/compiler/src/org.graalvm.compiler.api.directives/src/org/graalvm/compiler/api/directives/GraalDirectives.java +++ b/compiler/src/org.graalvm.compiler.api.directives/src/org/graalvm/compiler/api/directives/GraalDirectives.java @@ -207,6 +207,38 @@ public static boolean injectIterationCount(double iterations, boolean condition) return injectBranchProbability(1. - 1. / iterations, condition); } + /** + * Injects a probability into the profiling information of a branch of a switch instruction. The + * probability must be a value between 0.0 and 1.0 (inclusive). This directive should only be + * used as the first statement of each branch of a switch statement. Either all or none of the + * branches should contain a call to injectSwitchCaseProbability, and the sum of the values + * across all branches must be 1.0. + * + * Example usage: + * + * + * int a = ...; + * switch (a) { + * case 0: + * GraalDirectives.injectSwitchCaseProbability(0.3); + * // ... + * break; + * case 10: + * GraalDirectives.injectSwitchCaseProbability(0.2); + * // ... + * break; + * default: + * GraalDirectives.injectSwitchCaseProbability(0.5); + * // ... + * break; + * } + * + * + * @param probability the probability value between 0.0 and 1.0 that should be injected + */ + public static void injectSwitchCaseProbability(double probability) { + } + /** * Consume a value, making sure the compiler doesn't optimize away the computation of this * value, even if it is otherwise unused. diff --git a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchCaseProbabilityNode.java b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchCaseProbabilityNode.java new file mode 100644 index 00000000000..79c986972f6 --- /dev/null +++ b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchCaseProbabilityNode.java @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.graalvm.compiler.nodes.extended; + +import static org.graalvm.compiler.nodeinfo.NodeCycles.CYCLES_0; +import static org.graalvm.compiler.nodeinfo.NodeSize.SIZE_0; + +import org.graalvm.compiler.core.common.type.StampFactory; +import org.graalvm.compiler.debug.GraalError; +import org.graalvm.compiler.graph.NodeClass; +import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.AbstractBeginNode; +import org.graalvm.compiler.nodes.FixedWithNextNode; +import org.graalvm.compiler.nodes.ProfileData; +import org.graalvm.compiler.nodes.ProfileData.ProfileSource; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.spi.Lowerable; +import org.graalvm.compiler.nodes.spi.LoweringTool; +import org.graalvm.compiler.nodes.spi.Simplifiable; +import org.graalvm.compiler.nodes.spi.SimplifierTool; + +import jdk.vm.ci.meta.JavaKind; + +/** + * Injects profile data to a {@link SwitchNode}. Specifically, it sets the successor probability for + * the switch-case block this node is located in. For a given {@link SwitchNode}, either none or all + * of its successors should be followed by a SwitchCaseProbabilityNode, and their probabilities + * should sum to one. + * + * Instances of this node class will be simplified by their preceding switch node, injecting the + * given probability of each branch into the switch profile data. Then all probability nodes will be + * removed at once. + */ +@NodeInfo(cycles = CYCLES_0, cyclesRationale = "Artificial Node", size = SIZE_0) +public final class SwitchCaseProbabilityNode extends FixedWithNextNode implements Simplifiable, Lowerable { + + public static final NodeClass TYPE = NodeClass.create(SwitchCaseProbabilityNode.class); + + @Input ValueNode probability; + + public SwitchCaseProbabilityNode(ValueNode probability) { + super(TYPE, StampFactory.forKind(JavaKind.Void)); + this.probability = probability; + } + + public ValueNode getProbability() { + return probability; + } + + public void setProbability(ValueNode probability) { + updateUsages(this.probability, probability); + this.probability = probability; + } + + @Override + public void simplify(SimplifierTool tool) { + AbstractBeginNode switchSuccessor = AbstractBeginNode.prevBegin(this); + GraalError.guarantee(switchSuccessor != null && switchSuccessor.predecessor() instanceof SwitchNode, "SwitchCaseProbabilityNode can only be used within a switch statement"); + + /* + * We simplify all SwitchCaseProbabilityNodes at once to check whether all switch cases have + * an injected probability attached to them. + */ + SwitchNode switchNode = (SwitchNode) switchSuccessor.predecessor(); + double[] probabilities = switchNode.getKeyProbabilities(); + double sum = 0.0; + for (int i = 0; i < probabilities.length; ++i) { + GraalError.guarantee(switchNode.keySuccessor(i).next() instanceof SwitchCaseProbabilityNode, "All switch branches must be followed by a SwitchCaseProbabilityNode"); + SwitchCaseProbabilityNode caseProbabilityNode = (SwitchCaseProbabilityNode) switchNode.keySuccessor(i).next(); + ValueNode probabilityNode = caseProbabilityNode.getProbability(); + if (!probabilityNode.isConstant()) { + /* + * If any of the probabilities are not constant we bail out of simplification, which + * will cause compilation to fail later during lowering since the node will be left + * behind + */ + return; + } + + double probabilityValue = probabilityNode.asJavaConstant().asDouble(); + if (probabilityValue < 0.0) { + throw new GraalError("A negative probability of " + probabilityValue + " is not allowed!"); + } else if (probabilityValue > 1.0) { + throw new GraalError("A probability of more than 1.0 (" + probabilityValue + ") is not allowed!"); + } else if (Double.isNaN(probabilityValue)) { + /* + * We allow NaN if the node is in unreachable code that will eventually fall away, + * or else an error will be thrown during lowering since we keep the node around. + */ + return; + } + probabilities[i] = probabilityValue; + sum += probabilityValue; + caseProbabilityNode.replaceAtUsages(null); + graph().removeFixed(caseProbabilityNode); + } + + GraalError.guarantee(sum == 1.0, "Sum of all injected switch case probabilities must be 1"); + switchNode.setProfileData(ProfileData.SwitchProbabilityData.create(probabilities, ProfileSource.INJECTED)); + } + + @Override + public void lower(LoweringTool tool) { + throw new GraalError("Switch case probability could not be injected, because the probability value did not reduce to a constant value."); + } +} diff --git a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java index 5f4d2b0dd5c..2c2e6f67c2a 100644 --- a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java +++ b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java @@ -161,6 +161,11 @@ public boolean setProbability(AbstractBeginNode successor, BranchProbabilityData return true; } + public void setProfileData(SwitchProbabilityData profileData) { + this.profileData = profileData; + assert assertProbabilities(); + } + @Override public SwitchProbabilityData getProfileData() { return profileData; diff --git a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StandardGraphBuilderPlugins.java b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StandardGraphBuilderPlugins.java index 5439cafd59e..10c2b07339c 100644 --- a/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StandardGraphBuilderPlugins.java +++ b/compiler/src/org.graalvm.compiler.replacements/src/org/graalvm/compiler/replacements/StandardGraphBuilderPlugins.java @@ -133,6 +133,7 @@ import org.graalvm.compiler.nodes.extended.OpaqueNode; import org.graalvm.compiler.nodes.extended.RawLoadNode; import org.graalvm.compiler.nodes.extended.RawStoreNode; +import org.graalvm.compiler.nodes.extended.SwitchCaseProbabilityNode; import org.graalvm.compiler.nodes.extended.UnboxNode; import org.graalvm.compiler.nodes.extended.UnsafeMemoryLoadNode; import org.graalvm.compiler.nodes.extended.UnsafeMemoryStoreNode; @@ -1811,6 +1812,13 @@ public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Rec return false; } }); + r.register(new RequiredInlineOnlyInvocationPlugin("injectSwitchCaseProbability", double.class) { + @Override + public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode probability) { + b.add(new SwitchCaseProbabilityNode(probability)); + return true; + } + }); for (JavaKind kind : JavaKind.values()) { if ((kind.isPrimitive() && kind != JavaKind.Void) || kind == JavaKind.Object) { From 01a3eb454b48a89a2abbbf92f8470e2bbfd7444e Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Thu, 9 Feb 2023 15:17:04 +0100 Subject: [PATCH 06/43] Move simplification of SwitchCaseProbabilityNode to SwitchNode --- .../SwitchCaseProbabilityDirectiveTest.java | 37 +++++++++++ .../api/directives/GraalDirectives.java | 2 +- .../nodes/extended/IntegerSwitchNode.java | 3 + .../extended/SwitchCaseProbabilityNode.java | 55 +--------------- .../compiler/nodes/extended/SwitchNode.java | 64 ++++++++++++++++++- .../compiler/nodes/java/TypeSwitchNode.java | 3 + 6 files changed, 106 insertions(+), 58 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.api.directives.test/src/org/graalvm/compiler/api/directives/test/SwitchCaseProbabilityDirectiveTest.java b/compiler/src/org.graalvm.compiler.api.directives.test/src/org/graalvm/compiler/api/directives/test/SwitchCaseProbabilityDirectiveTest.java index e1071b1eed2..57168bcb4c5 100644 --- a/compiler/src/org.graalvm.compiler.api.directives.test/src/org/graalvm/compiler/api/directives/test/SwitchCaseProbabilityDirectiveTest.java +++ b/compiler/src/org.graalvm.compiler.api.directives.test/src/org/graalvm/compiler/api/directives/test/SwitchCaseProbabilityDirectiveTest.java @@ -64,9 +64,30 @@ public static int switchProbabilitySnippet1(int x) { } } + public static int keyHoleSwitchSnippet(int x) { + switch (x) { + case 3: + GraalDirectives.injectSwitchCaseProbability(0.2); + return 10; + case 4: + GraalDirectives.injectSwitchCaseProbability(0.2); + return 20; + case 6: + GraalDirectives.injectSwitchCaseProbability(0.2); + return 30; + case 7: + GraalDirectives.injectSwitchCaseProbability(0.2); + return 40; + default: + GraalDirectives.injectSwitchCaseProbability(0.2); + return 42; + } + } + @Test public void testSwitch() { test("switchProbabilitySnippet1", 1); + test("keyHoleSwitchSnippet", 4); } @Override @@ -89,6 +110,22 @@ protected void checkLowTierGraph(StructuredGraph graph) { case 2: expectedProbability = 0.1; break; + case 3: + expectedProbability = 0.2; + break; + case 4: + expectedProbability = 0.2; + break; + case 5: + // Default probability should fill the hole and be divided in two + expectedProbability = 0.1; + break; + case 6: + expectedProbability = 0.2; + break; + case 7: + expectedProbability = 0.2; + break; default: GraalError.shouldNotReachHereUnexpectedValue(switchNode.intKeyAt(i)); } diff --git a/compiler/src/org.graalvm.compiler.api.directives/src/org/graalvm/compiler/api/directives/GraalDirectives.java b/compiler/src/org.graalvm.compiler.api.directives/src/org/graalvm/compiler/api/directives/GraalDirectives.java index d97150407cd..5fa7107c4af 100644 --- a/compiler/src/org.graalvm.compiler.api.directives/src/org/graalvm/compiler/api/directives/GraalDirectives.java +++ b/compiler/src/org.graalvm.compiler.api.directives/src/org/graalvm/compiler/api/directives/GraalDirectives.java @@ -233,7 +233,7 @@ public static boolean injectIterationCount(double iterations, boolean condition) * break; * } * - * + * * @param probability the probability value between 0.0 and 1.0 that should be injected */ public static void injectSwitchCaseProbability(double probability) { diff --git a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/IntegerSwitchNode.java b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/IntegerSwitchNode.java index 9edaca261c3..1989bc8479f 100644 --- a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/IntegerSwitchNode.java +++ b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/IntegerSwitchNode.java @@ -182,6 +182,9 @@ public int successorIndexAtKey(int key) { @Override public void simplify(SimplifierTool tool) { + if (shouldInjectBranchProbabilities()) { + injectBranchProbabilities(); + } NodeView view = NodeView.from(tool); if (blockSuccessorCount() == 1) { tool.addToWorkList(defaultSuccessor()); diff --git a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchCaseProbabilityNode.java b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchCaseProbabilityNode.java index 79c986972f6..ab037bdd6e0 100644 --- a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchCaseProbabilityNode.java +++ b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchCaseProbabilityNode.java @@ -31,15 +31,10 @@ import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.NodeClass; import org.graalvm.compiler.nodeinfo.NodeInfo; -import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.FixedWithNextNode; -import org.graalvm.compiler.nodes.ProfileData; -import org.graalvm.compiler.nodes.ProfileData.ProfileSource; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.spi.Lowerable; import org.graalvm.compiler.nodes.spi.LoweringTool; -import org.graalvm.compiler.nodes.spi.Simplifiable; -import org.graalvm.compiler.nodes.spi.SimplifierTool; import jdk.vm.ci.meta.JavaKind; @@ -54,8 +49,7 @@ * removed at once. */ @NodeInfo(cycles = CYCLES_0, cyclesRationale = "Artificial Node", size = SIZE_0) -public final class SwitchCaseProbabilityNode extends FixedWithNextNode implements Simplifiable, Lowerable { - +public final class SwitchCaseProbabilityNode extends FixedWithNextNode implements Lowerable { public static final NodeClass TYPE = NodeClass.create(SwitchCaseProbabilityNode.class); @Input ValueNode probability; @@ -74,53 +68,6 @@ public void setProbability(ValueNode probability) { this.probability = probability; } - @Override - public void simplify(SimplifierTool tool) { - AbstractBeginNode switchSuccessor = AbstractBeginNode.prevBegin(this); - GraalError.guarantee(switchSuccessor != null && switchSuccessor.predecessor() instanceof SwitchNode, "SwitchCaseProbabilityNode can only be used within a switch statement"); - - /* - * We simplify all SwitchCaseProbabilityNodes at once to check whether all switch cases have - * an injected probability attached to them. - */ - SwitchNode switchNode = (SwitchNode) switchSuccessor.predecessor(); - double[] probabilities = switchNode.getKeyProbabilities(); - double sum = 0.0; - for (int i = 0; i < probabilities.length; ++i) { - GraalError.guarantee(switchNode.keySuccessor(i).next() instanceof SwitchCaseProbabilityNode, "All switch branches must be followed by a SwitchCaseProbabilityNode"); - SwitchCaseProbabilityNode caseProbabilityNode = (SwitchCaseProbabilityNode) switchNode.keySuccessor(i).next(); - ValueNode probabilityNode = caseProbabilityNode.getProbability(); - if (!probabilityNode.isConstant()) { - /* - * If any of the probabilities are not constant we bail out of simplification, which - * will cause compilation to fail later during lowering since the node will be left - * behind - */ - return; - } - - double probabilityValue = probabilityNode.asJavaConstant().asDouble(); - if (probabilityValue < 0.0) { - throw new GraalError("A negative probability of " + probabilityValue + " is not allowed!"); - } else if (probabilityValue > 1.0) { - throw new GraalError("A probability of more than 1.0 (" + probabilityValue + ") is not allowed!"); - } else if (Double.isNaN(probabilityValue)) { - /* - * We allow NaN if the node is in unreachable code that will eventually fall away, - * or else an error will be thrown during lowering since we keep the node around. - */ - return; - } - probabilities[i] = probabilityValue; - sum += probabilityValue; - caseProbabilityNode.replaceAtUsages(null); - graph().removeFixed(caseProbabilityNode); - } - - GraalError.guarantee(sum == 1.0, "Sum of all injected switch case probabilities must be 1"); - switchNode.setProfileData(ProfileData.SwitchProbabilityData.create(probabilities, ProfileSource.INJECTED)); - } - @Override public void lower(LoweringTool tool) { throw new GraalError("Switch case probability could not be injected, because the probability value did not reduce to a constant value."); diff --git a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java index 2c2e6f67c2a..9d6c100b6e2 100644 --- a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java +++ b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java @@ -42,16 +42,17 @@ import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeClass; import org.graalvm.compiler.graph.NodeSuccessorList; -import org.graalvm.compiler.nodes.spi.SimplifierTool; import org.graalvm.compiler.nodeinfo.NodeCycles; import org.graalvm.compiler.nodeinfo.NodeInfo; import org.graalvm.compiler.nodeinfo.NodeSize; import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.ControlSplitNode; import org.graalvm.compiler.nodes.NodeView; +import org.graalvm.compiler.nodes.ProfileData; import org.graalvm.compiler.nodes.ProfileData.BranchProbabilityData; import org.graalvm.compiler.nodes.ProfileData.SwitchProbabilityData; import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.spi.SimplifierTool; import jdk.vm.ci.meta.Constant; @@ -103,9 +104,9 @@ private boolean assertProbabilities() { double total = 0; for (double d : getKeyProbabilities()) { total += d; - assert d >= 0.0 : "Cannot have negative probabilities in switch node: " + d; + GraalError.guarantee(d >= 0.0, "Cannot have negative probabilities in switch node: %s", d); } - assert total > 0.999 && total < 1.001 : "Total " + total; + GraalError.guarantee(total > 0.999 && total < 1.001, "Total probability across branches not equal to one: %.4f", total); return true; } @@ -260,6 +261,63 @@ public AbstractBeginNode getPrimarySuccessor() { return null; } + protected boolean shouldInjectBranchProbabilities() { + for (AbstractBeginNode succ : successors) { + if (succ.next() instanceof SwitchCaseProbabilityNode) { + return true; + } + } + return false; + } + + protected void injectBranchProbabilities() { + /* + * Since multiple keys can point to the same block, we must divide each block's probability + * between all keys sharing it. + */ + int[] numKeysPerBlock = new int[blockSuccessorCount()]; + for (int i = 0; i < keySuccessors.length; ++i) { + numKeysPerBlock[keySuccessorIndex(i)]++; + } + + double[] nodeProbabilities = new double[keySuccessors.length]; + for (int i = 0; i < keySuccessors.length; ++i) { + GraalError.guarantee(keySuccessor(i).next() instanceof SwitchCaseProbabilityNode, "All switch branches must be followed by a SwitchCaseProbabilityNode"); + SwitchCaseProbabilityNode caseProbabilityNode = (SwitchCaseProbabilityNode) keySuccessor(i).next(); + + ValueNode probabilityNode = caseProbabilityNode.getProbability(); + if (!probabilityNode.isConstant()) { + /* + * If any of the probabilities are not constant we bail out of simplification, which + * will cause compilation to fail later during lowering since the node will be left + * behind + */ + return; + } + double probabilityValue = probabilityNode.asJavaConstant().asDouble(); + if (probabilityValue < 0.0) { + throw new GraalError("A negative probability of " + probabilityValue + " is not allowed!"); + } else if (probabilityValue > 1.0) { + throw new GraalError("A probability of more than 1.0 (" + probabilityValue + ") is not allowed!"); + } else if (Double.isNaN(probabilityValue)) { + /* + * We allow NaN if the node is in unreachable code that will eventually fall away, + * or else an error will be thrown during lowering since we keep the node around. + */ + return; + } + nodeProbabilities[i] = probabilityValue / numKeysPerBlock[keySuccessorIndex(i)]; + } + + for (AbstractBeginNode blockSuccessor : successors) { + SwitchCaseProbabilityNode caseProbabilityNode = (SwitchCaseProbabilityNode) blockSuccessor.next(); + caseProbabilityNode.replaceAtUsages(null); + graph().removeFixed(caseProbabilityNode); + } + + setProfileData(ProfileData.SwitchProbabilityData.create(nodeProbabilities, ProfileData.ProfileSource.INJECTED)); + } + /** * Delete all other successors except for the one reached by {@code survivingEdge}. * diff --git a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/TypeSwitchNode.java b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/TypeSwitchNode.java index 2eb31d8581f..8d999d47a39 100644 --- a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/TypeSwitchNode.java +++ b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/java/TypeSwitchNode.java @@ -128,6 +128,9 @@ public void generate(NodeLIRBuilderTool gen) { @Override public void simplify(SimplifierTool tool) { + if (shouldInjectBranchProbabilities()) { + injectBranchProbabilities(); + } NodeView view = NodeView.from(tool); if (value() instanceof ConstantNode) { Constant constant = value().asConstant(); From ccc481c557ea5847c2cd8874ba1a7828930f8b51 Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Tue, 14 Mar 2023 14:40:58 +0100 Subject: [PATCH 07/43] Add types to hightiercodegen, implement lowering of TypeSwitchNode --- .../compiler/hightiercodegen/CodeBuffer.java | 20 ++ .../compiler/hightiercodegen/CodeGenTool.java | 49 +++- .../compiler/hightiercodegen/Emitter.java | 215 ++++++++++++++++++ .../compiler/hightiercodegen/NodeLowerer.java | 25 +- .../irwalk/StackifierIRWalker.java | 205 +++++++++++++++-- .../lowerer/PhiResolveLowerer.java | 3 +- 6 files changed, 480 insertions(+), 37 deletions(-) create mode 100644 compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/Emitter.java diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeBuffer.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeBuffer.java index e34bb790fb4..78b2e3556be 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeBuffer.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeBuffer.java @@ -37,6 +37,7 @@ import jdk.vm.ci.common.JVMCIError; import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.ResolvedJavaType; import jdk.vm.ci.meta.Signature; /** @@ -150,12 +151,21 @@ public void emitStringLiteral(String s) { emitText(getStringLiteral(s)); } + public void emitBoolLiteral(boolean b) { + emitText(b ? "true" : "false"); + } + public void emitKeyword(Keyword keyword) { emitText(keyword.toString()); } public abstract void emitDeclarationPrefix(); + @SuppressWarnings("unused") + public void emitDeclarationPrefix(ResolvedJavaType type) { + emitDeclarationPrefix(); + } + public abstract void emitAnonymousClassHeader(String superClass); /** @@ -283,6 +293,14 @@ public void emitDeclPrefix(String name) { emitWhiteSpace(); } + public void emitDeclPrefix(ResolvedJavaType type, String name) { + emitDeclarationPrefix(type); + emitDeclarationName(name); + emitWhiteSpace(); + emitAssignmentSymbol(); + emitWhiteSpace(); + } + public abstract void emitAssignmentSymbol(); public void emitNew() { @@ -301,6 +319,8 @@ public void emitTry() { public abstract void emitCatch(String expName); + public abstract void emitCatch(String expName, ResolvedJavaType type); + /** * Emit a simple linebreak. */ diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java index 672b381b01f..d6091b9498b 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java @@ -29,6 +29,7 @@ import java.util.List; import org.graalvm.compiler.core.common.calc.CanonicalCondition; +import org.graalvm.compiler.core.common.type.Stamp; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.hightiercodegen.variables.ResolvedVar; import org.graalvm.compiler.hightiercodegen.variables.VariableAllocation; @@ -42,6 +43,7 @@ import jdk.vm.ci.meta.ResolvedJavaField; import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.meta.ResolvedJavaType; +import org.graalvm.compiler.nodes.spi.CoreProviders; public abstract class CodeGenTool { @@ -68,7 +70,7 @@ protected CodeGenTool(CodeBuffer codeBuffer, VariableAllocation variableAllocati /** * Generates an efficient representation of an integer value literal. */ - public static String getEfficientIntLiteral(int i) { + public static String getEfficientIntLiteral(long i) { StringBuilder sb = new StringBuilder(); /* * Once a number gets larger than 10^6, representing it in hex gets cheaper in terms of @@ -77,13 +79,13 @@ public static String getEfficientIntLiteral(int i) { * 10^6 takes 7 decimal digits and also 7 hex digits (0xF4240) to represent. After that * point, hex is more efficient. */ - int abs = Math.abs(i); + long abs = Math.abs(i); if (abs >= 1000000) { if (i < 0) { sb.append('-'); } sb.append("0x"); - sb.append(Integer.toHexString(abs)); + sb.append(Long.toHexString(abs)); } else { // Write in decimal sb.append(i); @@ -92,6 +94,8 @@ public static String getEfficientIntLiteral(int i) { return sb.toString(); } + public abstract CoreProviders getProviders(); + /** * Prepare the current object to be ready for use in generating code for a method. */ @@ -173,13 +177,17 @@ public void genArrayLoad(ValueNode index, ValueNode array) { genArrayAccessPostfix(); } - public void genArrayStore(IEmitter index, ValueNode array, ValueNode value) { - nodeLowerer().lowerValue(array); + public void genArrayStore(IEmitter index, IEmitter array, IEmitter value) { + lower(array); genArrayAccessPrefix(); lower(index); genArrayAccessPostfix(); genAssignment(); - nodeLowerer().lowerValue(value); + lower(value); + } + + public void genArrayStore(IEmitter index, ValueNode array, ValueNode value) { + genArrayStore(index, Emitter.of(array), Emitter.of(value)); } protected abstract void genArrayAccessPostfix(); @@ -202,9 +210,13 @@ public void genClassEnd() { codeBuffer.emitNewLine(); } - public void genIfHeader(LogicNode logicNode) { + public void genIfHeader(LogicNode condition) { + genIfHeader(Emitter.of(condition)); + } + + public void genIfHeader(IEmitter condition) { codeBuffer.emitIfHeaderLeft(); - lowerValue(logicNode); + lower(condition); codeBuffer.emitIfHeaderRight(); } @@ -290,6 +302,14 @@ public void genResolvedVarDeclPrefix(String name) { codeBuffer.emitDeclPrefix(name); } + public void genResolvedVarDeclPrefix(String name, Stamp stamp) { + genResolvedVarDeclPrefix(name, stamp.javaType(getProviders().getMetaAccess())); + } + + public void genResolvedVarDeclPrefix(String name, ResolvedJavaType javaType) { + codeBuffer.emitDeclPrefix(javaType, name); + } + public abstract void genResolvedVarDeclPostfix(String comment); public void genResolvedVarAssignmentPrefix(String name) { @@ -301,6 +321,11 @@ public void genResolvedVarAssignmentPrefix(String name) { public abstract void genNewInstance(ResolvedJavaType t); + /** + * @param arrayType the array type to be generated (not the component type) + */ + public abstract void genNewArray(ResolvedJavaType arrayType, IEmitter length); + /** * Generates a declaration without initialization. Marks the definition as lowered. * @@ -336,6 +361,14 @@ public void genCatchBlockPrefix(String expName) { codeBuffer.emitCatch(expName); } + public void genCatchBlockPrefix(String expName, Stamp stamp) { + genCatchBlockPrefix(expName, stamp.javaType(getProviders().getMetaAccess())); + } + + public void genCatchBlockPrefix(String expName, ResolvedJavaType javaType) { + codeBuffer.emitCatch(expName, javaType); + } + public abstract void genThrow(ValueNode exception); @SuppressWarnings("deprecation") diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/Emitter.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/Emitter.java new file mode 100644 index 00000000000..09a6ec10bbc --- /dev/null +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/Emitter.java @@ -0,0 +1,215 @@ +/* + * Copyright (c) 2022, 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ + +package org.graalvm.compiler.hightiercodegen; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; + +import org.graalvm.compiler.hightiercodegen.variables.ResolvedVar; +import org.graalvm.compiler.nodes.ValueNode; + +import jdk.vm.ci.meta.ResolvedJavaField; +import jdk.vm.ci.meta.ResolvedJavaMethod; +import jdk.vm.ci.meta.ResolvedJavaType; + +/** + * Facilitate lowering of various objects. + * + * Methods that want to accept multiple different types for lowering, especially as part of a list + * of arguments, can accept {@link Emitter} instead of having to creating many overloaded methods. + */ +public class Emitter implements IEmitter { + private static final Emitter NULL = new Emitter(CodeGenTool::genNull); + + private final Consumer lowerer; + + protected Emitter(Consumer lowerer) { + this.lowerer = lowerer; + } + + @Override + public void lower(CodeGenTool codeGenTool) { + lowerer.accept(codeGenTool); + } + + public static Emitter ofNull() { + return NULL; + } + + public static Emitter of(String s) { + Objects.requireNonNull(s); + return new Emitter((t) -> t.getCodeBuffer().emitText(s)); + } + + public static Emitter stringLiteral(String s) { + Objects.requireNonNull(s); + return new Emitter((t) -> t.getCodeBuffer().emitStringLiteral(s)); + } + + public static Emitter of(ResolvedJavaMethod m) { + Objects.requireNonNull(m); + return new Emitter((t) -> t.genMethodName(m)); + } + + public static Emitter of(ResolvedJavaField f) { + Objects.requireNonNull(f); + return new Emitter((t) -> t.genFieldName(f)); + } + + public static Emitter of(ResolvedJavaType type) { + Objects.requireNonNull(type); + return new Emitter((t) -> t.genTypeName(type)); + } + + public static Emitter of(ValueNode n) { + if (n == null) { + return ofNull(); + } + + return new Emitter((t) -> t.lowerValue(n)); + } + + public static Emitter of(ResolvedVar v) { + Objects.requireNonNull(v); + return new Emitter((t) -> t.getCodeBuffer().emitText(v.getName())); + } + + public static Emitter of(Class c) { + Objects.requireNonNull(c); + return new Emitter((t) -> t.genTypeName(c)); + } + + public static Emitter of(Method m) { + Objects.requireNonNull(m); + return new Emitter((t) -> t.genMethodName(m)); + } + + public static Emitter of(Field f) { + Objects.requireNonNull(f); + return new Emitter((t) -> t.genFieldName(f)); + } + + public static Emitter of(Integer i) { + Objects.requireNonNull(i); + return new Emitter((t) -> t.getCodeBuffer().emitIntLiteral(i)); + } + + public static Emitter of(Byte i) { + return Emitter.of(i.intValue()); + } + + public static Emitter of(Short i) { + return Emitter.of(i.intValue()); + } + + public static Emitter of(Character c) { + return Emitter.of((int) c); + } + + public static Emitter of(Float f) { + Objects.requireNonNull(f); + return new Emitter((t) -> t.getCodeBuffer().emitFloatLiteral(f)); + } + + public static Emitter of(Double d) { + Objects.requireNonNull(d); + return new Emitter((t) -> t.getCodeBuffer().emitDoubleLiteral(d)); + } + + public static Emitter of(Boolean b) { + Objects.requireNonNull(b); + return new Emitter((t) -> t.getCodeBuffer().emitBoolLiteral(b)); + } + + public static Emitter[] of(List l) { + Emitter[] lowerList = new Emitter[l.size()]; + + for (int i = 0; i < l.size(); i++) { + lowerList[i] = Emitter.of(l.get(i)); + } + + return lowerList; + } + + public static Emitter of(Emitter... emitters) { + return new Emitter((t) -> { + for (Emitter emitter : emitters) { + emitter.lower(t); + } + }); + } + + public static Emitter of(Keyword context) { + return new Emitter((t) -> t.getCodeBuffer().emitKeyword(context)); + } + + public static Emitter ofArray(Emitter... emitters) { + return new Emitter((t) -> { + boolean first = true; + t.getCodeBuffer().emitText("["); + for (Emitter emitter : emitters) { + if (first) { + first = false; + } else { + t.getCodeBuffer().emitText(","); + } + emitter.lower(t); + } + t.getCodeBuffer().emitText("]"); + }); + } + + /** + * Emits a dictionary, using a map of emitters, for which keys are either strings or emitters. + * + * If the key is a string, it is treated as a normal string property, and enclosed in double + * quotes. If the key is an emitter, the emitted code is enclosed in square brackets. + */ + public static Emitter ofObject(Map props) { + return new Emitter((t) -> { + t.getCodeBuffer().emitScopeBegin(); + for (Map.Entry entry : props.entrySet()) { + Object key = entry.getKey(); + if (key instanceof String) { + t.getCodeBuffer().emitStringLiteral((String) key); + } else { + t.getCodeBuffer().emitText("["); + ((Emitter) key).lower(t); + t.getCodeBuffer().emitText("]"); + } + t.getCodeBuffer().emitText(" : "); + entry.getValue().lower(t); + t.getCodeBuffer().emitText(","); + t.getCodeBuffer().emitNewLine(); + } + t.getCodeBuffer().emitScopeEnd(); + }); + } +} diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/NodeLowerer.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/NodeLowerer.java index 7b7ab937ed9..b6086063c97 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/NodeLowerer.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/NodeLowerer.java @@ -24,6 +24,7 @@ */ package org.graalvm.compiler.hightiercodegen; +import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.iterators.NodeIterable; import org.graalvm.compiler.hightiercodegen.variables.ResolvedVar; @@ -128,8 +129,7 @@ public void lowerStatement(Node node) { * Here we assume a value node is first visited in the scheduled order of basic blocks. * All nodes that depend on it come after in the schedule. */ - codeGenTool.genResolvedVarDeclPrefix(resolvedVar.getName()); - resolvedVar.setDefinitionLowered(); + lowerVarDeclPrefix(resolvedVar); } dispatch(node); @@ -137,6 +137,12 @@ public void lowerStatement(Node node) { codeGenTool.genResolvedVarDeclPostfix(nodeDebugInfo(node)); } + /** + * Generates the code for the declaration and assignment of a variable, but not the value it's + * assigned to. Also sets the variable's definition as lowered if it wasn't already. + */ + protected abstract void lowerVarDeclPrefix(ResolvedVar resolvedVar); + /** * Generates code that represents the value of the given node. *

@@ -150,7 +156,7 @@ public final void lowerValue(ValueNode node) { assert isActiveValueNode(node); ResolvedVar resolvedVar = codeGenTool.getAllocatedVariable(node); if (resolvedVar != null) { - assert resolvedVar.isDefinitionLowered(); + assert resolvedVar.isDefinitionLowered() : "Variable definition for node " + node + " was not lowered before use"; lower(resolvedVar); } else { dispatch(node); @@ -325,8 +331,9 @@ protected void dispatch(Node node) { } else if (node instanceof ClassIsAssignableFromNode) { lower((ClassIsAssignableFromNode) node); } else { - assert !isForbiddenNode(node) : reportForbiddenNode(node); - assert isIgnored(node) : "Cannot lower node: " + node; + if (!isIgnored(node)) { + handleUnknownNodeType(node); + } } } @@ -337,6 +344,14 @@ protected void dispatch(Node node) { protected abstract String reportForbiddenNode(Node node); + /** + * Called when a node is encountered that is neither handled by {@link #dispatch}, forbidden or + * ignored. + */ + protected void handleUnknownNodeType(Node node) { + throw GraalError.unimplemented("Could not lower node: " + node); + } + protected abstract void lower(BlackholeNode node); protected abstract void lower(ReachabilityFenceNode node); diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java index 585943a7850..c541950d5f6 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java @@ -31,6 +31,7 @@ import org.graalvm.compiler.core.common.cfg.BlockMap; import org.graalvm.compiler.core.common.cfg.Loop; import org.graalvm.compiler.debug.DebugContext; +import org.graalvm.compiler.debug.GraalError; import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeMap; import org.graalvm.compiler.hightiercodegen.CodeGenTool; @@ -55,13 +56,17 @@ import org.graalvm.compiler.nodes.LoopBeginNode; import org.graalvm.compiler.nodes.LoopEndNode; import org.graalvm.compiler.nodes.LoopExitNode; +import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.PhiNode; +import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.WithExceptionNode; import org.graalvm.compiler.nodes.cfg.ControlFlowGraph; import org.graalvm.compiler.nodes.cfg.HIRBlock; import org.graalvm.compiler.nodes.extended.IntegerSwitchNode; import org.graalvm.compiler.nodes.java.ExceptionObjectNode; -import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode; +import org.graalvm.compiler.nodes.java.TypeSwitchNode; + +import jdk.vm.ci.meta.ResolvedJavaType; public class StackifierIRWalker extends IRWalker { public static final String LABEL_PREFIX = "looplabel_"; @@ -186,15 +191,24 @@ protected void lower(DebugContext debugContext) { StackifierData stackifierData = (StackifierData) reconstructionData; stackifierData.debugDump(debugContext); - // * scan for all phi values needed in stacked scopes - for (AbstractMergeNode n : cfg.graph.getNodes(AbstractMergeNode.TYPE)) { + predeclareVariables(cfg.graph); + + lowerBlocks(stackifierData.getBlocks()); + } + + /** + * Declare all variables that should be located at the top of the function before lowering, e.g. + * PhiNodes. + */ + protected void predeclareVariables(StructuredGraph graph) { + // scan for all phi values needed in stacked scopes + for (AbstractMergeNode n : graph.getNodes(AbstractMergeNode.TYPE)) { for (PhiNode phi : n.phis()) { if (codeGenTool.nodeLowerer().actualUsageCount(phi) > 0) { codeGenTool.nodeLowerer().lower(phi); } } } - lowerBlocks(stackifierData.getBlocks()); } /** @@ -258,7 +272,7 @@ private void lowerBlocks(HIRBlock[] blocks) { continue; } - codeGenTool.genComment("Start of block " + currentBlock.getId()); + codeGenTool.genComment("Start of block " + currentBlock); labeledBlockStartsAfterLoop.forEach(this::genLabeledBlockHeader); @@ -286,9 +300,11 @@ private void lowerBlocks(HIRBlock[] blocks) { lowerIfStackifier(currentBlock, (IfNode) lastNode); } else if (lastNode instanceof IntegerSwitchNode) { lowerSwitch((IntegerSwitchNode) lastNode, stackifierData); - } else if (isWithExceptionNode(lastNode)) { + } else if (lastNode instanceof TypeSwitchNode) { + lowerTypeSwitch((TypeSwitchNode) lastNode, stackifierData); + } else if (lastNode instanceof WithExceptionNode) { lowerWithExceptionStackifier(currentBlock, (WithExceptionNode) lastNode); - } else if ((lastNode instanceof ControlSplitNode) && !(lastNode instanceof BasicArrayCopyNode)) { + } else if (lastNode instanceof ControlSplitNode) { // BasicArrayCopyNode is also a ControlSplitNode assert false : "Unsupported control split node " + lastNode + " is not implemented yet"; } else if (lastNode instanceof LoopEndNode) { @@ -310,7 +326,7 @@ private void lowerBlocks(HIRBlock[] blocks) { verifier.visitNode(lastNode, codeGenTool); blockHistory.visitBlock(currentBlock); - codeGenTool.genComment("End of block " + currentBlock.getId()); + codeGenTool.genComment("End of block " + currentBlock); } } @@ -443,7 +459,7 @@ private void lowerWithExceptionStackifier(HIRBlock currentBlock, WithExceptionNo codeGenTool.genTryBlock(); lowerNode(lastNode); generateForwardJump(currentBlock, normSucc, stackifierData); - codeGenTool.genCatchBlockPrefix(codeGenTool.getExceptionObjectId(excpObj)); + codeGenTool.genCatchBlockPrefix(codeGenTool.getExceptionObjectId(excpObj), excpObj.stamp(NodeView.DEFAULT)); if (catchScope != null) { lowerBlocks(catchScope.getSortedBlocks()); } else { @@ -452,6 +468,10 @@ private void lowerWithExceptionStackifier(HIRBlock currentBlock, WithExceptionNo codeGenTool.genScopeEnd(); } + protected void lowerIfHeader(IfNode ifNode) { + codeGenTool.genIfHeader(ifNode.condition()); + } + /** * Lower an IfNode. If there are {@link Scope} for the then/else branch, the scopes are lowered * in their own recursive call. @@ -466,7 +486,7 @@ private void lowerIfStackifier(HIRBlock currentBlock, IfNode lastNode) { IfScopeContainer ifScopeContainer = (IfScopeContainer) stackifierData.getScopeEntry(lastNode); Scope thenScope = ifScopeContainer.getThenScope(); Scope elseScope = ifScopeContainer.getElseScope(); - codeGenTool.genIfHeader(lastNode.condition()); + lowerIfHeader(lastNode); if (thenScope != null) { lowerBlocks(thenScope.getSortedBlocks()); } else { @@ -494,6 +514,22 @@ public static boolean labeledBlockEndsInLoop(LoopScopeContainer loopScopeContain return loopScopeContainer.getLoopScope().getBlocks().contains(labeledBlockEnd); } + /** + * Generates code for one case of an integer switch node. This only generates the condition + * (`case` statement); the block successor must be generated separately. + * + * @param switchNode the {@link IntegerSwitchNode} that is being lowered + * @param successor the block successor corresponding to the case in question. + * @param keys the switch keys that have `successor` as their block successor. + */ + protected void lowerSwitchCase(IntegerSwitchNode switchNode, AbstractBeginNode successor, int[] keys) { + codeGenTool.genSwitchCase(keys); + } + + protected void lowerSwitchDefaultCase(IntegerSwitchNode switchNode) { + codeGenTool.genSwitchDefaultCase(); + } + /** * Lowers the given {@link IntegerSwitchNode}. If multiple keys lead to the same successor, * multiple {@code case} statements need to be generated for the successor. @@ -593,39 +629,157 @@ public void lowerSwitch(IntegerSwitchNode switchNode, StackifierData stackifierD } } - assert succKeys.size() > 0; + assert succKeys.size() > 0 : "no keys of " + switchNode + " have " + succ + " as block successor"; int[] succk = new int[succKeys.size()]; for (int s = 0; s < succKeys.size(); s++) { succk[s] = succKeys.get(s); } - codeGenTool.genSwitchCase(succk); + lowerSwitchCase(switchNode, succ, succk); if (caseScopes[i] != null) { lowerBlocks(caseScopes[i].getSortedBlocks()); } else { generateForwardJump(cfg.blockFor(switchNode), cfg.blockFor(succ), stackifierData); } - codeGenTool.genBreak(); - codeGenTool.genScopeEnd(); } if (hasdefault) { - codeGenTool.genSwitchDefaultCase(); + lowerSwitchDefaultCase(switchNode); int defaultIndex = switchNode.defaultSuccessorIndex(); if (caseScopes[defaultIndex] != null) { lowerBlocks(caseScopes[defaultIndex].getSortedBlocks()); } else { generateForwardJump(cfg.blockFor(switchNode), cfg.blockFor(switchNode.defaultSuccessor()), stackifierData); } - codeGenTool.genBreak(); codeGenTool.genScopeEnd(); } codeGenTool.genScopeEnd(); } + /** + * Lower one case of a type switch (that is, an if or else-if with a type check condition). + * + * @param switchNode the {@link TypeSwitchNode} that is being lowered + * @param succ the block successor corresponding to the case in question. + * @param succNum the index in generation order of the case being lowered. Can be used to + * distinguish between if or else-if. + * @param succKeys the allowed types for the condition of this case. + */ + protected void lowerTypeSwitchCase(TypeSwitchNode switchNode, AbstractBeginNode succ, int succNum, List succKeys) { + GraalError.unimplementedParent(); + } + + /** + * Lower the default case of a type switch (which should be lowered as an else statement). + * + * @param switchNode the {@link TypeSwitchNode} that is being lowered. + */ + protected void lowerTypeSwitchDefaultCase(TypeSwitchNode switchNode) { + GraalError.unimplementedParent(); + } + + /** + * Lowers the given {@link TypeSwitchNode}. Since a type switch performs exact type checks + * instead of instanceof ones, a pattern-matching switch cannot be used. Therefore, the node is + * lowered as a cascade of if-else blocks, where each if checks the type of the input against a + * given class. + * + * Example: Suppose we have the following Java program where A, B, C and D roughly correspond to + * basic blocks (for simplicity assume that all these basic blocks end with a + * {@link ControlSinkNode}). Further, assume that the following type switch performs exact type + * checks: + * + * 

+     *  Object a;
+     *  ...
+     *  switch(a){
+     *      case Integer:
+     *      case Long:
+     *             A();
+     *             break;
+     *      case Float:
+     *              B();
+     *      case String:
+     *              C();
+     *              break;
+     *      default:
+     *              D();
+     *  }
+     *
+ * + * Witch the topological order A->D->B->C, this will generate the following pseudocode: + * + * 
+     * block3: {
+     *     block2: {
+     *         block1: {
+     *             block0: {
+     *                 if (Integer.class.equals(a.getClass()) || Long.class.equals(a.getClass())) {
+     *                 } else if (Float.class.equals(a.getClass()) {
+     *                         break block1;
+     *                 } else if (String.class.equals(b.getClass()) {
+     *                         break block2;
+     *                 } else { // default case
+     *                         break block0;
+     *                 } // end type switch
+     *                 A();
+     *             } // block0
+     *             D();
+     *         } // block1
+     *         B();
+     *         break block3;
+     *     } // block2
+     * } // block3
+     * C();
+     *
+ * + * Note that the {@link LabeledBlock} {@code block3} is not necessary, but will be produced + * because the Graal IR sometimes produces a {@link HIRBlock} that only contains a + * {@link BeginNode} and a {@link EndNode}. Since no code is emitted for such a + * {@link HIRBlock}, there is no generated code between the ends of {@code block2} and + * {@code block3}. + * + * Similar to if-then-else, an optimization could pull the code of A, B, C and D inside the case + * blocks. + * + * @param switchNode node to be lowered + * @param stackifierData data for forward jumps + */ + public void lowerTypeSwitch(TypeSwitchNode switchNode, StackifierData stackifierData) { + boolean hasdefault = switchNode.defaultSuccessor() != null; + for (int i = 0; i < switchNode.blockSuccessorCount(); i++) { + // one successor + AbstractBeginNode succ = switchNode.blockSuccessor(i); + // the default case must be lowered at the end + if (hasdefault) { + if (succ.equals(switchNode.defaultSuccessor())) { + continue; + } + } + ArrayList succKeys = new ArrayList<>(); + // query all keys that have the succ as block succ + for (int keyIndex = 0; keyIndex < switchNode.keyCount(); keyIndex++) { + ResolvedJavaType key = switchNode.typeAt(keyIndex); + AbstractBeginNode keySucc = switchNode.keySuccessor(keyIndex); + if (succ.equals(keySucc)) { + succKeys.add(key); + } + } + assert succKeys.size() > 0 : "no keys of " + switchNode + " have " + succ + " as block successor"; + lowerTypeSwitchCase(switchNode, succ, i, succKeys); + generateForwardJump(cfg.blockFor(switchNode), cfg.blockFor(succ), stackifierData); + codeGenTool.genScopeEnd(); + } + if (hasdefault) { + lowerTypeSwitchDefaultCase(switchNode); + generateForwardJump(cfg.blockFor(switchNode), cfg.blockFor(switchNode.defaultSuccessor()), stackifierData); + codeGenTool.genScopeEnd(); + } + } + private void genLabeledBlockHeader(LabeledBlock labeledBlock) { blockNestingVerifier.pushLabel(labeledBlock); codeGenTool.genLabeledBlockHeader(labeledBlock.getLabel()); @@ -652,20 +806,25 @@ private void genLoopEnd(HIRBlock header) { String label = getLabel(header); blockNestingVerifier.popLabel(label); /* - * A break statement is always emitted before the loop end in order to get rid of the - * implicit back edge. Example: If the Graal IR contains a loop with 2 back edges, it has 2 - * LoopEndNodes. The generated code will have 2 back-edges because of 2 continue statements - * for the LoopEndNodes and 1 back-edge that comes from the loop end. This back-edge needs - * to be suppressed (i.e. made unreachable) with the break statement to guarantee that the + * A break statement was originally emitted before the loop end in order to get rid of the + * implicit back edge. The explanation given was as follows: + * + * Example: If the Graal IR contains a loop with 2 back edges, it has 2 LoopEndNodes. The + * generated code will have 2 back-edges because of 2 continue statements for the + * LoopEndNodes and 1 back-edge that comes from the loop end. This back-edge needs to be + * suppressed (i.e. made unreachable) with the break statement to guarantee that the * generated loop has the same semantics as the Graal IR. + * + * However, all blocks contained within the loop will themselves contain either a break or a + * continue statement, meaning that the final break before the loop is always unreachable + * and thus unnecessary. */ - codeGenTool.genBreak(); codeGenTool.genScopeEnd(); codeGenTool.genComment("End of loop " + label); } private static String getLabel(HIRBlock block) { assert block.isLoopHeader(); - return LABEL_PREFIX + block.getId(); + return LABEL_PREFIX + (int) block.getId(); } } diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/PhiResolveLowerer.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/PhiResolveLowerer.java index 6f1a9948e44..ad4eb56c6aa 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/PhiResolveLowerer.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/PhiResolveLowerer.java @@ -26,6 +26,7 @@ import org.graalvm.compiler.nodes.AbstractEndNode; import org.graalvm.compiler.nodes.AbstractMergeNode; +import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.ValuePhiNode; @@ -74,7 +75,7 @@ public void lower(CodeGenTool codeGenTool) { if (tmpName == null) { // This is the first move into the temp variable, declare it first. tmpName = "TEMP_" + codeGenTool.genUniqueID(); - codeGenTool.genResolvedVarDeclPrefix(tmpName); + codeGenTool.genResolvedVarDeclPrefix(tmpName, move.source.stamp(NodeView.DEFAULT)); } else { codeGenTool.genResolvedVarAssignmentPrefix(tmpName); } From 896c5a7cb6c4541e338cb1ebf330fb51288839d5 Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Tue, 14 Mar 2023 14:41:27 +0100 Subject: [PATCH 08/43] Add CommitAllocationLowerer to hightiercodegen --- .../lowerer/CommitAllocationLowerer.java | 179 ++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java new file mode 100644 index 00000000000..8d4ba882b6d --- /dev/null +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java @@ -0,0 +1,179 @@ +/* + * Copyright (c) 2021, 2022, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Oracle designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package org.graalvm.compiler.hightiercodegen.lowerer; + +import java.util.ArrayList; +import java.util.List; + +import org.graalvm.compiler.hightiercodegen.CodeGenTool; +import org.graalvm.compiler.hightiercodegen.Emitter; +import org.graalvm.compiler.nodes.ValueNode; +import org.graalvm.compiler.nodes.virtual.CommitAllocationNode; +import org.graalvm.compiler.nodes.virtual.VirtualArrayNode; +import org.graalvm.compiler.nodes.virtual.VirtualInstanceNode; +import org.graalvm.compiler.nodes.virtual.VirtualObjectNode; + +import jdk.vm.ci.meta.JavaConstant; +import jdk.vm.ci.meta.ResolvedJavaField; + +public class CommitAllocationLowerer { + + static class Materialization { + public VirtualObjectNode object; + public List values; + public boolean containsVirtual; + } + + /** See the comment inside {@link #lower}. */ + protected void lowerInstance(VirtualInstanceNode node, CodeGenTool codeGenTool) { + assert codeGenTool.declared(node) : "variable for VirtualInstanceNode must be declared"; + codeGenTool.lowerValue(node); + codeGenTool.genAssignment(); + codeGenTool.genNewInstance(node.type()); + codeGenTool.getCodeBuffer().emitInsEnd(); + } + + /** See the comment inside {@link #lower}. */ + protected void lowerArray(VirtualArrayNode node, CodeGenTool codeGenTool) { + assert codeGenTool.declared(node) : "variable for VirtualArrayNode must be declared"; + codeGenTool.lowerValue(node); + codeGenTool.genAssignment(); + + codeGenTool.genNewArray(node.type(), Emitter.of(node.entryCount())); + codeGenTool.getCodeBuffer().emitInsEnd(); + } + + public static List computeMaterializations(CommitAllocationNode commit) { + List mats = new ArrayList<>(); + int valindex = 0; + for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) { + Materialization mat = new Materialization(); + mat.object = commit.getVirtualObjects().get(objIndex); + mat.values = new ArrayList<>(); + VirtualObjectNode virtual = commit.getVirtualObjects().get(objIndex); + int entryCount = virtual.entryCount(); + for (int i = 0; i < entryCount; i++) { + ValueNode value = commit.getValues().get(valindex++); + if (value instanceof VirtualObjectNode) { + mat.containsVirtual = true; + } + mat.values.add(value); + } + mats.add(mat); + } + return mats; + } + + /** + * Lower the initialization of a single field of a {@link VirtualInstanceNode} in a + * {@link CommitAllocationNode}. + */ + protected void lowerFieldAssignment(VirtualInstanceNode instanceNode, ResolvedJavaField field, ValueNode value, CodeGenTool codeGenTool) { + codeGenTool.genPropertyAccess(Emitter.of(instanceNode), Emitter.of(field)); + codeGenTool.genAssignment(); + codeGenTool.lowerValue(value); + } + + /** + * Lowering of the virtual objects is very special in the sense that we need to ignore them from + * the schedule yet they are not the same as inlined nodes. + * + * The logic that handles them is scattered in three different places: + * + * (1) The inline logic dictates that virtual objects need variable allocation; + * + * (2) StackifierIRWalker#lowerNode does not issue them to NodeLowerer when processing the + * schedule; + * + * (3) The binding is generated here, i.e., at the schedule point of the associated + * CommitAllocationNode. + * + * Note: The same virtual object might be associated with several different CommitAllocationNode + * in different branches, thus it is safe to share the same variable name. + * + * The following is one example: + * + * 
+     * class Hello {
+     *     static int[] holder1 = null;
+     *     static int[] holder2 = null;
+     *
+     *     public static void main(String[] args) {
+     *         int[] array = new int[10];
+     *         if (args.length > 5) {
+     *             holder1 = array;
+     *         } else if (args.length < 1) {
+     *             holder2 = array;
+     *         } else {
+     *             System.out.println(array.length);
+     *         }
+     *     }
+     * }
+     *
+ * + * In the code above, the scheduled point for VirtualArrayNode is at the beginning of the + * method. We need to ignore the VirtualArrayNode there. Instead, we instantiate the array at + * the two CommitAllocationNodes, i.e., the first two branches just before the leaking of the + * array. + */ + public void lower(CommitAllocationNode commit, CodeGenTool codeGenTool) { + List mats = computeMaterializations(commit); + for (Materialization materialization : mats) { + VirtualObjectNode virtual = materialization.object; + if (virtual instanceof VirtualInstanceNode) { + lowerInstance((VirtualInstanceNode) virtual, codeGenTool); + } else { + lowerArray((VirtualArrayNode) virtual, codeGenTool); + } + } + + for (Materialization mat : mats) { + VirtualObjectNode virtual = mat.object; + for (int propertyNum = 0; propertyNum < mat.values.size(); propertyNum++) { + ValueNode value = mat.values.get(propertyNum); + if (value != null && !JavaConstant.NULL_POINTER.equals(value.asJavaConstant())) { + if (virtual instanceof VirtualInstanceNode instanceNode) { + ResolvedJavaField field = instanceNode.field(propertyNum); + lowerFieldAssignment(instanceNode, field, value, codeGenTool); + codeGenTool.genResolvedVarDeclPostfix("Materialize virtual object property assignment"); + } else { + if (value.isJavaConstant()) { + JavaConstant javaConstant = value.asJavaConstant(); + if (javaConstant.isDefaultForKind()) { + /* + * Default values are already assigned in the array initialization + * function. + */ + continue; + } + } + codeGenTool.genArrayStore(Emitter.of(propertyNum), virtual, value); + codeGenTool.genResolvedVarDeclPostfix("Materialize virtual array assignment"); + } + } + } + } + } +} From 4b3e732eb8ffd92979267d31a4292c20803e8050 Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Wed, 15 Mar 2023 15:43:37 +0100 Subject: [PATCH 09/43] Support lowering of WithExceptionNodes with a non-ExceptionObjectNode exception edge --- .../compiler/hightiercodegen/CodeGenTool.java | 5 ++-- .../irwalk/StackifierIRWalker.java | 24 ++++++++++++++++--- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java index d6091b9498b..62b25bb423d 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java @@ -38,12 +38,11 @@ import org.graalvm.compiler.nodes.ParameterNode; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.ValueNode; -import org.graalvm.compiler.nodes.java.ExceptionObjectNode; +import org.graalvm.compiler.nodes.spi.CoreProviders; import jdk.vm.ci.meta.ResolvedJavaField; import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.meta.ResolvedJavaType; -import org.graalvm.compiler.nodes.spi.CoreProviders; public abstract class CodeGenTool { @@ -372,7 +371,7 @@ public void genCatchBlockPrefix(String expName, ResolvedJavaType javaType) { public abstract void genThrow(ValueNode exception); @SuppressWarnings("deprecation") - public String getExceptionObjectId(ExceptionObjectNode n) { + public String getExceptionObjectId(Node n) { return "exception_object_" + n.getId(); } diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java index c541950d5f6..2207269615c 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java @@ -450,16 +450,29 @@ private void lowerWithExceptionStackifier(HIRBlock currentBlock, WithExceptionNo HIRBlock excpSucc = cfg.blockFor(lastNode.exceptionEdge()); CatchScopeContainer scopeEntry = (CatchScopeContainer) stackifierData.getScopeEntry(lastNode); Scope catchScope = scopeEntry.getCatchScope(); + + codeGenTool.genTryBlock(); /* * Since the ExceptionObjectNode is needed inside the catch block already, the * ExceptionObjectNode is lowered here together with the InvokeWithExceptionNode and skipped * once it is encountered in its own basic block. */ - ExceptionObjectNode excpObj = (ExceptionObjectNode) excpSucc.getBeginNode(); - codeGenTool.genTryBlock(); lowerNode(lastNode); generateForwardJump(currentBlock, normSucc, stackifierData); - codeGenTool.genCatchBlockPrefix(codeGenTool.getExceptionObjectId(excpObj), excpObj.stamp(NodeView.DEFAULT)); + + String caughtObjectName = codeGenTool.getExceptionObjectId(excpSucc.getBeginNode()); + ResolvedJavaType caughtObjectType = codeGenTool.getProviders().getMetaAccess().lookupJavaType(Throwable.class); + /* + * The exception edge does not necessarily have to be an ExceptionObjectNode. It could also, + * for example, be an UnreachableBeginNode. To cover those instances, the default type of + * the caught object is set to Throwable, and only changed if the exception edge indeed is + * an ExceptionObjectNode + */ + if (excpSucc.getBeginNode()instanceof ExceptionObjectNode excpObj) { + caughtObjectType = excpObj.stamp(NodeView.DEFAULT).javaType(codeGenTool.getProviders().getMetaAccess()); + } + codeGenTool.genCatchBlockPrefix(caughtObjectName, caughtObjectType); + if (catchScope != null) { lowerBlocks(catchScope.getSortedBlocks()); } else { @@ -526,6 +539,11 @@ protected void lowerSwitchCase(IntegerSwitchNode switchNode, AbstractBeginNode s codeGenTool.genSwitchCase(keys); } + /** + * Generates code for the default case of an integer switch. + * + * @param switchNode the {@link IntegerSwitchNode} that is being lowered. + */ protected void lowerSwitchDefaultCase(IntegerSwitchNode switchNode) { codeGenTool.genSwitchDefaultCase(); } From e420f3ef9cf2f6c5132c8fd5e5e022923222e04d Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Mon, 20 Mar 2023 11:20:25 +0100 Subject: [PATCH 10/43] Add tests for incorrect usage of GraalDirectives.injectSwitchCaseProbability --- .../SwitchCaseProbabilityDirectiveTest.java | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/compiler/src/org.graalvm.compiler.api.directives.test/src/org/graalvm/compiler/api/directives/test/SwitchCaseProbabilityDirectiveTest.java b/compiler/src/org.graalvm.compiler.api.directives.test/src/org/graalvm/compiler/api/directives/test/SwitchCaseProbabilityDirectiveTest.java index 57168bcb4c5..699fed91b94 100644 --- a/compiler/src/org.graalvm.compiler.api.directives.test/src/org/graalvm/compiler/api/directives/test/SwitchCaseProbabilityDirectiveTest.java +++ b/compiler/src/org.graalvm.compiler.api.directives.test/src/org/graalvm/compiler/api/directives/test/SwitchCaseProbabilityDirectiveTest.java @@ -90,6 +90,60 @@ public void testSwitch() { test("keyHoleSwitchSnippet", 4); } + public static int missingProbability(int x) { + switch (x) { + case 1: + GraalDirectives.injectSwitchCaseProbability(0.25); + return 10; + case 2: + GraalDirectives.injectSwitchCaseProbability(0.25); + return 20; + case 3: + GraalDirectives.injectSwitchCaseProbability(0.25); + return 3; + case 4: + GraalDirectives.injectSwitchCaseProbability(0.25); + return 15; + default: + /* + * No probability usage here, which is an error, even though the other branches' + * probability adds up to 1 + */ + return 42; + } + } + + public static int incorrectTotalProbability(int x) { + /* + * Total probability across branches should add up to 1, and if it doesn't an error should + * be thrown. + */ + switch (x) { + case 1: + GraalDirectives.injectSwitchCaseProbability(0.20); + return 10; + case 2: + GraalDirectives.injectSwitchCaseProbability(0.20); + return 20; + case 3: + GraalDirectives.injectSwitchCaseProbability(0.20); + return 3; + default: + GraalDirectives.injectSwitchCaseProbability(0.20); + return 42; + } + } + + @Test(expected = GraalError.class) + public void testMissingProbability() { + test("missingProbability", 1); + } + + @Test(expected = GraalError.class) + public void testIncorrectTotalProbability() { + test("incorrectTotalProbability", 3); + } + @Override protected void checkLowTierGraph(StructuredGraph graph) { NodeIterable switchProbabilityInjectionNodes = graph.getNodes().filter(SwitchCaseProbabilityNode.class); From e1ea3a33445f811ee9da05de21fba5113422512f Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Tue, 28 Mar 2023 09:04:40 +0200 Subject: [PATCH 11/43] Fix VariableAllocation NullPointerException --- .../hightiercodegen/variables/VariableAllocation.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/variables/VariableAllocation.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/variables/VariableAllocation.java index 99b86220888..d16414d53d5 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/variables/VariableAllocation.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/variables/VariableAllocation.java @@ -33,6 +33,8 @@ import org.graalvm.compiler.graph.Node; import org.graalvm.compiler.graph.NodeInputList; import org.graalvm.compiler.graph.NodeMap; +import org.graalvm.compiler.hightiercodegen.CodeGenTool; +import org.graalvm.compiler.hightiercodegen.NodeLowerer; import org.graalvm.compiler.nodes.AbstractBeginNode; import org.graalvm.compiler.nodes.AbstractMergeNode; import org.graalvm.compiler.nodes.CallTargetNode; @@ -44,9 +46,6 @@ import org.graalvm.compiler.nodes.ValuePhiNode; import org.graalvm.compiler.nodes.cfg.HIRBlock; -import org.graalvm.compiler.hightiercodegen.CodeGenTool; -import org.graalvm.compiler.hightiercodegen.NodeLowerer; - /** * Performs variable allocation on SSA form. *

@@ -138,6 +137,10 @@ protected boolean isSafeToInline(ValueNode node, SafetyPolicy policy, CodeGenToo return false; } + if (thisNodeBlock == null) { + return false; + } + // Only safe if none of the block successors are merges. for (int i = 0; i < thisNodeBlock.getSuccessorCount(); i++) { AbstractBeginNode b = thisNodeBlock.getSuccessorAt(i).getBeginNode(); From 9c10062fc82da757b07ebd6019de8c3879a4139e Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Wed, 29 Mar 2023 11:03:28 +0200 Subject: [PATCH 12/43] Pass ValueNode to CodeGenTool#genResolvedVarDecl instead of Stamp --- .../org/graalvm/compiler/hightiercodegen/CodeBuffer.java | 2 +- .../org/graalvm/compiler/hightiercodegen/CodeGenTool.java | 6 +++--- .../compiler/hightiercodegen/lowerer/PhiResolveLowerer.java | 6 ++---- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeBuffer.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeBuffer.java index 78b2e3556be..e0261159257 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeBuffer.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeBuffer.java @@ -293,7 +293,7 @@ public void emitDeclPrefix(String name) { emitWhiteSpace(); } - public void emitDeclPrefix(ResolvedJavaType type, String name) { + public void emitDeclPrefix(String name, ResolvedJavaType type) { emitDeclarationPrefix(type); emitDeclarationName(name); emitWhiteSpace(); diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java index 62b25bb423d..97818f5e0b1 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java @@ -301,12 +301,12 @@ public void genResolvedVarDeclPrefix(String name) { codeBuffer.emitDeclPrefix(name); } - public void genResolvedVarDeclPrefix(String name, Stamp stamp) { - genResolvedVarDeclPrefix(name, stamp.javaType(getProviders().getMetaAccess())); + public void genResolvedVarDeclPrefix(String name, ValueNode node) { + genResolvedVarDeclPrefix(name); } public void genResolvedVarDeclPrefix(String name, ResolvedJavaType javaType) { - codeBuffer.emitDeclPrefix(javaType, name); + codeBuffer.emitDeclPrefix(name, javaType); } public abstract void genResolvedVarDeclPostfix(String comment); diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/PhiResolveLowerer.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/PhiResolveLowerer.java index ad4eb56c6aa..e2794483a86 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/PhiResolveLowerer.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/PhiResolveLowerer.java @@ -24,14 +24,12 @@ */ package org.graalvm.compiler.hightiercodegen.lowerer; +import org.graalvm.compiler.hightiercodegen.CodeGenTool; import org.graalvm.compiler.nodes.AbstractEndNode; import org.graalvm.compiler.nodes.AbstractMergeNode; -import org.graalvm.compiler.nodes.NodeView; import org.graalvm.compiler.nodes.ValueNode; import org.graalvm.compiler.nodes.ValuePhiNode; -import org.graalvm.compiler.hightiercodegen.CodeGenTool; - /** * * Resolves phi's on end node lowering. This component is responsible for scheduling phi-to-phi @@ -75,7 +73,7 @@ public void lower(CodeGenTool codeGenTool) { if (tmpName == null) { // This is the first move into the temp variable, declare it first. tmpName = "TEMP_" + codeGenTool.genUniqueID(); - codeGenTool.genResolvedVarDeclPrefix(tmpName, move.source.stamp(NodeView.DEFAULT)); + codeGenTool.genResolvedVarDeclPrefix(tmpName, move.source); } else { codeGenTool.genResolvedVarAssignmentPrefix(tmpName); } From 900d0f5968501c0a27aa5bab6d11d55e4a77950b Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Thu, 30 Mar 2023 11:39:44 +0200 Subject: [PATCH 13/43] Cast array accesses on java.lang.Object to array type --- .../org/graalvm/compiler/hightiercodegen/CodeGenTool.java | 8 ++++++-- .../org/graalvm/compiler/hightiercodegen/NodeLowerer.java | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java index 97818f5e0b1..7b50cb5e2c2 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java @@ -170,9 +170,13 @@ public void genFunctionEnd() { public abstract void genMethodHeader(StructuredGraph graph, ResolvedJavaMethod method, List parameters); public void genArrayLoad(ValueNode index, ValueNode array) { - nodeLowerer().lowerValue(array); + genArrayLoad(Emitter.of(index), Emitter.of(array)); + } + + public void genArrayLoad(IEmitter index, IEmitter array) { + lower(array); genArrayAccessPrefix(); - nodeLowerer().lowerValue(index); + lower(index); genArrayAccessPostfix(); } diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/NodeLowerer.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/NodeLowerer.java index b6086063c97..310cd7dfed3 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/NodeLowerer.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/NodeLowerer.java @@ -153,7 +153,7 @@ public void lowerStatement(Node node) { * {@link #lowerStatement(Node)}. */ public final void lowerValue(ValueNode node) { - assert isActiveValueNode(node); + assert isActiveValueNode(node) : "Attempted to lower " + node + " which is not an active value node"; ResolvedVar resolvedVar = codeGenTool.getAllocatedVariable(node); if (resolvedVar != null) { assert resolvedVar.isDefinitionLowered() : "Variable definition for node " + node + " was not lowered before use"; From 3e32282651daa9a717c7a67db8cf5695e8a09df7 Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Tue, 11 Apr 2023 21:05:54 +0200 Subject: [PATCH 14/43] Refactor array element assignment in CommitAllocationLowerer to own function for extensibility --- .../lowerer/CommitAllocationLowerer.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java index 8d4ba882b6d..21751f5d0e7 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java @@ -96,6 +96,14 @@ protected void lowerFieldAssignment(VirtualInstanceNode instanceNode, ResolvedJa codeGenTool.lowerValue(value); } + /** + * Lower the initialization of a single element of a {@link VirtualArrayNode} in a + * {@link CommitAllocationNode}. + */ + protected void lowerArrayElementAssignment(VirtualArrayNode array, int index, ValueNode value, CodeGenTool codeGenTool) { + codeGenTool.genArrayStore(Emitter.of(index), array, value); + } + /** * Lowering of the virtual objects is very special in the sense that we need to ignore them from * the schedule yet they are not the same as inlined nodes. @@ -169,7 +177,7 @@ public void lower(CommitAllocationNode commit, CodeGenTool codeGenTool) { continue; } } - codeGenTool.genArrayStore(Emitter.of(propertyNum), virtual, value); + lowerArrayElementAssignment((VirtualArrayNode) virtual, propertyNum, value, codeGenTool); codeGenTool.genResolvedVarDeclPostfix("Materialize virtual array assignment"); } } From 369938a452ca8003ba61dd420636f9900de4f782 Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Tue, 11 Apr 2023 21:20:03 +0200 Subject: [PATCH 15/43] Add IEmitter version of CodeGenTool#genReturn --- .../graalvm/compiler/hightiercodegen/CodeGenTool.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java index 7b50cb5e2c2..0f94f752698 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java @@ -262,9 +262,13 @@ public void genReturn(String returnValue) { codeBuffer.emitInsEnd(); } - public void genReturn(ValueNode returnValue) { + public void genReturn(IEmitter returnValue) { genReturnPrefix(); - lowerValue(returnValue); + lower(returnValue); + } + + public void genReturn(ValueNode returnValue) { + genReturn(Emitter.of(returnValue)); } private void genReturnPrefix() { @@ -305,6 +309,7 @@ public void genResolvedVarDeclPrefix(String name) { codeBuffer.emitDeclPrefix(name); } + @SuppressWarnings("unused") public void genResolvedVarDeclPrefix(String name, ValueNode node) { genResolvedVarDeclPrefix(name); } From dc23d839b7920ab0090d210adbca03fb6a6b9e11 Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Thu, 13 Apr 2023 22:12:07 +0200 Subject: [PATCH 16/43] Fix switch case probability injection when blockSuccessor is nested LoopExit --- .../compiler/nodes/extended/SwitchNode.java | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java index 9d6c100b6e2..e5b406fbbed 100644 --- a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java +++ b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java @@ -282,8 +282,18 @@ protected void injectBranchProbabilities() { double[] nodeProbabilities = new double[keySuccessors.length]; for (int i = 0; i < keySuccessors.length; ++i) { - GraalError.guarantee(keySuccessor(i).next() instanceof SwitchCaseProbabilityNode, "All switch branches must be followed by a SwitchCaseProbabilityNode"); - SwitchCaseProbabilityNode caseProbabilityNode = (SwitchCaseProbabilityNode) keySuccessor(i).next(); + AbstractBeginNode succ = keySuccessor(i); + /* + * When a switch case exits out of a nested loop the SwitchProbabilityNode will be + * placed after a series of LoopExits, one per noop lesting level. + */ + while (succ.next()instanceof AbstractBeginNode next) { + succ = next; + } + assertTrue(succ.next() instanceof SwitchCaseProbabilityNode, + "Cannot inject switch probability, since key successor %s is not a SwitchCaseProbabilityNode", + this, succ.next()); + SwitchCaseProbabilityNode caseProbabilityNode = (SwitchCaseProbabilityNode) succ.next(); ValueNode probabilityNode = caseProbabilityNode.getProbability(); if (!probabilityNode.isConstant()) { @@ -310,7 +320,11 @@ protected void injectBranchProbabilities() { } for (AbstractBeginNode blockSuccessor : successors) { - SwitchCaseProbabilityNode caseProbabilityNode = (SwitchCaseProbabilityNode) blockSuccessor.next(); + AbstractBeginNode succ = blockSuccessor; + while (succ.next()instanceof AbstractBeginNode next) { + succ = next; + } + SwitchCaseProbabilityNode caseProbabilityNode = (SwitchCaseProbabilityNode) succ.next(); caseProbabilityNode.replaceAtUsages(null); graph().removeFixed(caseProbabilityNode); } From 96927bfed314b173745c630a61425040de9aef55 Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Fri, 14 Apr 2023 12:25:56 +0200 Subject: [PATCH 17/43] Restore breaks at loop ends --- .../irwalk/StackifierIRWalker.java | 41 ++++++++++--------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java index 2207269615c..b3940099b4e 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java @@ -65,6 +65,7 @@ import org.graalvm.compiler.nodes.extended.IntegerSwitchNode; import org.graalvm.compiler.nodes.java.ExceptionObjectNode; import org.graalvm.compiler.nodes.java.TypeSwitchNode; +import org.graalvm.compiler.replacements.nodes.BasicArrayCopyNode; import jdk.vm.ci.meta.ResolvedJavaType; @@ -302,9 +303,9 @@ private void lowerBlocks(HIRBlock[] blocks) { lowerSwitch((IntegerSwitchNode) lastNode, stackifierData); } else if (lastNode instanceof TypeSwitchNode) { lowerTypeSwitch((TypeSwitchNode) lastNode, stackifierData); - } else if (lastNode instanceof WithExceptionNode) { + } else if (isWithExceptionNode(lastNode)) { lowerWithExceptionStackifier(currentBlock, (WithExceptionNode) lastNode); - } else if (lastNode instanceof ControlSplitNode) { + } else if ((lastNode instanceof ControlSplitNode) && !(lastNode instanceof BasicArrayCopyNode)) { // BasicArrayCopyNode is also a ControlSplitNode assert false : "Unsupported control split node " + lastNode + " is not implemented yet"; } else if (lastNode instanceof LoopEndNode) { @@ -634,9 +635,7 @@ public void lowerSwitch(IntegerSwitchNode switchNode, StackifierData stackifierD continue; } } - ArrayList succKeys = new ArrayList<>(); - // query all keys that have the succ as block succ for (int keyIndex = 0; keyIndex < switchNode.keyCount(); keyIndex++) { // the key @@ -646,24 +645,20 @@ public void lowerSwitch(IntegerSwitchNode switchNode, StackifierData stackifierD succKeys.add(key); } } - assert succKeys.size() > 0 : "no keys of " + switchNode + " have " + succ + " as block successor"; - int[] succk = new int[succKeys.size()]; for (int s = 0; s < succKeys.size(); s++) { succk[s] = succKeys.get(s); } - lowerSwitchCase(switchNode, succ, succk); - if (caseScopes[i] != null) { lowerBlocks(caseScopes[i].getSortedBlocks()); } else { generateForwardJump(cfg.blockFor(switchNode), cfg.blockFor(succ), stackifierData); } + genBlockEndBreak(); codeGenTool.genScopeEnd(); } - if (hasdefault) { lowerSwitchDefaultCase(switchNode); int defaultIndex = switchNode.defaultSuccessorIndex(); @@ -672,6 +667,7 @@ public void lowerSwitch(IntegerSwitchNode switchNode, StackifierData stackifierD } else { generateForwardJump(cfg.blockFor(switchNode), cfg.blockFor(switchNode.defaultSuccessor()), stackifierData); } + genBlockEndBreak(); codeGenTool.genScopeEnd(); } codeGenTool.genScopeEnd(); @@ -789,11 +785,13 @@ public void lowerTypeSwitch(TypeSwitchNode switchNode, StackifierData stackifier assert succKeys.size() > 0 : "no keys of " + switchNode + " have " + succ + " as block successor"; lowerTypeSwitchCase(switchNode, succ, i, succKeys); generateForwardJump(cfg.blockFor(switchNode), cfg.blockFor(succ), stackifierData); + genBlockEndBreak(); codeGenTool.genScopeEnd(); } if (hasdefault) { lowerTypeSwitchDefaultCase(switchNode); generateForwardJump(cfg.blockFor(switchNode), cfg.blockFor(switchNode.defaultSuccessor()), stackifierData); + genBlockEndBreak(); codeGenTool.genScopeEnd(); } } @@ -819,24 +817,27 @@ private void genLoopHeader(HIRBlock block) { codeGenTool.genWhileTrueHeader(); } + /** + * Generates a break statement at the end of a loop or a switch case statement, + * used to kill the implicit loop back-edge. See comment in {@link #genLoopEnd} for further explanation. + */ + protected void genBlockEndBreak() { + codeGenTool.genBreak(); + } + private void genLoopEnd(HIRBlock header) { assert header.isLoopHeader(); String label = getLabel(header); blockNestingVerifier.popLabel(label); /* - * A break statement was originally emitted before the loop end in order to get rid of the - * implicit back edge. The explanation given was as follows: - * - * Example: If the Graal IR contains a loop with 2 back edges, it has 2 LoopEndNodes. The - * generated code will have 2 back-edges because of 2 continue statements for the - * LoopEndNodes and 1 back-edge that comes from the loop end. This back-edge needs to be - * suppressed (i.e. made unreachable) with the break statement to guarantee that the + * A break statement is always emitted before the loop end in order to get rid of the + * implicit back edge. Example: If the Graal IR contains a loop with 2 back edges, it has 2 + * LoopEndNodes. The generated code will have 2 back-edges because of 2 continue statements + * for the LoopEndNodes and 1 back-edge that comes from the loop end. This back-edge needs + * to be suppressed (i.e. made unreachable) with the break statement to guarantee that the * generated loop has the same semantics as the Graal IR. - * - * However, all blocks contained within the loop will themselves contain either a break or a - * continue statement, meaning that the final break before the loop is always unreachable - * and thus unnecessary. */ + genBlockEndBreak(); codeGenTool.genScopeEnd(); codeGenTool.genComment("End of loop " + label); } From 94c74cfd57905bf9a6fb28ce613d6516e75c72f0 Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Fri, 14 Apr 2023 16:51:25 +0200 Subject: [PATCH 18/43] Reformat code with Eclipse --- .../compiler/hightiercodegen/irwalk/StackifierIRWalker.java | 6 +++--- .../src/org/graalvm/compiler/nodes/extended/SwitchNode.java | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java index b3940099b4e..b875d079e9c 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java @@ -469,7 +469,7 @@ private void lowerWithExceptionStackifier(HIRBlock currentBlock, WithExceptionNo * the caught object is set to Throwable, and only changed if the exception edge indeed is * an ExceptionObjectNode */ - if (excpSucc.getBeginNode()instanceof ExceptionObjectNode excpObj) { + if (excpSucc.getBeginNode() instanceof ExceptionObjectNode excpObj) { caughtObjectType = excpObj.stamp(NodeView.DEFAULT).javaType(codeGenTool.getProviders().getMetaAccess()); } codeGenTool.genCatchBlockPrefix(caughtObjectName, caughtObjectType); @@ -818,8 +818,8 @@ private void genLoopHeader(HIRBlock block) { } /** - * Generates a break statement at the end of a loop or a switch case statement, - * used to kill the implicit loop back-edge. See comment in {@link #genLoopEnd} for further explanation. + * Generates a break statement at the end of a loop or a switch case statement, used to kill the + * implicit loop back-edge. See comment in {@link #genLoopEnd} for further explanation. */ protected void genBlockEndBreak() { codeGenTool.genBreak(); diff --git a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java index e5b406fbbed..5c832ef875b 100644 --- a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java +++ b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java @@ -287,7 +287,7 @@ protected void injectBranchProbabilities() { * When a switch case exits out of a nested loop the SwitchProbabilityNode will be * placed after a series of LoopExits, one per noop lesting level. */ - while (succ.next()instanceof AbstractBeginNode next) { + while (succ.next() instanceof AbstractBeginNode next) { succ = next; } assertTrue(succ.next() instanceof SwitchCaseProbabilityNode, @@ -321,7 +321,7 @@ protected void injectBranchProbabilities() { for (AbstractBeginNode blockSuccessor : successors) { AbstractBeginNode succ = blockSuccessor; - while (succ.next()instanceof AbstractBeginNode next) { + while (succ.next() instanceof AbstractBeginNode next) { succ = next; } SwitchCaseProbabilityNode caseProbabilityNode = (SwitchCaseProbabilityNode) succ.next(); From 72cb36cae7334b4314c878a4a85ebeb58d8be642 Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Tue, 18 Apr 2023 13:13:37 +0200 Subject: [PATCH 19/43] Move gen*() methods in StackifierIRWalker to CodeGenTool --- .../api/directives/GraalDirectives.java | 9 ++++---- .../compiler/hightiercodegen/CodeGenTool.java | 8 +++++++ .../irwalk/StackifierIRWalker.java | 18 ++++----------- .../lowerer/CommitAllocationLowerer.java | 23 +++++++++++-------- .../compiler/nodes/extended/SwitchNode.java | 1 + 5 files changed, 32 insertions(+), 27 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.api.directives/src/org/graalvm/compiler/api/directives/GraalDirectives.java b/compiler/src/org.graalvm.compiler.api.directives/src/org/graalvm/compiler/api/directives/GraalDirectives.java index 5fa7107c4af..4f4672608c1 100644 --- a/compiler/src/org.graalvm.compiler.api.directives/src/org/graalvm/compiler/api/directives/GraalDirectives.java +++ b/compiler/src/org.graalvm.compiler.api.directives/src/org/graalvm/compiler/api/directives/GraalDirectives.java @@ -208,11 +208,10 @@ public static boolean injectIterationCount(double iterations, boolean condition) } /** - * Injects a probability into the profiling information of a branch of a switch instruction. The - * probability must be a value between 0.0 and 1.0 (inclusive). This directive should only be - * used as the first statement of each branch of a switch statement. Either all or none of the - * branches should contain a call to injectSwitchCaseProbability, and the sum of the values - * across all branches must be 1.0. + * Injects a probability into the profiling information of a switch branch. The probability must + * be a value between 0.0 and 1.0 (inclusive). This directive should only be used as the first + * statement of each switch branch. Either all or none of the branches should contain a call to + * injectSwitchCaseProbability, and the sum of the values across all branches must be 1.0. * * Example usage: * diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java index 0f94f752698..f439feb6907 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java @@ -293,6 +293,14 @@ public void genBreak(String label) { codeBuffer.emitBreakLabel(label); } + /** + * Generates a break statement at the end of a loop or a switch case statement, used to kill the + * implicit loop back-edge. + */ + public void genBlockEndBreak() { + genBreak(); + } + public void genLabel(String label) { codeBuffer.emitLabel(label); } diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java index b875d079e9c..22cb4ecc5d5 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java @@ -656,7 +656,7 @@ public void lowerSwitch(IntegerSwitchNode switchNode, StackifierData stackifierD } else { generateForwardJump(cfg.blockFor(switchNode), cfg.blockFor(succ), stackifierData); } - genBlockEndBreak(); + codeGenTool.genBlockEndBreak(); codeGenTool.genScopeEnd(); } if (hasdefault) { @@ -667,7 +667,7 @@ public void lowerSwitch(IntegerSwitchNode switchNode, StackifierData stackifierD } else { generateForwardJump(cfg.blockFor(switchNode), cfg.blockFor(switchNode.defaultSuccessor()), stackifierData); } - genBlockEndBreak(); + codeGenTool.genBlockEndBreak(); codeGenTool.genScopeEnd(); } codeGenTool.genScopeEnd(); @@ -785,13 +785,13 @@ public void lowerTypeSwitch(TypeSwitchNode switchNode, StackifierData stackifier assert succKeys.size() > 0 : "no keys of " + switchNode + " have " + succ + " as block successor"; lowerTypeSwitchCase(switchNode, succ, i, succKeys); generateForwardJump(cfg.blockFor(switchNode), cfg.blockFor(succ), stackifierData); - genBlockEndBreak(); + codeGenTool.genBlockEndBreak(); codeGenTool.genScopeEnd(); } if (hasdefault) { lowerTypeSwitchDefaultCase(switchNode); generateForwardJump(cfg.blockFor(switchNode), cfg.blockFor(switchNode.defaultSuccessor()), stackifierData); - genBlockEndBreak(); + codeGenTool.genBlockEndBreak(); codeGenTool.genScopeEnd(); } } @@ -817,14 +817,6 @@ private void genLoopHeader(HIRBlock block) { codeGenTool.genWhileTrueHeader(); } - /** - * Generates a break statement at the end of a loop or a switch case statement, used to kill the - * implicit loop back-edge. See comment in {@link #genLoopEnd} for further explanation. - */ - protected void genBlockEndBreak() { - codeGenTool.genBreak(); - } - private void genLoopEnd(HIRBlock header) { assert header.isLoopHeader(); String label = getLabel(header); @@ -837,7 +829,7 @@ private void genLoopEnd(HIRBlock header) { * to be suppressed (i.e. made unreachable) with the break statement to guarantee that the * generated loop has the same semantics as the Graal IR. */ - genBlockEndBreak(); + codeGenTool.genBlockEndBreak(); codeGenTool.genScopeEnd(); codeGenTool.genComment("End of loop " + label); } diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java index 21751f5d0e7..e6469780151 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java @@ -41,9 +41,15 @@ public class CommitAllocationLowerer { static class Materialization { - public VirtualObjectNode object; - public List values; - public boolean containsVirtual; + Materialization(VirtualObjectNode object, List values, boolean containsVirtual) { + this.object = object; + this.values = values; + this.containsVirtual = containsVirtual; + } + + public final VirtualObjectNode object; + public final List values; + public final boolean containsVirtual; } /** See the comment inside {@link #lower}. */ @@ -69,19 +75,18 @@ public static List computeMaterializations(CommitAllocationNode List mats = new ArrayList<>(); int valindex = 0; for (int objIndex = 0; objIndex < commit.getVirtualObjects().size(); objIndex++) { - Materialization mat = new Materialization(); - mat.object = commit.getVirtualObjects().get(objIndex); - mat.values = new ArrayList<>(); + List values = new ArrayList<>(); VirtualObjectNode virtual = commit.getVirtualObjects().get(objIndex); int entryCount = virtual.entryCount(); + boolean containsVirtual = false; for (int i = 0; i < entryCount; i++) { ValueNode value = commit.getValues().get(valindex++); if (value instanceof VirtualObjectNode) { - mat.containsVirtual = true; + containsVirtual = true; } - mat.values.add(value); + values.add(value); } - mats.add(mat); + mats.add(new Materialization(virtual, values, containsVirtual)); } return mats; } diff --git a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java index 5c832ef875b..dcdca26442b 100644 --- a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java +++ b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/extended/SwitchNode.java @@ -313,6 +313,7 @@ protected void injectBranchProbabilities() { /* * We allow NaN if the node is in unreachable code that will eventually fall away, * or else an error will be thrown during lowering since we keep the node around. + * See analogous case in BranchProbabilityNode. */ return; } From 50ae489d83535c6331e99aa73ade095f69125fcf Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Wed, 19 Apr 2023 14:29:37 +0200 Subject: [PATCH 20/43] More fixes for directives, turn injectProfile into instanceOf directive --- .../src/org/graalvm/compiler/nodes/SnippetAnchorNode.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/SnippetAnchorNode.java b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/SnippetAnchorNode.java index cabd8d82529..1f93a51304a 100644 --- a/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/SnippetAnchorNode.java +++ b/compiler/src/org.graalvm.compiler.nodes/src/org/graalvm/compiler/nodes/SnippetAnchorNode.java @@ -32,13 +32,14 @@ import org.graalvm.compiler.core.common.type.StampFactory; import org.graalvm.compiler.graph.NodeClass; -import org.graalvm.compiler.nodes.spi.Simplifiable; -import org.graalvm.compiler.nodes.spi.SimplifierTool; import org.graalvm.compiler.nodeinfo.NodeInfo; +import org.graalvm.compiler.nodes.extended.AnchoringNode; import org.graalvm.compiler.nodes.extended.GuardingNode; +import org.graalvm.compiler.nodes.spi.Simplifiable; +import org.graalvm.compiler.nodes.spi.SimplifierTool; @NodeInfo(allowedUsageTypes = {Value, Anchor, Guard}, cycles = CYCLES_0, size = SIZE_0) -public final class SnippetAnchorNode extends FixedWithNextNode implements Simplifiable, GuardingNode { +public final class SnippetAnchorNode extends FixedWithNextNode implements Simplifiable, GuardingNode, AnchoringNode { public static final NodeClass TYPE = NodeClass.create(SnippetAnchorNode.class); public SnippetAnchorNode() { From 5eee3ac9d7701c360b3912c703478297251cf965 Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Wed, 19 Apr 2023 14:41:46 +0200 Subject: [PATCH 21/43] fixup! Add high tier code generation API for StructuredGraph --- .../compiler/hightiercodegen/irwalk/StackifierIRWalker.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java index 22cb4ecc5d5..a2f96742bda 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java @@ -469,7 +469,7 @@ private void lowerWithExceptionStackifier(HIRBlock currentBlock, WithExceptionNo * the caught object is set to Throwable, and only changed if the exception edge indeed is * an ExceptionObjectNode */ - if (excpSucc.getBeginNode() instanceof ExceptionObjectNode excpObj) { + if (excpSucc.getBeginNode()instanceof ExceptionObjectNode excpObj) { caughtObjectType = excpObj.stamp(NodeView.DEFAULT).javaType(codeGenTool.getProviders().getMetaAccess()); } codeGenTool.genCatchBlockPrefix(caughtObjectName, caughtObjectType); @@ -836,6 +836,6 @@ private void genLoopEnd(HIRBlock header) { private static String getLabel(HIRBlock block) { assert block.isLoopHeader(); - return LABEL_PREFIX + (int) block.getId(); + return LABEL_PREFIX + block.getId(); } } From 2be228556f5fad35ce916118a5d846b0f89baa3b Mon Sep 17 00:00:00 2001 From: Christian Wimmer Date: Fri, 21 Apr 2023 15:16:35 -0700 Subject: [PATCH 22/43] Remove code for old JDK versions --- .../compiler/core/test/MarkUnsafeAccessTest.java | 2 -- .../hotspot/test/BoxDeoptimizationTest.java | 13 ------------- .../replacements/test/UnsafeBooleanAccessTest.java | 10 ---------- .../compiler/truffle/compiler/PartialEvaluator.java | 2 +- .../reflect/ReflectionMetadataEncoderImpl.java | 8 ++------ .../com/oracle/svm/truffle/TruffleJFRFeature.java | 9 ++++++--- 6 files changed, 9 insertions(+), 35 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/MarkUnsafeAccessTest.java b/compiler/src/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/MarkUnsafeAccessTest.java index 9b9b7e720b9..e0a05543652 100644 --- a/compiler/src/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/MarkUnsafeAccessTest.java +++ b/compiler/src/org.graalvm.compiler.core.test/src/org/graalvm/compiler/core/test/MarkUnsafeAccessTest.java @@ -40,7 +40,6 @@ import org.graalvm.compiler.phases.common.inlining.InliningPhase; import org.graalvm.compiler.phases.common.inlining.policy.InlineEverythingPolicy; import org.graalvm.compiler.phases.tiers.HighTierContext; -import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.junit.Assert; import org.junit.Assume; import org.junit.Test; @@ -120,7 +119,6 @@ public void testStandard() throws IOException { @Test public void testCompiled() throws IOException { Assume.assumeFalse("Crashes on AArch64 (GR-8351)", System.getProperty("os.arch").equalsIgnoreCase("aarch64")); - Assume.assumeTrue("JDK-8259360", JavaVersionUtil.JAVA_SPEC < 16); ResolvedJavaMethod getMethod = asResolvedJavaMethod(getMethod(ByteBuffer.class, "get", new Class[]{})); ResolvedJavaType mbbClass = getMetaAccess().lookupJavaType(MappedByteBuffer.class); AssumptionResult answer = mbbClass.findUniqueConcreteMethod(getMethod); diff --git a/compiler/src/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/BoxDeoptimizationTest.java b/compiler/src/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/BoxDeoptimizationTest.java index 347e6905e58..15b5f9cf6f8 100644 --- a/compiler/src/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/BoxDeoptimizationTest.java +++ b/compiler/src/org.graalvm.compiler.hotspot.test/src/org/graalvm/compiler/hotspot/test/BoxDeoptimizationTest.java @@ -24,20 +24,13 @@ */ package org.graalvm.compiler.hotspot.test; -import static org.graalvm.compiler.serviceprovider.JavaVersionUtil.JAVA_SPEC; - import org.graalvm.compiler.api.directives.GraalDirectives; import org.graalvm.compiler.core.test.GraalCompilerTest; import org.junit.Assert; -import org.junit.Assume; import org.junit.Test; public class BoxDeoptimizationTest extends GraalCompilerTest { - private static void checkJDK() { - Assume.assumeTrue(JAVA_SPEC >= 13); - } - public static void testIntegerSnippet() { Object[] values = {42, -42, new Exception()}; GraalDirectives.deoptimize(); @@ -47,7 +40,6 @@ public static void testIntegerSnippet() { @Test public void testInteger() { - checkJDK(); test("testIntegerSnippet"); } @@ -62,7 +54,6 @@ public static void testLongSnippet() { @Test public void testLong() { - checkJDK(); test("testLongSnippet"); } @@ -75,7 +66,6 @@ public static void testCharSnippet() { @Test public void testChar() { - checkJDK(); test("testCharSnippet"); } @@ -88,7 +78,6 @@ public static void testShortSnippet() { @Test public void testShort() { - checkJDK(); test("testShortSnippet"); } @@ -101,7 +90,6 @@ public static void testByteSnippet() { @Test public void testByte() { - checkJDK(); test("testByteSnippet"); } @@ -114,7 +102,6 @@ public static void testBooleanSnippet() { @Test public void testBoolean() { - checkJDK(); test("testBooleanSnippet"); } } diff --git a/compiler/src/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/UnsafeBooleanAccessTest.java b/compiler/src/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/UnsafeBooleanAccessTest.java index 0efa1f17c6a..ce94f2d4731 100644 --- a/compiler/src/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/UnsafeBooleanAccessTest.java +++ b/compiler/src/org.graalvm.compiler.replacements.test/src/org/graalvm/compiler/replacements/test/UnsafeBooleanAccessTest.java @@ -27,9 +27,6 @@ import java.lang.reflect.Field; import org.graalvm.compiler.core.test.GraalCompilerTest; -import org.graalvm.compiler.serviceprovider.JavaVersionUtil; -import org.junit.Assume; -import org.junit.Before; import org.junit.Test; public class UnsafeBooleanAccessTest extends GraalCompilerTest { @@ -49,13 +46,6 @@ public class UnsafeBooleanAccessTest extends GraalCompilerTest { } } - @Before - public void testJDK() { - // Prior to JDK-8250825, C2 does not support reading a short field via an unaligned - // Unsafe.getBoolean, so -Xcomp can crash this test. - Assume.assumeTrue(JavaVersionUtil.JAVA_SPEC >= 16); - } - public static boolean testGetBooleanSnippet() { UNSAFE.putShort(onHeapMemoryBase, onHeapMemoryOffset, (short) 0x0204); return UNSAFE.getBoolean(onHeapMemoryBase, onHeapMemoryOffset); diff --git a/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/PartialEvaluator.java b/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/PartialEvaluator.java index e1b82f26470..650653fd5c3 100644 --- a/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/PartialEvaluator.java +++ b/compiler/src/org.graalvm.compiler.truffle.compiler/src/org/graalvm/compiler/truffle/compiler/PartialEvaluator.java @@ -459,7 +459,7 @@ private GraphBuilderConfiguration createGraphBuilderConfig(GraphBuilderConfigura } protected void appendParsingNodePlugins(Plugins plugins) { - if (JavaVersionUtil.JAVA_SPEC >= 16 && JavaVersionUtil.JAVA_SPEC < 19) { + if (JavaVersionUtil.JAVA_SPEC < 19) { ResolvedJavaType memorySegmentProxyType = TruffleCompilerEnvironment.get().runtime().resolveType(config.lastTier().providers().getMetaAccess(), "jdk.internal.access.foreign.MemorySegmentProxy"); for (ResolvedJavaMethod m : memorySegmentProxyType.getDeclaredMethods()) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/reflect/ReflectionMetadataEncoderImpl.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/reflect/ReflectionMetadataEncoderImpl.java index 83954c1ccad..396a6ab6acc 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/reflect/ReflectionMetadataEncoderImpl.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/reflect/ReflectionMetadataEncoderImpl.java @@ -69,7 +69,6 @@ import org.graalvm.compiler.api.replacements.SnippetReflectionProvider; import org.graalvm.compiler.core.common.util.TypeConversion; import org.graalvm.compiler.core.common.util.UnsafeArrayTypeWriter; -import org.graalvm.compiler.serviceprovider.JavaVersionUtil; import org.graalvm.nativeimage.AnnotationAccess; import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.nativeimage.impl.RuntimeReflectionSupport; @@ -323,7 +322,7 @@ private void registerEnclosingMethodInfo(Object[] enclosingMethodInfo) { private static final Method getPermittedSubclasses = ReflectionUtil.lookupMethod(true, Class.class, "getPermittedSubclasses"); private Class[] getPermittedSubclasses(MetaAccessProvider metaAccess, Class clazz) { - if (JavaVersionUtil.JAVA_SPEC < 17 || (dataBuilder.getEnabledReflectionQueries(clazz) & ALL_PERMITTED_SUBCLASSES_FLAG) == 0) { + if ((dataBuilder.getEnabledReflectionQueries(clazz) & ALL_PERMITTED_SUBCLASSES_FLAG) == 0) { return null; } try { @@ -335,7 +334,7 @@ private Class[] getPermittedSubclasses(MetaAccessProvider metaAccess, Class[] getNestMembers(MetaAccessProvider metaAccess, Class clazz) { - if (JavaVersionUtil.JAVA_SPEC < 17 || (dataBuilder.getEnabledReflectionQueries(clazz) & ALL_NEST_MEMBERS_FLAG) == 0) { + if ((dataBuilder.getEnabledReflectionQueries(clazz) & ALL_NEST_MEMBERS_FLAG) == 0) { return null; } return filterDeletedClasses(metaAccess, clazz.getNestMembers()); @@ -433,9 +432,6 @@ public void addReflectionExecutableMetadata(MetaAccessProvider metaAccess, Hoste private static final Method isFieldTrustedFinal = ReflectionUtil.lookupMethod(true, Field.class, "isTrustedFinal"); private static boolean isTrustedFinal(Field field) { - if (JavaVersionUtil.JAVA_SPEC < 17) { - return false; - } try { return (boolean) isFieldTrustedFinal.invoke(field); } catch (IllegalAccessException | InvocationTargetException e) { diff --git a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleJFRFeature.java b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleJFRFeature.java index 815d60a45e7..2b9d186f67e 100644 --- a/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleJFRFeature.java +++ b/substratevm/src/com.oracle.svm.truffle/src/com/oracle/svm/truffle/TruffleJFRFeature.java @@ -32,9 +32,9 @@ import org.graalvm.compiler.truffle.runtime.serviceprovider.TruffleRuntimeServices; import org.graalvm.nativeimage.ImageSingletons; +import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.feature.InternalFeature; import com.oracle.svm.core.jfr.JfrFeature; -import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature; import com.oracle.svm.core.util.UserError; @AutomaticallyRegisteredFeature @@ -54,7 +54,10 @@ public void afterRegistration(AfterRegistrationAccess access) { } private static boolean isEnabled() { - // The substratevm JFR implementation is not yet stable on the JDK 17, see GR-38866. - return ImageSingletons.contains(TruffleFeature.class) && ImageSingletons.contains(JfrFeature.class) && JavaVersionUtil.JAVA_SPEC < 17; + /* + * GR-38866: Does not work on JDK 17 due to the Truffle module not being open to the JFR + * module, but the problematic code was removed for later JDKs. + */ + return ImageSingletons.contains(TruffleFeature.class) && ImageSingletons.contains(JfrFeature.class) && JavaVersionUtil.JAVA_SPEC > 17; } } From 31785dbfe77ffa9bdc8eed58e4596639b8301731 Mon Sep 17 00:00:00 2001 From: Josef Eisl Date: Mon, 24 Apr 2023 11:12:12 +0200 Subject: [PATCH 23/43] svm: libfdm was ported to Java in JDK 21 [GR-45492] See https://bugs.openjdk.org/browse/JDK-8134780 and follow up issues. --- .../src/com/oracle/svm/core/jdk/JavaLangSubstitutions.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JavaLangSubstitutions.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JavaLangSubstitutions.java index 94f55aefdc2..51d4c6f978a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JavaLangSubstitutions.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JavaLangSubstitutions.java @@ -479,7 +479,7 @@ public static double pow(double a, double b) { } } -@TargetClass(java.lang.StrictMath.class) +@TargetClass(value = StrictMath.class, onlyWith = JDK20OrEarlier.class) @Platforms(InternalPlatform.NATIVE_ONLY.class) final class Target_java_lang_StrictMath { From 4cf27f0ec4888e9659e87b66ddea1d8378dd56ba Mon Sep 17 00:00:00 2001 From: Roland Schatz Date: Fri, 21 Apr 2023 16:12:53 +0200 Subject: [PATCH 24/43] Smoke test for the dependencies of the LLVM_TOOLCHAIN component. --- sdk/mx.sdk/mx_sdk.py | 2 + sdk/mx.sdk/mx_sdk_toolchain.py | 154 ++++++++++++++++++++ sdk/mx.sdk/suite.py | 6 + sdk/src/org.graalvm.toolchain.test/hello.c | 46 ++++++ sdk/src/org.graalvm.toolchain.test/hello.cc | 46 ++++++ 5 files changed, 254 insertions(+) create mode 100644 sdk/mx.sdk/mx_sdk_toolchain.py create mode 100644 sdk/src/org.graalvm.toolchain.test/hello.c create mode 100644 sdk/src/org.graalvm.toolchain.test/hello.cc diff --git a/sdk/mx.sdk/mx_sdk.py b/sdk/mx.sdk/mx_sdk.py index e408ada03c1..9d6ce464256 100644 --- a/sdk/mx.sdk/mx_sdk.py +++ b/sdk/mx.sdk/mx_sdk.py @@ -55,6 +55,8 @@ from mx_gate import Task from mx_unittest import unittest +# re-export custom mx project classes so they can be used from suite.py +from mx_sdk_toolchain import ToolchainTestProject # pylint: disable=unused-import _suite = mx.suite('sdk') diff --git a/sdk/mx.sdk/mx_sdk_toolchain.py b/sdk/mx.sdk/mx_sdk_toolchain.py new file mode 100644 index 00000000000..1fb2b80635d --- /dev/null +++ b/sdk/mx.sdk/mx_sdk_toolchain.py @@ -0,0 +1,154 @@ +# +# Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +# + +from __future__ import print_function +import os +import os.path + +import mx + +class ToolchainTestProject(mx.Project): + def __init__(self, suite, name, deps, workingSets, theLicense, **kwArgs): + d = os.path.join(suite.dir, kwArgs['subDir'], name) + super(ToolchainTestProject, self).__init__(suite, name, srcDirs=[], deps=deps, workingSets=workingSets, d=d, theLicense=theLicense, **kwArgs) + + def getBuildTask(self, args): + return ToolchainTestBuildTask(self, args, 1) + + def isPlatformDependent(self): + return True + + def getOutput(self, name): + return os.path.join(self.get_output_root(), mx.exe_suffix(name)) + + def getSource(self, name): + return os.path.join(self.dir, name) + + def getArchivableResults(self, use_relpath=True, single=False): + return [] + + def getResults(self): + return [self.getOutput(n) for n in ["hello", "hello-cxx"]] + +class ToolchainTestBuildTask(mx.BuildTask): + def __str__(self): + return "Testing " + self.subject.name + + def newestOutput(self): + return mx.TimeStampFile.newest(self.subject.getResults()) + + def needsBuild(self, newestInput): + for result in self.subject.getResults(): + if not os.path.exists(result): + return True, f'{result} does not exist' + return False, 'up to date' + + def build(self): + mx.ensure_dir_exists(self.subject.get_output_root()) + + toolchainPath = mx.distribution('LLVM_TOOLCHAIN').output + clang = os.path.join(toolchainPath, 'bin', 'clang') + clangxx = os.path.join(toolchainPath, 'bin', 'clang++') + + def runTest(cmd, onError=None, rerunOnFailure=False): + out = mx.OutputCapture() + status = mx.run(cmd, nonZeroIsFatal=False, out=out, err=out) + if status != 0: + mx.log_error("Failed command: " + " ".join(cmd)) + if rerunOnFailure: + mx.log("rerunning with -v") + mx.run(cmd + ['-v'], nonZeroIsFatal=False) + if onError: + onError(status) + return str(out) + + def runCompile(compiler, src, binary, onError): + sourceFile = self.subject.getSource(src) + binFile = self.subject.getOutput(binary) + if mx.is_darwin(): + compileCmd = ["xcrun", compiler] + else: + compileCmd = [compiler, "-fuse-ld=lld"] + runTest(compileCmd + [sourceFile, '-o', binFile], onError=onError, rerunOnFailure=True) + out = runTest([binFile], onError=lambda status: + mx.abort(f"{os.path.basename(compiler)} could compile {src}, but the result doesn't work. It returned with exit code {status}.") + ) + expected = "Hello, World!" + result = out.strip() + if result != expected: + mx.abort(f"{os.path.basename(compiler)} could compile {src}, but the result does not match (expected: \"{expected}\", got: \"{result}\").") + + runCompile(clang, "hello.c", "hello", onError=lambda status: + mx.abort("The LLVM toolchain does not work. Do you have development packages installed?") + ) + + runCompile(clangxx, "hello.cc", "hello-cxx", onError=lambda status: check_multiple_gcc_issue(clang)) + + def clean(self, forBuild=False): + if os.path.exists(self.subject.get_output_root()): + mx.rmtree(self.subject.get_output_root()) + +def check_multiple_gcc_issue(clang): + mx.log_error("The LLVM C++ compiler does not work. Do you have the libstdc++ development package installed?") + + # If there is more than one GCC version installed, the LLVM toolchain will always pick the + # newest version. If that version does not have the libstdc++ development package installed, it + # will not work. This can lead to confusing errors for the user, especially if an older version + # of gcc is installed with the proper development headers available. + candidates = [] + selected = None + + def captureCandidates(line): + nonlocal selected + if line.startswith("Found candidate GCC installation: "): + candidates.append(line.split(':')[1].strip()) + elif line.startswith("Selected GCC installation: "): + selected = line.split(':')[1].strip() + mx.run([clang, '-v'], err=captureCandidates) + + if len(candidates) > 1: + mx.log("Note that LLVM found multiple installations of GCC:") + for c in candidates: + mx.log(f"\t{c}") + mx.log(f"It decided to use this version:\n\t{selected}") + mx.log("Make sure you have the libstdc++-dev package for that specific version installed.") + + mx.abort(1) diff --git a/sdk/mx.sdk/suite.py b/sdk/mx.sdk/suite.py index bba44c2b190..dc262852461 100644 --- a/sdk/mx.sdk/suite.py +++ b/sdk/mx.sdk/suite.py @@ -367,6 +367,11 @@ "javaCompliance" : "17+", "workingSets" : "API,SDK", }, + "org.graalvm.toolchain.test" : { + "class" : "ToolchainTestProject", + "subDir" : "src", + "buildDependencies" : ["LLVM_TOOLCHAIN"], + }, }, "licenses" : { "UPL" : { @@ -683,6 +688,7 @@ }, "dependencies": [ "LLVM_TOOLCHAIN", + "org.graalvm.toolchain.test", ], }, }, diff --git a/sdk/src/org.graalvm.toolchain.test/hello.c b/sdk/src/org.graalvm.toolchain.test/hello.c new file mode 100644 index 00000000000..675b51f3e06 --- /dev/null +++ b/sdk/src/org.graalvm.toolchain.test/hello.c @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include + +int main() { + printf("Hello, World!\n"); + return 0; +} diff --git a/sdk/src/org.graalvm.toolchain.test/hello.cc b/sdk/src/org.graalvm.toolchain.test/hello.cc new file mode 100644 index 00000000000..97a79f9fa26 --- /dev/null +++ b/sdk/src/org.graalvm.toolchain.test/hello.cc @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2023, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * The Universal Permissive License (UPL), Version 1.0 + * + * Subject to the condition set forth below, permission is hereby granted to any + * person obtaining a copy of this software, associated documentation and/or + * data (collectively the "Software"), free of charge and under any and all + * copyright rights in the Software, and any and all patent rights owned or + * freely licensable by each licensor hereunder covering either (i) the + * unmodified Software as contributed to or provided by such licensor, or (ii) + * the Larger Works (as defined below), to deal in both + * + * (a) the Software, and + * + * (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if + * one is included with the Software each a "Larger Work" to which the Software + * is contributed by such licensors), + * + * without restriction, including without limitation the rights to copy, create + * derivative works of, display, perform, and distribute the Software and make, + * use, sell, offer for sale, import, export, have made, and have sold the + * Software and the Larger Work(s), and to sublicense the foregoing rights on + * either these or other terms. + * + * This license is subject to the following condition: + * + * The above copyright notice and either this complete permission notice or at a + * minimum a reference to the UPL must be included in all copies or substantial + * portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ +#include + +int main() { + std::cout << "Hello, World!" << std::endl; + return 0; +} From ae2b8c938dad6e02be1ba47f9d55d2617ddc0610 Mon Sep 17 00:00:00 2001 From: Roland Schatz Date: Mon, 24 Apr 2023 16:51:04 +0200 Subject: [PATCH 25/43] Add suggestions for how to install C++ dependencies on common distributions. --- sdk/mx.sdk/mx_sdk_toolchain.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/sdk/mx.sdk/mx_sdk_toolchain.py b/sdk/mx.sdk/mx_sdk_toolchain.py index 1fb2b80635d..fc98e2f0ff2 100644 --- a/sdk/mx.sdk/mx_sdk_toolchain.py +++ b/sdk/mx.sdk/mx_sdk_toolchain.py @@ -42,6 +42,7 @@ from __future__ import print_function import os import os.path +import re import mx @@ -126,6 +127,12 @@ def clean(self, forBuild=False): if os.path.exists(self.subject.get_output_root()): mx.rmtree(self.subject.get_output_root()) +_known_gcc_packages = [ + (r"/usr/lib/gcc/x86_64-redhat-linux/([0-9]+)", "Oracle Linux or Redhat", r"yum install gcc-c++"), + (r"/opt/rh/gcc-toolset-([0-9]+)/.*", "Oracle Linux or Redhat with gcc-toolset", r"yum install gcc-toolset-\1-libstdc++-devel"), + (r"/usr/lib/gcc/x86_64-linux-gnu/([0-9]+)", "Ubuntu", r"apt install libstdc++-\1-dev") +] + def check_multiple_gcc_issue(clang): mx.log_error("The LLVM C++ compiler does not work. Do you have the libstdc++ development package installed?") @@ -151,4 +158,11 @@ def captureCandidates(line): mx.log(f"It decided to use this version:\n\t{selected}") mx.log("Make sure you have the libstdc++-dev package for that specific version installed.") + if selected: + for (regex, dist, suggestion) in _known_gcc_packages: + m = re.fullmatch(regex, selected) + if m: + mx.log(f"Based on the GCC path, I'm guessing you're running on {dist}.\nTry running '{m.expand(suggestion)}'.") + break + mx.abort(1) From 4d2d82251183f9bb857c9e8cea9b6db87ebc85af Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Mon, 24 Apr 2023 21:11:45 +0200 Subject: [PATCH 26/43] Add more comments to classes in hightiercodegen --- .../compiler/hightiercodegen/CodeGenTool.java | 13 +++++++++---- .../graalvm/compiler/hightiercodegen/Emitter.java | 6 ++---- .../graalvm/compiler/hightiercodegen/IEmitter.java | 6 ++++++ .../compiler/hightiercodegen/NodeLowerer.java | 3 +++ .../compiler/hightiercodegen/irwalk/IRWalker.java | 5 ++++- .../hightiercodegen/irwalk/StackifierIRWalker.java | 4 ++++ .../lowerer/CommitAllocationLowerer.java | 3 +++ .../hightiercodegen/lowerer/PhiResolveLowerer.java | 2 -- .../stackifier/StackifierReconstructionPhase.java | 8 ++++++-- .../variables/VariableAllocation.java | 4 ++++ 10 files changed, 41 insertions(+), 13 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java index f439feb6907..c5590071c1d 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeGenTool.java @@ -44,6 +44,11 @@ import jdk.vm.ci.meta.ResolvedJavaMethod; import jdk.vm.ci.meta.ResolvedJavaType; +/** + * Abstract interface for code generation. Its methods operate at a generally higher level of + * abstraction (e.g. {@link #genArrayLoad}, {@link #genFunctionCall}) compared to + * {@link CodeBuffer}, which is mainly concerned with emitting individual tokens. + */ public abstract class CodeGenTool { protected final CodeBuffer codeBuffer; @@ -56,7 +61,7 @@ public abstract class CodeGenTool { /** * Used for generating method-scoped unique IDs. - * + *

* Its value is reset to 0 in {@link CodeGenTool#prepareForMethod(StructuredGraph)}. */ private int methodScopeUniqueID = 0; @@ -67,7 +72,7 @@ protected CodeGenTool(CodeBuffer codeBuffer, VariableAllocation variableAllocati } /** - * Generates an efficient representation of an integer value literal. + * Generates an efficient representation of an integral literal. */ public static String getEfficientIntLiteral(long i) { StringBuilder sb = new StringBuilder(); @@ -109,7 +114,7 @@ public boolean declared(Node n) { /** * Generate a unique ID within the scope of the current method. - * + *

* Invariant: each call of this method will return a different value for the current method * under lowering. */ @@ -351,7 +356,7 @@ public void genResolvedVarAssignmentPrefix(String name) { /** * Generates a comma-separated list of the given inputs. - * + *

* If an input is a {@link Node}, it is lowered in place, otherwise the object is converted to a * string. */ diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/Emitter.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/Emitter.java index 09a6ec10bbc..660de43c32b 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/Emitter.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/Emitter.java @@ -40,10 +40,8 @@ import jdk.vm.ci.meta.ResolvedJavaType; /** - * Facilitate lowering of various objects. - * - * Methods that want to accept multiple different types for lowering, especially as part of a list - * of arguments, can accept {@link Emitter} instead of having to creating many overloaded methods. + * This class holds utility methods to create instances of {@link IEmitter} to generate various code + * constructs, such as identifiers and literals of various types. */ public class Emitter implements IEmitter { private static final Emitter NULL = new Emitter(CodeGenTool::genNull); diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/IEmitter.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/IEmitter.java index a21b3a37f6c..5abb83c2500 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/IEmitter.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/IEmitter.java @@ -25,6 +25,12 @@ package org.graalvm.compiler.hightiercodegen; +/** + * Abstract interface to facilitate lowering of various objects. + * + * Methods that want to accept multiple different types for lowering, especially as part of a list + * of arguments, can accept {@link IEmitter} instead of having to create many overloaded methods. + */ public interface IEmitter { void lower(CodeGenTool codeGenTool); } diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/NodeLowerer.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/NodeLowerer.java index 310cd7dfed3..5893529ff40 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/NodeLowerer.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/NodeLowerer.java @@ -109,6 +109,9 @@ import org.graalvm.compiler.replacements.nodes.UnaryMathIntrinsicNode; import org.graalvm.compiler.word.WordCastNode; +/** + * This class is responsible for generating code for individual {@link Node}s in the graph. + */ public abstract class NodeLowerer { protected final CodeGenTool codeGenTool; diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/IRWalker.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/IRWalker.java index 8112d710398..dfed1ff10ec 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/IRWalker.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/IRWalker.java @@ -36,9 +36,12 @@ import org.graalvm.compiler.hightiercodegen.lowerer.PhiResolveLowerer; import org.graalvm.compiler.hightiercodegen.reconstruction.ReconstructionData; import org.graalvm.compiler.nodes.LoopEndNode; -import org.graalvm.compiler.nodes.cfg.HIRBlock; import org.graalvm.compiler.nodes.cfg.ControlFlowGraph; +import org.graalvm.compiler.nodes.cfg.HIRBlock; +/** + * Abstract interface for generating textual control flow in code from a {@link ControlFlowGraph}s. + */ @SuppressWarnings("try") public abstract class IRWalker { diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java index a2f96742bda..3cc1b7d0a83 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/irwalk/StackifierIRWalker.java @@ -69,6 +69,10 @@ import jdk.vm.ci.meta.ResolvedJavaType; +/** + * Generates code by using the Stackifier algorithm to handle control flow. See + * {@link #lower(DebugContext)} for an example. + */ public class StackifierIRWalker extends IRWalker { public static final String LABEL_PREFIX = "looplabel_"; protected final BlockNestingVerifier blockNestingVerifier; diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java index e6469780151..fddb55161fa 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/CommitAllocationLowerer.java @@ -38,6 +38,9 @@ import jdk.vm.ci.meta.JavaConstant; import jdk.vm.ci.meta.ResolvedJavaField; +/** + * Generates code for a {@link CommitAllocationNode} and its associated {@link VirtualObjectNode}s. + */ public class CommitAllocationLowerer { static class Materialization { diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/PhiResolveLowerer.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/PhiResolveLowerer.java index e2794483a86..d8e457917c4 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/PhiResolveLowerer.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/lowerer/PhiResolveLowerer.java @@ -31,12 +31,10 @@ import org.graalvm.compiler.nodes.ValuePhiNode; /** - * * Resolves phi's on end node lowering. This component is responsible for scheduling phi-to-phi * assignments at control flow merges. If there is a situation like a=b,b=a, no valid scheduling * without tmp vars exists that satisfies this relation. Therefore, phi resolving introduces * additional variables during code gen. - * */ public class PhiResolveLowerer { private final AbstractEndNode end; diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/reconstruction/stackifier/StackifierReconstructionPhase.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/reconstruction/stackifier/StackifierReconstructionPhase.java index f0d99c11a54..a507cbd6fbf 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/reconstruction/stackifier/StackifierReconstructionPhase.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/reconstruction/stackifier/StackifierReconstructionPhase.java @@ -26,9 +26,11 @@ import java.util.Optional; +import org.graalvm.compiler.hightiercodegen.irwalk.StackifierIRWalker; import org.graalvm.compiler.hightiercodegen.reconstruction.ReconstructionPhase; import org.graalvm.compiler.hightiercodegen.reconstruction.ScheduleWithReconstructionResult; import org.graalvm.compiler.hightiercodegen.reconstruction.StackifierData; +import org.graalvm.compiler.hightiercodegen.reconstruction.stackifier.blocks.LabeledBlockGeneration; import org.graalvm.compiler.nodes.GraphState; import org.graalvm.compiler.nodes.StructuredGraph; import org.graalvm.compiler.nodes.cfg.ControlFlowGraph; @@ -36,8 +38,10 @@ import org.graalvm.compiler.phases.BasePhase; import org.graalvm.compiler.phases.schedule.SchedulePhase; -import org.graalvm.compiler.hightiercodegen.reconstruction.stackifier.blocks.LabeledBlockGeneration; - +/** + * Computes a {@link ScheduleWithReconstructionResult} for the graph, meant specifically for use by + * {@link StackifierIRWalker}. + */ public class StackifierReconstructionPhase extends ReconstructionPhase { @Override protected void run(StructuredGraph graph, CoreProviders providers) { diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/variables/VariableAllocation.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/variables/VariableAllocation.java index d16414d53d5..c519cfd7afb 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/variables/VariableAllocation.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/variables/VariableAllocation.java @@ -137,6 +137,10 @@ protected boolean isSafeToInline(ValueNode node, SafetyPolicy policy, CodeGenToo return false; } + /* + * Unscheduled nodes such as ValueProxies are not in the nodeToBlockMap, so for + * safety in this case we disable inlining. + */ if (thisNodeBlock == null) { return false; } From d9cba7bbe0af26a9a1d24b980026f883151082f5 Mon Sep 17 00:00:00 2001 From: Carlo Refice Date: Tue, 25 Apr 2023 11:33:43 +0200 Subject: [PATCH 27/43] Refactor CodeBuffer to allow implementations to override indentation string --- .../compiler/hightiercodegen/CodeBuffer.java | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeBuffer.java b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeBuffer.java index e0261159257..89d09563302 100644 --- a/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeBuffer.java +++ b/compiler/src/org.graalvm.compiler.hightiercodegen/src/org/graalvm/compiler/hightiercodegen/CodeBuffer.java @@ -45,11 +45,6 @@ */ public abstract class CodeBuffer { - /** - * Prefix for indent. - */ - public static final String INDENT_PREFIX = "\t"; - /** * Current indent level. */ @@ -129,9 +124,19 @@ public void emitCode(byte[] b) { codeBytes.write(b, 0, b.length); } + /** + * Emit whitespace for one level of indentation. + */ + protected void emitSingleIndent() { + emitText("\t"); + } + + /** + * Emit whitespace for the current level of indentation. + */ private void emitIndent() { for (int i = 0; i < scopeIndent; i++) { - emitText(INDENT_PREFIX); + emitSingleIndent(); } } From 578810cfd41ff616f27a6ac520d4048627801aca Mon Sep 17 00:00:00 2001 From: Francois Farquet Date: Tue, 25 Apr 2023 16:14:23 +0200 Subject: [PATCH 28/43] Fix metric.name in NI compile-time memory extraction --- vm/mx.vm/mx_vm_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm/mx.vm/mx_vm_benchmark.py b/vm/mx.vm/mx_vm_benchmark.py index 224d0a18f0e..fae9e05fbbc 100644 --- a/vm/mx.vm/mx_vm_benchmark.py +++ b/vm/mx.vm/mx_vm_benchmark.py @@ -808,7 +808,7 @@ def __call__(self, *args, **kwargs): rules.append( mx_benchmark.JsonFixedFileRule(self.config.image_build_stats_file, { "benchmark": benchmarks[0], - "metric.name": "analysis-stats", + "metric.name": "compile-time", "metric.type": "numeric", "metric.unit": "B", "metric.value": ("<" + value_name + ">", NativeImageTimeToInt()), From becc043601e52098c0d5cc221a1256185bebca25 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 3 Apr 2023 18:13:22 +0200 Subject: [PATCH 29/43] Make the core part of the serial GC uninterruptible. --- .../genscavenge/AbstractCollectionPolicy.java | 2 + .../core/genscavenge/AlignedHeapChunk.java | 4 ++ .../core/genscavenge/AuxiliaryImageHeap.java | 1 + .../genscavenge/BasicCollectionPolicies.java | 5 ++- .../core/genscavenge/ChunksAccounting.java | 5 +++ .../core/genscavenge/CollectionPolicy.java | 3 ++ .../svm/core/genscavenge/GCAccounting.java | 2 + .../oracle/svm/core/genscavenge/GCImpl.java | 41 +++++++++++++++---- .../svm/core/genscavenge/Generation.java | 6 +++ .../core/genscavenge/GreyObjectsWalker.java | 11 ++--- .../genscavenge/GreyToBlackObjRefVisitor.java | 21 ++++++++++ .../genscavenge/GreyToBlackObjectVisitor.java | 2 + .../svm/core/genscavenge/HeapChunk.java | 9 +++- .../core/genscavenge/HeapChunkProvider.java | 30 +++----------- .../oracle/svm/core/genscavenge/HeapImpl.java | 4 ++ .../svm/core/genscavenge/HeapParameters.java | 1 + .../svm/core/genscavenge/ImageHeapInfo.java | 7 +++- .../svm/core/genscavenge/ImageHeapWalker.java | 39 ++++++++++++++++-- .../core/genscavenge/JfrGCEventSupport.java | 3 ++ .../svm/core/genscavenge/JfrGCEvents.java | 3 ++ .../core/genscavenge/ObjectHeaderImpl.java | 7 ++++ .../svm/core/genscavenge/OldGeneration.java | 12 ++++-- .../core/genscavenge/PinnedObjectImpl.java | 6 +++ .../ReferenceObjectProcessing.java | 6 +++ .../oracle/svm/core/genscavenge/Space.java | 17 ++++++++ .../oracle/svm/core/genscavenge/Timers.java | 6 +++ .../core/genscavenge/UnalignedHeapChunk.java | 1 + .../svm/core/genscavenge/YoungGeneration.java | 14 +++++-- .../remset/AlignedChunkRememberedSet.java | 12 ++++++ .../core/genscavenge/remset/CardTable.java | 10 +++++ .../remset/CardTableBasedRememberedSet.java | 11 +++++ .../genscavenge/remset/FirstObjectTable.java | 22 ++++++++++ .../genscavenge/remset/NoRememberedSet.java | 12 ++++++ .../genscavenge/remset/RememberedSet.java | 12 ++++++ .../remset/UnalignedChunkRememberedSet.java | 7 ++++ .../posix/pthread/PthreadVMLockSupport.java | 1 + .../core/windows/WindowsVMLockSupport.java | 1 + .../src/com/oracle/svm/core/MemoryWalker.java | 5 ++- .../oracle/svm/core/c/NonmovableArrays.java | 11 ++++- .../oracle/svm/core/code/CodeInfoTable.java | 1 + .../svm/core/code/RuntimeCodeInfoAccess.java | 2 + .../oracle/svm/core/config/ObjectLayout.java | 2 + .../core/heap/CodeReferenceMapDecoder.java | 2 +- .../heap/InstanceReferenceMapDecoder.java | 9 +++- .../oracle/svm/core/heap/ObjectHeader.java | 1 + .../oracle/svm/core/heap/OutOfMemoryUtil.java | 2 + .../svm/core/heap/PodReferenceMapDecoder.java | 14 +++++-- .../svm/core/heap/ReferenceInternals.java | 8 +++- .../svm/core/heap/ReferenceMapIndex.java | 3 ++ .../com/oracle/svm/core/hub/DynamicHub.java | 2 + .../svm/core/hub/InteriorObjRefWalker.java | 14 ++++++- .../oracle/svm/core/hub/LayoutEncoding.java | 5 +++ .../svm/core/jdk/UninterruptibleUtils.java | 7 ++++ .../locks/SingleThreadedVMLockSupport.java | 1 + .../oracle/svm/core/locks/VMCondition.java | 1 + .../os/AbstractCommittedMemoryProvider.java | 1 + .../threadlocal/VMThreadLocalMTSupport.java | 1 + 57 files changed, 389 insertions(+), 59 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java index bee7b2871e1..1f8e4b4308f 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AbstractCollectionPolicy.java @@ -243,6 +243,7 @@ public UnsignedWord getCurrentHeapCapacity() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnsignedWord getSurvivorSpacesCapacity() { assert VMOperation.isGCInProgress() : "use only during GC"; guaranteeSizeParametersInitialized(); @@ -290,6 +291,7 @@ public UnsignedWord getMaximumFreeAlignedChunksSize() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getTenuringAge() { assert VMOperation.isGCInProgress() : "use only during GC"; return tenuringThreshold; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java index bc15be90bdc..86dd5c3e0c4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AlignedHeapChunk.java @@ -83,6 +83,7 @@ private AlignedHeapChunk() { // all static public interface AlignedHeader extends HeapChunk.Header { } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void initialize(AlignedHeader chunk, UnsignedWord chunkSize) { HeapChunk.initialize(chunk, AlignedHeapChunk.getObjectsStart(chunk), chunkSize); } @@ -101,6 +102,7 @@ public static Pointer getObjectsEnd(AlignedHeader that) { } /** Allocate uninitialized memory within this AlignedHeapChunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static Pointer allocateMemory(AlignedHeader that, UnsignedWord size) { Pointer result = WordFactory.nullPointer(); UnsignedWord available = HeapChunk.availableObjectMemory(that); @@ -129,6 +131,7 @@ public static AlignedHeader getEnclosingChunkFromObjectPointer(Pointer ptr) { } /** Return the offset of an object within the objects part of a chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getObjectOffset(AlignedHeader that, Pointer objectPointer) { Pointer objectsStart = getObjectsStart(that); return objectPointer.subtract(objectsStart); @@ -139,6 +142,7 @@ static boolean walkObjects(AlignedHeader that, ObjectVisitor visitor) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static boolean walkObjectsInline(AlignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFromInline(that, getObjectsStart(that), visitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AuxiliaryImageHeap.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AuxiliaryImageHeap.java index 2d18a356ece..3ac40d5a385 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AuxiliaryImageHeap.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/AuxiliaryImageHeap.java @@ -48,6 +48,7 @@ static AuxiliaryImageHeap singleton() { boolean walkObjects(ObjectVisitor visitor); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean walkRegions(MemoryWalker.ImageHeapRegionVisitor visitor); ImageHeapInfo getImageHeapInfo(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java index ae89d374067..150aa514aac 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/BasicCollectionPolicies.java @@ -32,6 +32,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.heap.PhysicalMemory; import com.oracle.svm.core.heap.ReferenceAccess; @@ -49,7 +50,7 @@ static int getMaxSurvivorSpaces(Integer userValue) { private BasicCollectionPolicies() { } - abstract static class BasicPolicy implements CollectionPolicy { + public abstract static class BasicPolicy implements CollectionPolicy { protected static UnsignedWord m(long bytes) { assert 0 <= bytes; return WordFactory.unsigned(bytes).multiply(1024).multiply(1024); @@ -167,6 +168,7 @@ public UnsignedWord getMaximumSurvivorSize() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnsignedWord getSurvivorSpacesCapacity() { return WordFactory.zero(); } @@ -202,6 +204,7 @@ public final UnsignedWord getMaximumFreeAlignedChunksSize() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getTenuringAge() { return 1; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java index 17c24b57683..68c036cb640 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ChunksAccounting.java @@ -54,12 +54,14 @@ final class ChunksAccounting { reset(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void reset() { alignedCount = 0L; unalignedCount = 0L; unalignedChunkBytes = WordFactory.zero(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnsignedWord getChunkBytes() { return getAlignedChunkBytes().add(getUnalignedChunkBytes()); } @@ -96,6 +98,7 @@ void noteAlignedHeapChunk() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void unnoteAlignedHeapChunk() { alignedCount--; if (parent != null) { @@ -117,10 +120,12 @@ private void noteUnaligned(UnsignedWord size) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void unnoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk) { unnoteUnaligned(UnalignedHeapChunk.getCommittedObjectMemory(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void unnoteUnaligned(UnsignedWord size) { unalignedCount--; unalignedChunkBytes = unalignedChunkBytes.subtract(size); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java index 1d11025ef0f..ba738c3df22 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CollectionPolicy.java @@ -30,6 +30,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.SubstrateOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; import com.oracle.svm.core.heap.PhysicalMemory; import com.oracle.svm.core.util.UserError; @@ -176,6 +177,7 @@ static boolean shouldCollectYoungGenSeparately(boolean defaultValue) { * survivor-to spaces of all ages. In other words, when copying during a collection, up to 2x * this amount can be used for surviving objects. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) UnsignedWord getSurvivorSpacesCapacity(); /** The capacity of the young generation, comprising the eden and survivor spaces. */ @@ -200,6 +202,7 @@ static boolean shouldCollectYoungGenSeparately(boolean defaultValue) { * 1 (straight from eden) and the {@linkplain HeapParameters#getMaxSurvivorSpaces() number of * survivor spaces + 1}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getTenuringAge(); /** Called at the beginning of a collection, in the safepoint operation. */ diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java index 128df06b608..c006ab6faa5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCAccounting.java @@ -30,6 +30,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.log.Log; /** @@ -154,6 +155,7 @@ void beforeCollection(boolean completeCollection) { /** Called after an object has been promoted from the young generation to the old generation. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void onSurvivorOverflowed() { lastIncrementalCollectionOverflowedSurvivors = true; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index 6dd5aff78aa..c84521e6841 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -525,6 +525,7 @@ public void collectCompletely(GCCause cause) { collect(cause, true); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isCompleteCollection() { return completeCollection; } @@ -538,11 +539,7 @@ private void scavenge(boolean incremental) { try { startTicks = JfrGCEvents.startGCPhasePause(); try { - if (incremental) { - cheneyScanFromDirtyRoots(); - } else { - cheneyScanFromRoots(); - } + cheneyScan(incremental); } finally { JfrGCEvents.emitGCPhasePauseEvent(getCollectionEpoch(), incremental ? "Incremental Scan" : "Scan", startTicks); } @@ -620,6 +617,7 @@ private void scavenge(boolean incremental) { * compiled code to the Java heap must be consider as either strong or weak references, * depending on whether the code is currently on the execution stack. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void walkRuntimeCodeCache() { Timer walkRuntimeCodeCacheTimer = timers.walkRuntimeCodeCache.open(); try { @@ -638,6 +636,16 @@ private void cleanRuntimeCodeCache() { } } + @Uninterruptible(reason = "We don't want any safepoint checks in the core part of the GC.") + private void cheneyScan(boolean incremental) { + if (incremental) { + cheneyScanFromDirtyRoots(); + } else { + cheneyScanFromRoots(); + } + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void cheneyScanFromRoots() { Timer cheneyScanFromRootsTimer = timers.cheneyScanFromRoots.open(); try { @@ -707,6 +715,7 @@ private void cheneyScanFromRoots() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void cheneyScanFromDirtyRoots() { Timer cheneyScanFromDirtyRootsTimer = timers.cheneyScanFromDirtyRoots.open(); try { @@ -798,6 +807,7 @@ private void cheneyScanFromDirtyRoots() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void promoteChunksWithPinnedObjects() { Timer promotePinnedObjectsTimer = timers.promotePinnedObjects.open(); try { @@ -821,6 +831,7 @@ private void promoteChunksWithPinnedObjects() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static PinnedObjectImpl removeClosedPinnedObjects(PinnedObjectImpl list) { PinnedObjectImpl firstOpen = null; PinnedObjectImpl lastOpen = null; @@ -849,7 +860,7 @@ private static PinnedObjectImpl removeClosedPinnedObjects(PinnedObjectImpl list) @NeverInline("Starting a stack walk in the caller frame. " + "Note that we could start the stack frame also further down the stack, because GC stack frames must not access any objects that are processed by the GC. " + "But we don't store stack frame information for the first frame we would need to process.") - @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.", calleeMustBe = false) + @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.") private void blackenStackRoots() { Timer blackenStackRootsTimer = timers.blackenStackRoots.open(); try { @@ -891,7 +902,7 @@ private void blackenStackRoots() { * {@link SimpleCodeInfoQueryResult} twice per frame, and also ensures that there are no virtual * calls to a stack frame visitor. */ - @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.", calleeMustBe = false) + @Uninterruptible(reason = "Required by called JavaStackWalker methods. We are at a safepoint during GC, so it does not change anything for this method.") private void walkStack(JavaStackWalk walk) { assert VMOperation.isGCInProgress() : "This methods accesses a CodeInfo without a tether"; @@ -943,6 +954,7 @@ private void walkStack(JavaStackWalk walk) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void walkThreadLocals() { if (SubstrateOptions.MultiThreaded.getValue()) { Timer walkThreadLocalsTimer = timers.walkThreadLocals.open(); @@ -956,6 +968,7 @@ private void walkThreadLocals() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void blackenDirtyImageHeapRoots() { if (!HeapImpl.usesImageHeapCardMarking()) { blackenImageHeapRoots(); @@ -978,6 +991,7 @@ private void blackenDirtyImageHeapRoots() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void blackenDirtyImageHeapChunkRoots(AlignedHeader firstAligned, UnalignedHeader firstUnaligned) { /* * We clean and remark cards of the image heap only during complete collections when we also @@ -999,6 +1013,7 @@ private void blackenDirtyImageHeapChunkRoots(AlignedHeader firstAligned, Unalign } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void blackenImageHeapRoots() { if (HeapImpl.usesImageHeapCardMarking()) { // Avoid scanning the entire image heap even for complete collections: its remembered @@ -1017,6 +1032,7 @@ private void blackenImageHeapRoots() { private class BlackenImageHeapRootsVisitor implements MemoryWalker.ImageHeapRegionVisitor { @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitNativeImageHeapRegion(T region, MemoryWalker.NativeImageHeapRegionAccess access) { if (access.containsReferences(region) && access.isWritable(region)) { access.visitObjects(region, greyToBlackObjectVisitor); @@ -1025,6 +1041,7 @@ public boolean visitNativeImageHeapRegion(T region, MemoryWalker.NativeImage } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void blackenDirtyCardRoots() { Timer blackenDirtyCardRootsTimer = timers.blackenDirtyCardRoots.open(); try { @@ -1039,6 +1056,7 @@ private void blackenDirtyCardRoots() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void prepareForPromotion(boolean isIncremental) { HeapImpl heap = HeapImpl.getHeapImpl(); heap.getOldGeneration().prepareForPromotion(); @@ -1047,6 +1065,7 @@ private static void prepareForPromotion(boolean isIncremental) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void scanGreyObjects(boolean isIncremental) { Timer scanGreyObjectsTimer = timers.scanGreyObjects.open(); try { @@ -1060,6 +1079,7 @@ private void scanGreyObjects(boolean isIncremental) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void scanGreyObjectsLoop() { HeapImpl heap = HeapImpl.getHeapImpl(); YoungGeneration youngGen = heap.getYoungGeneration(); @@ -1072,6 +1092,7 @@ private static void scanGreyObjectsLoop() { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") Object promoteObject(Object original, UnsignedWord header) { HeapImpl heap = HeapImpl.getHeapImpl(); @@ -1105,6 +1126,7 @@ Object promoteObject(Object original, UnsignedWord header) { return result; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Header getChunk(Object obj, boolean isAligned) { if (isAligned) { return AlignedHeapChunk.getEnclosingChunk(obj); @@ -1113,6 +1135,7 @@ private static Header getChunk(Object obj, boolean isAligned) { return UnalignedHeapChunk.getEnclosingChunk(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void promotePinnedObject(PinnedObjectImpl pinned) { HeapImpl heap = HeapImpl.getHeapImpl(); Object referent = pinned.getObject(); @@ -1217,6 +1240,7 @@ public static boolean hasNeverCollectPolicy() { return getPolicy() instanceof NeverCollect; } + @Fold GreyToBlackObjectVisitor getGreyToBlackObjectVisitor() { return greyToBlackObjectVisitor; } @@ -1326,6 +1350,7 @@ public boolean isEmpty() { return firstAligned.isNull() && firstUnaligned.isNull(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void add(AlignedHeader chunks) { if (chunks.isNonNull()) { assert HeapChunk.getPrevious(chunks).isNull() : "prev must be null"; @@ -1338,6 +1363,7 @@ public void add(AlignedHeader chunks) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void add(UnalignedHeader chunks) { if (chunks.isNonNull()) { assert HeapChunk.getPrevious(chunks).isNull() : "prev must be null"; @@ -1361,6 +1387,7 @@ void release(boolean keepAllAlignedChunks) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static > T getLast(T chunks) { T prev = chunks; T next = HeapChunk.getNext(prev); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java index 35b694891c1..e487a4c7f1d 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Generation.java @@ -27,6 +27,8 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; +import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.log.Log; @@ -66,6 +68,8 @@ public String getName() { * promotion was done by copying, or {@code null} if there was insufficient capacity in * this generation. */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected abstract Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace); /** @@ -79,6 +83,8 @@ public String getName() { * was promoted through HeapChunk motion, or {@code null} if there was insufficient * capacity in this generation. */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected abstract Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace); /** diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java index 3435527236a..09ad12fd356 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyObjectsWalker.java @@ -31,7 +31,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.NeverInline; -import com.oracle.svm.core.log.Log; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.util.VMError; /** @@ -57,26 +57,25 @@ final class GreyObjectsWalker { * Take a snapshot of a Space, such that all Objects in the Space are now black, and any new * Objects in the Space will be grey, and can have an ObjectVisitor applied to them. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void setScanStart(Space s) { - Log trace = Log.noopLog().string("[Space.GreyObjectsWalker.setScanStart:").string(" s: ").string(s.getName()); space = s; AlignedHeapChunk.AlignedHeader aChunk = s.getLastAlignedHeapChunk(); alignedHeapChunk = aChunk; - trace.string(" alignedHeapChunk: ").zhex(alignedHeapChunk).string(" isNull: ").bool(aChunk.isNull()); alignedTop = (aChunk.isNonNull() ? HeapChunk.getTopPointer(aChunk) : WordFactory.nullPointer()); - trace.string(" alignedTop: ").zhex(alignedTop); unalignedHeapChunk = s.getLastUnalignedHeapChunk(); - trace.string(" unalignedChunkPointer: ").zhex(unalignedHeapChunk).string("]").newline(); } /** Compare the snapshot to the current state of the Space to see if there are grey Objects. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean haveGreyObjects() { return alignedHeapChunk.notEqual(space.getLastAlignedHeapChunk()) || alignedHeapChunk.isNonNull() && alignedTop.notEqual(HeapChunk.getTopPointer(alignedHeapChunk)) || unalignedHeapChunk.notEqual(space.getLastUnalignedHeapChunk()); } @NeverInline("Split the GC into reasonable compilation units") + @Uninterruptible(reason = "Called from uninterruptible code.") void walkGreyObjects() { while (haveGreyObjects()) { walkAlignedGreyObjects(); @@ -85,6 +84,7 @@ void walkGreyObjects() { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void walkAlignedGreyObjects() { AlignedHeapChunk.AlignedHeader aChunk; if (alignedHeapChunk.isNull() && alignedTop.isNull()) { @@ -113,6 +113,7 @@ private void walkAlignedGreyObjects() { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void walkUnalignedGreyObjects() { /* Visit the Objects in the UnalignedChunk after the snapshot UnalignedChunk. */ UnalignedHeapChunk.UnalignedHeader uChunk; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java index ac1be2f4d2f..3281a53ac41 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjRefVisitor.java @@ -30,6 +30,7 @@ import org.graalvm.word.Pointer; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.ObjectHeader; import com.oracle.svm.core.heap.ObjectReferenceVisitor; @@ -59,12 +60,14 @@ final class GreyToBlackObjRefVisitor implements ObjectReferenceVisitor { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectReference(Pointer objRef, boolean compressed, Object holderObject) { return visitObjectReferenceInline(objRef, 0, compressed, holderObject); } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectReferenceInline(Pointer objRef, int innerOffset, boolean compressed, Object holderObject) { assert innerOffset >= 0; assert !objRef.isNull(); @@ -131,16 +134,22 @@ public interface Counters extends AutoCloseable { @Override void close(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteObjRef(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteNullReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteForwardedReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteNonHeapReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteCopiedReferent(); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void noteUnmodifiedReference(); void toLog(); @@ -185,31 +194,37 @@ public void close() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteObjRef() { objRef += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNullReferent() { nullReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteForwardedReferent() { forwardedReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNonHeapReferent() { nonHeapReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteCopiedReferent() { copiedReferent += 1L; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteUnmodifiedReference() { unmodifiedReference += 1L; } @@ -244,26 +259,32 @@ public void close() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteObjRef() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNullReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteForwardedReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteNonHeapReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteCopiedReferent() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void noteUnmodifiedReference() { } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java index 9a8b48845d4..bc432b86626 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GreyToBlackObjectVisitor.java @@ -29,6 +29,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.hub.InteriorObjRefWalker; import com.oracle.svm.core.util.VMError; @@ -55,6 +56,7 @@ public boolean visitObject(Object o) { @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean visitObjectInline(Object o) { ReferenceObjectProcessing.discoverIfReference(o, objRefVisitor); InteriorObjRefWalker.walkObjectInline(o, objRefVisitor); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java index 6014037a109..02512d65b1e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunk.java @@ -177,6 +177,7 @@ public interface Header> extends HeaderPadding { void setIdentityHashSalt(UnsignedWord value, LocationIdentity identity); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void initialize(Header chunk, Pointer objectsStart, UnsignedWord chunkSize) { HeapChunk.setEndOffset(chunk, chunkSize); HeapChunk.setTopPointer(chunk, objectsStart); @@ -306,11 +307,12 @@ public static boolean walkObjectsFrom(Header that, Pointer offset, ObjectVisi } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkObjectsFromInline(Header that, Pointer startOffset, ObjectVisitor visitor) { Pointer offset = startOffset; while (offset.belowThan(getTopPointer(that))) { // crucial: top can move, so always re-read Object obj = offset.toObject(); - if (!visitor.visitObjectInline(obj)) { + if (!callVisitor(visitor, obj)) { return false; } offset = offset.add(LayoutEncoding.getSizeFromObjectInlineInGC(obj)); @@ -318,6 +320,11 @@ public static boolean walkObjectsFromInline(Header that, Pointer startOffset, return true; } + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static boolean callVisitor(ObjectVisitor visitor, Object obj) { + return visitor.visitObjectInline(obj); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord availableObjectMemory(Header that) { return that.getEndOffset().subtract(that.getTopOffset()); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java index 4df6532b8e5..f94172566a4 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapChunkProvider.java @@ -24,7 +24,6 @@ */ package com.oracle.svm.core.genscavenge; -import com.oracle.svm.core.heap.OutOfMemoryUtil; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; @@ -35,11 +34,11 @@ import com.oracle.svm.core.FrameAccess; import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.SubstrateOptions; -import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.HeapChunk.Header; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; +import com.oracle.svm.core.heap.OutOfMemoryUtil; import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.jdk.UninterruptibleUtils.AtomicUnsigned; import com.oracle.svm.core.log.Log; @@ -89,23 +88,15 @@ public UnsignedWord getBytesInUnusedChunks() { return bytesInUnusedAlignedChunks.get(); } - @AlwaysInline("Remove all logging when noopLog is returned by this method") - private static Log log() { - return Log.noopLog(); - } - private static final OutOfMemoryError ALIGNED_OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Could not allocate an aligned heap chunk"); private static final OutOfMemoryError UNALIGNED_OUT_OF_MEMORY_ERROR = new OutOfMemoryError("Could not allocate an unaligned heap chunk"); /** Acquire a new AlignedHeapChunk, either from the free list or from the operating system. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) AlignedHeader produceAlignedChunk() { UnsignedWord chunkSize = HeapParameters.getAlignedHeapChunkSize(); - log().string("[HeapChunkProvider.produceAlignedChunk chunk size: ").unsigned(chunkSize).newline(); - AlignedHeader result = popUnusedAlignedChunk(); - log().string(" unused chunk: ").zhex(result).newline(); - if (result.isNull()) { /* Unused list was empty, need to allocate memory. */ noteFirstAllocationTime(); @@ -113,7 +104,6 @@ AlignedHeader produceAlignedChunk() { if (result.isNull()) { throw OutOfMemoryUtil.reportOutOfMemoryError(ALIGNED_OUT_OF_MEMORY_ERROR); } - log().string(" new chunk: ").zhex(result).newline(); AlignedHeapChunk.initialize(result, chunkSize); } @@ -123,8 +113,6 @@ AlignedHeader produceAlignedChunk() { if (HeapParameters.getZapProducedHeapChunks()) { zap(result, HeapParameters.getProducedHeapChunkZapWord()); } - - log().string(" result chunk: ").zhex(result).string(" ]").newline(); return result; } @@ -200,13 +188,10 @@ private void pushUnusedAlignedChunk(AlignedHeader chunk) { if (SubstrateOptions.MultiThreaded.getValue()) { VMThreads.guaranteeOwnsThreadMutex("Should hold the lock when pushing to the global list."); } - log().string(" old list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); HeapChunk.setNext(chunk, unusedAlignedChunks.get()); unusedAlignedChunks.set(chunk); bytesInUnusedAlignedChunks.addAndGet(HeapParameters.getAlignedHeapChunkSize()); - - log().string(" new list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); } /** @@ -218,15 +203,13 @@ private void pushUnusedAlignedChunk(AlignedHeader chunk) { * garbage collections, I avoid the ABA problem by making the kernel of this method * uninterruptible so it can not be interrupted by a safepoint. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private AlignedHeader popUnusedAlignedChunk() { - log().string(" old list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); - AlignedHeader result = popUnusedAlignedChunkUninterruptibly(); if (result.isNull()) { return WordFactory.nullPointer(); } else { bytesInUnusedAlignedChunks.subtractAndGet(HeapParameters.getAlignedHeapChunkSize()); - log().string(" new list top: ").zhex(unusedAlignedChunks.get()).string(" list bytes ").signed(bytesInUnusedAlignedChunks.get()).newline(); return result; } } @@ -268,7 +251,6 @@ private void freeUnusedAlignedChunksAtSafepoint(UnsignedWord count) { /** Acquire an UnalignedHeapChunk from the operating system. */ UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) { UnsignedWord chunkSize = UnalignedHeapChunk.getChunkSizeForObject(objectSize); - log().string("[HeapChunkProvider.produceUnalignedChunk objectSize: ").unsigned(objectSize).string(" chunkSize: ").zhex(chunkSize).newline(); noteFirstAllocationTime(); UnalignedHeader result = (UnalignedHeader) CommittedMemoryProvider.get().allocateUnalignedChunk(chunkSize); @@ -282,8 +264,6 @@ UnalignedHeader produceUnalignedChunk(UnsignedWord objectSize) { if (HeapParameters.getZapProducedHeapChunks()) { zap(result, HeapParameters.getProducedHeapChunkZapWord()); } - - log().string(" returns ").zhex(result).string(" ]").newline(); return result; } @@ -300,10 +280,10 @@ static void consumeUnalignedChunks(UnalignedHeader firstChunk) { freeUnalignedChunkList(firstChunk); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void zap(Header chunk, WordBase value) { Pointer start = HeapChunk.getTopPointer(chunk); Pointer limit = HeapChunk.getEndPointer(chunk); - log().string(" zap chunk: ").zhex(chunk).string(" start: ").zhex(start).string(" limit: ").zhex(limit).string(" value: ").zhex(value).newline(); for (Pointer p = start; p.belowThan(limit); p = p.add(FrameAccess.wordSize())) { p.writeWord(0, value); } @@ -332,12 +312,14 @@ boolean walkHeapChunks(MemoryWalker.Visitor visitor) { return continueVisiting; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void noteFirstAllocationTime() { if (firstAllocationTime == 0L) { firstAllocationTime = System.nanoTime(); } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) long getFirstAllocationTime() { return firstAllocationTime; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index b5f84cedaaa..db5ddbd1bea 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -235,6 +235,7 @@ public HeapAccounting getAccounting() { return accounting; } + @Fold GCImpl getGCImpl() { return gcImpl; } @@ -262,6 +263,7 @@ public OldGeneration getOldGeneration() { return oldGeneration; } + @Fold AtomicReference getPinHead() { return pinHead; } @@ -468,6 +470,7 @@ public boolean walkCollectedHeapObjects(ObjectVisitor visitor) { return getYoungGeneration().walkObjects(visitor) && getOldGeneration().walkObjects(visitor); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean walkNativeImageHeapRegions(MemoryWalker.ImageHeapRegionVisitor visitor) { return ImageHeapWalker.walkRegions(imageHeapInfo, visitor) && (!AuxiliaryImageHeap.isPresent() || AuxiliaryImageHeap.singleton().walkRegions(visitor)); @@ -717,6 +720,7 @@ public long getIdentityHashSalt(Object obj) { return HeapChunk.getIdentityHashSalt(chunk).rawValue(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static Pointer getImageHeapStart() { int imageHeapOffsetInAddressSpace = Heap.getHeap().getImageHeapOffsetInAddressSpace(); if (imageHeapOffsetInAddressSpace > 0) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java index 9c2be3f9a44..e5817b56068 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapParameters.java @@ -137,6 +137,7 @@ public static UnsignedWord getLargeArrayThreshold() { * Zapping */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean getZapProducedHeapChunks() { return SerialAndEpsilonGCOptions.ZapChunks.getValue() || SerialAndEpsilonGCOptions.ZapProducedHeapChunks.getValue(); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java index 44d04a7cdd1..4be1beea5a6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapInfo.java @@ -30,10 +30,10 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; -import com.oracle.svm.core.heap.UnknownObjectField; -import com.oracle.svm.core.heap.UnknownPrimitiveField; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.UnalignedHeapChunk.UnalignedHeader; +import com.oracle.svm.core.heap.UnknownObjectField; +import com.oracle.svm.core.heap.UnknownPrimitiveField; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.log.Log; import com.oracle.svm.core.snippets.KnownIntrinsics; @@ -187,10 +187,12 @@ public boolean isInImageHeap(Pointer objectPointer) { return result; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public AlignedHeader getFirstWritableAlignedChunk() { return asImageHeapChunk(offsetOfFirstWritableAlignedChunk); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public UnalignedHeader getFirstWritableUnalignedChunk() { return asImageHeapChunk(offsetOfFirstWritableUnalignedChunk); } @@ -204,6 +206,7 @@ private static Pointer getObjectEnd(Object obj) { } @SuppressWarnings("unchecked") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static > T asImageHeapChunk(long offsetInImageHeap) { if (offsetInImageHeap < 0) { return (T) WordFactory.nullPointer(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java index 2c17d3bbf73..c2ef95c399a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java @@ -31,8 +31,9 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.MemoryWalker; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.ObjectVisitor; import com.oracle.svm.core.hub.LayoutEncoding; import com.oracle.svm.core.os.CommittedMemoryProvider; @@ -50,6 +51,7 @@ public final class ImageHeapWalker { private ImageHeapWalker() { } + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) public static boolean walkRegions(ImageHeapInfo heapInfo, MemoryWalker.ImageHeapRegionVisitor visitor) { return visitor.visitNativeImageHeapRegion(heapInfo, READ_ONLY_PRIMITIVE_WALKER) && visitor.visitNativeImageHeapRegion(heapInfo, READ_ONLY_REFERENCE_WALKER) && @@ -75,11 +77,13 @@ static boolean walkPartition(Object firstObject, Object lastObject, ObjectVisito } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static boolean walkPartitionInline(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks) { return walkPartitionInline(firstObject, lastObject, visitor, alignedChunks, true); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkPartitionInline(Object firstObject, Object lastObject, ObjectVisitor visitor, boolean alignedChunks, boolean inlineObjectVisit) { if (firstObject == null || lastObject == null) { assert firstObject == null && lastObject == null; @@ -113,10 +117,10 @@ private static boolean walkPartitionInline(Object firstObject, Object lastObject while (current.belowOrEqual(limit)) { Object currentObject = current.toObject(); if (inlineObjectVisit) { - if (!visitor.visitObjectInline(currentObject)) { + if (!visitObjectInline(visitor, currentObject)) { return false; } - } else if (!visitor.visitObject(currentObject)) { + } else if (!visitObject(visitor, currentObject)) { return false; } current = LayoutEncoding.getImageHeapObjectEnd(current.toObject()); @@ -130,6 +134,16 @@ private static boolean walkPartitionInline(Object firstObject, Object lastObject } while (current.belowOrEqual(lastPointer)); return true; } + + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static boolean visitObject(ObjectVisitor visitor, Object currentObject) { + return visitor.visitObject(currentObject); + } + + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static boolean visitObjectInline(ObjectVisitor visitor, Object currentObject) { + return visitor.visitObjectInline(currentObject); + } } abstract class MemoryWalkerAccessBase implements MemoryWalker.NativeImageHeapRegionAccess { @@ -167,24 +181,29 @@ public String getRegionName(ImageHeapInfo region) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean containsReferences(ImageHeapInfo region) { return containsReferences; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isWritable(ImageHeapInfo region) { return isWritable; } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public final boolean visitObjects(ImageHeapInfo region, ObjectVisitor visitor) { boolean alignedChunks = !hasHugeObjects; return ImageHeapWalker.walkPartitionInline(getFirstObject(region), getLastObject(region), visitor, alignedChunks); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected abstract Object getFirstObject(ImageHeapInfo info); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected abstract Object getLastObject(ImageHeapInfo info); } @@ -195,11 +214,13 @@ final class ReadOnlyPrimitiveMemoryWalkerAccess extends MemoryWalkerAccessBase { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getFirstObject(ImageHeapInfo info) { return info.firstReadOnlyPrimitiveObject; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getLastObject(ImageHeapInfo info) { return info.lastReadOnlyPrimitiveObject; } @@ -212,11 +233,13 @@ final class ReadOnlyReferenceMemoryWalkerAccess extends MemoryWalkerAccessBase { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getFirstObject(ImageHeapInfo info) { return info.firstReadOnlyReferenceObject; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getLastObject(ImageHeapInfo info) { return info.lastReadOnlyReferenceObject; } @@ -229,11 +252,13 @@ final class ReadOnlyRelocatableMemoryWalkerAccess extends MemoryWalkerAccessBase } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getFirstObject(ImageHeapInfo info) { return info.firstReadOnlyRelocatableObject; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getLastObject(ImageHeapInfo info) { return info.lastReadOnlyRelocatableObject; } @@ -246,11 +271,13 @@ final class WritablePrimitiveMemoryWalkerAccess extends MemoryWalkerAccessBase { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getFirstObject(ImageHeapInfo info) { return info.firstWritablePrimitiveObject; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getLastObject(ImageHeapInfo info) { return info.lastWritablePrimitiveObject; } @@ -263,11 +290,13 @@ final class WritableReferenceMemoryWalkerAccess extends MemoryWalkerAccessBase { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getFirstObject(ImageHeapInfo info) { return info.firstWritableReferenceObject; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getLastObject(ImageHeapInfo info) { return info.lastWritableReferenceObject; } @@ -280,11 +309,13 @@ final class WritableHugeMemoryWalkerAccess extends MemoryWalkerAccessBase { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getFirstObject(ImageHeapInfo info) { return info.firstWritableHugeObject; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getLastObject(ImageHeapInfo info) { return info.lastWritableHugeObject; } @@ -297,11 +328,13 @@ final class ReadOnlyHugeMemoryWalkerAccess extends MemoryWalkerAccessBase { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getFirstObject(ImageHeapInfo info) { return info.firstReadOnlyHugeObject; } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getLastObject(ImageHeapInfo info) { return info.lastReadOnlyHugeObject; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java index 47126cddf9e..6a2a962d5ae 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEventSupport.java @@ -54,11 +54,13 @@ class JfrGCEventSupport { this.gcName = gcName; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public long startGCPhasePause() { pushPhase(); return JfrTicks.elapsedTicks(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int stopGCPhasePause() { return popPhase(); } @@ -124,6 +126,7 @@ private static JfrEvent getGCPhasePauseEvent(int level) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void pushPhase() { assert currentPhase < MAX_PHASE_LEVEL; currentPhase++; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEvents.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEvents.java index 63eb13e1ee8..9bcd47d1cac 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEvents.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/JfrGCEvents.java @@ -28,9 +28,11 @@ import org.graalvm.nativeimage.ImageSingletons; import org.graalvm.word.UnsignedWord; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.heap.GCCause; class JfrGCEvents { + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long startGCPhasePause() { if (hasJfrSupport()) { return jfrSupport().startGCPhasePause(); @@ -38,6 +40,7 @@ public static long startGCPhasePause() { return 0; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void emitGCPhasePauseEvent(UnsignedWord gcEpoch, String name, long start) { if (hasJfrSupport()) { int level = jfrSupport().stopGCPhasePause(); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 85a64165988..74feb1659f7 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -166,6 +166,7 @@ public boolean hasOptionalIdentityHashField(Word header) { return header.and(IDHASH_STATE_BITS).equal(inFieldState); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void setIdentityHashInField(Object o) { assert VMOperation.isGCInProgress(); VMError.guarantee(!hasFixedIdentityHashField()); @@ -320,6 +321,7 @@ public static boolean isAlignedObject(Object o) { return !isUnalignedObject(o); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isAlignedHeader(UnsignedWord header) { return !isUnalignedHeader(header); } @@ -335,12 +337,14 @@ public static boolean isUnalignedHeader(UnsignedWord header) { return header.and(UNALIGNED_BIT).notEqual(0); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setRememberedSetBit(Object o) { UnsignedWord oldHeader = readHeaderFromObject(o); UnsignedWord newHeader = oldHeader.or(REMEMBERED_SET_BIT); writeHeaderToObject(o, newHeader); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean hasRememberedSet(UnsignedWord header) { return header.and(REMEMBERED_SET_BIT).notEqual(0); } @@ -356,10 +360,12 @@ public static boolean isForwardedHeader(UnsignedWord header) { return header.and(FORWARDED_BIT).notEqual(0); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object getForwardedObject(Pointer ptr) { return getForwardedObject(ptr, readHeaderFromPointer(ptr)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object getForwardedObject(Pointer ptr, UnsignedWord header) { assert isForwardedHeader(header); if (ReferenceAccess.singleton().haveCompressedReferences()) { @@ -380,6 +386,7 @@ Object getForwardedObject(Pointer ptr, UnsignedWord header) { /** In an Object, install a forwarding pointer to a different Object. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void installForwardingPointer(Object original, Object copy) { assert !isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); UnsignedWord forwardHeader; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java index 395e39ef03f..8c77d261b9b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/OldGeneration.java @@ -31,8 +31,8 @@ import org.graalvm.nativeimage.Platforms; import org.graalvm.word.UnsignedWord; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.GCImpl.ChunkReleaser; import com.oracle.svm.core.genscavenge.remset.RememberedSet; @@ -73,6 +73,7 @@ public boolean walkObjects(ObjectVisitor visitor) { /** Promote an Object to ToSpace if it is not already in ToSpace. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -80,6 +81,7 @@ public Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHead } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -88,6 +90,7 @@ protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.Unal } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAligned, Space originalSpace) { assert originalSpace.isFromSpace(); if (isAligned) { @@ -102,10 +105,12 @@ void releaseSpaces(ChunkReleaser chunkReleaser) { getFromSpace().releaseChunks(chunkReleaser); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void prepareForPromotion() { toGreyObjectsWalker.setScanStart(getToSpace()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean scanGreyObjects() { if (!toGreyObjectsWalker.haveGreyObjects()) { return false; @@ -139,6 +144,7 @@ void swapSpaces() { } /* Extract all the HeapChunks from FromSpace and append them to ToSpace. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void emptyFromSpaceIntoToSpace() { getToSpace().absorb(getFromSpace()); } @@ -158,13 +164,13 @@ UnsignedWord getChunkBytes() { return fromBytes.add(toBytes); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") AlignedHeapChunk.AlignedHeader requestAlignedChunk() { assert VMOperation.isGCInProgress() : "Should only be called from the collector."; AlignedHeapChunk.AlignedHeader chunk = HeapImpl.getChunkProvider().produceAlignedChunk(); if (probability(EXTREMELY_SLOW_PATH_PROBABILITY, chunk.isNull())) { - Log.log().string("[! OldGeneration.requestAlignedChunk: failure to allocate aligned chunk!]"); - throw VMError.shouldNotReachHere("Promotion failure"); + throw VMError.shouldNotReachHere("OldGeneration.requestAlignedChunk: failure to allocate aligned chunk"); } RememberedSet.get().enableRememberedSetForChunk(chunk); return chunk; diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java index dc945e0be14..9f85d0afa7b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/PinnedObjectImpl.java @@ -77,12 +77,14 @@ static void pushPinnedObject(PinnedObjectImpl newHead) { } while (!pinHead.compareAndSet(sampleHead, newHead)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static PinnedObjectImpl getPinnedObjects() { assert VMOperation.isGCInProgress(); UninterruptibleUtils.AtomicReference pinHead = HeapImpl.getHeapImpl().getPinHead(); return pinHead.get(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) static void setPinnedObjects(PinnedObjectImpl list) { assert VMOperation.isGCInProgress(); UninterruptibleUtils.AtomicReference pinHead = HeapImpl.getHeapImpl().getPinHead(); @@ -108,6 +110,7 @@ public void close() { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getObject() { assert open : "Should not call getObject() on a closed PinnedObject."; return referent; @@ -133,14 +136,17 @@ public T addressOfArrayElement(int index) { return (T) addressOfObject().add(offsetOfArrayElement); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isOpen() { return open; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public PinnedObjectImpl getNext() { return next; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void setNext(PinnedObjectImpl value) { // Avoid useless writes as those would dirty the card table unnecessarily. if (value != next) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java index e3a4ee9b894..e2b2df83407 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ReferenceObjectProcessing.java @@ -36,6 +36,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.remset.RememberedSet; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.heap.ObjectHeader; @@ -79,6 +80,7 @@ public static void setSoftReferencesAreWeak(boolean enabled) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void discoverIfReference(Object object, ObjectReferenceVisitor refVisitor) { assert object != null; DynamicHub hub = KnownIntrinsics.readHub(object); @@ -87,6 +89,8 @@ public static void discoverIfReference(Object object, ObjectReferenceVisitor ref } } + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void discover(Object obj, ObjectReferenceVisitor refVisitor) { Reference dr = (Reference) obj; // The discovered field might contain an object with a forwarding header @@ -218,6 +222,7 @@ private static boolean processRememberedRef(Reference dr) { return false; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean maybeUpdateForwardedReference(Reference dr, Pointer referentAddr) { ObjectHeaderImpl ohi = ObjectHeaderImpl.getObjectHeaderImpl(); UnsignedWord header = ObjectHeader.readHeaderFromPointer(referentAddr); @@ -229,6 +234,7 @@ private static boolean maybeUpdateForwardedReference(Reference dr, Pointer re return false; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean willSurviveThisCollection(Object obj) { HeapChunk.Header chunk = HeapChunk.getEnclosingHeapChunk(obj); Space space = HeapChunk.getSpace(chunk); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java index 1aff988d39a..281dcbfc8e8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Space.java @@ -94,6 +94,7 @@ public String getName() { return name; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isEmpty() { return (getFirstAlignedHeapChunk().isNull() && getFirstUnalignedHeapChunk().isNull()); } @@ -109,10 +110,12 @@ boolean isEdenSpace() { return age == 0; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isYoungSpace() { return age <= HeapParameters.getMaxSurvivorSpaces(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean isSurvivorSpace() { return age > 0 && age <= HeapParameters.getMaxSurvivorSpaces(); } @@ -122,14 +125,17 @@ public boolean isOldSpace() { return age == (HeapParameters.getMaxSurvivorSpaces() + 1); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getAge() { return age; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) int getNextAgeForPromotion() { return age + 1; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean isFromSpace() { return isFromSpace; } @@ -168,6 +174,7 @@ public Log report(Log log, boolean traceHeapChunks) { * Allocate memory from an AlignedHeapChunk in this Space. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateMemory(UnsignedWord objectSize) { Pointer result = WordFactory.nullPointer(); /* Fast-path: try allocating in the last chunk. */ @@ -182,6 +189,7 @@ private Pointer allocateMemory(UnsignedWord objectSize) { return allocateInNewChunk(objectSize); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Pointer allocateInNewChunk(UnsignedWord objectSize) { AlignedHeapChunk.AlignedHeader newChunk = requestAlignedHeapChunk(); if (newChunk.isNonNull()) { @@ -190,6 +198,7 @@ private Pointer allocateInNewChunk(UnsignedWord objectSize) { return WordFactory.nullPointer(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void releaseChunks(ChunkReleaser chunkReleaser) { chunkReleaser.add(firstAlignedHeapChunk); chunkReleaser.add(firstUnalignedHeapChunk); @@ -229,6 +238,7 @@ private void appendAlignedHeapChunkUninterruptibly(AlignedHeapChunk.AlignedHeade } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void extractAlignedHeapChunk(AlignedHeapChunk.AlignedHeader aChunk) { assert VMOperation.isGCInProgress() : "Should only be called by the collector."; extractAlignedHeapChunkUninterruptibly(aChunk); @@ -282,6 +292,7 @@ private void appendUnalignedHeapChunkUninterruptibly(UnalignedHeapChunk.Unaligne } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void extractUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader uChunk) { assert VMOperation.isGCInProgress() : "Trying to extract an unaligned chunk but not in a VMOperation."; extractUnalignedHeapChunkUninterruptibly(uChunk); @@ -350,6 +361,7 @@ private void setLastUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk) /** Promote an aligned Object to this Space. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Object promoteAlignedObject(Object original, Space originalSpace) { assert ObjectHeaderImpl.isAlignedObject(original); assert this != originalSpace && originalSpace.isFromSpace(); @@ -362,6 +374,7 @@ Object promoteAlignedObject(Object original, Space originalSpace) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private Object copyAlignedObject(Object originalObj) { assert VMOperation.isGCInProgress(); assert ObjectHeaderImpl.isAlignedObject(originalObj); @@ -408,6 +421,7 @@ private Object copyAlignedObject(Object originalObj) { } /** Promote an AlignedHeapChunk by moving it to this space. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); @@ -425,6 +439,7 @@ void promoteAlignedHeapChunk(AlignedHeapChunk.AlignedHeader chunk, Space origina } /** Promote an UnalignedHeapChunk by moving it to this Space. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space originalSpace) { assert this != originalSpace && originalSpace.isFromSpace(); @@ -441,6 +456,7 @@ void promoteUnalignedHeapChunk(UnalignedHeapChunk.UnalignedHeader chunk, Space o } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private AlignedHeapChunk.AlignedHeader requestAlignedHeapChunk() { AlignedHeapChunk.AlignedHeader chunk; if (isYoungSpace()) { @@ -455,6 +471,7 @@ private AlignedHeapChunk.AlignedHeader requestAlignedHeapChunk() { return chunk; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void absorb(Space src) { /* * Absorb the chunks of a source into this Space. I cannot just copy the lists, because each diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java index b86261133cc..a62552c9231 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/Timers.java @@ -24,6 +24,7 @@ */ package com.oracle.svm.core.genscavenge; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.log.Log; /** @@ -46,10 +47,12 @@ public String getName() { return name; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Timer open() { return openAt(System.nanoTime()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Timer openAt(long nanoTime) { openNanos = nanoTime; wasOpened = true; @@ -59,10 +62,12 @@ Timer openAt(long nanoTime) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void close() { closeAt(System.nanoTime()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void closeAt(long nanoTime) { closeNanos = nanoTime; wasClosed = true; @@ -77,6 +82,7 @@ public void reset() { collectedNanos = 0L; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public long getOpenedTime() { if (!wasOpened) { /* If a timer was not opened, pretend it was opened at the start of the VM. */ diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java index b6673242aec..8a1ef2eaa98 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/UnalignedHeapChunk.java @@ -145,6 +145,7 @@ public static boolean walkObjects(UnalignedHeader that, ObjectVisitor visitor) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkObjectsInline(UnalignedHeader that, ObjectVisitor visitor) { return HeapChunk.walkObjectsFromInline(that, getObjectStart(that), visitor); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java index 1c997255d9a..d7f337a6a0b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/YoungGeneration.java @@ -132,6 +132,7 @@ Space getSurvivorFromSpaceAt(int index) { return survivorFromSpaces[index]; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private GreyObjectsWalker getSurvivorGreyObjectsWalker(int index) { return survivorGreyObjectsWalkers[index]; } @@ -164,6 +165,7 @@ boolean walkHeapChunks(MemoryWalker.Visitor visitor) { return false; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void prepareForPromotion() { for (int i = 0; i < maxSurvivorSpaces; i++) { assert getSurvivorToSpaceAt(i).isEmpty() : "SurvivorToSpace should be empty."; @@ -171,8 +173,8 @@ void prepareForPromotion() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean scanGreyObjects() { - Log trace = Log.noopLog().string("[YoungGeneration.scanGreyObjects:"); boolean needScan = false; for (int i = 0; i < maxSurvivorSpaces; i++) { if (getSurvivorGreyObjectsWalker(i).haveGreyObjects()) { @@ -184,10 +186,8 @@ boolean scanGreyObjects() { return false; } for (int i = 0; i < maxSurvivorSpaces; i++) { - trace.string("[Scanning survivor-").signed(i).string("]").newline(); getSurvivorGreyObjectsWalker(i).walkGreyObjects(); } - trace.string("]").newline(); return true; } @@ -238,6 +238,7 @@ UnsignedWord computeSurvivorObjectBytes() { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @SuppressWarnings("static-method") public boolean contains(Object object) { if (!HeapImpl.usesImageHeapCardMarking()) { @@ -251,6 +252,7 @@ public boolean contains(Object object) { } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override protected Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -266,6 +268,7 @@ protected Object promoteAlignedObject(Object original, AlignedHeapChunk.AlignedH } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @Override protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.UnalignedHeader originalChunk, Space originalSpace) { assert originalSpace.isFromSpace(); @@ -281,6 +284,7 @@ protected Object promoteUnalignedObject(Object original, UnalignedHeapChunk.Unal } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAligned, Space originalSpace) { assert originalSpace.isFromSpace(); assert originalSpace.getAge() < maxSurvivorSpaces; @@ -298,6 +302,7 @@ protected boolean promoteChunk(HeapChunk.Header originalChunk, boolean isAlig return true; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean fitsInSurvivors(HeapChunk.Header chunk, boolean isAligned) { if (isAligned) { return alignedChunkFitsInSurvivors(); @@ -305,17 +310,20 @@ private boolean fitsInSurvivors(HeapChunk.Header chunk, boolean isAligned) { return unalignedChunkFitsInSurvivors((UnalignedHeapChunk.UnalignedHeader) chunk); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean alignedChunkFitsInSurvivors() { UnsignedWord sum = survivorsToSpacesAccounting.getChunkBytes().add(HeapParameters.getAlignedHeapChunkSize()); return sum.belowOrEqual(GCImpl.getPolicy().getSurvivorSpacesCapacity()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private boolean unalignedChunkFitsInSurvivors(UnalignedHeapChunk.UnalignedHeader chunk) { UnsignedWord size = UnalignedHeapChunk.getCommittedObjectMemory(chunk); UnsignedWord sum = survivorsToSpacesAccounting.getChunkBytes().add(size); return sum.belowOrEqual(GCImpl.getPolicy().getSurvivorSpacesCapacity()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) AlignedHeapChunk.AlignedHeader requestAlignedSurvivorChunk() { assert VMOperation.isGCInProgress() : "Should only be called from the collector."; if (!alignedChunkFitsInSurvivors()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java index d244d47c7cb..8c678b33e9a 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/AlignedChunkRememberedSet.java @@ -37,6 +37,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; @@ -86,6 +87,7 @@ public static void enableRememberedSet(HostedByteBufferPointer chunk, int chunkP } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { Pointer fotStart = getFirstObjectTableStart(chunk); Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); @@ -95,6 +97,7 @@ public static void enableRememberedSetForObject(AlignedHeader chunk, Object obj) ObjectHeaderImpl.setRememberedSetBit(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSet(AlignedHeader chunk) { // Completely clean the card table and the first object table as further objects may be // added later on to this chunk. @@ -110,6 +113,7 @@ public static void enableRememberedSet(AlignedHeader chunk) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void clearRememberedSet(AlignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); } @@ -118,6 +122,7 @@ public static void clearRememberedSet(AlignedHeader chunk) { * Dirty the card corresponding to the given Object. This has to be fast, because it is used by * the post-write barrier. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void dirtyCardForObject(Object object, boolean verifyOnly) { Pointer objectPointer = Word.objectToUntrackedPointer(object); AlignedHeader chunk = AlignedHeapChunk.getEnclosingChunkFromObjectPointer(objectPointer); @@ -130,6 +135,7 @@ public static void dirtyCardForObject(Object object, boolean verifyOnly) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) { Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); Pointer objectsLimit = HeapChunk.getTopPointer(chunk); @@ -183,6 +189,7 @@ public static void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisito } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void walkObjects(AlignedHeader chunk, Pointer start, Pointer end, GreyToBlackObjectVisitor visitor) { Pointer fotStart = getFirstObjectTableStart(chunk); Pointer objectsStart = AlignedHeapChunk.getObjectsStart(chunk); @@ -203,6 +210,7 @@ public static boolean verify(AlignedHeader chunk) { } /** Return the index of an object within the tables of a chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord getObjectIndex(AlignedHeader chunk, Pointer objectPointer) { UnsignedWord offset = AlignedHeapChunk.getObjectOffset(chunk, objectPointer); return CardTable.memoryOffsetToIndex(offset); @@ -260,18 +268,22 @@ static UnsignedWord getCardTableLimitOffset() { return UnsignedUtils.roundUp(tableLimit, alignment); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(AlignedHeader chunk) { return getCardTableStart(HeapChunk.asPointer(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(Pointer chunk) { return chunk.add(getCardTableStartOffset()); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getFirstObjectTableStart(AlignedHeader chunk) { return getFirstObjectTableStart(HeapChunk.asPointer(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getFirstObjectTableStart(Pointer chunk) { return chunk.add(getFirstObjectTableStartOffset()); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java index fa707695db9..d49928864d3 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTable.java @@ -33,6 +33,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.genscavenge.HeapChunk; import com.oracle.svm.core.genscavenge.HeapImpl; @@ -83,10 +84,12 @@ final class CardTable { private CardTable() { } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void cleanTable(Pointer tableStart, UnsignedWord size) { UnmanagedMemoryUtil.fill(tableStart, size, CLEAN_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setDirty(Pointer table, UnsignedWord index) { byte valueBefore = table.readByte(index, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); // Using a likely probability should typically avoid placing the write below at a separate @@ -96,10 +99,12 @@ public static void setDirty(Pointer table, UnsignedWord index) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setClean(Pointer table, UnsignedWord index) { table.writeByte(index, CLEAN_ENTRY, BarrierSnippets.CARD_REMEMBERED_SET_LOCATION); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean isDirty(Pointer table, UnsignedWord index) { int entry = readEntry(table, index); return entry == DIRTY_ENTRY; @@ -110,23 +115,28 @@ private static boolean isClean(Pointer table, UnsignedWord index) { return entry == CLEAN_ENTRY; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int readEntry(Pointer table, UnsignedWord index) { return table.readByte(index); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord memoryOffsetToIndex(UnsignedWord offset) { return offset.unsignedDivide(BYTES_COVERED_BY_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer cardToHeapAddress(Pointer cardTableStart, Pointer cardAddr, Pointer objectsStart) { UnsignedWord offset = cardAddr.subtract(cardTableStart).multiply(CardTable.BYTES_COVERED_BY_ENTRY); return objectsStart.add(offset); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord tableSizeForMemorySize(UnsignedWord memorySize) { return indexLimitForMemorySize(memorySize); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord indexLimitForMemorySize(UnsignedWord memorySize) { UnsignedWord roundedMemory = UnsignedUtils.roundUp(memorySize, WordFactory.unsigned(BYTES_COVERED_BY_ENTRY)); return CardTable.memoryOffsetToIndex(roundedMemory); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java index 57d0cf1f614..ca4bd4b7409 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/CardTableBasedRememberedSet.java @@ -32,6 +32,7 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.GCImpl; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; @@ -87,33 +88,39 @@ public void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk) } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(AlignedHeader chunk) { AlignedChunkRememberedSet.enableRememberedSet(chunk); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(UnalignedHeader chunk) { UnalignedChunkRememberedSet.enableRememberedSet(chunk); } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { AlignedChunkRememberedSet.enableRememberedSetForObject(chunk, obj); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(AlignedHeader chunk) { AlignedChunkRememberedSet.clearRememberedSet(chunk); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(UnalignedHeader chunk) { UnalignedChunkRememberedSet.clearRememberedSet(chunk); } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean hasRememberedSet(UnsignedWord header) { return ObjectHeaderImpl.hasRememberedSet(header); } @@ -132,6 +139,7 @@ public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) { @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void dirtyCardIfNecessary(Object holderObject, Object object) { if (holderObject == null || object == null) { return; @@ -164,16 +172,19 @@ public void dirtyCardIfNecessary(Object holderObject, Object object) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) { AlignedChunkRememberedSet.walkDirtyObjects(chunk, visitor, clean); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) { UnalignedChunkRememberedSet.walkDirtyObjects(chunk, visitor, clean); } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walkDirtyObjects(Space space, GreyToBlackObjectVisitor visitor, boolean clean) { AlignedHeader aChunk = space.getFirstAlignedHeapChunk(); while (aChunk.isNonNull()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java index 2c5f3ff3b14..6c927f47cf0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/FirstObjectTable.java @@ -30,6 +30,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.SubstrateUtil; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.hub.LayoutEncoding; @@ -163,6 +164,7 @@ final class FirstObjectTable { private FirstObjectTable() { } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void initializeTable(Pointer table, UnsignedWord size) { if (SubstrateUtil.HOSTED) { // Initialize this table unconditionally as this simplifies a few things. @@ -173,12 +175,14 @@ public static void initializeTable(Pointer table, UnsignedWord size) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean doInitializeTable(Pointer table, UnsignedWord size) { UnmanagedMemoryUtil.fill(table, size, (byte) UNINITIALIZED_ENTRY); return true; } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setTableForObject(Pointer table, UnsignedWord startOffset, UnsignedWord endOffset) { assert startOffset.belowThan(endOffset); UnsignedWord startIndex = memoryOffsetToIndex(startOffset); @@ -232,6 +236,7 @@ public static void setTableForObject(Pointer table, UnsignedWord startOffset, Un * outside the current card. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer objectsStart, UnsignedWord index) { Pointer result; Pointer firstObject = getFirstObject(tableStart, objectsStart, index); @@ -249,6 +254,7 @@ public static Pointer getFirstObjectImprecise(Pointer tableStart, Pointer object } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getFirstObject(Pointer tableStart, Pointer objectsStart, UnsignedWord index) { UnsignedWord currentIndex = index; int currentEntry = getEntryAtIndex(tableStart, currentIndex); @@ -278,6 +284,8 @@ private static Pointer getFirstObject(Pointer tableStart, Pointer objectsStart, return result; } + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord entryToMemoryOffset(UnsignedWord index, int entry) { assert isMemoryOffsetEntry(entry) : "Entry out of bounds."; UnsignedWord entryOffset = WordFactory.unsigned(-entry).multiply(memoryOffsetScale()); @@ -328,69 +336,83 @@ private static UnsignedWord getTableSizeForMemoryRange(Pointer memoryStart, Poin * The multiplier from memory offsets to byte offsets into the previous card. This is the * granularity to which I can point to the start of an object. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int memoryOffsetScale() { return ConfigurationValues.getObjectLayout().getAlignment(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int getEntryAtIndex(Pointer table, UnsignedWord index) { return table.readByte(indexToTableOffset(index)); } /** Set the table entry at a given index. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static void setEntryAtIndex(Pointer table, UnsignedWord index, int value) { assert isValidEntry(value) : "Invalid entry"; assert isUninitializedIndex(table, index) || getEntryAtIndex(table, index) == value : "Overwriting!"; table.writeByte(indexToTableOffset(index), (byte) value); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isUninitializedIndex(Pointer table, UnsignedWord index) { int entry = getEntryAtIndex(table, index); return isUninitializedEntry(entry); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isValidEntry(int entry) { return ENTRY_MIN <= entry && entry <= ENTRY_MAX; } /** May only be used for assertions. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isUninitializedEntry(int entry) { assert isValidEntry(entry) : "Invalid entry"; return entry == UNINITIALIZED_ENTRY; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean isMemoryOffsetEntry(int entry) { assert isValidEntry(entry) : "Invalid entry"; return MEMORY_OFFSET_MIN <= entry && entry <= MEMORY_OFFSET_MAX; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int biasExponent(int exponent) { assert EXPONENT_MIN <= exponent && exponent <= EXPONENT_MAX : "Exponent out of bounds."; return exponent + EXPONENT_BIAS; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int unbiasExponent(int entry) { int exponent = entry - EXPONENT_BIAS; assert EXPONENT_MIN <= exponent && exponent <= EXPONENT_MAX : "Exponent out of bounds."; return exponent; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord exponentToOffset(int n) { assert 0 <= n && n <= 63 : "Exponent out of bounds."; return WordFactory.unsigned(1L << n); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord indexToTableOffset(UnsignedWord index) { return index.multiply(ENTRY_SIZE_BYTES); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord indexToMemoryOffset(UnsignedWord index) { return index.multiply(BYTES_COVERED_BY_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static UnsignedWord memoryOffsetToIndex(UnsignedWord offset) { return offset.unsignedDivide(BYTES_COVERED_BY_ENTRY); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static int memoryOffsetToEntry(UnsignedWord memoryOffset) { assert memoryOffset.belowThan(BYTES_COVERED_BY_ENTRY) : "Offset out of bounds."; UnsignedWord scaledOffset = memoryOffset.unsignedDivide(memoryOffsetScale()); diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java index 5f61e5c4d38..8f7b870b487 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/NoRememberedSet.java @@ -34,6 +34,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; @@ -85,32 +86,39 @@ public void enableRememberedSetForUnalignedChunk(HostedByteBufferPointer chunk) } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(AlignedHeader chunk) { // Nothing to do. } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForChunk(UnalignedHeader chunk) { // Nothing to do. } @Override + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void enableRememberedSetForObject(AlignedHeader chunk, Object obj) { // Nothing to do. } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(AlignedHeader chunk) { // Nothing to do. } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void clearRememberedSet(UnalignedHeader chunk) { // Nothing to do. } @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean hasRememberedSet(UnsignedWord header) { return false; } @@ -127,21 +135,25 @@ public void dirtyCardForUnalignedObject(Object object, boolean verifyOnly) { @Override @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void dirtyCardIfNecessary(Object holderObject, Object object) { // Nothing to do. } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) { throw VMError.shouldNotReachHereAtRuntime(); // ExcludeFromJacocoGeneratedReport } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) { throw VMError.shouldNotReachHereAtRuntime(); // ExcludeFromJacocoGeneratedReport } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walkDirtyObjects(Space space, GreyToBlackObjectVisitor visitor, boolean clean) { throw VMError.shouldNotReachHereAtRuntime(); // ExcludeFromJacocoGeneratedReport } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java index d6b1725a296..d641f6569a8 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/RememberedSet.java @@ -33,6 +33,7 @@ import org.graalvm.word.UnsignedWord; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.genscavenge.AlignedHeapChunk.AlignedHeader; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.Space; @@ -76,28 +77,35 @@ static RememberedSet get() { * Enables remembered set tracking for an aligned chunk and its objects. Must be called when * adding a new chunk to the image heap or old generation. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void enableRememberedSetForChunk(AlignedHeader chunk); /** * Enables remembered set tracking for an unaligned chunk and its objects. Must be called when * adding a new chunk to the image heap or old generation. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void enableRememberedSetForChunk(UnalignedHeader chunk); /** * Enables remembered set tracking for a single object in an aligned chunk. Must be called when * an object is added to the image heap or old generation. */ + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void enableRememberedSetForObject(AlignedHeader chunk, Object obj); /** Clears the remembered set of an aligned chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void clearRememberedSet(AlignedHeader chunk); /** Clears the remembered set of an unaligned chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void clearRememberedSet(UnalignedHeader chunk); /** Checks if remembered set tracking is enabled for an object. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean hasRememberedSet(UnsignedWord header); /** @@ -122,21 +130,25 @@ static RememberedSet get() { * tracking is enabled. */ @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void dirtyCardIfNecessary(Object holderObject, Object object); /** * Walks all dirty objects in an aligned chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void walkDirtyObjects(AlignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean); /** * Walks all dirty objects in an unaligned chunk. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean); /** * Walks all dirty objects in a {@link Space}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void walkDirtyObjects(Space space, GreyToBlackObjectVisitor visitor, boolean clean); /** diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java index 1e993fb3f30..c59c995c714 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/remset/UnalignedChunkRememberedSet.java @@ -33,6 +33,7 @@ import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.genscavenge.GreyToBlackObjectVisitor; import com.oracle.svm.core.genscavenge.HeapChunk; @@ -59,6 +60,7 @@ public static void enableRememberedSet(HostedByteBufferPointer chunk) { // The remembered set bit in the header will be set by the code that writes the objects. } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void enableRememberedSet(UnalignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); // Unaligned chunks don't have a first object table. @@ -67,6 +69,7 @@ public static void enableRememberedSet(UnalignedHeader chunk) { ObjectHeaderImpl.setRememberedSetBit(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void clearRememberedSet(UnalignedHeader chunk) { CardTable.cleanTable(getCardTableStart(chunk), getCardTableSize()); } @@ -75,6 +78,7 @@ public static void clearRememberedSet(UnalignedHeader chunk) { * Dirty the card corresponding to the given Object. This has to be fast, because it is used by * the post-write barrier. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void dirtyCardForObject(Object obj, boolean verifyOnly) { UnalignedHeader chunk = UnalignedHeapChunk.getEnclosingChunk(obj); Pointer cardTableStart = getCardTableStart(chunk); @@ -86,6 +90,7 @@ public static void dirtyCardForObject(Object obj, boolean verifyOnly) { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void walkDirtyObjects(UnalignedHeader chunk, GreyToBlackObjectVisitor visitor, boolean clean) { Pointer rememberedSetStart = getCardTableStart(chunk); UnsignedWord objectIndex = getObjectIndex(); @@ -132,10 +137,12 @@ static UnsignedWord getObjectIndex() { return WordFactory.zero(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(UnalignedHeader chunk) { return getCardTableStart(HeapChunk.asPointer(chunk)); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static Pointer getCardTableStart(Pointer chunk) { return chunk.add(getCardTableStartOffset()); } diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java index 1386cf5461a..a7a165ac0a7 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/pthread/PthreadVMLockSupport.java @@ -371,6 +371,7 @@ private static long remainingNanos(long waitNanos, long startNanos) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { PthreadVMLockSupport.checkResult(Pthread.pthread_cond_signal(getStructPointer()), "pthread_cond_signal"); } diff --git a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java index 55980c4a1f5..4fd44af9e3f 100644 --- a/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core.windows/src/com/oracle/svm/core/windows/WindowsVMLockSupport.java @@ -359,6 +359,7 @@ public long blockNoTransition(long waitNanos) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { Process.NoTransitions.WakeConditionVariable(getStructPointer()); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java index a480231284e..7770445438a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java @@ -27,9 +27,9 @@ import org.graalvm.word.PointerBase; import org.graalvm.word.UnsignedWord; -import com.oracle.svm.core.heap.RestrictHeapAccess; import com.oracle.svm.core.code.CodeInfo; import com.oracle.svm.core.heap.ObjectVisitor; +import com.oracle.svm.core.heap.RestrictHeapAccess; /** A walker over different kinds of allocated memory. */ public final class MemoryWalker { @@ -64,10 +64,13 @@ public interface NativeImageHeapRegionAccess { String getRegionName(T region); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean containsReferences(T region); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean isWritable(T region); + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean visitObjects(T region, ObjectVisitor visitor); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java index cb8b929c1db..a63a9c01afc 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/c/NonmovableArrays.java @@ -43,8 +43,8 @@ import com.oracle.svm.core.JavaMemoryUtil; import com.oracle.svm.core.SubstrateUtil; -import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.UnmanagedMemoryUtil; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.heap.Heap; import com.oracle.svm.core.heap.ObjectHeader; @@ -443,6 +443,7 @@ public static void setObject(NonmovableObjectArray array, int index, T va /** * Visits all array elements with the provided {@link ObjectReferenceVisitor}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkUnmanagedObjectArray(NonmovableObjectArray array, ObjectReferenceVisitor visitor) { if (array.isNonNull()) { return walkUnmanagedObjectArray(array, visitor, 0, lengthOf(array)); @@ -453,6 +454,7 @@ public static boolean walkUnmanagedObjectArray(NonmovableObjectArray array, O /** * Visits all array elements with the provided {@link ObjectReferenceVisitor}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkUnmanagedObjectArray(NonmovableObjectArray array, ObjectReferenceVisitor visitor, int startIndex, int count) { if (array.isNonNull()) { assert startIndex >= 0 && count <= lengthOf(array) - startIndex; @@ -460,7 +462,7 @@ public static boolean walkUnmanagedObjectArray(NonmovableObjectArray array, O assert refSize == (1 << readElementShift(array)); Pointer p = ((Pointer) array).add(readArrayBase(array)).add(startIndex * refSize); for (int i = 0; i < count; i++) { - if (!visitor.visitObjectReference(p, true, null)) { + if (!callVisitor(visitor, p)) { return false; } p = p.add(refSize); @@ -469,6 +471,11 @@ public static boolean walkUnmanagedObjectArray(NonmovableObjectArray array, O return true; } + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static boolean callVisitor(ObjectReferenceVisitor visitor, Pointer p) { + return visitor.visitObjectReference(p, true, null); + } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void tearDown() { assert runtimeArraysInExistence.get() == 0 : "All runtime-allocated NonmovableArrays must have been freed"; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java index ffdddf3856d..1ce82946f0c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/CodeInfoTable.java @@ -152,6 +152,7 @@ public static boolean visitObjectReferences(Pointer sp, CodePointer ip, CodeInfo return CodeReferenceMapDecoder.walkOffsetsFromPointer(sp, referenceMapEncoding, referenceMapIndex, visitor, null); } + @Uninterruptible(reason = "Not really uninterruptible, but we are about to fail.", calleeMustBe = false) public static RuntimeException reportNoReferenceMap(Pointer sp, CodePointer ip, CodeInfo info) { Log.log().string("ip: ").hex(ip).string(" sp: ").hex(sp).string(" info:"); CodeInfoAccess.log(info, Log.log()).newline(); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java index 996f1089c4a..7332fbffbef 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoAccess.java @@ -153,6 +153,7 @@ public static boolean areAllObjectsOnImageHeap(CodeInfo info) { /** * Walks all strong references in a {@link CodeInfo} object. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkStrongReferences(CodeInfo info, ObjectReferenceVisitor visitor) { return NonmovableArrays.walkUnmanagedObjectArray(cast(info).getObjectFields(), visitor, CodeInfoImpl.FIRST_STRONGLY_REFERENCED_OBJFIELD, CodeInfoImpl.STRONGLY_REFERENCED_OBJFIELD_COUNT); } @@ -161,6 +162,7 @@ public static boolean walkStrongReferences(CodeInfo info, ObjectReferenceVisitor * Walks all weak references in a {@link CodeInfo} object. */ @DuplicatedInNativeCode + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkWeakReferences(CodeInfo info, ObjectReferenceVisitor visitor) { CodeInfoImpl impl = cast(info); boolean continueVisiting = true; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java index febd8215da9..abf3f93fc04 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/config/ObjectLayout.java @@ -75,11 +75,13 @@ public ObjectLayout(SubstrateTargetDescription target, int referenceSize, int ob } /** The minimum alignment of objects (instances and arrays). */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getAlignment() { return objectAlignment; } /** Tests if the given offset or address is aligned according to {@link #getAlignment()}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isAligned(final long value) { return (value % getAlignment() == 0L); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/CodeReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/CodeReferenceMapDecoder.java index a85773cdd5e..6eba6c4dfc5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/CodeReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/CodeReferenceMapDecoder.java @@ -186,7 +186,7 @@ public static long decodeSign(long value) { } @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true, calleeMustBe = false) + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) private static boolean callVisitObjectReferenceInline(ObjectReferenceVisitor visitor, Pointer derivedRef, int innerOffset, boolean compressed, Object holderObject) { return visitor.visitObjectReferenceInline(derivedRef, innerOffset, compressed, holderObject); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java index 0c8034d5114..d7a10f67ec7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/InstanceReferenceMapDecoder.java @@ -29,6 +29,7 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.util.DuplicatedInNativeCode; @@ -38,6 +39,7 @@ @DuplicatedInNativeCode public class InstanceReferenceMapDecoder { @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkOffsetsFromPointer(Pointer baseAddress, NonmovableArray referenceMapEncoding, long referenceMapIndex, ObjectReferenceVisitor visitor, Object holderObject) { assert ReferenceMapIndex.denotesValidReferenceMap(referenceMapIndex); assert referenceMapEncoding.isNonNull(); @@ -61,7 +63,7 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, NonmovableArra Pointer objRef = baseAddress.add(offset); for (int c = 0; c < count; c++) { - final boolean visitResult = visitor.visitObjectReferenceInline(objRef, 0, compressed, holderObject); + final boolean visitResult = callVisitor(visitor, holderObject, compressed, objRef); if (!visitResult) { return false; } @@ -70,4 +72,9 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, NonmovableArra } return true; } + + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static boolean callVisitor(ObjectReferenceVisitor visitor, Object holderObject, boolean compressed, Pointer objRef) { + return visitor.visitObjectReferenceInline(objRef, 0, compressed, holderObject); + } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java index 25df0155520..d74bd5435bf 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ObjectHeader.java @@ -68,6 +68,7 @@ public DynamicHub dynamicHubFromObjectHeader(Word header) { return (DynamicHub) extractPotentialDynamicHubFromHeader(header).toObject(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static DynamicHub readDynamicHubFromObject(Object o) { return KnownIntrinsics.readHub(o); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java index 86fb9c99d2d..2daabb05d6e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/OutOfMemoryUtil.java @@ -25,6 +25,7 @@ package com.oracle.svm.core.heap; import com.oracle.svm.core.SubstrateGCOptions; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.headers.LibC; import com.oracle.svm.core.jdk.JDKUtils; import com.oracle.svm.core.log.Log; @@ -38,6 +39,7 @@ public static OutOfMemoryError heapSizeExceeded() { return reportOutOfMemoryError(OUT_OF_MEMORY_ERROR); } + @Uninterruptible(reason = "Not uninterruptible but it doesn't matter for the callers.", calleeMustBe = false) @RestrictHeapAccess(access = RestrictHeapAccess.Access.NO_ALLOCATION, reason = "Can't allocate while out of memory.") public static OutOfMemoryError reportOutOfMemoryError(OutOfMemoryError error) { if (SubstrateGCOptions.ExitOnOutOfMemoryError.getValue()) { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java index bd10eb301a5..913455f9615 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/PodReferenceMapDecoder.java @@ -34,16 +34,19 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.JavaMemoryUtil; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.graal.nodes.NewPodInstanceNode; import com.oracle.svm.core.hub.DynamicHub; import com.oracle.svm.core.hub.LayoutEncoding; +import com.oracle.svm.core.jdk.UninterruptibleUtils; import com.oracle.svm.core.util.DuplicatedInNativeCode; import com.oracle.svm.core.util.UnsignedUtils; public final class PodReferenceMapDecoder { @DuplicatedInNativeCode @AlwaysInline("de-virtualize calls to ObjectReferenceVisitor") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEncoding, ObjectReferenceVisitor visitor, Object obj) { int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); boolean isCompressed = ReferenceAccess.singleton().haveCompressedReferences(); @@ -55,11 +58,11 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEnco int gap; do { mapOffset = mapOffset.subtract(2); - gap = Byte.toUnsignedInt(baseAddress.readByte(mapOffset)); - nrefs = Byte.toUnsignedInt(baseAddress.readByte(mapOffset.add(1))); + gap = UninterruptibleUtils.Byte.toUnsignedInt(baseAddress.readByte(mapOffset)); + nrefs = UninterruptibleUtils.Byte.toUnsignedInt(baseAddress.readByte(mapOffset.add(1))); for (int i = 0; i < nrefs; i++) { - if (!visitor.visitObjectReferenceInline(baseAddress.add(refOffset), 0, isCompressed, obj)) { + if (!callVisitor(baseAddress, visitor, obj, isCompressed, refOffset)) { return false; } refOffset = refOffset.add(referenceSize); @@ -70,6 +73,11 @@ public static boolean walkOffsetsFromPointer(Pointer baseAddress, int layoutEnco return true; } + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static boolean callVisitor(Pointer baseAddress, ObjectReferenceVisitor visitor, Object obj, boolean isCompressed, UnsignedWord refOffset) { + return visitor.visitObjectReferenceInline(baseAddress.add(refOffset), 0, isCompressed, obj); + } + /** * Implements the allocation and the copying of the reference map and data stored in the hybrid * array for {@link Object#clone()}. diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java index 68605e09b18..f0c879cd838 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceInternals.java @@ -69,6 +69,7 @@ static Reference uncast(Target_java_lang_ref_Reference instance) { } /** Barrier-less read of {@link Target_java_lang_ref_Reference#referent} as a pointer. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getReferentPointer(Reference instance) { return Word.objectToUntrackedPointer(ObjectAccess.readObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.referentFieldOffset))); } @@ -79,7 +80,7 @@ public static T getReferent(Reference instance) { } /** Write {@link Target_java_lang_ref_Reference#referent}. */ - @SuppressWarnings("unchecked") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setReferent(Reference instance, Object value) { BarrieredAccess.writeObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.referentFieldOffset), value); } @@ -105,6 +106,7 @@ public static void clear(Reference instance) { ObjectAccess.writeObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.referentFieldOffset), null); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getReferentFieldAddress(Reference instance) { return Word.objectToUntrackedPointer(instance).add(WordFactory.unsigned(Target_java_lang_ref_Reference.referentFieldOffset)); } @@ -119,6 +121,7 @@ public static Reference getNextDiscovered(Reference instance) { } /** Barrier-less read of {@link Target_java_lang_ref_Reference#discovered} as a pointer. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getDiscoveredPointer(Reference instance) { return Word.objectToUntrackedPointer(ObjectAccess.readObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.discoveredFieldOffset))); } @@ -140,6 +143,7 @@ public static boolean isAnyReferenceFieldOffset(long offset) { } /** Write {@link Target_java_lang_ref_Reference#discovered}. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static void setNextDiscovered(Reference instance, Reference newNext) { BarrieredAccess.writeObject(instance, WordFactory.signed(Target_java_lang_ref_Reference.discoveredFieldOffset), newNext); } @@ -251,6 +255,7 @@ public static boolean waitForReferenceProcessing() throws InterruptedException { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long getSoftReferenceClock() { return Target_java_lang_ref_SoftReference.clock; } @@ -262,6 +267,7 @@ public static void updateSoftReferenceClock() { } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static long getSoftReferenceTimestamp(SoftReference instance) { Target_java_lang_ref_SoftReference ref = SubstrateUtil.cast(instance, Target_java_lang_ref_SoftReference.class); return ref.timestamp; diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java index 4989c633cbb..e80b89687ee 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/heap/ReferenceMapIndex.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.core.heap; +import com.oracle.svm.core.Uninterruptible; + public class ReferenceMapIndex { /** * Marker value returned by @@ -44,6 +46,7 @@ public static boolean denotesEmptyReferenceMap(long referenceMapIndex) { return referenceMapIndex == EMPTY_REFERENCE_MAP || referenceMapIndex == NO_REFERENCE_MAP; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean denotesValidReferenceMap(long referenceMapIndex) { return referenceMapIndex != NO_REFERENCE_MAP; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java index d228ddb2166..7be91179acd 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/DynamicHub.java @@ -618,6 +618,7 @@ public DynamicHub getArrayHub() { return arrayHub; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getReferenceMapIndex() { return referenceMapIndex; } @@ -662,6 +663,7 @@ public String getName() { return name; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public int getHubType() { return hubType; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java index fe2a8f6c450..989fad267fa 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/InteriorObjRefWalker.java @@ -33,6 +33,7 @@ import com.oracle.svm.core.AlwaysInline; import com.oracle.svm.core.NeverInline; +import com.oracle.svm.core.Uninterruptible; import com.oracle.svm.core.c.NonmovableArray; import com.oracle.svm.core.config.ConfigurationValues; import com.oracle.svm.core.heap.InstanceReferenceMapDecoder; @@ -66,6 +67,7 @@ public static boolean walkObject(final Object obj, final ObjectReferenceVisitor } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static boolean walkObjectInline(final Object obj, final ObjectReferenceVisitor visitor) { final DynamicHub objHub = ObjectHeader.readDynamicHubFromObject(obj); final Pointer objPointer = Word.objectToUntrackedPointer(obj); @@ -107,6 +109,7 @@ public boolean visitObjectReference(Pointer objRef, boolean compressed, Object h } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkInstance(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { NonmovableArray referenceMapEncoding = DynamicHubSupport.getReferenceMapEncoding(); long referenceMapIndex = objHub.getReferenceMapIndex(); @@ -116,6 +119,7 @@ private static boolean walkInstance(Object obj, ObjectReferenceVisitor visitor, } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkPod(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { if (!Pod.RuntimeSupport.isPresent()) { throw VMError.shouldNotReachHere("Pod objects cannot be in the heap if the pod support is disabled."); @@ -124,6 +128,7 @@ private static boolean walkPod(Object obj, ObjectReferenceVisitor visitor, Dynam } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkStoredContinuation(Object obj, ObjectReferenceVisitor visitor) { if (!Continuation.isSupported()) { throw VMError.shouldNotReachHere("Stored continuation objects cannot be in the heap if the continuation support is disabled."); @@ -132,11 +137,13 @@ private static boolean walkStoredContinuation(Object obj, ObjectReferenceVisitor } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkOther() { throw VMError.shouldNotReachHere("Unexpected object with hub type 'other' in the heap."); } @AlwaysInline("Performance critical version") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private static boolean walkObjectArray(Object obj, ObjectReferenceVisitor visitor, DynamicHub objHub, Pointer objPointer) { int length = ArrayLengthNode.arrayLength(obj); int referenceSize = ConfigurationValues.getObjectLayout().getReferenceSize(); @@ -145,7 +152,7 @@ private static boolean walkObjectArray(Object obj, ObjectReferenceVisitor visito Pointer pos = objPointer.add(LayoutEncoding.getArrayBaseOffset(objHub.getLayoutEncoding())); Pointer end = pos.add(WordFactory.unsigned(referenceSize).multiply(length)); while (pos.belowThan(end)) { - final boolean visitResult = visitor.visitObjectReferenceInline(pos, 0, isCompressed, obj); + final boolean visitResult = callVisitor(obj, visitor, isCompressed, pos); if (!visitResult) { return false; } @@ -153,4 +160,9 @@ private static boolean walkObjectArray(Object obj, ObjectReferenceVisitor visito } return true; } + + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static boolean callVisitor(Object obj, ObjectReferenceVisitor visitor, boolean isCompressed, Pointer pos) { + return visitor.visitObjectReferenceInline(pos, 0, isCompressed, obj); + } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java index e13f5e27379..b1f30bc5a7d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/hub/LayoutEncoding.java @@ -329,16 +329,19 @@ public static UnsignedWord getMomentarySizeFromObject(Object obj) { return getSizeFromObject(obj); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInGC(Object obj) { return getSizeFromObjectInlineInGC(obj); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInlineInGC(Object obj) { return getSizeFromObjectInlineInGC(obj, false); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord getSizeFromObjectInlineInGC(Object obj, boolean addOptionalIdHashField) { boolean withOptionalIdHashField = addOptionalIdHashField || (!ConfigurationValues.getObjectLayout().hasFixedIdentityHashField() && checkOptionalIdentityHashField(obj)); @@ -365,11 +368,13 @@ private static boolean checkOptionalIdentityHashField(Object obj) { return oh.hasOptionalIdentityHashField(header); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getObjectEndInGC(Object obj) { return getObjectEndInlineInGC(obj); } @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static Pointer getObjectEndInlineInGC(Object obj) { UnsignedWord size = getSizeFromObjectInlineInGC(obj, false); return Word.objectToUntrackedPointer(obj).add(size); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java index edb8c09bb17..838d452afd3 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java @@ -435,6 +435,13 @@ public static long abs(long a) { } } + public static class Byte { + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public static int toUnsignedInt(byte x) { + return ((int) x) & 0xff; + } + } + public static class Long { /** Uninterruptible version of {@link java.lang.Long#numberOfLeadingZeros(long)}. */ @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java index cb6a1753960..ce0e926ebee 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/SingleThreadedVMLockSupport.java @@ -177,6 +177,7 @@ public long blockNoTransition(long nanos) { } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { /* Nothing to do. */ } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java index 6a587217130..f85d789026d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/locks/VMCondition.java @@ -98,6 +98,7 @@ public void blockNoTransitionUnspecifiedOwner() { /** * Wakes up a single thread that is waiting on this condition. */ + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void signal() { throw VMError.shouldNotReachHere("VMCondition cannot be used during native image generation"); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java index f381b6ed1b4..9e2be0270d7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/AbstractCommittedMemoryProvider.java @@ -103,6 +103,7 @@ public int protect(PointerBase start, UnsignedWord nbytes, EnumSet acces } @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment) { return allocate(nbytes, alignment, false); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalMTSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalMTSupport.java index bb192cc5a5b..874bd588bc1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalMTSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/threadlocal/VMThreadLocalMTSupport.java @@ -67,6 +67,7 @@ public int getThreadLocalsReferenceMapIndex() { return (int) result; } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walk(IsolateThread isolateThread, ObjectReferenceVisitor referenceVisitor) { NonmovableArray threadRefMapEncoding = NonmovableArrays.fromImageHeap(vmThreadReferenceMapEncoding); InstanceReferenceMapDecoder.walkOffsetsFromPointer((Pointer) isolateThread, threadRefMapEncoding, vmThreadReferenceMapIndex, referenceVisitor, null); From 6d361f5fbc081e37d6d6c78e59b5286342f89901 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 3 Apr 2023 18:14:15 +0200 Subject: [PATCH 30/43] Minor fixes for fatal error diagnostics. --- .../src/com/oracle/svm/core/SubstrateDiagnostics.java | 4 ++-- .../src/com/oracle/svm/core/SubstrateSegfaultHandler.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java index c6a4372a42d..0d9e14f7d4e 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateDiagnostics.java @@ -988,11 +988,11 @@ public int maxInvocationCount() { public void printDiagnostics(Log log, ErrorContext context, int maxDiagnosticLevel, int invocationCount) { String[] args = ImageSingletons.lookup(JavaMainWrapper.JavaMainSupport.class).mainArgs; if (args != null) { - log.string("Command line: ").newline(); + log.string("Command line: "); for (String arg : args) { log.string("'").string(arg).string("' "); } - log.newline(); + log.newline().newline(); } } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateSegfaultHandler.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateSegfaultHandler.java index 980cd605c99..a931b4b974d 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateSegfaultHandler.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/SubstrateSegfaultHandler.java @@ -192,7 +192,7 @@ protected static void printSegfaultAddressInfo(Log log, long addr) { if (addr != 0) { long delta = addr - CurrentIsolate.getIsolate().rawValue(); String sign = (delta >= 0 ? "+" : "-"); - log.string("(heapBase ").string(sign).string(" ").signed(Math.abs(delta)).string(")"); + log.string(" (heapBase ").string(sign).string(" ").signed(Math.abs(delta)).string(")"); } } From d494087d4c60768debccdb5c0723f2fc2821a9c4 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 3 Apr 2023 18:19:53 +0200 Subject: [PATCH 31/43] Add support for unmanaged thread locals. --- .../posix/PosixSubstrateSigprofHandler.java | 37 ------------------- .../posix/thread/PosixPlatformThreads.java | 34 +++++++++++++++++ .../core/sampler/SubstrateSigprofHandler.java | 29 +++++---------- .../svm/core/thread/PlatformThreads.java | 33 +++++++++++++++++ 4 files changed, 76 insertions(+), 57 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java index dc4a803d0cd..b0ed8c238fc 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixSubstrateSigprofHandler.java @@ -31,18 +31,14 @@ import org.graalvm.compiler.options.Option; import org.graalvm.nativeimage.ImageSingletons; -import org.graalvm.nativeimage.IsolateThread; import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; -import org.graalvm.nativeimage.StackValue; import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.function.CEntryPointLiteral; import org.graalvm.nativeimage.c.function.CodePointer; import org.graalvm.nativeimage.c.struct.SizeOf; -import org.graalvm.nativeimage.c.type.VoidPointer; import org.graalvm.nativeimage.hosted.Feature; import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.IsolateListenerSupport; @@ -59,7 +55,6 @@ import com.oracle.svm.core.jfr.JfrFeature; import com.oracle.svm.core.jfr.sampler.JfrExecutionSampler; import com.oracle.svm.core.option.HostedOptionKey; -import com.oracle.svm.core.posix.headers.Pthread; import com.oracle.svm.core.posix.headers.Signal; import com.oracle.svm.core.posix.headers.Time; import com.oracle.svm.core.sampler.SubstrateSigprofHandler; @@ -136,38 +131,6 @@ protected void uninstallSignalHandler() { updateInterval(0); } - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected UnsignedWord createNativeThreadLocal() { - Pthread.pthread_key_tPointer key = StackValue.get(Pthread.pthread_key_tPointer.class); - PosixUtils.checkStatusIs0(Pthread.pthread_key_create(key, WordFactory.nullPointer()), "pthread_key_create(key, keyDestructor): failed."); - return key.read(); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected void deleteNativeThreadLocal(UnsignedWord key) { - int resultCode = Pthread.pthread_key_delete((Pthread.pthread_key_t) key); - PosixUtils.checkStatusIs0(resultCode, "pthread_key_delete(key): failed."); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected void setNativeThreadLocalValue(UnsignedWord key, IsolateThread value) { - int resultCode = Pthread.pthread_setspecific((Pthread.pthread_key_t) key, (VoidPointer) value); - PosixUtils.checkStatusIs0(resultCode, "pthread_setspecific(key, value): wrong arguments."); - } - - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected IsolateThread getNativeThreadLocalValue(UnsignedWord key) { - /* - * Although this method is not async-signal-safe in general we rely on - * implementation-specific behavior here. - */ - return (IsolateThread) Pthread.pthread_getspecific((Pthread.pthread_key_t) key); - } - public static class Options { @Option(help = "Determines if JFR uses a signal handler for execution sampling.")// public static final HostedOptionKey SignalHandlerBasedExecutionSampler = new HostedOptionKey<>(false); diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java index fdc2cc34125..ed9d99e28d4 100644 --- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/thread/PosixPlatformThreads.java @@ -39,6 +39,7 @@ import org.graalvm.nativeimage.c.type.CCharPointer; import org.graalvm.nativeimage.c.type.CTypeConversion; import org.graalvm.nativeimage.c.type.CTypeConversion.CCharPointerHolder; +import org.graalvm.nativeimage.c.type.VoidPointer; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.nativeimage.impl.UnmanagedMemorySupport; import org.graalvm.word.Pointer; @@ -267,6 +268,39 @@ public boolean joinThreadUnmanaged(OSThreadHandle threadHandle, WordPointer thre return status == 0; } + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public ThreadLocalKey createUnmanagedThreadLocal() { + Pthread.pthread_key_tPointer key = StackValue.get(Pthread.pthread_key_tPointer.class); + PosixUtils.checkStatusIs0(Pthread.pthread_key_create(key, WordFactory.nullPointer()), "pthread_key_create(key, keyDestructor): failed."); + return (ThreadLocalKey) key.read(); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void deleteUnmanagedThreadLocal(ThreadLocalKey key) { + int resultCode = Pthread.pthread_key_delete((Pthread.pthread_key_t) key); + PosixUtils.checkStatusIs0(resultCode, "pthread_key_delete(key): failed."); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @SuppressWarnings("unchecked") + public T getUnmanagedThreadLocalValue(ThreadLocalKey key) { + /* + * Although this method is not async-signal-safe in general we rely on + * implementation-specific behavior here. + */ + return (T) Pthread.pthread_getspecific((Pthread.pthread_key_t) key); + } + + @Override + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void setUnmanagedThreadLocalValue(ThreadLocalKey key, WordBase value) { + int resultCode = Pthread.pthread_setspecific((Pthread.pthread_key_t) key, (VoidPointer) value); + PosixUtils.checkStatusIs0(resultCode, "pthread_setspecific(key, value): wrong arguments."); + } + @Override @SuppressWarnings("unused") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java index 9ca0aea7e20..2aa121f33fb 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/sampler/SubstrateSigprofHandler.java @@ -32,7 +32,6 @@ import org.graalvm.nativeimage.Platform; import org.graalvm.nativeimage.Platforms; import org.graalvm.word.Pointer; -import org.graalvm.word.UnsignedWord; import org.graalvm.word.WordFactory; import com.oracle.svm.core.IsolateListenerSupport.IsolateListener; @@ -44,6 +43,8 @@ import com.oracle.svm.core.graal.nodes.WriteHeapBaseNode; import com.oracle.svm.core.jfr.SubstrateJVM; import com.oracle.svm.core.jfr.sampler.AbstractJfrExecutionSampler; +import com.oracle.svm.core.thread.PlatformThreads; +import com.oracle.svm.core.thread.PlatformThreads.ThreadLocalKey; import com.oracle.svm.core.thread.ThreadListener; import com.oracle.svm.core.thread.VMOperation; import com.oracle.svm.core.thread.VMThreads; @@ -61,7 +62,7 @@ */ public abstract class SubstrateSigprofHandler extends AbstractJfrExecutionSampler implements IsolateListener, ThreadListener { private static final CGlobalData SIGNAL_HANDLER_ISOLATE = CGlobalDataFactory.createWord(); - private UnsignedWord keyForNativeThreadLocal; + private ThreadLocalKey keyForNativeThreadLocal; @Platforms(Platform.HOSTED_ONLY.class) protected SubstrateSigprofHandler() { @@ -75,15 +76,15 @@ public static SubstrateSigprofHandler singleton() { @Override @Uninterruptible(reason = "Thread state not set up yet.") public void afterCreateIsolate(Isolate isolate) { - keyForNativeThreadLocal = createNativeThreadLocal(); + keyForNativeThreadLocal = PlatformThreads.singleton().createUnmanagedThreadLocal(); } @Override @Uninterruptible(reason = "The isolate teardown is in progress.") public void onIsolateTeardown() { - UnsignedWord oldKey = keyForNativeThreadLocal; + ThreadLocalKey oldKey = keyForNativeThreadLocal; keyForNativeThreadLocal = WordFactory.nullPointer(); - deleteNativeThreadLocal(oldKey); + PlatformThreads.singleton().deleteUnmanagedThreadLocal(oldKey); } @Override @@ -175,18 +176,6 @@ private static void uninstall(IsolateThread thread) { } } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected abstract UnsignedWord createNativeThreadLocal(); - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected abstract void deleteNativeThreadLocal(UnsignedWord key); - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected abstract void setNativeThreadLocalValue(UnsignedWord key, IsolateThread value); - - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - protected abstract IsolateThread getNativeThreadLocalValue(UnsignedWord key); - /** * Called from the platform dependent sigprof handler to enter isolate. */ @@ -205,8 +194,8 @@ protected static boolean tryEnterIsolate() { /* We are keeping reference to isolate thread inside OS thread local area. */ if (SubstrateOptions.MultiThreaded.getValue()) { - UnsignedWord key = singleton().keyForNativeThreadLocal; - IsolateThread thread = singleton().getNativeThreadLocalValue(key); + ThreadLocalKey key = singleton().keyForNativeThreadLocal; + IsolateThread thread = PlatformThreads.singleton().getUnmanagedThreadLocalValue(key); if (thread.isNull()) { /* Thread is not yet initialized or already detached from isolate. */ return false; @@ -220,6 +209,6 @@ protected static boolean tryEnterIsolate() { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) private void storeIsolateThreadInNativeThreadLocal(IsolateThread isolateThread) { - setNativeThreadLocalValue(keyForNativeThreadLocal, isolateThread); + PlatformThreads.singleton().setUnmanagedThreadLocalValue(keyForNativeThreadLocal, isolateThread); } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java index 3b6076c42a2..9815a26d16f 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java @@ -64,10 +64,12 @@ import org.graalvm.nativeimage.c.function.CEntryPoint; import org.graalvm.nativeimage.c.function.CFunctionPointer; import org.graalvm.nativeimage.c.struct.RawField; +import org.graalvm.nativeimage.c.struct.RawPointerTo; import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.type.WordPointer; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; +import org.graalvm.word.WordBase; import org.graalvm.word.WordFactory; import com.oracle.svm.core.NeverInline; @@ -559,6 +561,26 @@ public boolean joinThreadUnmanaged(OSThreadHandle threadHandle, WordPointer thre throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.joinThreadUnmanaged directly."); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public ThreadLocalKey createUnmanagedThreadLocal() { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.createNativeThreadLocal directly."); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void deleteUnmanagedThreadLocal(ThreadLocalKey key) { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.deleteNativeThreadLocal directly."); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public T getUnmanagedThreadLocalValue(ThreadLocalKey key) { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.getNativeThreadLocalValue directly."); + } + + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + public void setUnmanagedThreadLocalValue(ThreadLocalKey key, WordBase value) { + throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.setNativeThreadLocalValue directly."); + } + @SuppressWarnings("unused") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void closeOSThreadHandle(OSThreadHandle threadHandle) { @@ -1189,8 +1211,19 @@ static void blockedOn(Target_sun_nio_ch_Interruptible b) { } } + @RawStructure public interface OSThreadHandle extends PointerBase { } + + @RawPointerTo(OSThreadHandle.class) + public interface OSThreadHandlePointer extends PointerBase { + void write(int index, OSThreadHandle value); + + OSThreadHandle read(int index); + } + + public interface ThreadLocalKey extends PointerBase { + } } @TargetClass(value = ThreadPoolExecutor.class, innerClass = "Worker") From f6c9936c1a7a61746edae65716b23639da6a65f7 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 3 Apr 2023 18:25:32 +0200 Subject: [PATCH 32/43] Use a single store when installing the forwarding header. --- .../core/genscavenge/ObjectHeaderImpl.java | 38 ++++++++++++------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 74feb1659f7..6481fbdf2d6 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -95,7 +95,7 @@ public final class ObjectHeaderImpl extends ObjectHeader { } else { VMError.guarantee(ReferenceAccess.singleton().haveCompressedReferences(), "Ensures hubs (at the start of the image heap) remain addressable"); numReservedBits = numMinimumReservedBits + 2; - VMError.guarantee(numReservedBits <= numAlignmentBits || ReferenceAccess.singleton().getCompressEncoding().hasShift(), + VMError.guarantee(numReservedBits <= numAlignmentBits || hasShift(), "With no shift, forwarding references are stored directly in the header (with 64-bit, must be) and we cannot use non-alignment header bits"); } numReservedExtraBits = numReservedBits - numAlignmentBits; @@ -298,8 +298,7 @@ public long encodeAsImageHeapObjectHeader(ImageHeapObject obj, long hubOffsetFro VMError.guarantee((header >>> numReservedExtraBits) == hubOffsetFromHeapBase, "Hub is too far from heap base for encoding in object header"); assert (header & reservedBitsMask) == 0 : "Object header bits must be zero initially"; if (HeapImpl.usesImageHeapCardMarking()) { - if (obj.getPartition() instanceof ChunkedImageHeapPartition) { - ChunkedImageHeapPartition partition = (ChunkedImageHeapPartition) obj.getPartition(); + if (obj.getPartition() instanceof ChunkedImageHeapPartition partition) { if (partition.isWritable()) { header |= REMEMBERED_SET_BIT.rawValue(); } @@ -369,7 +368,7 @@ Object getForwardedObject(Pointer ptr) { Object getForwardedObject(Pointer ptr, UnsignedWord header) { assert isForwardedHeader(header); if (ReferenceAccess.singleton().haveCompressedReferences()) { - if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + if (hasShift()) { // References compressed with shift have no bits to spare, so the forwarding // reference is stored separately, after the object header ObjectLayout layout = ConfigurationValues.getObjectLayout(); @@ -389,22 +388,30 @@ Object getForwardedObject(Pointer ptr, UnsignedWord header) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) void installForwardingPointer(Object original, Object copy) { assert !isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); - UnsignedWord forwardHeader; + UnsignedWord forwardHeader = getForwardHeader(copy); + ObjectAccess.writeLong(original, getHubOffset(), forwardHeader.rawValue()); + assert isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); + } + + @AlwaysInline("GC performance") + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private UnsignedWord getForwardHeader(Object copy) { + UnsignedWord result; if (ReferenceAccess.singleton().haveCompressedReferences()) { - if (ReferenceAccess.singleton().getCompressEncoding().hasShift()) { + UnsignedWord compressedCopy = ReferenceAccess.singleton().getCompressedRepresentation(copy); + if (hasShift()) { // Compression with a shift uses all bits of a reference, so store the forwarding // pointer in the location following the hub pointer. - forwardHeader = WordFactory.unsigned(0xe0e0e0e0e0e0e0e0L); - ObjectAccess.writeObject(original, getHubOffset() + getReferenceSize(), copy); + result = compressedCopy.shiftLeft(32).or(WordFactory.unsigned(0x00000000e0e0e0e0L)); } else { - forwardHeader = ReferenceAccess.singleton().getCompressedRepresentation(copy); + result = compressedCopy; } } else { - forwardHeader = Word.objectToUntrackedPointer(copy); + result = Word.objectToUntrackedPointer(copy); } - assert getHeaderBitsFromHeader(forwardHeader).equal(0); - writeHeaderToObject(original, forwardHeader.or(FORWARDED_BIT)); - assert isPointerToForwardedObject(Word.objectToUntrackedPointer(original)); + + assert getHeaderBitsFromHeader(result).equal(0); + return result.or(FORWARDED_BIT); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) @@ -413,6 +420,11 @@ private UnsignedWord getHeaderBitsFromHeader(UnsignedWord header) { assert !isConsumedHeapChunkZapped(header) : "Consumed chunk zap value"; return header.and(reservedBitsMask); } + } + + @Fold + static boolean hasShift() { + return ReferenceAccess.singleton().getCompressEncoding().hasShift(); @Fold static boolean hasFixedIdentityHashField() { From 7808dd4b630478f321c6df491c70f79057d4c90f Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 3 Apr 2023 19:15:58 +0200 Subject: [PATCH 33/43] Minor cleanups. --- .../CompleteGarbageCollectorMXBean.java | 4 ++-- .../oracle/svm/core/genscavenge/GCImpl.java | 6 +++--- .../oracle/svm/core/genscavenge/HeapImpl.java | 19 ++++++++++--------- .../IncrementalGarbageCollectorMXBean.java | 4 ++-- .../core/genscavenge/ObjectHeaderImpl.java | 2 +- 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompleteGarbageCollectorMXBean.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompleteGarbageCollectorMXBean.java index 3515aab4fb2..e991441a0f1 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompleteGarbageCollectorMXBean.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/CompleteGarbageCollectorMXBean.java @@ -48,12 +48,12 @@ public CompleteGarbageCollectorMXBean() { @Override public long getCollectionCount() { - return HeapImpl.getHeapImpl().getGCImpl().getAccounting().getCompleteCollectionCount(); + return HeapImpl.getGCImpl().getAccounting().getCompleteCollectionCount(); } @Override public long getCollectionTime() { - long nanos = HeapImpl.getHeapImpl().getGCImpl().getAccounting().getCompleteCollectionTotalNanos(); + long nanos = HeapImpl.getGCImpl().getAccounting().getCompleteCollectionTotalNanos(); return TimeUtils.roundNanosToMillis(nanos); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index c84521e6841..a8ca95b5c0b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -515,7 +515,7 @@ static boolean runtimeAssertions() { @Fold public static GCImpl getGCImpl() { - GCImpl gcImpl = HeapImpl.getHeapImpl().getGCImpl(); + GCImpl gcImpl = HeapImpl.getGCImpl(); assert gcImpl != null; return gcImpl; } @@ -1290,7 +1290,7 @@ protected void operate(NativeVMOperationData data) { */ ImplicitExceptions.activateImplicitExceptionsAreFatal(); try { - HeapImpl.getHeapImpl().getGCImpl().collectOperation((CollectionVMOperationData) data); + HeapImpl.getGCImpl().collectOperation((CollectionVMOperationData) data); } catch (Throwable t) { throw VMError.shouldNotReachHere(t); } finally { @@ -1301,7 +1301,7 @@ protected void operate(NativeVMOperationData data) { @Override protected boolean hasWork(NativeVMOperationData data) { CollectionVMOperationData d = (CollectionVMOperationData) data; - return HeapImpl.getHeapImpl().getGCImpl().getCollectionEpoch().equal(d.getRequestingEpoch()); + return HeapImpl.getGCImpl().getCollectionEpoch().equal(d.getRequestingEpoch()); } } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index db5ddbd1bea..aeed806222e 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -214,14 +214,20 @@ public ObjectHeader getObjectHeader() { return objectHeaderImpl; } - ObjectHeaderImpl getObjectHeaderImpl() { - return objectHeaderImpl; + @Fold + static ObjectHeaderImpl getObjectHeaderImpl() { + return getHeapImpl().objectHeaderImpl; } @Fold @Override public GC getGC() { - return getGCImpl(); + return getHeapImpl().gcImpl; + } + + @Fold + static GCImpl getGCImpl() { + return getHeapImpl().gcImpl; } @Fold @@ -235,11 +241,6 @@ public HeapAccounting getAccounting() { return accounting; } - @Fold - GCImpl getGCImpl() { - return gcImpl; - } - @Override @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isAllocationDisallowed() { @@ -707,7 +708,7 @@ public void dirtyAllReferencesOf(Object obj) { @Override public long getMillisSinceLastWholeHeapExamined() { - return getGCImpl().getMillisSinceLastWholeHeapExamined(); + return HeapImpl.getGCImpl().getMillisSinceLastWholeHeapExamined(); } @Override diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/IncrementalGarbageCollectorMXBean.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/IncrementalGarbageCollectorMXBean.java index 7c08b40086b..f11508e80d0 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/IncrementalGarbageCollectorMXBean.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/IncrementalGarbageCollectorMXBean.java @@ -48,12 +48,12 @@ public IncrementalGarbageCollectorMXBean() { @Override public long getCollectionCount() { - return HeapImpl.getHeapImpl().getGCImpl().getAccounting().getIncrementalCollectionCount(); + return HeapImpl.getGCImpl().getAccounting().getIncrementalCollectionCount(); } @Override public long getCollectionTime() { - long nanos = HeapImpl.getHeapImpl().getGCImpl().getAccounting().getIncrementalCollectionTotalNanos(); + long nanos = HeapImpl.getGCImpl().getAccounting().getIncrementalCollectionTotalNanos(); return TimeUtils.roundNanosToMillis(nanos); } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 6481fbdf2d6..14e6cb87780 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -104,7 +104,7 @@ public final class ObjectHeaderImpl extends ObjectHeader { @Fold public static ObjectHeaderImpl getObjectHeaderImpl() { - ObjectHeaderImpl oh = HeapImpl.getHeapImpl().getObjectHeaderImpl(); + ObjectHeaderImpl oh = HeapImpl.getObjectHeaderImpl(); assert oh != null; return oh; } From 763e72c35cf8a5f6a8757bae62a11fcb7dbe51f5 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Mon, 3 Apr 2023 19:47:38 +0200 Subject: [PATCH 34/43] Improve UninterruptibleAnnotationChecker. --- .../UninterruptibleAnnotationChecker.java | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/UninterruptibleAnnotationChecker.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/UninterruptibleAnnotationChecker.java index c372b8e1591..15e58be1f46 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/UninterruptibleAnnotationChecker.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/code/UninterruptibleAnnotationChecker.java @@ -45,6 +45,7 @@ import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.SubstrateOptions; import com.oracle.svm.core.Uninterruptible; +import com.oracle.svm.core.classinitialization.EnsureClassInitializedNode; import com.oracle.svm.core.feature.AutomaticallyRegisteredImageSingleton; import com.oracle.svm.core.option.HostedOptionKey; import com.oracle.svm.core.os.RawFileOperationSupport; @@ -73,8 +74,7 @@ private static UninterruptibleAnnotationChecker singleton() { public static void checkAfterParsing(ResolvedJavaMethod method, StructuredGraph graph) { if (Uninterruptible.Utils.isUninterruptible(method) && graph != null) { - singleton().checkNoAllocation(method, graph); - singleton().checkNoSynchronization(method, graph); + singleton().checkGraph(method, graph); } } @@ -134,12 +134,6 @@ private void checkSpecifiedOptions(HostedMethod method, Uninterruptible annotati } if (annotation.mayBeInlined()) { - if (!annotation.reason().equals(Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE) && !AnnotationAccess.isAnnotationPresent(method, AlwaysInline.class)) { - violations.add("Method " + method.format("%H.%n(%p)") + " is annotated with @Uninterruptible('mayBeInlined = true') which allows the method to be inlined into interruptible code. " + - "If the method has an inherent reason for being uninterruptible, besides being called from uninterruptible code, then please remove 'mayBeInlined = true'. " + - "Otherwise, use the following reason: '" + Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE + "'"); - } - if (AnnotationAccess.isAnnotationPresent(method, NeverInline.class)) { violations.add("Method " + method.format("%H.%n(%p)") + " is annotated with conflicting annotations: @Uninterruptible('mayBeInlined = true') and @NeverInline"); @@ -151,6 +145,14 @@ private void checkSpecifiedOptions(HostedMethod method, Uninterruptible annotati } } + if (annotation.mayBeInlined() && annotation.calleeMustBe()) { + if (!annotation.reason().equals(Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE) && !AnnotationAccess.isAnnotationPresent(method, AlwaysInline.class)) { + violations.add("Method " + method.format("%H.%n(%p)") + " is annotated with @Uninterruptible('mayBeInlined = true') which allows the method to be inlined into interruptible code. " + + "If the method has an inherent reason for being uninterruptible, besides being called from uninterruptible code, then please remove 'mayBeInlined = true'. " + + "Otherwise, use the following reason: '" + Uninterruptible.CALLED_FROM_UNINTERRUPTIBLE_CODE + "'"); + } + } + if (!annotation.mayBeInlined() && !annotation.callerMustBe() && AnnotationAccess.isAnnotationPresent(method, AlwaysInline.class)) { violations.add("Method " + method.format("%H.%n(%p)") + " is annotated with @Uninterruptible and @AlwaysInline. If the method may be inlined into interruptible code, please specify 'mayBeInlined = true'. Otherwise, specify 'callerMustBe = true'."); @@ -259,18 +261,14 @@ private void checkCallers(HostedMethod caller, Uninterruptible callerAnnotation, } } - private void checkNoAllocation(ResolvedJavaMethod method, StructuredGraph graph) { + private void checkGraph(ResolvedJavaMethod method, StructuredGraph graph) { for (Node node : graph.getNodes()) { if (isAllocationNode(node)) { violations.add("Uninterruptible method " + method.format("%H.%n(%p)") + " is not allowed to allocate."); - } - } - } - - private void checkNoSynchronization(ResolvedJavaMethod method, StructuredGraph graph) { - for (Node node : graph.getNodes()) { - if (node instanceof MonitorEnterNode) { + } else if (node instanceof MonitorEnterNode) { violations.add("Uninterruptible method " + method.format("%H.%n(%p)") + " is not allowed to use 'synchronized'."); + } else if (node instanceof EnsureClassInitializedNode) { + violations.add("Uninterruptible method " + method.format("%H.%n(%p)") + " not allowed to do class initialization."); } } } From 04e6759f93c543bfbd6bd226fa343bbffa8ee208 Mon Sep 17 00:00:00 2001 From: Christian Haeubl Date: Tue, 4 Apr 2023 09:50:50 +0200 Subject: [PATCH 35/43] A few more fixes regarding code style and uninterruptible. --- .../oracle/svm/core/genscavenge/GCImpl.java | 23 +++++++-------- .../oracle/svm/core/genscavenge/HeapImpl.java | 12 -------- .../svm/core/genscavenge/ImageHeapWalker.java | 20 ------------- .../core/genscavenge/ObjectHeaderImpl.java | 2 +- .../src/com/oracle/svm/core/MemoryWalker.java | 3 -- .../svm/core/code/RuntimeCodeInfoMemory.java | 29 ++++++++++--------- .../svm/core/jdk/UninterruptibleUtils.java | 1 + .../svm/core/os/CommittedMemoryProvider.java | 1 + .../svm/core/thread/PlatformThreads.java | 6 +++- .../svm/core/util/coder/NativeCoder.java | 3 +- 10 files changed, 37 insertions(+), 63 deletions(-) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java index a8ca95b5c0b..e998229ffe5 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/GCImpl.java @@ -44,7 +44,6 @@ import org.graalvm.word.WordFactory; import com.oracle.svm.core.AlwaysInline; -import com.oracle.svm.core.MemoryWalker; import com.oracle.svm.core.NeverInline; import com.oracle.svm.core.RuntimeAssertionsSupport; import com.oracle.svm.core.SubstrateGCOptions; @@ -100,7 +99,6 @@ public final class GCImpl implements GC { private final GreyToBlackObjRefVisitor greyToBlackObjRefVisitor = new GreyToBlackObjRefVisitor(); private final GreyToBlackObjectVisitor greyToBlackObjectVisitor = new GreyToBlackObjectVisitor(greyToBlackObjRefVisitor); - private final BlackenImageHeapRootsVisitor blackenImageHeapRootsVisitor = new BlackenImageHeapRootsVisitor(); private final RuntimeCodeCacheWalker runtimeCodeCacheWalker = new RuntimeCodeCacheWalker(greyToBlackObjRefVisitor); private final RuntimeCodeCacheCleaner runtimeCodeCacheCleaner = new RuntimeCodeCacheCleaner(); @@ -1024,21 +1022,22 @@ private void blackenImageHeapRoots() { Timer blackenImageHeapRootsTimer = timers.blackenImageHeapRoots.open(); try { - HeapImpl.getHeapImpl().walkNativeImageHeapRegions(blackenImageHeapRootsVisitor); + blackenImageHeapRoots(HeapImpl.getImageHeapInfo()); + if (AuxiliaryImageHeap.isPresent()) { + ImageHeapInfo auxImageHeapInfo = AuxiliaryImageHeap.singleton().getImageHeapInfo(); + if (auxImageHeapInfo != null) { + blackenImageHeapRoots(auxImageHeapInfo); + } + } } finally { blackenImageHeapRootsTimer.close(); } } - private class BlackenImageHeapRootsVisitor implements MemoryWalker.ImageHeapRegionVisitor { - @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - public boolean visitNativeImageHeapRegion(T region, MemoryWalker.NativeImageHeapRegionAccess access) { - if (access.containsReferences(region) && access.isWritable(region)) { - access.visitObjects(region, greyToBlackObjectVisitor); - } - return true; - } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + private void blackenImageHeapRoots(ImageHeapInfo imageHeapInfo) { + ImageHeapWalker.walkPartitionInline(imageHeapInfo.firstWritableReferenceObject, imageHeapInfo.lastWritableReferenceObject, greyToBlackObjectVisitor, true); + ImageHeapWalker.walkPartitionInline(imageHeapInfo.firstWritableHugeObject, imageHeapInfo.lastWritableHugeObject, greyToBlackObjectVisitor, false); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java index aeed806222e..e7fa7e9e294 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/HeapImpl.java @@ -192,12 +192,6 @@ public boolean walkObjects(ObjectVisitor visitor) { return walkImageHeapObjects(visitor) && walkCollectedHeapObjects(visitor); } - /** Walk the regions of the heap. */ - boolean walkMemory(MemoryWalker.Visitor visitor) { - VMOperation.guaranteeInProgressAtSafepoint("must only be executed at a safepoint"); - return walkNativeImageHeapRegions(visitor) && getYoungGeneration().walkHeapChunks(visitor) && getOldGeneration().walkHeapChunks(visitor) && getChunkProvider().walkHeapChunks(visitor); - } - /** Tear down the heap and release its memory. */ @Override @Uninterruptible(reason = "Tear-down in progress.") @@ -471,12 +465,6 @@ public boolean walkCollectedHeapObjects(ObjectVisitor visitor) { return getYoungGeneration().walkObjects(visitor) && getOldGeneration().walkObjects(visitor); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) - boolean walkNativeImageHeapRegions(MemoryWalker.ImageHeapRegionVisitor visitor) { - return ImageHeapWalker.walkRegions(imageHeapInfo, visitor) && - (!AuxiliaryImageHeap.isPresent() || AuxiliaryImageHeap.singleton().walkRegions(visitor)); - } - @Override public void doReferenceHandling() { if (ReferenceHandler.isExecutedManually()) { diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java index c2ef95c399a..9f781524dcd 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ImageHeapWalker.java @@ -51,7 +51,6 @@ public final class ImageHeapWalker { private ImageHeapWalker() { } - @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) public static boolean walkRegions(ImageHeapInfo heapInfo, MemoryWalker.ImageHeapRegionVisitor visitor) { return visitor.visitNativeImageHeapRegion(heapInfo, READ_ONLY_PRIMITIVE_WALKER) && visitor.visitNativeImageHeapRegion(heapInfo, READ_ONLY_REFERENCE_WALKER) && @@ -181,29 +180,24 @@ public String getRegionName(ImageHeapInfo region) { } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean containsReferences(ImageHeapInfo region) { return containsReferences; } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public boolean isWritable(ImageHeapInfo region) { return isWritable; } @Override @AlwaysInline("GC performance") - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public final boolean visitObjects(ImageHeapInfo region, ObjectVisitor visitor) { boolean alignedChunks = !hasHugeObjects; return ImageHeapWalker.walkPartitionInline(getFirstObject(region), getLastObject(region), visitor, alignedChunks); } - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected abstract Object getFirstObject(ImageHeapInfo info); - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) protected abstract Object getLastObject(ImageHeapInfo info); } @@ -214,13 +208,11 @@ final class ReadOnlyPrimitiveMemoryWalkerAccess extends MemoryWalkerAccessBase { } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getFirstObject(ImageHeapInfo info) { return info.firstReadOnlyPrimitiveObject; } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getLastObject(ImageHeapInfo info) { return info.lastReadOnlyPrimitiveObject; } @@ -233,13 +225,11 @@ final class ReadOnlyReferenceMemoryWalkerAccess extends MemoryWalkerAccessBase { } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getFirstObject(ImageHeapInfo info) { return info.firstReadOnlyReferenceObject; } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getLastObject(ImageHeapInfo info) { return info.lastReadOnlyReferenceObject; } @@ -252,13 +242,11 @@ final class ReadOnlyRelocatableMemoryWalkerAccess extends MemoryWalkerAccessBase } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getFirstObject(ImageHeapInfo info) { return info.firstReadOnlyRelocatableObject; } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getLastObject(ImageHeapInfo info) { return info.lastReadOnlyRelocatableObject; } @@ -271,13 +259,11 @@ final class WritablePrimitiveMemoryWalkerAccess extends MemoryWalkerAccessBase { } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getFirstObject(ImageHeapInfo info) { return info.firstWritablePrimitiveObject; } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getLastObject(ImageHeapInfo info) { return info.lastWritablePrimitiveObject; } @@ -290,13 +276,11 @@ final class WritableReferenceMemoryWalkerAccess extends MemoryWalkerAccessBase { } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getFirstObject(ImageHeapInfo info) { return info.firstWritableReferenceObject; } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getLastObject(ImageHeapInfo info) { return info.lastWritableReferenceObject; } @@ -309,13 +293,11 @@ final class WritableHugeMemoryWalkerAccess extends MemoryWalkerAccessBase { } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getFirstObject(ImageHeapInfo info) { return info.firstWritableHugeObject; } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getLastObject(ImageHeapInfo info) { return info.lastWritableHugeObject; } @@ -328,13 +310,11 @@ final class ReadOnlyHugeMemoryWalkerAccess extends MemoryWalkerAccessBase { } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getFirstObject(ImageHeapInfo info) { return info.firstReadOnlyHugeObject; } @Override - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public Object getLastObject(ImageHeapInfo info) { return info.lastReadOnlyHugeObject; } diff --git a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java index 14e6cb87780..be0def8590b 100644 --- a/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java +++ b/substratevm/src/com.oracle.svm.core.genscavenge/src/com/oracle/svm/core/genscavenge/ObjectHeaderImpl.java @@ -420,11 +420,11 @@ private UnsignedWord getHeaderBitsFromHeader(UnsignedWord header) { assert !isConsumedHeapChunkZapped(header) : "Consumed chunk zap value"; return header.and(reservedBitsMask); } - } @Fold static boolean hasShift() { return ReferenceAccess.singleton().getCompressEncoding().hasShift(); + } @Fold static boolean hasFixedIdentityHashField() { diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java index 7770445438a..85104becd0a 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/MemoryWalker.java @@ -64,13 +64,10 @@ public interface NativeImageHeapRegionAccess { String getRegionName(T region); - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean containsReferences(T region); - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean isWritable(T region); - @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) boolean visitObjects(T region, ObjectVisitor visitor); } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java index a2bc227f563..cdb27e1dbc5 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/code/RuntimeCodeInfoMemory.java @@ -201,14 +201,16 @@ private void rehashAfterUnregisterAt(int index) { // from IdentityHashMap: Knuth } } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void walkRuntimeMethodsDuringGC(CodeInfoVisitor visitor) { assert VMOperation.isGCInProgress() : "otherwise, we would need to make sure that the CodeInfo is not freeded by the GC"; if (table.isNonNull()) { int length = NonmovableArrays.lengthOf(table); for (int i = 0; i < length;) { - UntetheredCodeInfo info = NonmovableArrays.getWord(table, i); - if (info.isNonNull()) { - visitor.visitCode(CodeInfoAccess.convert(info)); + UntetheredCodeInfo untetheredInfo = NonmovableArrays.getWord(table, i); + if (untetheredInfo.isNonNull()) { + CodeInfo info = CodeInfoAccess.convert(untetheredInfo); + callVisitor(visitor, info); } /* @@ -216,7 +218,7 @@ public void walkRuntimeMethodsDuringGC(CodeInfoVisitor visitor) { * visit the now updated entry one more time. However, this could have the effect * that some entries are visited more than once. */ - if (info == NonmovableArrays.getWord(table, i)) { + if (untetheredInfo == NonmovableArrays.getWord(table, i)) { i++; } } @@ -228,23 +230,24 @@ public void walkRuntimeMethods(CodeInfoVisitor visitor) { if (table.isNonNull()) { int length = NonmovableArrays.lengthOf(table); for (int i = 0; i < length; i++) { - UntetheredCodeInfo info = NonmovableArrays.getWord(table, i); - if (info.isNonNull()) { - Object tether = CodeInfoAccess.acquireTether(info); + UntetheredCodeInfo untetheredInfo = NonmovableArrays.getWord(table, i); + if (untetheredInfo.isNonNull()) { + Object tether = CodeInfoAccess.acquireTether(untetheredInfo); try { - callVisitor(visitor, info, tether); + CodeInfo info = CodeInfoAccess.convert(untetheredInfo, tether); + callVisitor(visitor, info); } finally { - CodeInfoAccess.releaseTether(info, tether); + CodeInfoAccess.releaseTether(untetheredInfo, tether); } - assert info == NonmovableArrays.getWord(table, i); + assert untetheredInfo == NonmovableArrays.getWord(table, i); } } } } - @Uninterruptible(reason = "Call the visitor, which may execute interruptible code.", calleeMustBe = false) - private static void callVisitor(CodeInfoVisitor visitor, UntetheredCodeInfo info, Object tether) { - visitor.visitCode(CodeInfoAccess.convert(info, tether)); + @Uninterruptible(reason = "Bridge between uninterruptible and potentially interruptible code.", mayBeInlined = true, calleeMustBe = false) + private static void callVisitor(CodeInfoVisitor visitor, CodeInfo info) { + visitor.visitCode(info); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java index 838d452afd3..143954b7ed1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/UninterruptibleUtils.java @@ -437,6 +437,7 @@ public static long abs(long a) { public static class Byte { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) + @SuppressWarnings("cast") public static int toUnsignedInt(byte x) { return ((int) x) & 0xff; } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java index d54b95287d5..11b30062e60 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/os/CommittedMemoryProvider.java @@ -89,6 +89,7 @@ default UnsignedWord getGranularity() { return VirtualMemoryProvider.get().getGranularity(); } + @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) Pointer allocateAlignedChunk(UnsignedWord nbytes, UnsignedWord alignment); Pointer allocateUnalignedChunk(UnsignedWord nbytes); diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java index 9815a26d16f..f2cb98519e7 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/thread/PlatformThreads.java @@ -67,6 +67,7 @@ import org.graalvm.nativeimage.c.struct.RawPointerTo; import org.graalvm.nativeimage.c.struct.RawStructure; import org.graalvm.nativeimage.c.type.WordPointer; +import org.graalvm.word.ComparableWord; import org.graalvm.word.Pointer; import org.graalvm.word.PointerBase; import org.graalvm.word.WordBase; @@ -566,16 +567,19 @@ public ThreadLocalKey createUnmanagedThreadLocal() { throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.createNativeThreadLocal directly."); } + @SuppressWarnings("unused") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void deleteUnmanagedThreadLocal(ThreadLocalKey key) { throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.deleteNativeThreadLocal directly."); } + @SuppressWarnings("unused") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public T getUnmanagedThreadLocalValue(ThreadLocalKey key) { throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.getNativeThreadLocalValue directly."); } + @SuppressWarnings("unused") @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public void setUnmanagedThreadLocalValue(ThreadLocalKey key, WordBase value) { throw VMError.shouldNotReachHere("Shouldn't call PlatformThreads.setNativeThreadLocalValue directly."); @@ -1222,7 +1226,7 @@ public interface OSThreadHandlePointer extends PointerBase { OSThreadHandle read(int index); } - public interface ThreadLocalKey extends PointerBase { + public interface ThreadLocalKey extends ComparableWord { } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/coder/NativeCoder.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/coder/NativeCoder.java index bc00b091f32..bc616f0a878 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/coder/NativeCoder.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/util/coder/NativeCoder.java @@ -26,6 +26,7 @@ import org.graalvm.word.Pointer; import org.graalvm.word.UnsignedWord; +import org.graalvm.word.WordFactory; import com.oracle.svm.core.Uninterruptible; @@ -48,7 +49,7 @@ public static long readU4(Pointer ptr) { @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) public static UnsignedWord readU8(Pointer ptr) { - return ptr.readWord(0); + return WordFactory.unsigned(ptr.readLong(0)); } @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true) From 2f54f483c5abcdfe20b699a8d2ec722073529653 Mon Sep 17 00:00:00 2001 From: Matthias Neugschwandtner Date: Wed, 5 Oct 2022 14:01:09 +0200 Subject: [PATCH 36/43] Restructure the security guide and update with Polyglot sandboxing --- .../embedding/sandbox-options.md | 335 ------------ .../native-image/Compatibility.md | 4 +- .../native-image/InspectTool.md | 55 +- docs/security/native-image.md | 104 ++++ docs/security/polyglot-sandbox.md | 512 ++++++++++++++++++ docs/security/sandbox_security_boundary.png | Bin 0 -> 9869 bytes docs/security/security-guide.md | 111 +--- .../org/graalvm/polyglot/SandboxPolicy.java | 5 +- 8 files changed, 647 insertions(+), 479 deletions(-) delete mode 100644 docs/reference-manual/embedding/sandbox-options.md create mode 100644 docs/security/native-image.md create mode 100644 docs/security/polyglot-sandbox.md create mode 100644 docs/security/sandbox_security_boundary.png diff --git a/docs/reference-manual/embedding/sandbox-options.md b/docs/reference-manual/embedding/sandbox-options.md deleted file mode 100644 index a82464c015c..00000000000 --- a/docs/reference-manual/embedding/sandbox-options.md +++ /dev/null @@ -1,335 +0,0 @@ ---- -layout: docs -toc_group: embedding -link_title: Sandbox Resource Limits -permalink: /reference-manual/embed-languages/sandbox-resource-limits/ ---- - -## Oracle GraalVM Sandbox Resource Limits - -Oracle GraalVM provides the Sandbox Resource Limits feature that allows for the limiting of resources used by guest applications. -These resource limits are not available in GraalVM Community Edition. -The following document describes how to configure sandbox resource limits using options in GraalVM Polyglot API. - -In general all resource limit options are prefixed with `sandbox` option group and they can be listed using the help of any language launcher provided in GraalVM, e.g., `js --help:tools`. -Polyglot options can be provided through the language launcher, using the polyglot embedding API of the Graal SDK, or on the JVM using a system property. -For better understanding of the examples it is recommended to read the [polyglot embedding guide](embed-languages.md) of the reference manual first. - -Currently all sandbox options are experimental therefore in these examples it is assumed that experimental options are enabled (e.g., with `--experimental-options`). -The options are a best effort approach to limiting resource usage of guest applications. - -The resource limits may be configured using the following options: - - -- `--sandbox.AllocatedBytesCheckFactor=[0.0, inf)` : Specifies a factor of MaxHeapMemory the allocation of which triggers retained heap memory computation. When allocated bytes for an execution context reach the specified factor, computation of bytes retained in the heap by the context is initiated. Is set to '1.0' by default. -- `--sandbox.AllocatedBytesCheckInterval=[1, inf)ms|s|m|h|d` : Time interval to check allocated bytes for an execution context. Exceeding certain number of allocated bytes triggers computation of bytes retained in the heap by the context. Is set to '10ms' by default. Maximum interval is 1h. -- `--sandbox.MaxASTDepth=[1, inf)` : Maximum AST depth of a function (default: no limit). -- `--sandbox.MaxCPUTime=[1, inf)ms|s|m|h|d` : Limits the total maximum CPU time that was spent running the application. No limit is set by default. Example value: '100ms'. -- `--sandbox.MaxCPUTimeCheckInterval=[1, inf)ms|s|m|h|d` : Time interval to check the active CPU time for an execution context. Is set to '10ms' by default. Maximum interval is 1h. -- `--sandbox.MaxErrorStreamSize=[0, inf)B|KB|MB|GB` : Specifies the maximum size that the guest application can write to stderr. No limit is set by default. Example value: '10MB'. -- `--sandbox.MaxHeapMemory=[1, inf)B|KB|MB|GB` : Specifies the maximum heap memory that can be retained by the application during its run. Includes only data retained by the guest application, runtime allocated data is not included. No limit is set by default and setting the related expert options has no effect. Example value: '100MB'. -- `--sandbox.MaxOutputStreamSize=[0, inf)B|KB|MB|GB` : Specifies the maximum size that the guest application can write to stdout. No limit is set by default. Example value: '10MB'. -- `--sandbox.MaxStackFrames=[1, inf)` : Limits the maximum number of guest stack frames (default: no limit). -- `--sandbox.MaxStatements=[1, inf)` : Limits the maximum number of guest language statements executed. The execution is cancelled with an resource exhausted error when it is exceeded. -- `--sandbox.MaxStatementsIncludeInternal` : Configures whether to include internal sources in the max statements computation. -- `--sandbox.MaxThreads=[1, inf)` : Limits the number of threads that can be entered by a context at the same point in time (default: no limit). -- `--sandbox.RetainedBytesCheckFactor=[0.0, inf)` : Specifies a factor of total heap memory of the host VM the exceeding of which stops the world. When the total number of bytes allocated in the heap for the whole host VM exceeds the factor, the following process is initiated. Execution for all engines with at least one memory-limited execution context is paused. Retained bytes in the heap for each memory-limited context are computed. Contexts exceeding their limits are cancelled. The execution is resumed. Is set to '0.7' by default. Has no effect if 'UseLowMemoryTrigger' is 'false'. -- `--sandbox.RetainedBytesCheckInterval=[1, inf)ms|s|m|h|d` : Specifies the minimum time interval between two computations of retained bytes in the heap for a single execution context. Is set to '10ms' by default. Maximum value for the minimum interval is 1h. -- `--sandbox.ReuseLowMemoryTriggerThreshold=true|false` : Specifies whether an already set heap memory notification limit can be reused for the low memory trigger. When reusing is allowed and the usage threshold or the collection usage threshold of a heap memory pool has already been set, then the value of 'RetainedBytesCheckFactor' is ignored for that memory pool and threshold type and whatever threshold value has already been set is used. Is set to 'false' by default. -- `--sandbox.TraceLimits=true|false` : Records the maximum amount of resources used during execution, and reports a summary of resource limits to the log file upon application exit. Users may also provide limits to enforce while tracing. This flag can be used to estimate an application's optimal sandbox parameters, either by tracing the limits of a stress test or peak usage. -- `--sandbox.UseLowMemoryTrigger=true|false` : Specifies whether stopping the world is enabled. When enabled, engines with at least one memory limited execution context are paused when the total number of bytes allocated in the heap for the whole host VM exceeds the specified factor of total heap memory of the host VM. Is set to 'true' by default. - - -Different configurations may be provided for each polyglot embedding `Context` instance. -In addition to that the limits may be reset at any point of time during the execution. Resetting is only aplicable to `sandbox.MaxStatements` and `sandbox.MaxCPUTime`. - -A guest language might choose to create an inner context within the outer execution context. The limits are applied to the outer context and all inner contexts it spawns. -It is not possible to specify a separate limit for inner contexts and it is also not possible to escape any limit by creating an inner context. - -## Limiting the active CPU time - -The `sandbox.MaxCPUTime` option allows you to specify the maximum CPU time spent running the application. -The maximum [CPU time](https://docs.oracle.com/en/java/javase/11/docs/api/java.management/java/lang/management/ThreadMXBean.html#getThreadCpuTime\(long\)) specifies how long a context can be active until it is automatically cancelled and the context is closed. -By default the time limit is checked every 10 milliseconds. -This can be customized using the `sandbox.MaxCPUTimeCheckInterval` option. -Both maximum CPU time limit and check interval must be positive. -By default no CPU time limit is enforced. -If the time limit is exceeded then the polyglot context is cancelled and the execution stops by throwing a [`PolyglotException`](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/PolyglotException.html) which returns `true` for `isResourceExhausted()`. -As soon as the time limit is triggered, no further application code can be executed with this context. -It will continuously throw a `PolyglotException` for any method of the polyglot context that will be invoked. - -The used CPU time of a context includes time spent in callbacks to host code. -This is also the case when running with [Polyglot Isolates]. - -The used CPU time of a context typically does not include time spent waiting for synchronization or IO. -The CPU time of all threads will be added and checked against the CPU time limit. -This can mean that if two threads execute the same context then the time limit will be exceeded twice as fast. - -The time limit is enforced by a separate high-priority thread that will be woken regularly. -There is no guarantee that the context will be cancelled within the accuracy specified. -The accuracy may be significantly missed, e.g. if the host VM causes a full garbage collection. -If the time limit is never exceeded then the throughput of the guest context is not affected. -If the time limit is exceeded for one context then it may slow down the throughput for other contexts with the same explicit engine temporarily. - -Available units to specify time durations are `ms` for milliseconds, `s` for seconds, `m` for minutes, `h` for hours and `d` for days. -It is not allowed specify negative values or no time unit with CPU time limit options. - -### Example Usage - -```java -try (Context context = Context.newBuilder("js") - .allowExperimentalOptions(true) - .option("sandbox.MaxCPUTime", "500ms") - .option("sandbox.MaxCPUTimeCheckInterval", "5ms") - .build();) { - context.eval("js", "while(true);"); - assert false; -} catch (PolyglotException e) { - // triggered after 500ms; - // context is closed and can no longer be used - // error message: Maximum CPU time limit of 500ms exceeded. - assert e.isCancelled(); - assert e.isResourceExhausted(); -} -``` - -## Limiting the number of executed statements - -Specifies the maximum number of statements a context may execute until the the context will be cancelled. -After the statement limit was triggered for a context, it is no longer usable and every use of the context will throw a `PolyglotException` that returns `true` for `PolyglotException.isCancelled()`. -The statement limit is independent of the number of threads executing and is applied per context. -It is also possible to specify this limit using the `[ResourceLimits]()` API of the polyglot embedding API. - -By default there is no statement limit applied. -The limit may be set to a negative number to disable it. -Whether this limit is applied internal sources only can be configured using `sandbox.MaxStatementsIncludeInternal`. -By default the limit does not include statements of sources that are marked internal. -If a shared engine is used then the same internal configuration must be used for all contexts of an engine. -The maximum statement limit can be configured for each context of an engine separately. - -Attaching a statement limit to a context reduces the throughput of all guest applications with the same engine. -The statement counter needs to be updated with every statement that is executed. -It is recommended to benchmark the use of the statement limit before it is used in production. - -The complexity of a single statement may not be constant time depending on the guest language. -For example, statements that execute JavaScript builtins, like `Array.sort`, may account for a single statement, but its execution time is dependent on the size of the array. -The statement count limit is therefore not suitable to perform time boxing and must be combined with other more reliable measures like the CPU time limit. - -```java -try (Context context = Context.newBuilder("js") - .allowExperimentalOptions(true) - .option("sandbox.MaxStatements", "2") - .option("sandbox.MaxStatementsIncludeInternal", "false") - .build();) { - context.eval("js", "purpose = 41"); - context.eval("js", "purpose++"); - context.eval("js", "purpose++"); // triggers max statements - assert false; -} catch (PolyglotException e) { - // context is closed and can no longer be used - // error message: Maximum statements limit of 2 exceeded. - assert e.isCancelled(); - assert e.isResourceExhausted(); -} -``` - -## Limiting the AST depth of functions - -A limit on the maximum expression depth of a guest language function. -Only instrumentable nodes count towards the limit. -If the limit is exceeded, evaluation of the code fails and the context is canceled. - -The AST depth can give an estimate of the complexity of a function as well as its stack frame size. -Limiting the AST depth can serve as a safeguard against arbitrary stack space usage by a single function. - -## Limiting the number of stack frames - -Specifies the maximum number of frames a context can push on the stack. -Exceeding the limit results in cancellation of the context. -A thread-local stack frame counter is incremented on function enter and decremented on function return. -Resetting resource limits does not affect the stack frame counter. - -The stack frame limit in itself can serve as a safeguard against infinite recursion. -If used together with the AST depth limit it can be used to estimate total stack space usage. - -## Limiting the number of active threads - -Limits the number of threads that can be used by a context at the same point in time. -By default, an arbitrary number of threads can be used. -If a set limit is exceeded, entering the context fails with a `PolyglotException` and the polyglot context is canceled. -Resetting resource limits does not affect thread limits. - -## Limiting the maximum heap memory - -The `sandbox.MaxHeapMemory` option allows you to specify the maximum heap memory the application is allowed to retain during its run. -`sandbox.MaxHeapMemory` must be positive. This option is only supported on a HotSpot-based VM. -Enabling this option in a native executable will result in a `PolyglotException`. -The option is also not supported with [Polyglot Isolates], which have different means of controlling memory consumption. -When exceeding of the limit is detected, the corresponding context is automatically cancelled and then closed. - -Only objects residing in the guest application count towards the limit - memory allocated during callbacks to host code does not. -The efficacy of this option (also) depends on the garbage collector used. - -#### Example Usage - -```java -try (Context context = Context.newBuilder("js") - .allowExperimentalOptions(true) - .option("sandbox.MaxHeapMemory", "100MB") - .build()) { - context.eval("js", "var r = {}; var o = r; while(true) { o.o = {}; o = o.o; };"); - assert false; -} catch (PolyglotException e) { - // triggered after the retained size is greater than 100MB; - // context is closed and can no longer be used - // error message: Maximum heap memory limit of 104857600 bytes exceeded. Current memory at least... - assert e.isCancelled(); - assert e.isResourceExhausted(); -} -``` - -#### Implementation details and expert options - -The limit is checked by retained size computation triggered either based on [allocated](https://docs.oracle.com/en/java/javase/11/docs/api/jdk.management/com/sun/management/ThreadMXBean.html#getThreadAllocatedBytes\(long\)) bytes or on [low memory notification](https://docs.oracle.com/en/java/javase/11/docs/api/java.management/java/lang/management/MemoryMXBean.html). - -The allocated bytes are checked by a separate high-priority thread that will be woken regularly. -There is one such thread for each memory-limited context (one with `sandbox.MaxHeapMemory` set). -The retained bytes computation is done by yet another high-priority thread that is started from the allocated bytes checking thread as needed. -The retained bytes computation thread also cancels the context if the heap memory limit is exeeded. -Additionaly, when low memory trigger is invoked, all contexts on engines with at least one memory-limited context are paused together with their allocation checkers. -All individual retained size computations are cancelled. -Retained bytes in the heap for each memory-limited context are computed by a single high-priority thread. -Contexts exceeding their limits are cancelled, and then the execution is resumed. - -The main goal of the heap memory limits is to prevent heap memory depletion related errors in most cases and thus enable the host VM to run smoothly even in the presence of misbehaving contexts. -The implementation is best effort. This means that there is no guarantee on the accuracy of the heap memory limit. -There is also no guarantee that setting a heap memory limit will prevent the context from causing `OutOfMemory` errors. -Guest applications that allocate many objects in quick succession have a lower accuracy than applications which allocate objects rarely. -The guest code execution will only be paused if the host heap memory is exhausted and a low memory trigger of the host VM is invoked. -Note that the scope of the pause is an engine, so a context without the `sandbox.MaxHeapMemory` option set is also paused in case it shares the engine with other context that is memory-limited. -Also note that if one context is cancelled other contexts with the same explicit engine may be slowed down. How the size retained by a context is computed can be -customized using the expert options `sandbox.AllocatedBytesCheckInterval`, `sandbox.AllocatedBytesCheckEnabled`, `sandbox.AllocatedBytesCheckFactor`, `sandbox.RetainedBytesCheckInterval`, `sandbox.RetainedBytesCheckFactor`, and `sandbox.UseLowMemoryTrigger` described below. - -Retained size computation for a context is triggered when a retained bytes estimate exceeds a certain factor of specified `sandbox.MaxHeapMemory`. -The estimate is based on heap memory -[allocated](https://docs.oracle.com/en/java/javase/11/docs/api/jdk.management/com/sun/management/ThreadMXBean.html#getThreadAllocatedBytes\(long\)) by threads where the context has been active. -More precisely, the estimate is the result of previous retained bytes computation, if available, plus bytes allocated since the start of the previous computation. -By default the factor of `sandbox.MaxHeapMemory` is 1.0 and it can be customized by the `sandbox.AllocatedBytesCheckFactor` option. -The factor must be positive. -For example, let `sandbox.MaxHeapMemory` be 100MB and `sandbox.AllocatedBytesCheckFactor` be 0.5. -The retained size computation is first triggered when allocated bytes reach 50MB. -Let the computed retained size be 25MB, then the next retained size computation is triggered when additional 25MB is allocated, etc. - -By default, allocated bytes are checked every 10 milliseconds. This can be configured by `sandbox.AllocatedBytesCheckInterval`. -The smallest possible interval is 1ms. Any smaller value is interpreted as 1ms. - -The beginnings of two retained size computations of the same context must be by default at least 10 milliseconds apart. -This can be configured by the `sandbox.RetainedBytesCheckInterval` option. The interval must be positive. - -The allocated bytes checking for a context can be disabled by the `sandbox.AllocatedBytesCheckEnabled` option. -By default it is enabled ("true"). If disabled ("false"), retained size checking for the context can be triggered only by the low memory trigger. - -When the total number of bytes allocated in the heap for the whole host VM exceeds a certain factor of the total heap memory of the VM, [low memory notification](https://docs.oracle.com/en/java/javase/11/docs/api/java.management/java/lang/management/MemoryMXBean.html) is invoked and initiates the following process. -The execution for all engines with at least one execution context which has the `sandbox.MaxHeapMemory` option set is paused, retained bytes in the heap for each memory-limited context are computed, contexts exceeding their limits are cancelled, and then the execution is resumed. -The default factor is 0.7. This can be configuted by the `sandbox.RetainedBytesCheckFactor` option. -The factor must be between 0.0 and 1.0. All contexts using the `sandbox.MaxHeapMemory` option must use the same value for `sandbox.RetainedBytesCheckFactor`. - -When the usage threshold or the collection usage threshold of any heap memory pool has already been set, then the low memory trigger cannot be used by default, because the limit specified by the `sandbox.RetainedBytesCheckFactor` cannot be implemented. -However, when `sandbox.ReuseLowMemoryTriggerThreshold` is set to true and the usage threshold or the collection usage threshold of a heap memory pool has already been set, then the value of `sandbox.RetainedBytesCheckFactor` is ignored for that memory pool and whatever limit has already been set is used. -That way the low memory trigger can be used together with libraries that also set the usage threshold or the collection usage threshold of heap memory pools. - -The described low memory trigger can be disabled by the `sandbox.UseLowMemoryTrigger` option. -By default it is enabled ("true"). If disabled ("false"), retained size checking for the execution context can be triggered only by the allocated bytes checker. -All contexts using the `sandbox.MaxHeapMemory` option must use the same value for `sandbox.UseLowMemoryTrigger`. - -If exceeding of the heap memory limit is detected then the polyglot context is cancelled and the execution stops by throwing a [`PolyglotException`](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/PolyglotException.html) which returns `true` for `isResourceExhausted()`. -As soon as the memory limit is triggered, no further application code can be executed with this context. -It will continuously throw a `PolyglotException` for any method of the polyglot context that will be invoked. - -Available units to specify time durations are `ms` for milliseconds, `s` for seconds, `m` for minutes, `h` for hours and `d` for days. -It is not allowed to specify negative values or no time unit with max heap memory options. - -Available units to specify sizes are `B` for bytes, `KB` for kilobytes, `MB` for megabytes, and `GB` for gigabytes. -It is not allowed to specify negative values or no size unit with max heap memory options. - -Resetting resource limits using `Context.resetLimits` does not affect the heap memory limit. - -## Limiting the size of the output written to stdout/stderr - -Limits the size of the output that the application writes to standard output or standard error output during runtime. -If the limit is exceeded, evaluation of the code fails and the context is canceled. -Limiting the size of the output can serve as protection against denial-of-service attacks that flood the output. - -#### Example Usage -```java -try (Context context = Context.newBuilder("js") - .option("sandbox.MaxOutputStreamSize", "100KB") - .build()) { - context.eval("js", "while(true) { console.log('Log message') };"); - assert false; -} catch (PolyglotException e) { - // triggered after writing more than 100KB to stdout - // context is closed and can no longer be used - // error message: Maximum output stream size of 102400 exceeded. Bytes written 102408. - assert e.isCancelled(); - assert e.isResourceExhausted(); -} -``` -```java -try (Context context = Context.newBuilder("js") - .option("sandbox.MaxErrorStreamSize", "100KB") - .build()) { - context.eval("js", "while(true) { console.error('Error message') };"); - assert false; -} catch (PolyglotException e) { - // triggered after writing more than 100KB to stderr - // context is closed and can no longer be used - // error message: Maximum error stream size of 102400 exceeded. Bytes written 102410. - assert e.isCancelled(); - assert e.isResourceExhausted(); -} -``` - -## Determining Sandbox Resource Limits - -The sandbox.TraceLimits option allows you to trace a running process and record the maximum resource utilization. This can be used to estimate parameters to define a sandbox for the workload. For example, a web server's sandbox parameters could be obtained by enabling this option and either stress-testing the server, or letting the server run during peak usage. When this option is enabled, the report is saved to the log file after the workload completes. Users can change the location of the log file by using --log.file= with a language launcher or -Dpolyglot.log.file= when using a java launcher. Each resource limit in the report can be passed directly to a sandbox option in order to enforce the limit. - -See, for example, how to trace limits for a Python workload: - -``` -graalpy --log.file=limits.log --experimental-options --sandbox.TraceLimits=true workload.py - -limits.log: -[trace-limits] Sandbox Limits Statistics: -HEAP 12MB -CPU 7s -STATEMENTS 9441565 -STACKFRAMES 29 -THREADS 1 -ASTDEPTH 15 - -Sandbox Command Line Options: ---sandbox.MaxHeapMemory=12MB --sandbox.MaxCPUTime=7s --sandbox.MaxStatements=9441565 --sandbox.MaxStackFrames=29 --sandbox.MaxThreads=1 --sandbox.MaxASTDepth=15 -``` - -## Resetting Resource Limits - -With the polyglot embedding API it is possible to reset the limits at any point in time using the [`Context.resetLimits`](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/Context.html#resetLimits--) method. -This can be useful if a known and trusted initialization script should be excluded from limit. Resetting the limits is not applicable to all limits. - -### Example Usage - -```java -try (Context context = Context.newBuilder("js") - .allowExperimentalOptions(true) - .option("sandbox.MaxCPUTime", "500ms") - .build();) { - context.eval("js", /*... initialization script ...*/); - context.resetLimits(); - context.eval("js", /*... user script ...*/); - assert false; -} catch (PolyglotException e) { - assert e.isCancelled(); - assert e.isResourceExhausted(); -} -``` diff --git a/docs/reference-manual/native-image/Compatibility.md b/docs/reference-manual/native-image/Compatibility.md index ee4ae1a6752..633be781d3a 100644 --- a/docs/reference-manual/native-image/Compatibility.md +++ b/docs/reference-manual/native-image/Compatibility.md @@ -41,9 +41,7 @@ Note that `invokedynamic` use cases generated by `javac` for, for example, Java ### Security Manager -The Java security manager is no longer recommended as a way to isolate less trusted code from more trusted code in the same process. -This is because almost all typical hardware architectures are susceptible to side-channel attacks to access data that is restricted via the security manager. -Using separate processes is now recommended for these cases. +Native Image will not allow a Java Security Manager to be enabled because this functionality has deprecated since Java 17. ## Features That May Operate Differently in a Native Image diff --git a/docs/reference-manual/native-image/InspectTool.md b/docs/reference-manual/native-image/InspectTool.md index cf4b264edd3..62c420b43bc 100644 --- a/docs/reference-manual/native-image/InspectTool.md +++ b/docs/reference-manual/native-image/InspectTool.md @@ -24,60 +24,7 @@ Native Image provides the `--enable-sbom` option to embed an SBOM into a native > Note: Embedding a Software Bill of Materials (SBOM) is not available in GraalVM Community Edition. The feature is currently experimental. -The CycloneDX format is supported and the default. -To embed a CycloneDX SBOM into a native executable, pass the `--enable-sbom` option to the `native-image` command. - -The implementation constructs the SBOM by recovering all version information observable in external library manifests for classes included in a native executable. -The SBOM is also compressed in order to limit the SBOM's impact on the native executable size. -Even though the tool is not yet supported on Windows, Windows users can still embed the SBOM with this experimental option. -The SBOM is stored in the `gzip` format with the exported `sbom` symbol referencing its start address and the `sbom_length` symbol its size. - -After embedding the compressed SBOM into the executable, the tool is able to extract the compressed SBOM using an optional `--sbom` parameter accessible through `$JAVA_HOME/bin/native-image-inspect --sbom ` and outputs the SBOM in the following format: - -```json -{ - "bomFormat": "CycloneDX", - "specVersion": "1.4", - "version": 1, - "components": [ - { - "type": "library", - "group": "io.netty", - "name": "netty-codec-http2", - "version": "4.1.76.Final", - "properties": [ - { - "name": "syft:cpe23", - "value": "cpe:2.3:a:codec:codec:4.1.76.Final:*:*:*:*:*:*:*" - }, - { - "name": "syft:cpe23", - "value": "cpe:2.3:a:codec:netty-codec-http2:4.1.76.Final:*:*:*:*:*:*:*" - }, - { - "name": "syft:cpe23", - "value": "cpe:2.3:a:codec:netty_codec_http2:4.1.76.Final:*:*:*:*:*:*:*" - }, - ... - ] - }, - ... - ], - "serialNumber": "urn:uuid:51ec305f-616e-4139-a033-a094bb94a17c" -} -``` - -The tool can extract the SBOM from both executables and shared libraries. -To scan for any vulnerable libraries, submit the SBOM to a vulnerability scanner. -For example, the popular [Anchore software supply chain management platform](https://anchore.com/) makes the `grype` scanner freely available. -You can check whether the libraries given in your SBOMs have known vulnerabilities documented in Anchore's database. -For this purpose, the output of the tool can be fed directly to the `grype` scanner to check for vulnerable libraries, using the command `$JAVA_HOME/bin/native-image-inspect --sbom | grype` which produces the following output: -```shell -NAME INSTALLED VULNERABILITY SEVERITY -netty-codec-http2 4.1.76.Final CVE-2022-24823 Medium -``` - -You can then use this report to update any vulnerable dependencies found in your executable. +The tool is able to extract the compressed SBOM using an optional `--sbom` parameter accessible through `$JAVA_HOME/bin/native-image-inspect --sbom `. ### Further Reading diff --git a/docs/security/native-image.md b/docs/security/native-image.md new file mode 100644 index 00000000000..0b14fb1033e --- /dev/null +++ b/docs/security/native-image.md @@ -0,0 +1,104 @@ +--- +layout: docs +toc_group: security-guide +link_title: Security Considerations in Native Image +permalink: /security-guide/native-image/ +--- +# Native Image + +The `native-image` builder generates a snapshot of an application after startup and bundles it in a binary executable. + +## Class Initialization + +The `native-image` builder may execute the static initializers of certain classes at build time (see [class initialization](../reference-manual/native-image/ClassInitialization.md) for more details). +Executing static initializers at build time persists the state after initialization in the image heap. +This means that any information that is obtained or computed in static initializers becomes part of a native executable. +This can either result in sensitive data ending up in the snapshot or fixing initialization data that is supposed to be obtained at startup, such as random number seeds. + +Developers can request static initializers that process sensitive information to be executed at run time by specifying the `--initialize-at-run-time` CLI parameter when building a native executable, followed by a comma-separated list of packages and classes (and implicitly all of their subclasses) that must be initialized at runtime and not during image building. +Alternatively developers can make use of the [`RuntimeClassInitialization` API](https://www.graalvm.org/sdk/javadoc/org/graalvm/nativeimage/hosted/RuntimeClassInitialization.html). + +Developers should run the `native-image` builder in a dedicated environment, such as a container, that does not contain any sensitive information in the first place. + +## Software Bill of Materials + +GraalVM Native Image can embed a Software Bill of Materials (SBOM) at build time to detect any libraries that may be susceptible to known security vulnerabilities. +Native Image provides the `--enable-sbom` option to embed an SBOM into a native executable. + +> Note: Embedding a Software Bill of Materials (SBOM) is not available in GraalVM Community Edition. + +The CycloneDX format is supported and the default. +To embed a CycloneDX SBOM into a native executable, pass the `--enable-sbom` option to the `native-image` command. + +The implementation constructs the SBOM by recovering all version information observable in external library manifests for classes included in a native executable. +The SBOM is also compressed in order to limit the SBOM's impact on the native executable size. +The SBOM is stored in the `gzip` format with the exported `sbom` symbol referencing its start address and the `sbom_length` symbol its size. + +After embedding the compressed SBOM into the executable, the [native image inspect tool](../reference-manual/native-image/InspectTool.md) is able to extract the compressed SBOM using an optional `--sbom` parameter accessible through `$JAVA_HOME/bin/native-image-inspect --sbom ` from both executables and shared libraries. +It outputs the SBOM in the following format: + +```json +{ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "version": 1, + "components": [ + { + "type": "library", + "group": "io.netty", + "name": "netty-codec-http2", + "version": "4.1.76.Final", + "properties": [ + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:codec:codec:4.1.76.Final:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:codec:netty-codec-http2:4.1.76.Final:*:*:*:*:*:*:*" + }, + { + "name": "syft:cpe23", + "value": "cpe:2.3:a:codec:netty_codec_http2:4.1.76.Final:*:*:*:*:*:*:*" + }, + ... + ] + }, + ... + ], + "serialNumber": "urn:uuid:51ec305f-616e-4139-a033-a094bb94a17c" +} +``` + +To scan for any vulnerable libraries, submit the SBOM to a vulnerability scanner. +For example, the popular [Anchore software supply chain management platform](https://anchore.com/) makes the `grype` scanner freely available. +You can check whether the libraries given in your SBOMs have known vulnerabilities documented in Anchore's database. +For this purpose, the output of the tool can be fed directly to the `grype` scanner to check for vulnerable libraries, using the command `$JAVA_HOME/bin/native-image-inspect --sbom | grype` which produces the following output: +```shell +NAME INSTALLED VULNERABILITY SEVERITY +netty-codec-http2 4.1.76.Final CVE-2022-24823 Medium +``` + +You can then use this report to update any vulnerable dependencies found in your executable. + +## Java serialization in Native Image + +Native Image supports Serialization to help users deserialize the constructors for classes, contained in a native executable. +Unless picked up by native image analysis automatically, [these classes have to be pre-specified](https://www.graalvm.org/22.0/reference-manual/native-image/Reflection/#manual-configuration), as classes not contained in a native executable cannot be deserialized. +Native Image cannot prevent exploitation of deserialization vulnerabilities in isolation. +The [serialization and deserialization Secure Coding Guidelines for Java SE](https://www.oracle.com/java/technologies/javase/seccodeguide.html#8) should be followed. + +## Miscellaneous + +Native Image provides multiple ways to specify a certificate file used to define the default TrustStore. +While the default behavior for `native-image` is to capture and use the default TrustStore from the build-time host environment, this can be changed at run time by setting the "javax.net.ssl.trustStore\*" system properties. +See the [documentation](../reference-manual/native-image/CertificateManagement.md) for more details. + +The directory containing the native executable is part of the search path when loading native libraries using `System.loadLibrary()` at run time. + +Native Image will not allow a Java Security Manager to be enabled because this functionality has now deprecated since Java 17. +Attempting to set a security manager will trigger a runtime error. + +## Related Documentation +- [Security Guide](security-guide.md) +- [Polyglot Sandboxing](polyglot-sandbox.md) \ No newline at end of file diff --git a/docs/security/polyglot-sandbox.md b/docs/security/polyglot-sandbox.md new file mode 100644 index 00000000000..3c552d263e0 --- /dev/null +++ b/docs/security/polyglot-sandbox.md @@ -0,0 +1,512 @@ +--- +layout: docs +toc_group: security-guide +link_title: Polyglot Sandboxing +permalink: /security-guide/polyglot-sandbox/ +--- +# Polyglot Sandboxing + +GraalVM allows a host application written in a JVM-based language to execute guest code written in Javascript via the [Polyglot Embedding API](../reference-manual/embedding/embed-languages.md). +Configured with a [sandbox policy](#sandbox-policies), a security boundary between a host application and guest code can be established. +For example, host code can execute untrusted guest code using the [UNTRUSTED](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/SandboxPolicy.html#UNTRUSTED) policy. +Host code can also execute multiple mutually distrusting instances of guest code that will be protected from one another. +Used this way, polyglot sandboxing supports a multi-tenant scenario: + +![](sandbox_security_boundary.png) + +Use cases that benefit from introducing a security boundary are: +* Usage of third party code, i.e., pulling in a dependency. Third party code is typically trusted and scanned for vulnerabilities before use, but sandboxing them is an additional precaution against supply-chain attacks. +* User plugins. Complex applications might allow users to install community-written plugins. Traditionally, those plugins are considered trusted and often run with full privileges, but ideally they should not be able to interfere with the application except when intended. +* Server scripting. Allowing users to customize a server application with their own logic expressed in a general-purpose scripting language, for example, to implement custom data processing on a shared data source. + +## Sandbox Policies + +Depending on the use case and the associated acceptable security risk, a [SandboxPolicy](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/SandboxPolicy.html) can be chosen, ranging from [TRUSTED](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/SandboxPolicy.html#TRUSTED) to [UNTRUSTED](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/SandboxPolicy.html#UNTRUSTED), enabling and configuring an increasing range of restrictions and mitigations. +A `SandboxPolicy` serves two purposes: preconfiguration and validation of the final configuration. +It preconfigures context and engine to comply to a policy by default. +In case the configuration is further customized, validation of the policy will ensure that the custom configuration does not unacceptably weaken the policy. + +### Trusted Policy + +The [TRUSTED](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/SandboxPolicy.html#TRUSTED) sandboxing policy is intended for guest code that is entirely trusted. +This is the default mode. +There are no restrictions to the context or engine configuration. + +Example: +```java +try (Context context = Context.newBuilder("js") + .sandbox(SandboxPolicy.TRUSTED) + .build();) { + context.eval("js", "print('Hello JavaScript!');"); +} +``` + +### Constrained Policy + +The [CONSTRAINED](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/SandboxPolicy.html#CONSTRAINED) sandboxing policy is intended for trusted applications whose access to host resources should be regulated. +The CONSTRAINED policy: +* Requires the languages for a context to be set. +* Disallows [native access](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/Context.Builder.html#allowNativeAccess-boolean-). +* Disallows [process creation](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/Context.Builder.html#allowCreateProcess-boolean-). +* Disallows [system exit](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/Context.Builder.html#useSystemExit-boolean-), prohibiting the guest code from terminating the entire VM where this is supported by the language. +* Requires redirection of the standard [output](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/Context.Builder.html#out-java.io.OutputStream-) and [error](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/Context.Builder.html#err-java.io.OutputStream-) streams. This is to mitigate risks where external components, such as log processing, may be confused by unexpected writes to output streams by guest code. +* Disallows [host file](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/io/IOAccess.Builder.html#allowHostFileAccess-boolean-) or [socket](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/io/IOAccess.Builder.html#allowHostSocketAccess-boolean-) access. Only custom [polyglot file system](https://www.graalvm.org/sdk/javadoc/index.html?org/graalvm/polyglot/io/FileSystem.html) implementations are allowed. +* Disallows [environment access](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/Context.Builder.html#allowEnvironmentAccess-org.graalvm.polyglot.EnvironmentAccess-). +* Restricts host access: + * Disallows [host class loading](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/Context.Builder.html#allowHostClassLoading-boolean-). + * Disallows [to all public host classes and methods by default](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/HostAccess.Builder.html#allowPublicAccess-boolean-). + * Disallows [access inheritance](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/HostAccess.Builder.html#allowAccessInheritance-boolean-). + * Disallows implementation of arbitrary host [classes](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/HostAccess.Builder.html#allowAllClassImplementations-boolean-) and [interfaces](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/HostAccess.Builder.html#allowAllImplementations-boolean-). + * Disallows implementation of `java.lang.FunctionalInterface`. + * Disallows host object mappings of [mutable target types](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/HostAccess.Builder.html#allowMutableTargetMappings-org.graalvm.polyglot.HostAccess.MutableTargetMapping...-). +The [HostAccess.CONSTRAINED](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/HostAccess.html#CONSTRAINED) host access policy is preconfigured to fulfil the requirements for the CONSTRAINED sandboxing policy. + +Example: +```java +try (Context context = Context.newBuilder("js") + .sandbox(SandboxPolicy.CONSTRAINED) + .out(new ByteArrayOutputStream()) + .err(new ByteArrayOutputStream()) + .build()) { + context.eval("js", "print('Hello JavaScript!');"); +} +``` + +### Isolated Policy + +The [ISOLATED](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/SandboxPolicy.html#ISOLATED) sandboxing policy builds on top of the CONSTRAINED policy and is intended for trusted applications that may misbehave because of implementation bugs or processing of untrusted input. +As the name already suggests, the ISOLATED policy enforces deeper isolation between host and guest code. +In particular, guest code running with the ISOLATED policy will be executed in their own virtual machine, on a separate heap. +This means that they no longer share runtime elements such as the JIT compiler or the garbage collector with the host application, making the host VM significantly more resilient against faults in the guest VM. + +In addition to the restrictions of the CONSTRAINED policy, the ISOLATED policy: +* Requires [method scoping](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/HostAccess.Builder.html#methodScoping-boolean-) to be enabled. This avoids cyclic dependencies between host and guest objects. The [HostAccess.ISOLATED](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/HostAccess.html#ISOLATED) host access policy is preconfigured to fulfil the requirements for the ISOLATED sandboxing policy. +* Requires setting the maximum isolate heap size. This is the heap size that will be used by the guest VM. If the engine is shared by multiple contexts, execution of these contexts will share the isolate heap. +* Requires setting the host call stack headroom. This protects against host stack starving on upcalls to the host: the guest will be prohibited from performing an upcall if the remaining stack size drops below the specified value. +* Requires setting the maximum CPU time limit. This restricts the workload to execute within the given time frame. + +Example: +```java +try (Context context = Context.newBuilder("js") + .sandbox(SandboxPolicy.ISOLATED) + .out(new ByteArrayOutputStream()) + .err(new ByteArrayOutputStream()) + .option("engine.MaxIsolateMemory", "256MB") + .option("sandbox.MaxCPUTime", "2s") + .build()) { + context.eval("js", "print('Hello JavaScript!');"); +} +``` + +### Untrusted Policy + +The [UNTRUSTED](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/SandboxPolicy.html#UNTRUSTED) sandboxing policy builds on top of the ISOLATED policy and is intended to mitigate risks from running actual untrusted code. +The attack surface of GraalVM when running untrusted code consists of the entire guest VM that executes the code as well as the host entry points made available to guest code. + +In addition to the restrictions of the ISOLATED policy, the UNTRUSTED policy: +* Requires redirection of the standard [input](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/Context.Builder.html#in-java.io.InputStream-) stream. +* Requires setting the maximum memory consumption of the guest code. This is a limit in addition to the maximum isolate heap size backed by a mechanism that keeps track of the size of objects allocated by the guest code on the guest VM heap. This limit can be thought of as a "soft" memory limit, whereas the isolate heap size is the "hard" limit. +* Requires setting the maximum number of stack frames that can be pushed on the stack by guest code. This limit can protect against unbounded recursion to exhaust the stack. +* Requires setting the maximum AST depth of the guest code. Together with the stack frame limit, this puts a bound on the stack space consumed by guest code. +* Requires setting the maximum output and error stream sizes. As output and error streams have to be redirected, the receiving ends are on the host side. Limiting the output and error stream sizes protects against availability issues on the host. +* Requires untrusted code mitigations to be enabled. Untrusted code mitigations address risks from JIT spraying and speculative execution attacks. They include constant blinding as well as comprehensive use of speculative execution barriers. +* Further restricts host access to ensure there are no implicit entry points to host code. This means that guest-code access to host arrays, lists, maps, buffers, iterables and iterators is disallowed. The reason is that there may be various implementations of these APIs on the host side, resulting in implicit entry points. In addition, direct mappings of guest implementations to host interfaces via [HostAccess.Builder#allowImplementationsAnnotatedBy()] are disallowed. The [HostAccess.UNTRUSTED](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/HostAccess.html#UNTRUSTED) host access policy is preconfigured to fulfil the requirements for the UNTRUSTED sandboxing policy. + +Example: +```java +try (Context context = Context.newBuilder("js") + .sandbox(SandboxPolicy.UNTRUSTED) + .in(new ByteArrayInputStream("foobar".getBytes())) + .out(new ByteArrayOutputStream()) + .err(new ByteArrayOutputStream()) + .allowHostAccess(HostAccess.UNTRUSTED) + .option("engine.MaxIsolateMemory", "8MB") + .option("sandbox.MaxHeapMemory", "128MB") + .option("sandbox.MaxCPUTime","2s") + .option("sandbox.MaxStatements","50000") + .option("sandbox.MaxStackFrames","2") + .option("sandbox.MaxThreads","1") + .option("sandbox.MaxASTDepth","10") + .option("sandbox.MaxOutputStreamSize","32B") + .option("sandbox.MaxErrorStreamSize","0B"); + .build()) { + context.eval("js", "print('Hello JavaScript!');"); +} +``` + +For further information on how to set the resource limits, please refer to the corresponding [guidance](#resource-limits). + +## Host Access + +GraalVM allows exchanging objects between host and guest code and exposing host methods to guest code. +When exposing host methods to less privileged guest code, these methods become part of the attack surface of the more privileged host code. +Therefore the sandboxing policies already restrict host access in the CONSTRAINED policy to make host entry points explicit. + +`HostAccess.CONSTRAINED` is the pre-defined host access policy for the CONSTRAINED sandbox policy. +To expose a host class method, it has to be annotated with `@HostAccess.Export`. +This annotation is not inherited. +Service providers such as [polyglot file system](https://www.graalvm.org/sdk/javadoc/index.html?org/graalvm/polyglot/io/FileSystem.html) implementations or output stream recipients for standard output and error stream redirections are exposed to guest code invocations. + +Guest code can also implement a Java interface that has been annotated with `@Implementable`. +Host code using such an interface directly interacts with guest code. + +Host code that interacts with guest code has to be implemented in a robust manner: +* Input validation. All data passed from the guest, e.g. via parameters to an exposed method, is untrusted and should be thorougly validated by host code where applicable. +* Reentrancy. Exposed host code should be reentrant as guest code may invoke it at any time. Do note that simply applying the `synchronized` keyword to a code block does not necessarily make it reentrant. +* Thread-safety. Exposed host code should be thread-safe as guest code may invoke them from multiple threads at the same time. +* Resource consumption. Exposed host code should be aware of its resource consumption. In particular, constructs that allocate memory based on untrusted input data, either directly or indirectly, e.g. through recursion, should either be avoided altogether or implement limits. +* Privileged functionality. Restrictions enforced by the sandbox can be entirely bypassed by exposing host methods that provide restricted functionality. For example, guest code with a CONSTRAINED sandbox policy cannot perform host file IO operations. However, exposing a host method to the context that allows writing to arbitrary files effectively bypasses this restriction. +* Side channels. Depending on the guest language, guest code may have access to timing information. For example, in Javascript the `Date()` object provides fine-grained timing information. In the UNTRUSTED sandbox policy the granularity of Javascript timers is pre-configured to one second and can be lowered to 100 milliseconds. However, host code should be aware that guest code may time its execution, potentially discovering secret information if the host code performs secret-depending processing. + +Host code that is unaware of interacting with untrusted guest code should never be directly exposed to guest code without taking the aforementioned aspects into account. +As an example, an antipattern would be to implement a third party interface and forwarding all method invocations to guest code. + +## Resource Limits + +The ISOLATED and UNTRUSTED sandbox policies require setting resource limits for a context. +Different configurations can be provided for each context. +If a limit is exceeded, evaluation of the code fails and the context is canceled with a [`PolyglotException`](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/PolyglotException.html) which returns `true` for `isResourceExhausted()`. +At this point, no more guest code can be executed in the context + +The `--sandbox.TraceLimits` option allows you to trace guest code and record the maximum resource utilization. +This can be used to estimate the parameters for the sandbox. +For example, a web server's sandbox parameters could be obtained by enabling this option and either stress-testing the server, or letting the server run during peak usage. +When this option is enabled, the report is saved to the log file after the workload completes. +Users can change the location of the log file by using `--log.file=` with a language launcher or `-Dpolyglot.log.file=` when using a `java` launcher. +Each resource limit in the report can be passed directly to a sandbox option to enforce the limit. + +See, for example, how to trace limits for a Python workload: + +``` +graalpy --log.file=limits.log --sandbox.TraceLimits=true workload.py + +limits.log: +Traced Limits: +Maximum Heap Memory: 12MB +CPU Time: 7s +Number of statements executed: 9441565 +Maximum active stack frames: 29 +Maximum number of threads: 1 +Maximum AST Depth: 15 +Size written to standard output: 4B +Size written to standard error output: 0B + +Recommended Programmatic Limits: +Context.newBuilder() + .option("sandbox.MaxHeapMemory", "2MB") + .option("sandbox.MaxCPUTime","10ms") + .option("sandbox.MaxStatements","1000") + .option("sandbox.MaxStackFrames","64") + .option("sandbox.MaxThreads","1") + .option("sandbox.MaxASTDepth","64") + .option("sandbox.MaxOutputStreamSize","1024KB") + .option("sandbox.MaxErrorStreamSize","1024KB") + .build(); + +Recommended Command Line Limits: +--sandbox.MaxHeapMemory=12MB --sandbox.MaxCPUTime=7s --sandbox.MaxStatements=9441565 --sandbox.MaxStackFrames=64 --sandbox.MaxThreads=1 --sandbox.MaxASTDepth=64 --sandbox.MaxOutputStreamSize=1024KB --sandbox.MaxErrorStreamSize=1024KB +``` + +Re-profiling may be required if the workload changes or when switching to a different major GraalVM version. + +Certain limits can be [reset](#resetting-resource-limits) at any point of time during the execution. + +### Limiting active CPU time + +The `sandbox.MaxCPUTime` option allows you to specify the maximum CPU time spent running guest code. +CPU time spent depends on the underlying hardware. +The maximum [CPU time](https://docs.oracle.com/en/java/javase/17/docs/api/java.management/java/lang/management/ThreadMXBean.html#getThreadCpuTime\(long\)) specifies how long a context can be active until it is automatically cancelled and the context is closed. +By default the time limit is checked every 10 milliseconds. +This can be customized using the `sandbox.MaxCPUTimeCheckInterval` option. + +As soon as the time limit is triggered, no further guest code can be executed with this context. +It will continuously throw a `PolyglotException` for any method of the polyglot context that will be invoked. + +The used CPU time of a context includes time spent in callbacks to host code. + +The used CPU time of a context typically does not include time spent waiting for synchronization or IO. +The CPU time of all threads will be added and checked against the CPU time limit. +This can mean that if two threads execute the same context then the time limit will be exceeded twice as fast. + +The time limit is enforced by a separate high-priority thread that will be woken regularly. +There is no guarantee that the context will be cancelled within the accuracy specified. +The accuracy may be significantly missed, e.g. if the host VM causes a full garbage collection. +If the time limit is never exceeded then the throughput of the guest context is not affected. +If the time limit is exceeded for one context then it may slow down the throughput for other contexts with the same explicit engine temporarily. + +Available units to specify time durations are `ms` for milliseconds, `s` for seconds, `m` for minutes, `h` for hours and `d` for days. +Both maximum CPU time limit and check interval must be positive followed by a time unit. + +```java +try (Context context = Context.newBuilder("js") + .option("sandbox.MaxCPUTime", "500ms") + .build();) { + context.eval("js", "while(true);"); + assert false; +} catch (PolyglotException e) { + // triggered after 500ms; + // context is closed and can no longer be used + // error message: Maximum CPU time limit of 500ms exceeded. + assert e.isCancelled(); + assert e.isResourceExhausted(); +} +``` + +### Limiting the number of executed statements + +Specifies the maximum number of statements a context may execute until it is cancelled. +After the statement limit was triggered for a context, it is no longer usable and every use of the context will throw a `PolyglotException` that returns `true` for `PolyglotException.isCancelled()`. +The statement limit is independent of the number of threads executing. + +The limit may be set to a negative number to disable it. +Whether this limit is applied internal sources only can be configured using `sandbox.MaxStatementsIncludeInternal`. +By default the limit does not include statements of sources that are marked internal. +If a shared engine is used then the same internal configuration must be used for all contexts of an engine. + +The complexity of a single statement may not be constant time depending on the guest language. +For example, statements that execute Javascript builtins, like `Array.sort`, may account for a single statement, but its execution time is dependent on the size of the array. + +```java +try (Context context = Context.newBuilder("js") + .option("sandbox.MaxStatements", "2") + .option("sandbox.MaxStatementsIncludeInternal", "false") + .build();) { + context.eval("js", "purpose = 41"); + context.eval("js", "purpose++"); + context.eval("js", "purpose++"); // triggers max statements + assert false; +} catch (PolyglotException e) { + // context is closed and can no longer be used + // error message: Maximum statements limit of 2 exceeded. + assert e.isCancelled(); + assert e.isResourceExhausted(); +} +``` + +### AST depth limit + +A limit on the maximum expression depth of a guest language function. +Only instrumentable nodes count towards the limit. + +The AST depth can give an estimate of the complexity of a function as well as its stack frame size. + +### Limiting the number of stack frames + +Specifies the maximum number of frames a context can push on the stack. +A thread-local stack frame counter is incremented on function enter and decremented on function return. + +The stack frame limit in itself serves as a safeguard against infinite recursion. +Together with the AST depth limit it can restrict total stack space usage. + +### Limiting the number of active threads + +Limits the number of threads that can be used by a context at the same point in time. +Multi-threading is not supported in the UNTRUSTED sandbox policy. + +### Heap memory limits + +The `sandbox.MaxHeapMemory` option specifies the maximum heap memory guest code is allowed to retain during its run. +Only objects residing in guest code count towards the limit - memory allocated during callbacks to host code does not. +This is not a hard limit as the efficacy of this option (also) depends on the garbage collector used. +This means that the limit may be exceeded by guest code. + +```java +try (Context context = Context.newBuilder("js") + .option("sandbox.MaxHeapMemory", "100MB") + .build()) { + context.eval("js", "var r = {}; var o = r; while(true) { o.o = {}; o = o.o; };"); + assert false; +} catch (PolyglotException e) { + // triggered after the retained size is greater than 100MB; + // context is closed and can no longer be used + // error message: Maximum heap memory limit of 104857600 bytes exceeded. Current memory at least... + assert e.isCancelled(); + assert e.isResourceExhausted(); +} +``` + +The limit is checked by retained size computation triggered either based on [allocated](https://docs.oracle.com/en/java/javase/11/docs/api/jdk.management/com/sun/management/ThreadMXBean.html#getThreadAllocatedBytes\(long\)) bytes or on [low memory notification](https://docs.oracle.com/en/java/javase/11/docs/api/java.management/java/lang/management/MemoryMXBean.html). + +The allocated bytes are checked by a separate high-priority thread that will be woken regularly. +There is one such thread for each memory-limited context (one with `sandbox.MaxHeapMemory` set). +The retained bytes computation is done by yet another high-priority thread that is started from the allocated bytes checking thread as needed. +The retained bytes computation thread also cancels the context if the heap memory limit is exeeded. +Additionaly, when the low memory trigger is invoked, all contexts on engines with at least one memory-limited context are paused together with their allocation checkers. +All individual retained size computations are cancelled. +Retained bytes in the heap for each memory-limited context are computed by a single high-priority thread. + +The heap memory limit will not prevent the context from causing `OutOfMemory` errors. +Guest code that allocates many objects in quick succession has a lower accuracy compared to code that allocates objects rarely. + +Retained size computation for a context can be customized using the expert options `sandbox.AllocatedBytesCheckInterval`, `sandbox.AllocatedBytesCheckEnabled`, `sandbox.AllocatedBytesCheckFactor`, `sandbox.RetainedBytesCheckInterval`, `sandbox.RetainedBytesCheckFactor`, and `sandbox.UseLowMemoryTrigger` described below. + +Retained size computation for a context is triggered when a retained bytes estimate exceeds a certain factor of specified `sandbox.MaxHeapMemory`. +The estimate is based on heap memory +[allocated](https://docs.oracle.com/en/java/javase/17/docs/api/jdk.management/com/sun/management/ThreadMXBean.html#getThreadAllocatedBytes\(long\)) by threads where the context has been active. +More precisely, the estimate is the result of previous retained bytes computation, if available, plus bytes allocated since the start of the previous computation. +By default the factor of `sandbox.MaxHeapMemory` is 1.0 and it can be customized by the `sandbox.AllocatedBytesCheckFactor` option. +The factor must be positive. +For example, let `sandbox.MaxHeapMemory` be 100MB and `sandbox.AllocatedBytesCheckFactor` be 0.5. +The retained size computation is first triggered when allocated bytes reach 50MB. +Let the computed retained size be 25MB, then the next retained size computation is triggered when additional 25MB is allocated, etc. + +By default, allocated bytes are checked every 10 milliseconds. This can be configured by `sandbox.AllocatedBytesCheckInterval`. +The smallest possible interval is 1ms. Any smaller value is interpreted as 1ms. + +The beginnings of two retained size computations of the same context must be by default at least 10 milliseconds apart. +This can be configured by the `sandbox.RetainedBytesCheckInterval` option. The interval must be positive. + +The allocated bytes checking for a context can be disabled by the `sandbox.AllocatedBytesCheckEnabled` option. +By default it is enabled ("true"). If disabled ("false"), retained size checking for the context can be triggered only by the low memory trigger. + +When the total number of bytes allocated in the heap for the whole host VM exceeds a certain factor of the total heap memory of the VM, [low memory notification](https://docs.oracle.com/en/java/javase/17/docs/api/java.management/java/lang/management/MemoryMXBean.html) is invoked and initiates the following process. +The execution for all engines with at least one execution context which has the `sandbox.MaxHeapMemory` option set is paused, retained bytes in the heap for each memory-limited context are computed, contexts exceeding their limits are cancelled, and then the execution is resumed. +The default factor is 0.7. This can be configured by the `sandbox.RetainedBytesCheckFactor` option. +The factor must be between 0.0 and 1.0. All contexts using the `sandbox.MaxHeapMemory` option must use the same value for `sandbox.RetainedBytesCheckFactor`. + +When the usage threshold or the collection usage threshold of any heap memory pool has already been set, then the low memory trigger cannot be used by default, because the limit specified by the `sandbox.RetainedBytesCheckFactor` cannot be implemented. +However, when `sandbox.ReuseLowMemoryTriggerThreshold` is set to true and the usage threshold or the collection usage threshold of a heap memory pool has already been set, then the value of `sandbox.RetainedBytesCheckFactor` is ignored for that memory pool and whatever limit has already been set is used. +That way the low memory trigger can be used together with libraries that also set the usage threshold or the collection usage threshold of heap memory pools. + +The described low memory trigger can be disabled by the `sandbox.UseLowMemoryTrigger` option. +By default it is enabled ("true"). If disabled ("false"), retained size checking for the execution context can be triggered only by the allocated bytes checker. +All contexts using the `sandbox.MaxHeapMemory` option must use the same value for `sandbox.UseLowMemoryTrigger`. + +### Limiting the amount of data written to standard output and error streams + +Limits the size of the output that guest code writes to standard output or standard error output during runtime. +Limiting the size of the output can serve as protection against denial-of-service attacks that flood the output. + +```java +try (Context context = Context.newBuilder("js") + .option("sandbox.MaxOutputStreamSize", "100KB") + .build()) { + context.eval("js", "while(true) { console.log('Log message') };"); + assert false; +} catch (PolyglotException e) { + // triggered after writing more than 100KB to stdout + // context is closed and can no longer be used + // error message: Maximum output stream size of 102400 exceeded. Bytes written 102408. + assert e.isCancelled(); + assert e.isResourceExhausted(); +} +``` +```java +try (Context context = Context.newBuilder("js") + .option("sandbox.MaxErrorStreamSize", "100KB") + .build()) { + context.eval("js", "while(true) { console.error('Error message') };"); + assert false; +} catch (PolyglotException e) { + // triggered after writing more than 100KB to stderr + // context is closed and can no longer be used + // error message: Maximum error stream size of 102400 exceeded. Bytes written 102410. + assert e.isCancelled(); + assert e.isResourceExhausted(); +} +``` + +### Resetting Resource Limits + +It is possible to reset the limits at any point in time using the [`Context.resetLimits`](https://www.graalvm.org/sdk/javadoc/org/graalvm/polyglot/Context.html#resetLimits--) method. +This can be useful if a known and trusted initialization script should be excluded from limit. +Only the statement, cpu time and output / error stream limits can be reset. + +```java +try (Context context = Context.newBuilder("js") + .option("sandbox.MaxCPUTime", "500ms") + .build();) { + context.eval("js", /*... initialization script ...*/); + context.resetLimits(); + context.eval("js", /*... user script ...*/); + assert false; +} catch (PolyglotException e) { + assert e.isCancelled(); + assert e.isResourceExhausted(); +} +``` + +## Runtime Defenses + +The main defense enforced by the ISOLATED and UNTRUSTED sandbox policy through the `engine.SpawnIsolate` option is that the Polyglot engine runs in a dedicated `native-image` isolate, moving execution of guest code to a VM-level fault domain separate from the host application, with its own heap, garbage collector and JIT compiler. + +Apart from setting a hard limit for the memory consumption of guest code via the guest's heap size, it also allows to focus runtime defenses just on guest code and not cause performance degredation of host code. +The runtime defenses are enabled by the `engine.UntrustedCodeMitigation` option. + +### Constant Blinding + +JIT compilers allow users to provide source code and, given the source code is valid, compile it to machine code. +From an attacker's perspective, JIT compilers compile attacker-controlled inputs to predictable bytes in executable memory. +In an attack called JIT spraying an attacker leverages the predictable compilation by feeding malicious input programs into the JIT compiler, thereby forcing it to emit code containing Return-Oriented Programming (ROP) gadgets. + +Constants in the input program are a particularly attractive target for such an attack, since JIT compilers often include them verbatim in the machine code. +Constant blinding aims to invalidate an attacker's predictions by introducing randomness into the compilation process. +Specifically, constant blinding encrypts constants with a random key at compile time and decrypts them at runtime at each occurrence. +Only the encrypted version of the constant appears verbatim in the machine code. +Absent knowledge of the random key, the attacker cannot predict the encrypted constant value and, therefore, can no longer predict the resulting bytes in executable memory. + +GraalVM blinds all immediate values and data embedded in code pages of runtime compiled guest code down to a size of four bytes. + +### Speculative Execution Attack Mitigations + +Speculative execution attacks such as Spectre exploit the fact that a CPU may transiently execute instructions based on branch prediction information. +In the case of a misprediction, the result of these instructions is discarded. +However, the execution may have caused side effects in the micro-architectural state of a CPU. +For example, data may have been pulled into the cache during transient execution - a side-channel that can be read by timing data access. + +GraalVM protects against Spectre attacks by inserting speculative execution barrier instructions in runtime compiled guest code to prevent attackers from crafting speculative execution gadgets. +A speculative execution barrier is placed at each target of a conditional branch to stop speculative execution based on the pattern history table (Spectre V1). +Speculative execution barriers are also placed at each possible indirect branch target to stop speculative execution based on the branch target buffer (Spectre V2). + +## Sharing Execution Engines + +Guest code of different trust domains has to be separated at the Polylgot engine level, i.e. only guest code of the same trust domain should share an engine. +When multiple context share an engine, all of them must have the same sandbox policy (the engine's sandbox policy). +Application developers may choose to share execution engines among execution contexts for performance reasons. +While the context holds the state of the executed code, the engine holds the code itself. +Sharing of an execution engine among multiple contexts needs to be set up explicitly and can increase performance in scenarios where a number of contexts execute the same code. In scenarios where contexts that share an execution engine for common code also execute sensitive (i.e., private) code, the corresponding source objects can opt out from code sharing with: +```java +Source.newBuilder(…).cached(false).build() +``` + +## Compatibility and Limitations + +Polyglot sandboxing is not available in GraalVM Community Edition. + +Depending on the sandboxing policy, only a subset of Truffle languages, instruments, and options are available. +In particular, sandboxing is currently only supported for the runtime's [default version](https://github.com/oracle/graaljs/blob/master/docs/user/JavaScriptCompatibility.md) of ECMAScript, i.e. ECMAScript 2022. +Sandboxing is also not supported from within GraalVM's Node.js. + +Polyglot sandboxing is not compatible with modifications to the VM setup via e.g. system properties that change the behavior of the VM. + +The sandboxing policy is subject to incompatible changes across major GraalVM releases to maintain a secure-by-default posture. + +Polyglot sandboxing cannot protect against vulnerabilities in its operating environment, i.e. vulnerabilities in the operating system or the underlying hardware. +We recommend to adopt the appropriate external isolation primitives to protect against corresponding risks. + +## Differentiation with Java Security Manager + +The Java Security Manager is deprecated in Java 17 with JEP-411. +The purpose of the security manager is stated as follows: "It allows an application to determine, before performing a possibly unsafe or sensitive operation, what the operation is and whether it is being attempted in a security context that allows the operation to be performed." + +The goal of the GraalVM sandbox is to allow the execution of untrusted guest code in a secure manner, meaning untrusted guest code should not be able to compromise the confidentiality, integrity or availability of the host code and its environment. + +The GraalVM sandbox differs from Security Managers in the following aspects: + +* *Security boundary*: The Java Security Manager features a flexible security boundary that depends on the actual calling context of a method. This makes "drawing the line" complex and error prone. A security-critical code block first needs to inspect the current calling stack to determine whether all frames on the stack have the authority to invoke the code. In the GraalVM sandbox, there is a straightforward, clear security boundary: the boundary between host and guest code, with guest code running on top of the Truffle framework, similar to how typical computer architectures distinguish between user mode and (privileged) kernel mode. +* *Isolation*: With the Java Security Manager, privileged code is almost on "equal footing" as untrusted code with respect to the language and runtime: + * *Shared language*: With the Java Security Manager, untrusted code is written in the same language as privileged code, with the advantage of straightforward interoperability between the two. In contrast, the GraalVM sandbox a guest application written in a Truffle language needs to pass an explicit boundary to host code written in Java. + * *Shared runtime*: With the Java Security Manager, untrusted code executes in the same JVM environment as trusted code, sharing JDK classes and runtime services such as the garbage collector or the compiler. In the GraalVM sandbox, untrusted code runs in dedicated VM instances (GraalVM isolates), separating services and JDK classes of host and guest by design. +* *Resource limits*: The Java Security Manager cannot restrict the usage of computational resources such as CPU time or memory, allowing untrusted code to DoS the JVM. The GraalVM sandbox offers controls to set limits on several computational resources (CPU time, memory, threads, processes), guest code may consume to address availability concerns. +* *Configuration*: Crafting a Java Security Manager policy was often found to be a complex and error-prone task, requiring a subject matter expert that knows exactly which parts of the program require what level of access. Configuring the GraalVM sandbox provides security profiles that focus on common sandboxing use cases and threat models. + +## Reporting vulnerabilities + +If you believe you have found a security vulnerability, please submit a report to secalert_us@oracle.com preferably with a proof of concept. +Please refer to [Reporting Vulnerabilities](https://www.oracle.com/corporate/security-practices/assurance/vulnerability/reporting.html) for additional information including our public encryption key for secure email. +We ask that you do not contact project contributors directly or through other channels about a report. + + +### Related Documentation +- [Security Guide](security-guide.md) +- [Native Image Security Aspects](native-image.md) diff --git a/docs/security/sandbox_security_boundary.png b/docs/security/sandbox_security_boundary.png new file mode 100644 index 0000000000000000000000000000000000000000..3e36f080dbf5538ce3e7976ca58e972ba538d429 GIT binary patch literal 9869 zcmeHtc{G$^{O>bkFt!>bOJR_;LZQeuvL%(hvJF`xDk8fWQuY+0#a^;y4Uxf+C6pyZ z*2%8Opt5tHQNQy$_uPB#a?U;H{`2ch>UG}teYWrO*`5j4zo5;+$ioN#V9`0FVF&;c z{ui{O>EO@q_*De_2jOn0eHs+BADjgM3v@KpjJ@&m$@K9Yn)_D@u{!k{siu zdcTW%)X%+Y-Q;&|N7t2++oQSb!{#PCH711{e1Z`|2eAP8_Y)+5;pEg_DFWm`GC*Md zeq2*Y$}2c!69KQw@*D|{?`Kx2t#0sLP)KT56=#Jjb_ddzU{7F8PU~@ z+@I++;uD&CF+cZ9Ft4Jg{_s1G*+G}u%3O2W#n)## zRCkolrzkyrCg<$gr$EcJ|79qjZ&=Ox_rqoH>M&>mK@x!mmMKMa$P~=w5<3RDYo9|@ z)@OurP2Y%Q>crhw*_f00-d`+~kFS2#8ZYjCsq*d3;f$UfqfpC4Z$wexPJr+4-b=H^ z&3rFbMgw;qBNP*$jcZZKvHdmERiaVkvwXERPIT|BFKG@PDX~q{5Ag49 zUyBG2Pp4zy(QkOpcK-$?Yt#teO~=s{R%G`u(!y=I2b^@D)Z%UEJbiTcWtpt8+_!&rjU2 z%lO%w$GP=q&?WxF4YQVmNlHG3DzyPy>y}7D;p}kL$z+u~&qU5W!Fv5BmIrRG-fB%! zSms5L;h<)~1p1>s}I)V6-UX&>rvJo}{4eQvG}X7Mz4w1UcS6bPuncO^S@e zo;yn|qBi4GpPnCIY~G2oLRuf#_%k2(u?|jSm<;piofY2msVbIp%?@DIVS97!+Sd|$ zw?3;}WAPkO6yd?n6op?HDf~D!PFNjesDIp%CEE>GiL$p-LHETUI6D1J>+K-3zU1RqQ`LG5AUH z8LqxH-c)kGgPRb7BrHsr%HRI+<-Ew`DUa`+s@scu6;e-bE;e-NGA0$oL`OfK3|990 zGxQb3HAn=;rXLr!ziA{DGfhYPm_`cee2={Nc2ursxCqar}k9-Fa#~=Lg(YRBsH(K2r@; zDXxBP`lfYav^FrXxTOurkO=yJHb!!a`~8_)ibjsuaRnZ-K$1JWZ3QA3YWR?(JTZbL zrc($Y(wf;4{-GL)2e$vEf{}yvA5JR6Yu_VR82*9WYXN|;j?UAEMZ9~%|L@1GqX9W$ z>~CMtbU_sr71pvn%@@st0S2R@#TY?H?WH##&28jPS=65TExSl@?LX4<|PpIqU%ouZ7aKZi@(?E&REX!4Wc!_kqL9pcBhf-I{% z4SI7;X8Vh6C61cExyr(ETiEq>Hq%#>j36-gd_eEvrsE zmacuI&2)FgIJ^u z?@e(2I#PF+Ui`*D=>s41r{bq@#km*Mz>{xLh6jZpdW!Z&a!Q?TtJ&H1Ye7>c+JgzA z(?5r++D74;X+qR^38}#F!9fK_3c1d>5U=b@siR-e*EtJ{vM*P-3P2)0({5k)@RWzs zvtzb&TN|T0uQGKLTc`c|QU%oMP6ci&m0jz;5BXgkPW_|W#n1I{(1#8$Hn8iRNjAzn z>mjOGXBr;Caro4ffNb+sxXG^)=@HiVWzOD;!i^~dhQMii0+sicr#@x&`e#{YUEuvH zU2$i_lNID$DokKTj#f@XHg91DQVc~m_9GegmKXc2{H#*xcydhW4xIN`12VJPo2#ED z*vg9?I{2Ih%k%BH*lx;BFe4n8zkkX9@g%hK;B&Y?o-FeHN6oe{fOfF&s98LuZ-IP# zu?_9-+3rv%gFHi#g@eoM$_{(>OW~7V0&;lZ&!Ngv|24PDKadPcILXT$s-hzYFgpqk zbe92T_)gx%{6q7jmF^Ce_hndW$;2fCgFV0&elR;xYyQlNgdoxYMhc5m}4aIfDsx} zUkTQ+zbO3()9vp)IwcsQ0EQl|!34LZObV>-_uQHCT37|dLy9@rbj@evkm&sINXR}q zMbxUsecEFavT&QoX?!BTyek%xcNAPA4JaFlk^3W2?byxrABD!5XCEt$BA=)R)(rJp z)dnQHO|}ZN`=luO*o|9tzdEleOPTIa**KC6#li{RrMJL4hSc>!+nyiSS72#>4Zz;8 zQ+qS=w>N*4aL5^`r8V@$l~|D2d*GJBF1r0wnmVR;cl-Unzw-mE|8T>w>G^8UNsV9~%Lj z5g=itn%Qt+X|6nfJH<_{Wd&OX60=J@au;tYqSUvBppXC0px#s%81`SZ*z*^e8G;-~ z;%_q4tLDFwLz4B(gW3P_B0!xMfn;!8A_@IZ(*MK|fNOK2Bg1a9038DB3TtO9&-#xk zhagjuA4qkv{T&$w4KV=sA=_2(-~AQpunu{xGKBwnhW(hZ=;+MhDz9!RgRgyhCTCn^ zt>!!NNIRDU_hP?4aPqhV0W9q zm14+Mi%_q0LAN0e)t>B~_1jc|J3m~Nh!5QIfU0GmeQSKURy2?1Wb5NNC#lOt1mBgP z=MyC@GoVvxOWyo5KbT|s=Bk@RM>7<13!5tbtF8l;?paAjP{aF@gXkxS{%cDHP;-ho zv?ndSxiMHy73aRI-$EO;U`Jo#`S1LCBwB$5N_C7c{*JWk0y)ttAboRfxf@ynardc@ zvi7a}Aow$uxs0ZqjTcj!A1HhACEvUo{>_03=eWYM9MmycN6pJSU&N?Lq$WaOh8j8k zCE!x_m95_RStZDL*#e3iF^J`8bUPq!F15jZz~Q_6=_D<0s-p9kBON6)7|_L+p3VQa zOhi!u8}lby9|_U3@F_ikY^1$cWi>6{ec}VF?^w9#3}gbxE)r{VIaw*O;TxUl>ZPvV zPMeiFKJsr57V!Aq({dH4Y_Hmyy}K=*Z&7Jj0I~VwQ<+e$2M2YaG|zyW^1~8HY-TUx zq(>1rjjyop)&+?XnUvswDDwU71V^whnqM`gJw^Gj*D%A$Da#uFr%exd51SGo@pnNk zTX3?fk5}Ch__lBacpIiFxL++n5*krb>XeC&Kc;ES+{YVa9$-L9`n@ZEi_9E7RB`Jr z9WxgWIydELZkhY?a)D4=kD8>4PF1>3e_onwn{4sbIV*kbQrA0T)*2k)w46v5|?Aw5}P6r0w~N!neXoUVhtHYO8nZM=c0qqHguO> z>xP=!C$?ZZ{4HdFo}w%2r`)HQ;S}kV&_LiztG~z84S7EYf@M#d8jZr+Yu&GwHX!EA zZf`7Powza3`Of3VwlPpzn}~}I$~G&@O*1RM{&Y_5Oq>YI{Alf{kv|;d*u7i3gA8&M zCMnMO#7w;w~+)bWDAjra&{k!TCYDsA{y{vk>LTefjHyuQVWSU}-+ zwl!$zz{!8_Sk+ClQWZ>zbLC}Q+HXiLe#fHrYWD6^fh*c3hMAGZeGk;q!JzBW);(IjLvC}Eo4_4e! z2{cHonG)-SuQhIO$;x{d#W7+3s}-lw{fEG6S>5J!q|{z0FlXyM9Z}M0p-l`9TUN+0 zPL0zcP=---16T9?QUlg=l!&3JKbs4GO@<d2-^?k?e zw9xiSYT))$yx+WBuE7hse5=}&3n@zCdv|;m8#!nHQy1PP$~_#F{LryJ%?xTSACAg? zh+(~IzL{-Z@c!*+z~Mg13qlOPs^ngt?_b|!*&SUahce&dUE8ANf%}b4v;Pb?f_X-K zoU=5KRZ4BS!pzSUP#W6f0-&Ioyr;NXsvPj0zcxIEpXnp}8q_PI9TU@uh&vFtbQlS% zmuQw>++8!Mxqg=x=&f&#VFMZ{P}+rC9WpMzZYpu9APEfuZcD40J>ln9^80h$haOQY zV+}W!+sI3uxNImQWFbwQBcQsG+BTi)uPnvyx~{tI{pYJgj!vA&Qa}(~(&y#Z zzh>OKiv>OU_4*C4%(SBo%Vx9oGw5LWNj-Lb>^F&WZ7p!gT%22Gi#m$G`LILz1XtfH}Wj4N>6X=;le=Cg!iJ&0e!m z({2y&7Y;AWZ25!}8bRdV{V3wM7}0INh3ZQQbT+oO_Ng#}q5sLYZ|LfX*avo1DFo$5 zr2AxRW)KpIixy}+psum{e&Kmvi;#E>_>+`71KOk){Ghl>-ruBip5!fq(omt#ygJ7w?n zs=?B0LI?Ii2rX;{NF0Y~l%@u}L!%$*$2ullRIu0PecSuDW`!yPR>N&={i-gpftQ5ze)v{jw2r~Ve!FpcSTbaA{fSDm2|>=a@2`^HWp@Tt7PS_OGCl&o}lYC`Q7A$>*SG%R8v z6lZ4}MhfAf^eq28?BjE>xqEuBkEO@q} zqGS09gfj|U+NXV+m7HatbC8dDCXAD#Bb!H!m_2rT?U37?0cj7XaeQww+NKWBf}s#3&F>pMQ87s2}^YJEIT`F zJg%9kb(1FuxONw&yggfX*9NyTWEH?k@Ln9G@qAp!BjqVK?07=}M2&f7N%!tyM3667 zAL?Rjh+LMucq7e;nEmDBQCkeSxz>Hz*;r8wML1TwW`Y@Olv?sk$b(M{N*g+(fzXrT z!y09GPlF0hY`9s7_pOn}To&VK&faI0_RL7q(Lf&OXeau6KzizAE#qn8s^&<$o5^_N z1(cQ(3&I-p#_wLXr#>32AW__4`p$QI8k1&Dsl!f{Fg0uAow@Yegxa#zP1%c*V~t!+ zqKt;ooT6dg-Dos$sW|0gLTwp~uNNSAGlV8Qp3Z$Bb&*Zl=o;0^!ANKuXizf&?7T- zz*RSg7s%z^ha^=4H9C?{s_$1^?_y(HgxSplNRk`Z3927qi3kS=cYG&g96L2A0!A`30L^ zI@oQePeaG)5aa~a=rS$plyEISx1trmZwk6NROT+)h$4l`?gQ!Iuhpk_SJDp=o2?nf zIc@oR0N3knkpI-Y2@8gnvWma#E{1KuavruJ(rRM<$R#=aoQCzwFBqUDU3Dppx{|I^ zxQ7VWku3ZPeGmV|l8l7G`=DqjD8R&u+Vbk8mhGe&?AGyzcV=Ph5P zt|S-}K8S=Pb2RlS_`Yf*jF5Q@`}O{7r6KGW-R&`px{^ao7(0-(l9R|{%xgt_ zojqAD0GQ<|(3r@2{j{K-TJ-7;7TePI=9?$}Sr{$6si<(5`dYGNPgtTVgM9HMo;ltHykFk8w^q(e}e&zR6gyz7fS8Ri`X(4rF@9V+0q+Bf!XB1E?Bn(JSWKZU_^?FFBuiFC+o27l(h zPHw(B|NL`HoaijHGd=G-oWFOb({M;yJb~$nHnd;qa?W3dID%9+DKuqHy%7)ilqJev zT}VC)<@zf~TZV(>H)f%+F#H4gLL1&2?@Yg!@0@M(gG`wA8dm%Mc3hfh@j4FtM_^aN zc{hgKJGv;}-czHe?*@gZ>df4owsxLsI}{o2x`%|^1$0z(_p`WgVj( z*Ol*j!6Wr;-@zhxnxnwmZBj=yAb3LYCO+`|1>g_+FAD{kS+UK%kX9Ip%>>c&VQUmO zsfRHo)8Rzp97C$FQ=lF>1d0KwenOI=sCOHIiQ5d+c)wthWVATeI13%1n`j4p-y7;_ zkR`dqr)1g|rm?@84!DE?$F2-o4l#YgNWjL8wF3?kkjAe;+xq=V{(#z1R)`EA-lG>! z$9BD6Wiw;WY0`Dnq+5E8V~wXu=~Px59i zkSN0{qmPyyl)kxL43c3~V)dH+EUbf|9*kA$ItV^mvihS7j(cRPSO-vUIjB# z(^urXHz*Aa$TVo@V_C>mfhykI!EaEb{>#z6u6dsNy40P$Bz>A_?av_UYdkKT6inlJ zGmRD$re4Z5*7=#X=Ga$|Jao^U4r|*Kbv$QjK&9~a@~O>mc2PT))P)(h_~G zcVYvqh~A6529LZM2%-TOe!n4*uOkxQBM-qCHWm@Klf*Q(MMErgo$EhV`6EUu5XQhg zFo?@JydZ4tw^`K}>O^0r0VzQ+MxueKDbY1UTMl0@E^j?eX9k#gPsPh;cCbrieSznW z=cxx^XaY4Vfa!BKJS1%J+A6I6!^5i@U}3lfPfs6v?7#q`g;(C?-2pkL_@VG+ACtU` zkA;KSgwJ+DXF72T&qpp>Ta8y~=3YQpo5O%53)XxW4Aq3IJ9ZW|MykAAoO-fZ7cT^k zbT-FrJ`&Q_xJ;=b)NcPSG_DP}v!=Kc-Pi;Ln(!pmfb*yZ5U&Hf!GZrAgy8=$2nz2B zy&~dxil_xXW@j?d%CSC#LDpqN>8pO?;Y*Uc(c-J=78RJ7l$5uRA}e6-eXbP7hOZn_ ze6+UPO+#SGwel*vbdFC3!!ySgUZt#wwLL1|>g<=36&+|U-#)axAM}YrqtFny*F}Hg zh0AD-VRP-~q*O_PyiEgojSk6AjH){!A6NGW4kOS22FX4D%U#sp2M=Lu-k8RSI;Zsb zu=78dV1P99w`A}^-U5}}r2oE&ZqlM^LLjJQl55nCxUjx*U7=IE6zQ6vB!P zzo7X)6k67VqUWHmq#OWriq!7{%$Tq}% zXOz@8jTk3lh>^CFvcXjTzq(AfHS75)k+Lfgpiq8qzx zqOPYIdaXhC_G(>%S$0$8(da+vDHx!F^GZLnOGc;992%tiFD><7TI#>F6x34rY#`x{ zXTkcdTenW#SULk{jQ3nDh4Qxa4Bh(&W<(UtC5`-2EuhhJ%EFEwu@^<)Wzjw=BuMZM zM<0+x1~pK>|H8fv4cX7yz1MRQV~rv%=>-PVP&^`;{YmZ>+}@}>6P^ffgF3Vj79a>* zlvF@>d*_a|5er7@gb$NPv<-A`26IUyV-^};Ym5BP`VEncT3iTt7|}r3fyQuSf#$gr z!MnJ{JllC6bbw(t)2LTYT%9!IPLJ;k;*F*2(Ys&J*%T!ph6DNgjcFPp&ySq8eDMS( z@B0)*rA^)w__!|R$?jXs2xis7!7nTwBdfxcXnTY7iU*6ozO?^4Pa_;hM!!N(4iDWg zQiWM&I<$UK(n#2~FvJe@i7TV@Cwjr6P25}kJ~8T%Kw~?@xnAis`len97ENNdbqFgw z7{D>FLa+I5N2Mr-E{yXz@!mTvc=AdR^Sof4dmo`JU2yT581#ukQSb`~z=jkE{p!|A z(0Mr=v47%dk6Z*arW(cWNFzDsnV?%`XVqDyB)G1is8dYye>cUXcODv60XoJxY%nU^ z?xU9ry9vep%%8nZ4lHnDoh|M~1n*|L1OgwtR!%5F{YxXiz+n_MnPiLoOCV=hq35<` z(ES&OTK~U(Jg5pqNHM*P+&y%M^BU%fgr#^}@8AwU{l6{{aB1jG{ub2S#arSB#Q^-H Nqj^E2;Iwt{e*;y29-aUI literal 0 HcmV?d00001 diff --git a/docs/security/security-guide.md b/docs/security/security-guide.md index d39a6ff723e..8dc877f9055 100644 --- a/docs/security/security-guide.md +++ b/docs/security/security-guide.md @@ -15,13 +15,6 @@ It assumes that readers are familiar with the GraalVM architecture. This guide does not replace but rather supplements the Java security documentation with aspects unique to GraalVM. It also provides security researchers with information on GraalVM's security model. -* [Security Model](#security-model) -* [Language Launchers](#language-launchers) -* [Guest Applications](#guest-applications) -* [Native Image](#native-image) -* [Security Manager and Untrusted Code](#security-manager-and-untrusted-code) -* [Oracle GraalVM to GraalVM Community Downgrade](#graalvm-enterprise-to-graalvm-community-downgrade) - ## Security Model GraalVM is a shared runtime. It accepts instructions in a higher-level @@ -29,73 +22,43 @@ programming language (or an intermediate representation thereof) as input, which Developers that implement security controls for their applications (such as access control) in code that is being run by GraalVM can rely on the correct execution of instructions. Incorrect execution of security-critical code running on top of GraalVM that allows to bypass such a security control is regarded a security vulnerability. -GraalVM does not support execution of untrusted code. -If untrusted and potentially malicious code is to be executed, we recommend GraalVM customers who have an immediate requirement to execute untrusted and potentially adversarial code, adopt the appropriate external isolation primitives to ensure the confidentiality and integrity of their application data. - Debug features should only be used in a trusted environment as they provide privileged access to an application, allowing to inspect and change its state and behavior. They may further open network sockets to allow debug clients to connect. Experimental features in GraalVM are not for production use and may have security limitations not covered in the Security Guide. +GraalVM enables execution of untrusted code in an appropriately configured polyglot execution context (see [Polyglot Sandboxing](polyglot-sandbox.md)). + We appreciate reports of bugs that break the security model via the process outlined in the [Reporting Vulnerabilities guide](https://www.oracle.com/corporate/security-practices/assurance/vulnerability/reporting.html). -## Language Launchers +## Polyglot Languages -For every language implemented with the Truffle framework, and shipped with GraalVM, a launcher, e.g., interactive shell, is provided. +For every Polyglot language shipped with GraalVM, a launcher, e.g., interactive shell, is provided. These launchers behave in the same way and come with the same security guarantees as their "original" counterparts. -## Guest Applications - -GraalVM allows a host application written in a JVM-based language to execute guest applications written in a Truffle language via the [Polyglot API](../reference-manual/embedding/embed-languages.md). -When creating a context, the host application can control which resources the guest can access. -This mechanism is only fully supported for Javascript. -By default, access to all managed resources is denied and needs to be granted explicitly, following the principle of least privilege. - -### Host Interoperability - -GraalVM allows exchanging objects between the host and the guest application. -By default only methods of host classes that are explicitly annotated by the embedder are exposed to guest applications. - -By exposing security critical host methods, access restrictions can be bypassed. -For example, a guest application in a context that is created with `allowIO=false` cannot perform IO operations via the guest language's native API. -However, exposing a host method to the context that allows writing to arbitrary files effectively bypasses this restriction. - -### Sharing Execution Engines - -Application developers may choose to share execution engines among execution contexts for performance reasons. -While the context holds the state of the executed code, the engine holds the code itself. -Sharing of an execution engine among multiple contexts needs to be set up explicitly and can increase performance in scenarios where a number of contexts execute the same code. In scenarios where contexts that share an execution engine for common code also execute sensitive (i.e., private) code, the corresponding source objects can opt out from code sharing with: -```java -Source.newBuilder(…).cached(false).build() -``` - -### Computational Resource Limits - -> Note: Available with Oracle GraalVM. +### Polyglot Sandboxing -Oracle GraalVM allows restricting certain computational resources used by guest applications, such as CPU time, heap memory or the number of threads that can be concurrently used by a context. -These [sandboxing options](../reference-manual/embedding/sandbox-options.md) are also available via the Polyglot embedding API. +Polyglot sandboxing can establish a security boundary between privileged host code and unprivileged guest code. +For further information please refer to the [Polyglot Sandboxing guide](polyglot-sandbox.md). ### ScriptEngine Compatibility -For reasons of backward compatibility, certain guest languages also support Java's ScriptEngine interface. -For example, this allows GraalVM JavaScript to be used as a drop-in replacement for Nashorn. -However, to maintain compatibility, the Nashorn GraalVM JavaScript ScriptEngine interface will create a context with **all privileges** granted to the script and **should be used with extreme caution** and only for trusted code. +For reasons of backward compatibility, certain Polyglot languages also support the [Java Scripting API](https://docs.oracle.com/javase/9/scripting/java-scripting-api.htm). +For example, this allows the GraalVM Javascript runtime to be used as a drop-in replacement for Nashorn. +However, to maintain compatibility, the Nashorn GraalVM JavaScript ScriptEngine interface will create a context with all privileges granted to the script and should be used with extreme caution and only for trusted code. ### Managed Execution of Native Code -> Note: Available with Oracle GraalVM. +Polyglot embedding also supports LLVM intermediate representation (IR) guest code. +Several native system programming languages, above all C/C++, can be compiled to LLVM IR with the LLVM compiler toolchain. +Typically, these languages are not memory-safe unless using managed execution and it must be remembered that violations of memory safety are a frequent cause of security vulnerabilities. -The Truffle framework also supports the LLVM intermediate representation (IR) as a guest language. Several native system programming languages, above all C/C++, can be compiled to LLVM IR with the LLVM compiler toolchain. Typically, these -languages are not memory-safe by themselves and it must be remembered that violations of memory safety are a frequent cause of security vulnerabilities. - -In managed mode, all ties to the native level are abstracted and routed through Oracle GraalVM. In particular this means that: +In managed mode, all access to unmanaged code including the operating system is mediated by the language runtime. In particular this means that: * In regards to temporal and spatial memory safety, memory is allocated from the Java heap. This means that memory allocations are managed objects and all accesses are performed in a memory-safe manner (no arbitrary pointer arithmetics and no unchecked out-of-bounds accesses). * Regarding type safety, it is not possible to reinterpret a data pointer into a function pointer and execute arbitrary instructions (since these are distinct pointer types for LLVM runtime). -* System calls are intercepted and routed to the corresponding Truffle -APIs. For example, file IO is mapped to the Truffle `FileSystem` API. +* System calls are intercepted and routed to the corresponding Truffle APIs. For example, file IO is mapped to the Truffle `FileSystem` API. The set of currently supported system calls is very limited -- only syscalls that can safely be mapped to the Truffle API level are available. Since LLVM Runtime in managed mode always runs bitcode compiled for Linux/x86, it only needs to implement system calls for this platform. * All dependent libraries are executed in managed mode as well, removing all references to natively executed system libraries. This includes libraries that are provided by the LLVM Runtime, such as muslibc. @@ -103,47 +66,23 @@ Managed mode can be selected when creating a context `(Context.create())` or whe ## Native Image -The `native-image` builder generates a snapshot of an application after startup and bundles it in a binary executable. - -By default, the `native-image` builder executes the static initializers of classes at build time and persists the state in the image heap. -This means that any information that is obtained or computed in static initializers becomes part of a native executable. -This can lead to unintentionally including properties of the build environment, such as environment variables in the image heap. -This can either result in sensitive data ending up in the snapshot or fixing initialization data that is supposed to be obtained at startup, such as random number seeds. - -Developers can request static initializers that process sensitive information to be instead executed at runtime by either specifying the `--initialize-at-run-time` CLI parameter when building a native executable, or making use of the `RuntimeClassInitialization` API. - -Native-image provides multiple ways to specify the certificate file used to define the default TrustStore. -While the default behavior for native-image is to capture and use the default TrustStore from the buildtime host environment, this can be changed at runtime by setting the "javax.net.ssl.trustStore\*" system properties. -See the [documentation](../reference-manual/native-image/CertificateManagement.md) for more details. - -In addition, developers can run the `native-image` builder in a dedicated environment, such as a container, that does not contain any sensitive information in the first place. - -The directory containing the native image is part of the search path when loading native libraries using `System.loadLibrary()` at runtime. +With GraalVM native image, an application's state is captured after startup and all reachable code is compiled ahead of time to be bundled as a native executable. +For further information please refer to the [native image security guide](native-image.md). -### Serialization in Native Image - -Native Image supports Serialization to help users deserialize the constructors for classes, contained in a native executable in the first place. -These classes should be whitelisted in an additional specific configuration file, as other classes cannot be deserialized. -Deserialization support also adds optional object checksums, and only classes with the same checksum can be deserialized at runtime. -The checksum mechanism must not be used for security purposes and the deserialization of untrusted data is not supported. - -## Security Manager and Untrusted Code +## Security Manager +Security manager has been deprecated in [JEP-411](https://openjdk.java.net/jeps/411). The OpenJDK vulnerability group strongly discourages running untrusted code under a security manager. This also applies to GraalVM, which does not support untrusted code execution in Java. -While GraalVM's ability to restrict the execution of guest language applications to a certain extent is not dependent on a security manager, it is not suited to be used as a sandbox for running untrusted code. - -Note that security manager deprecation is an option in [JEP-411](https://openjdk.java.net/jeps/411). - -Native Image does not support a security manager in general. Attempting to set a security manager will trigger a runtime error. - -The Truffle framework needs to be invoked with all permissions to make full use of its functionality - it provides its own controls to manage resources. -## Oracle GraalVM to GraalVM Community Edition Downgrade +## GraalVM Community Edition Downgrade -> Note: Managed execution of native code is not available in GraalVM Community Edition. +Polyglot sandboxing is not available in GraalVM Community Edition. +Managed execution of native code is not available with GraalVM Community Edition. When downgrading to GraalVM Community Edition, native code execution is only possible with the `allowNativeAccess` privilege. This also applies to languages implemented with Truffle that allow for native code extensions, such as Python and Ruby. -Computational resource limit options are not recognized by GraalVM Community Edition. +### Related Documentation +- [Polyglot Sandboxing](polyglot-sandbox.md) +- [Native Image Security Aspects](native-image.md) diff --git a/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/SandboxPolicy.java b/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/SandboxPolicy.java index 613ffb75f17..4c97248f554 100644 --- a/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/SandboxPolicy.java +++ b/sdk/src/org.graalvm.polyglot/src/org/graalvm/polyglot/SandboxPolicy.java @@ -86,6 +86,9 @@ * changelog. *

* + * For further information on Polyglot Sandboxing, please refer to the + * security guide. + * * @see Context.Builder#sandbox(SandboxPolicy) * @see Engine.Builder#sandbox(SandboxPolicy) * @@ -153,7 +156,7 @@ public enum SandboxPolicy { *

*

* Constrained Context building example: - * + * * 

      * ByteArrayOutputStream output = new ByteArrayOutputStream();
      * ByteArrayOutputStream errorOutput = new ByteArrayOutputStream();

From 701d508ab4f5e3179510aceb104f1681436b0aa3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Alfonso=C2=B2=20Peterssen?= 
Date: Fri, 3 Mar 2023 12:32:37 +0100
Subject: [PATCH 37/43] Avoid MagicAccessorImpl access checks bypass during
 class definition. Only classes defined by
 {sun|jdk.internal}.reflect.DelegatingClassLoader , in the
 {sun|jdk.internal}.reflect package can inherit from MagicAccessorImpl.

---
 .../classfile/constantpool/ClassConstant.java |  4 ++--
 .../truffle/espresso/impl/ClassRegistry.java  |  4 ++--
 .../oracle/truffle/espresso/impl/Klass.java   | 19 +++++++++++++++++--
 .../oracle/truffle/espresso/meta/Meta.java    |  2 +-
 4 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/classfile/constantpool/ClassConstant.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/classfile/constantpool/ClassConstant.java
index 40092695b4e..56df4f7e6dc 100644
--- a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/classfile/constantpool/ClassConstant.java
+++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/classfile/constantpool/ClassConstant.java
@@ -124,7 +124,7 @@ public Resolved resolve(RuntimeConstantPool pool, int thisIndex, Klass accessing
                 EspressoContext context = pool.getContext();
                 Symbol type = context.getTypes().fromName(klassName);
                 Klass klass = context.getMeta().resolveSymbolOrFail(type, accessingKlass.getDefiningClassLoader(), accessingKlass.protectionDomain());
-                if (!Klass.checkAccess(klass.getElementalType(), accessingKlass)) {
+                if (!Klass.checkAccess(klass.getElementalType(), accessingKlass, false)) {
                     Meta meta = context.getMeta();
                     context.getLogger().log(Level.FINE,
                                     "Access check of: " + klass.getType() + " from " + accessingKlass.getType() + " throws IllegalAccessError");
@@ -201,7 +201,7 @@ public Resolved resolve(RuntimeConstantPool pool, int thisIndex, Klass accessing
                 EspressoContext context = pool.getContext();
                 Meta meta = context.getMeta();
                 Klass klass = meta.resolveSymbolOrFail(context.getTypes().fromName(klassName), accessingKlass.getDefiningClassLoader(), accessingKlass.protectionDomain());
-                if (!Klass.checkAccess(klass.getElementalType(), accessingKlass)) {
+                if (!Klass.checkAccess(klass.getElementalType(), accessingKlass, false)) {
                     context.getLogger().log(Level.FINE,
                                     "Access check of: " + klass.getType() + " from " + accessingKlass.getType() + " throws IllegalAccessError");
                     throw meta.throwExceptionWithMessage(meta.java_lang_IllegalAccessError, meta.toGuestString(klassName));
diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/impl/ClassRegistry.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/impl/ClassRegistry.java
index 87e7d2408d8..000e9c00bef 100644
--- a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/impl/ClassRegistry.java
+++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/impl/ClassRegistry.java
@@ -448,7 +448,7 @@ private ObjectKlass createKlass(EspressoContext context, ParserKlass parserKlass
         }
 
         if (superKlass != null) {
-            if (!Klass.checkAccess(superKlass, klass)) {
+            if (!Klass.checkAccess(superKlass, klass, true)) {
                 throw EspressoClassLoadingException.illegalAccessError("class " + type + " cannot access its superclass " + superKlassType);
             }
             if (!superKlass.permittedSubclassCheck(klass)) {
@@ -458,7 +458,7 @@ private ObjectKlass createKlass(EspressoContext context, ParserKlass parserKlass
 
         for (ObjectKlass interf : superInterfaces) {
             if (interf != null) {
-                if (!Klass.checkAccess(interf, klass)) {
+                if (!Klass.checkAccess(interf, klass, true)) {
                     throw EspressoClassLoadingException.illegalAccessError("class " + type + " cannot access its superinterface " + interf.getType());
                 }
                 if (!interf.permittedSubclassCheck(klass)) {
diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/impl/Klass.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/impl/Klass.java
index 5c4992cdc7c..11bd059f5f0 100644
--- a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/impl/Klass.java
+++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/impl/Klass.java
@@ -636,7 +636,7 @@ protected static boolean hasFinalInstanceField(Class clazz) {
      * 
  • C is not public, and C and D are members of the same run-time package.
      * 
      */
-    public static boolean checkAccess(Klass klass, Klass accessingKlass) {
+    public static boolean checkAccess(Klass klass, Klass accessingKlass, boolean ignoreMagicAccessor) {
         if (accessingKlass == null) {
             return true;
         }
@@ -661,8 +661,23 @@ public static boolean checkAccess(Klass klass, Klass accessingKlass) {
                 return true;
             }
         }
-        return (context.getMeta().sun_reflect_MagicAccessorImpl.isAssignableFrom(accessingKlass));
 
+        if (ignoreMagicAccessor) {
+            /*
+             * Prevents any class inheriting from MagicAccessorImpl to have access to
+             * MagicAccessorImpl just because it implements MagicAccessorImpl.
+             * 
+             * Only generated accessors in the {sun|jdk.internal}.reflect package, defined by
+             * {sun|jdk.internal}.reflect.DelegatingClassLoader(s) have access to MagicAccessorImpl.
+             */
+            ObjectKlass magicAccessorImpl = context.getMeta().sun_reflect_MagicAccessorImpl;
+            return !StaticObject.isNull(accessingKlass.getDefiningClassLoader()) &&
+                            context.getMeta().sun_reflect_DelegatingClassLoader.equals(accessingKlass.getDefiningClassLoader().getKlass()) &&
+                            magicAccessorImpl.getRuntimePackage().equals(accessingKlass.getRuntimePackage()) &&
+                            magicAccessorImpl.isAssignableFrom(accessingKlass);
+        }
+
+        return (context.getMeta().sun_reflect_MagicAccessorImpl.isAssignableFrom(accessingKlass));
     }
 
     public static boolean doModuleAccessChecks(Klass klass, Klass accessingKlass, EspressoContext context) {
diff --git a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/meta/Meta.java b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/meta/Meta.java
index 04ca9529ebb..75f6197443f 100644
--- a/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/meta/Meta.java
+++ b/espresso/src/com.oracle.truffle.espresso/src/com/oracle/truffle/espresso/meta/Meta.java
@@ -2009,7 +2009,7 @@ public Klass resolveSymbolOrFail(Symbol type, @JavaType(ClassLoader.class)
     public Klass resolveSymbolAndAccessCheck(Symbol type, Klass accessingKlass) {
         assert accessingKlass != null;
         Klass klass = resolveSymbolOrFail(type, accessingKlass.getDefiningClassLoader(), java_lang_NoClassDefFoundError, accessingKlass.protectionDomain());
-        if (!Klass.checkAccess(klass.getElementalType(), accessingKlass)) {
+        if (!Klass.checkAccess(klass.getElementalType(), accessingKlass, false)) {
             throw throwException(java_lang_IllegalAccessError);
         }
         return klass;

From 07ce2db573fb2cd612cea5bcf8673e3743da10c2 Mon Sep 17 00:00:00 2001
From: Josef Haider 
Date: Mon, 12 Dec 2022 12:39:32 +0100
Subject: [PATCH 38/43] Fix GR-42846 and GR-42847

---
 .../src/com/oracle/truffle/api/strings/TStringOps.java          | 2 +-
 .../src/com/oracle/truffle/api/ArrayUtils.java                  | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/truffle/src/com.oracle.truffle.api.strings/src/com/oracle/truffle/api/strings/TStringOps.java b/truffle/src/com.oracle.truffle.api.strings/src/com/oracle/truffle/api/strings/TStringOps.java
index 54767d285af..fa67d86769c 100644
--- a/truffle/src/com.oracle.truffle.api.strings/src/com/oracle/truffle/api/strings/TStringOps.java
+++ b/truffle/src/com.oracle.truffle.api.strings/src/com/oracle/truffle/api/strings/TStringOps.java
@@ -1621,7 +1621,7 @@ static long byteLength(Object array) {
     }
 
     private static boolean rangeInBounds(int rangeStart, int rangeLength, int arrayLength) {
-        return Integer.toUnsignedLong(rangeStart + rangeLength) <= arrayLength;
+        return Integer.toUnsignedLong(rangeStart) + Integer.toUnsignedLong(rangeLength) <= arrayLength;
     }
 
     private static boolean isNativePointer(Object arrayB) {
diff --git a/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/ArrayUtils.java b/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/ArrayUtils.java
index 7c896adced8..99168e4a43b 100644
--- a/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/ArrayUtils.java
+++ b/truffle/src/com.oracle.truffle.api/src/com/oracle/truffle/api/ArrayUtils.java
@@ -529,7 +529,7 @@ private static void checkArgsIndexOf(int hayStackLength, int fromIndex, int leng
         if (fromIndex < 0 || length < 0) {
             illegalArgumentException("fromIndex and length must be positive");
         }
-        if (fromIndex + length > hayStackLength) {
+        if (Integer.toUnsignedLong(fromIndex) + Integer.toUnsignedLong(length) > hayStackLength) {
             illegalArgumentException("length out of range");
         }
     }

From 5ae350aed47913ff759b111f239c770c6c899b76 Mon Sep 17 00:00:00 2001
From: Christian Haeubl 
Date: Tue, 13 Dec 2022 18:46:02 +0100
Subject: [PATCH 39/43] Fixes related to the jvmstat performance data
 initialization.

---
 .../com/oracle/svm/core/posix/PosixStat.java  | 113 +++-
 .../oracle/svm/core/posix/headers/Dirent.java |  58 ++
 .../oracle/svm/core/posix/headers/Fcntl.java  |  19 +-
 .../core/posix/headers/PosixDirectives.java   |   4 +-
 .../oracle/svm/core/posix/headers/Pwd.java    |  11 +
 .../oracle/svm/core/posix/headers/Signal.java |   6 +
 .../oracle/svm/core/posix/headers/Unistd.java |   9 +-
 .../core/posix/headers/darwin/DarwinStat.java |  41 +-
 .../core/posix/headers/linux/LinuxStat.java   |  21 +-
 .../jvmstat/PosixPerfMemoryProvider.java      | 578 +++++++++++++-----
 .../oracle/svm/core/jvmstat/PerfMemory.java   |   1 +
 .../svm/core/jvmstat/SystemCounters.java      |  27 +-
 12 files changed, 697 insertions(+), 191 deletions(-)
 create mode 100644 substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Dirent.java

diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixStat.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixStat.java
index dacbd156b2e..652b9655a71 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixStat.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixStat.java
@@ -24,15 +24,21 @@
  */
 package com.oracle.svm.core.posix;
 
+import static org.graalvm.nativeimage.c.function.CFunction.Transition.NO_TRANSITION;
+
+import org.graalvm.compiler.api.replacements.Fold;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.nativeimage.c.CContext;
 import org.graalvm.nativeimage.c.constant.CConstant;
+import org.graalvm.nativeimage.c.function.CFunction;
+import org.graalvm.nativeimage.c.struct.SizeOf;
+import org.graalvm.nativeimage.c.type.CCharPointer;
+import org.graalvm.nativeimage.c.type.CConst;
+import org.graalvm.word.WordBase;
 
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.graal.stackvalue.UnsafeStackValue;
-import com.oracle.svm.core.headers.LibC;
-import com.oracle.svm.core.posix.headers.Errno;
 import com.oracle.svm.core.posix.headers.PosixDirectives;
 import com.oracle.svm.core.posix.headers.darwin.DarwinStat;
 import com.oracle.svm.core.posix.headers.linux.LinuxStat;
@@ -74,21 +80,6 @@ public final class PosixStat {
     @CConstant
     public static native int S_IXOTH();
 
-    public static boolean isOpen(int fd) {
-        int result;
-        if (Platform.includedIn(Platform.LINUX.class)) {
-            LinuxStat.stat64 stat = UnsafeStackValue.get(LinuxStat.stat64.class);
-            result = LinuxStat.fstat64(fd, stat);
-        } else if (Platform.includedIn(Platform.DARWIN.class)) {
-            DarwinStat.stat stat = UnsafeStackValue.get(DarwinStat.stat.class);
-            result = DarwinStat.fstat(fd, stat);
-        } else {
-            throw VMError.shouldNotReachHere("Unsupported platform");
-        }
-
-        return result == 0 || LibC.errno() != Errno.EBADF();
-    }
-
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static long getSize(int fd) {
         long size = -1;
@@ -102,11 +93,99 @@ public static long getSize(int fd) {
             if (DarwinStat.NoTransitions.fstat(fd, stat) == 0) {
                 size = stat.st_size();
             }
+        } else {
+            throw VMError.shouldNotReachHere("Unsupported platform");
         }
         return size;
     }
 
+    @Fold
+    public static int sizeOfStatStruct() {
+        if (Platform.includedIn(Platform.LINUX.class)) {
+            return SizeOf.get(LinuxStat.stat64.class);
+        } else if (Platform.includedIn(Platform.DARWIN.class)) {
+            return SizeOf.get(DarwinStat.stat.class);
+        } else {
+            throw VMError.shouldNotReachHere("Unsupported platform");
+        }
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static int st_uid(stat buf) {
+        if (Platform.includedIn(Platform.LINUX.class)) {
+            return ((LinuxStat.stat64) buf).st_uid();
+        } else if (Platform.includedIn(Platform.DARWIN.class)) {
+            return ((DarwinStat.stat) buf).st_uid();
+        } else {
+            throw VMError.shouldNotReachHere("Unsupported platform");
+        }
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static boolean S_ISLNK(stat buf) {
+        return (st_mode(buf) & S_IFLNK()) == S_IFLNK();
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static boolean S_ISDIR(stat buf) {
+        return (st_mode(buf) & S_IFDIR()) == S_IFDIR();
+    }
+
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    public static int st_mode(stat buf) {
+        if (Platform.includedIn(Platform.LINUX.class)) {
+            return ((LinuxStat.stat64) buf).st_mode();
+        } else if (Platform.includedIn(Platform.DARWIN.class)) {
+            return ((DarwinStat.stat) buf).st_mode();
+        } else {
+            throw VMError.shouldNotReachHere("Unsupported platform");
+        }
+    }
+
+    public static long st_nlink(stat buf) {
+        if (Platform.includedIn(Platform.LINUX.class)) {
+            return ((LinuxStat.stat64) buf).st_nlink();
+        } else if (Platform.includedIn(Platform.DARWIN.class)) {
+            return ((DarwinStat.stat) buf).st_nlink();
+        } else {
+            throw VMError.shouldNotReachHere("Unsupported platform");
+        }
+    }
+
+    /**
+     * Pointer to the OS-specific stat struct.
+     */
+    public interface stat extends WordBase {
+    }
+
     @Platforms(Platform.HOSTED_ONLY.class)
     private PosixStat() {
     }
+
+    public static class NoTransitions {
+        @CFunction(transition = NO_TRANSITION)
+        public static native int mkdir(@CConst CCharPointer pathname, int mode);
+
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        public static int fstat(int fd, stat buf) {
+            if (Platform.includedIn(Platform.LINUX.class)) {
+                return LinuxStat.NoTransitions.fstat64(fd, (LinuxStat.stat64) buf);
+            } else if (Platform.includedIn(Platform.DARWIN.class)) {
+                return DarwinStat.NoTransitions.fstat(fd, (DarwinStat.stat) buf);
+            } else {
+                throw VMError.shouldNotReachHere("Unsupported platform");
+            }
+        }
+
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        public static int lstat(CCharPointer path, stat buf) {
+            if (Platform.includedIn(Platform.LINUX.class)) {
+                return LinuxStat.NoTransitions.lstat64(path, (LinuxStat.stat64) buf);
+            } else if (Platform.includedIn(Platform.DARWIN.class)) {
+                return DarwinStat.NoTransitions.lstat(path, (DarwinStat.stat) buf);
+            } else {
+                throw VMError.shouldNotReachHere("Unsupported platform");
+            }
+        }
+    }
 }
diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Dirent.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Dirent.java
new file mode 100644
index 00000000000..208a2cfda78
--- /dev/null
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Dirent.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright (c) 2023, 2023, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.svm.core.posix.headers;
+
+import org.graalvm.nativeimage.c.CContext;
+import org.graalvm.nativeimage.c.function.CFunction;
+import org.graalvm.nativeimage.c.struct.CFieldAddress;
+import org.graalvm.nativeimage.c.struct.CStruct;
+import org.graalvm.nativeimage.c.type.CCharPointer;
+import org.graalvm.word.PointerBase;
+
+// Checkstyle: stop
+
+/**
+ * Definitions manually translated from the C header file dirent.h.
+ */
+@CContext(PosixDirectives.class)
+public class Dirent {
+    @CFunction
+    public static native DIR fdopendir(int fd);
+
+    @CFunction
+    public static native dirent readdir(DIR dir);
+
+    @CFunction
+    public static native int closedir(DIR dir);
+
+    public interface DIR extends PointerBase {
+    }
+
+    @CStruct(addStructKeyword = true)
+    public interface dirent extends PointerBase {
+        @CFieldAddress
+        CCharPointer d_name();
+    }
+}
diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Fcntl.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Fcntl.java
index 10ce569238f..185b9a2b9c7 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Fcntl.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Fcntl.java
@@ -24,11 +24,13 @@
  */
 package com.oracle.svm.core.posix.headers;
 
+import static org.graalvm.nativeimage.c.function.CFunction.Transition.NO_TRANSITION;
+
 import org.graalvm.nativeimage.c.CContext;
 import org.graalvm.nativeimage.c.constant.CConstant;
 import org.graalvm.nativeimage.c.function.CFunction;
-import org.graalvm.nativeimage.c.function.CFunction.Transition;
 import org.graalvm.nativeimage.c.type.CCharPointer;
+import org.graalvm.nativeimage.c.type.CConst;
 
 // Checkstyle: stop
 
@@ -41,6 +43,9 @@ public class Fcntl {
     @CConstant
     public static native int O_RDONLY();
 
+    @CConstant
+    public static native int O_NOFOLLOW();
+
     @CConstant
     public static native int O_RDWR();
 
@@ -57,7 +62,17 @@ public class Fcntl {
     public static native int O_EXCL();
 
     public static class NoTransitions {
-        @CFunction(value = "openSII", transition = Transition.NO_TRANSITION)
+        @CFunction(value = "openSII", transition = NO_TRANSITION)
         public static native int open(CCharPointer pathname, int flags, int mode);
+
+        @CFunction(transition = NO_TRANSITION)
+        public static native int openat(int dirfd, @CConst CCharPointer pathname, int flags, int mode);
+
+        @CFunction(transition = NO_TRANSITION)
+        public static native int unlink(@CConst CCharPointer pathname);
+
+        @CFunction(transition = NO_TRANSITION)
+        public static native int unlinkat(int dirfd, CCharPointer pathname, int flags);
+
     }
 }
diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/PosixDirectives.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/PosixDirectives.java
index 51bb598caa7..65089674639 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/PosixDirectives.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/PosixDirectives.java
@@ -37,12 +37,13 @@
 public class PosixDirectives implements CContext.Directives {
     private static final String[] commonLibs = new String[]{
                     "",
+                    "",
                     "",
                     "",
                     "",
                     "",
-                    "",
                     "",
+                    "",
                     "",
                     "",
                     "",
@@ -50,6 +51,7 @@ public class PosixDirectives implements CContext.Directives {
                     "",
                     "",
                     "",
+                    "",
                     "",
                     "",
                     "",
diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pwd.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pwd.java
index 5c629df96c5..9d627b26324 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pwd.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Pwd.java
@@ -27,9 +27,12 @@
 import org.graalvm.nativeimage.c.CContext;
 import org.graalvm.nativeimage.c.function.CFunction;
 import org.graalvm.nativeimage.c.struct.CField;
+import org.graalvm.nativeimage.c.struct.CPointerTo;
 import org.graalvm.nativeimage.c.struct.CStruct;
 import org.graalvm.nativeimage.c.type.CCharPointer;
+import org.graalvm.word.Pointer;
 import org.graalvm.word.PointerBase;
+import org.graalvm.word.UnsignedWord;
 
 // Checkstyle: stop
 
@@ -48,6 +51,14 @@ public interface passwd extends PointerBase {
         CCharPointer pw_dir();
     }
 
+    @CPointerTo(passwd.class)
+    public interface passwdPointer extends Pointer {
+        passwd read();
+    }
+
     @CFunction
     public static native passwd getpwuid(int __uid);
+
+    @CFunction
+    public static native int getpwuid_r(int __uid, passwd pwd, CCharPointer buf, UnsignedWord buflen, passwdPointer result);
 }
diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Signal.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Signal.java
index f94d14ba937..404a228892f 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Signal.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Signal.java
@@ -24,6 +24,8 @@
  */
 package com.oracle.svm.core.posix.headers;
 
+import static org.graalvm.nativeimage.c.function.CFunction.Transition.NO_TRANSITION;
+
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.nativeimage.c.CContext;
@@ -446,4 +448,8 @@ public interface AArch64DarwinMContext64 extends PointerBase {
         long pc();
     }
 
+    public static class NoTransitions {
+        @CFunction(transition = NO_TRANSITION)
+        public static native int kill(int pid, int sig);
+    }
 }
diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Unistd.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Unistd.java
index a7f4294e533..2942a16f71d 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Unistd.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/Unistd.java
@@ -77,6 +77,9 @@ public class Unistd {
     @CConstant
     public static native int _SC_PAGE_SIZE();
 
+    @CConstant
+    public static native int _SC_GETPW_R_SIZE_MAX();
+
     @CConstant
     @Platforms(Platform.LINUX.class)
     public static native int _SC_PHYS_PAGES();
@@ -97,9 +100,6 @@ public class Unistd {
     @CFunction
     public static native int getuid();
 
-    @CFunction
-    public static native int geteuid();
-
     @CFunction
     public static native int getgid();
 
@@ -139,5 +139,8 @@ public static class NoTransitions {
 
         @CFunction(transition = Transition.NO_TRANSITION)
         public static native int getpid();
+
+        @CFunction(transition = Transition.NO_TRANSITION)
+        public static native int geteuid();
     }
 }
diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinStat.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinStat.java
index 368c5e7be9c..fca031dc296 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinStat.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinStat.java
@@ -24,17 +24,19 @@
  */
 package com.oracle.svm.core.posix.headers.darwin;
 
-import com.oracle.svm.core.util.VMError;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.nativeimage.c.CContext;
 import org.graalvm.nativeimage.c.function.CFunction;
 import org.graalvm.nativeimage.c.struct.CField;
 import org.graalvm.nativeimage.c.struct.CStruct;
+import org.graalvm.nativeimage.c.type.CCharPointer;
+import org.graalvm.nativeimage.c.type.CConst;
 import org.graalvm.word.PointerBase;
 
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.posix.headers.PosixDirectives;
+import com.oracle.svm.core.util.VMError;
 
 // Checkstyle: stop
 
@@ -52,17 +54,29 @@ public class DarwinStat {
 
     @CStruct(addStructKeyword = true)
     public interface stat extends PointerBase {
+        @CField
+        long st_ino();
+
+        @CField
+        int st_mode();
+
+        @CField
+        int st_uid();
+
         @CField
         long st_size();
+
+        @CField
+        long st_nlink();
     }
 
     @CFunction("fstat$INODE64")
     @Platforms(Platform.DARWIN_AMD64.class)
-    public static native int fstat_amd64(int fd, stat buf);
+    private static native int fstat_amd64(int fd, stat buf);
 
     @CFunction("fstat")
     @Platforms(Platform.DARWIN_AARCH64.class)
-    public static native int fstat_aarch64(int fd, stat buf);
+    private static native int fstat_aarch64(int fd, stat buf);
 
     @Platforms(Platform.DARWIN.class)
     public static int fstat(int fd, stat buf) {
@@ -95,5 +109,26 @@ public static int fstat(int fd, stat buf) {
                 throw VMError.unsupportedPlatform(); // ExcludeFromJacocoGeneratedReport
             }
         }
+
+        @CFunction(value = "lstat$INODE64", transition = CFunction.Transition.NO_TRANSITION)
+        @Platforms(Platform.DARWIN_AMD64.class)
+        private static native int lstat_amd64(@CConst CCharPointer path, stat buf);
+
+        @CFunction(value = "lstat", transition = CFunction.Transition.NO_TRANSITION)
+        @Platforms(Platform.DARWIN_AARCH64.class)
+        private static native int lstat_aarch64(@CConst CCharPointer path, stat buf);
+
+        @Platforms(Platform.DARWIN.class)
+        @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+        public static int lstat(CCharPointer path, stat buf) {
+            if (Platform.includedIn(Platform.AMD64.class)) {
+                return lstat_amd64(path, buf);
+            } else if (Platform.includedIn(Platform.AARCH64.class)) {
+                return lstat_aarch64(path, buf);
+            } else {
+                throw VMError.shouldNotReachHere();
+            }
+        }
+
     }
 }
diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/linux/LinuxStat.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/linux/LinuxStat.java
index 725439ffefd..73d7a316e71 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/linux/LinuxStat.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/linux/LinuxStat.java
@@ -24,10 +24,14 @@
  */
 package com.oracle.svm.core.posix.headers.linux;
 
+import static org.graalvm.nativeimage.c.function.CFunction.Transition.NO_TRANSITION;
+
 import org.graalvm.nativeimage.c.CContext;
 import org.graalvm.nativeimage.c.function.CFunction;
 import org.graalvm.nativeimage.c.struct.CField;
 import org.graalvm.nativeimage.c.struct.CStruct;
+import org.graalvm.nativeimage.c.type.CCharPointer;
+import org.graalvm.nativeimage.c.type.CConst;
 import org.graalvm.word.PointerBase;
 
 import com.oracle.svm.core.posix.headers.PosixDirectives;
@@ -45,15 +49,24 @@ public interface stat64 extends PointerBase {
         @CField
         long st_ino();
 
+        @CField
+        int st_mode();
+
+        @CField
+        int st_uid();
+
         @CField
         long st_size();
-    }
 
-    @CFunction
-    public static native int fstat64(int fd, stat64 buf);
+        @CField
+        long st_nlink();
+    }
 
     public static class NoTransitions {
-        @CFunction(transition = CFunction.Transition.NO_TRANSITION)
+        @CFunction(transition = NO_TRANSITION)
         public static native int fstat64(int fd, stat64 buf);
+
+        @CFunction(transition = NO_TRANSITION)
+        public static native int lstat64(@CConst CCharPointer path, stat64 buf);
     }
 }
diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java
index d6db3441610..4766dd8baaf 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java
@@ -25,64 +25,73 @@
 package com.oracle.svm.core.posix.jvmstat;
 
 import static com.oracle.svm.core.jvmstat.PerfManager.Options.PerfDataMemoryMappedFile;
-import static java.nio.file.LinkOption.NOFOLLOW_LINKS;
-import static java.nio.file.StandardOpenOption.CREATE_NEW;
-import static java.nio.file.StandardOpenOption.READ;
-import static java.nio.file.StandardOpenOption.WRITE;
-import static java.nio.file.attribute.PosixFilePermission.GROUP_EXECUTE;
-import static java.nio.file.attribute.PosixFilePermission.GROUP_READ;
-import static java.nio.file.attribute.PosixFilePermission.GROUP_WRITE;
-import static java.nio.file.attribute.PosixFilePermission.OTHERS_EXECUTE;
-import static java.nio.file.attribute.PosixFilePermission.OTHERS_READ;
-import static java.nio.file.attribute.PosixFilePermission.OTHERS_WRITE;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_EXECUTE;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_READ;
-import static java.nio.file.attribute.PosixFilePermission.OWNER_WRITE;
-
-import java.io.File;
-import java.io.IOException;
-import java.nio.ByteOrder;
-import java.nio.MappedByteBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
-import java.nio.file.attribute.FileAttribute;
-import java.nio.file.attribute.PosixFilePermission;
-import java.nio.file.attribute.PosixFilePermissions;
-import java.nio.file.attribute.UserPrincipal;
-import java.util.EnumSet;
-import java.util.Set;
+import static com.oracle.svm.core.posix.PosixStat.S_IRGRP;
+import static com.oracle.svm.core.posix.PosixStat.S_IROTH;
+import static com.oracle.svm.core.posix.PosixStat.S_IRWXU;
+import static com.oracle.svm.core.posix.PosixStat.S_IXGRP;
+import static com.oracle.svm.core.posix.PosixStat.S_IXOTH;
+import static com.oracle.svm.core.posix.headers.Errno.EEXIST;
+import static com.oracle.svm.core.posix.headers.Errno.EPERM;
+import static com.oracle.svm.core.posix.headers.Errno.ESRCH;
+import static com.oracle.svm.core.posix.headers.Fcntl.O_CREAT;
+import static com.oracle.svm.core.posix.headers.Fcntl.O_NOFOLLOW;
+import static com.oracle.svm.core.posix.headers.Fcntl.O_RDONLY;
+import static com.oracle.svm.core.posix.headers.Fcntl.O_RDWR;
+import static com.oracle.svm.core.posix.headers.Unistd._SC_GETPW_R_SIZE_MAX;
+
+import java.nio.ByteBuffer;
 
 import org.graalvm.compiler.core.common.NumUtil;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
-
+import org.graalvm.nativeimage.StackValue;
+import org.graalvm.nativeimage.UnmanagedMemory;
+import org.graalvm.nativeimage.c.type.CCharPointer;
+import org.graalvm.nativeimage.c.type.CTypeConversion;
+import org.graalvm.nativeimage.impl.UnmanagedMemorySupport;
+import org.graalvm.word.Pointer;
+import org.graalvm.word.WordFactory;
+
+import com.oracle.svm.core.SubstrateUtil;
+import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.VMInspectionOptions;
 import com.oracle.svm.core.annotate.Alias;
 import com.oracle.svm.core.annotate.TargetClass;
+import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature;
 import com.oracle.svm.core.feature.InternalFeature;
 import com.oracle.svm.core.headers.LibC;
+import com.oracle.svm.core.jdk.Target_java_nio_DirectByteBuffer;
 import com.oracle.svm.core.jvmstat.PerfManager;
 import com.oracle.svm.core.jvmstat.PerfMemoryPrologue;
 import com.oracle.svm.core.jvmstat.PerfMemoryProvider;
+import com.oracle.svm.core.os.RawFileOperationSupport;
+import com.oracle.svm.core.os.RawFileOperationSupport.RawFileDescriptor;
+import com.oracle.svm.core.os.VirtualMemoryProvider;
+import com.oracle.svm.core.posix.PosixStat;
+import com.oracle.svm.core.posix.headers.Dirent;
+import com.oracle.svm.core.posix.headers.Dirent.DIR;
+import com.oracle.svm.core.posix.headers.Dirent.dirent;
 import com.oracle.svm.core.posix.headers.Errno;
+import com.oracle.svm.core.posix.headers.Fcntl;
+import com.oracle.svm.core.posix.headers.Mman;
+import com.oracle.svm.core.posix.headers.Pwd;
+import com.oracle.svm.core.posix.headers.Pwd.passwd;
+import com.oracle.svm.core.posix.headers.Pwd.passwdPointer;
 import com.oracle.svm.core.posix.headers.Signal;
 import com.oracle.svm.core.posix.headers.Unistd;
-import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature;
 
 /**
  * This class uses high-level JDK features at the moment. In the future, we will need to rewrite
  * this code so that it can be executed during the isolate startup (i.e., in uninterruptible code),
  * see GR-40601.
+ * 
    
+ * Based on JDK 19 (git commit hash: 967a28c3d85fdde6d5eb48aa0edd8f7597772469, JDK tag: jdk-19+36.
  */
 class PosixPerfMemoryProvider implements PerfMemoryProvider {
-    // Prefix of performance data file.
     private static final String PERFDATA_NAME = "hsperfdata";
 
-    private Path backingStoreFile;
+    private String backingFilePath;
 
     @Platforms(Platform.HOSTED_ONLY.class)
     PosixPerfMemoryProvider() {
@@ -90,147 +99,271 @@ class PosixPerfMemoryProvider implements PerfMemoryProvider {
 
     /**
      * Create a named shared memory region. Returns the address of the memory region on success or
-     * NULL on failure. A return value of NULL will ultimately disable the shared memory feature.
-     *
-     * On Solaris and Bsd, the name space for shared memory objects is the file system name space.
-     *
-     * A monitoring application attaching to a SubstrateVM does not need to know the file system
-     * name of the shared memory object. However, it may be convenient for applications to discover
-     * the existence of newly created and terminating SubstrateVMs by watching the file system name
-     * space for files being created or removed.
+     * null on failure. A return value of null will ultimately disable the shared memory feature.
      */
     @Override
-    public MappedByteBuffer create() {
-        assert backingStoreFile == null;
+    public ByteBuffer create() {
+        assert backingFilePath == null;
 
         int size = NumUtil.roundUp(PerfManager.Options.PerfDataMemorySize.getValue(), Unistd.getpagesize());
         if (size <= PerfMemoryPrologue.getPrologueSize()) {
             return null;
         }
 
-        String shortName = String.valueOf(Unistd.getpid());
-        String user = System.getProperty("user.name");
-        String tmpdir = Target_jdk_internal_vm_VMSupport.getVMTemporaryDirectory();
+        int vmId = Unistd.getpid();
+        String userName = getUserName(Unistd.NoTransitions.geteuid());
+        if (userName == null) {
+            return null;
+        }
+
+        String dirName = getUserTmpDir(userName, vmId, -1);
+        String fileName = getSharedMemFileName(vmId, -1);
+
+        int fd;
+        try (CTypeConversion.CCharPointerHolder d = CTypeConversion.toCString(dirName)) {
+            cleanupSharedMemResources(d.get(), vmId);
+
+            try (CTypeConversion.CCharPointerHolder f = CTypeConversion.toCString(fileName)) {
+                fd = createSharedMemResources(d.get(), f.get(), size);
+            }
+        }
 
-        if (user.equals("?")) {
+        if (fd == -1) {
             return null;
         }
-        String dirname = String.format("%s_%s", PERFDATA_NAME, user);
-        Path perfDir = Paths.get(tmpdir, dirname);
-        Path filename = perfDir.resolve(shortName);
 
-        // cleanup any stale jmvstat files
-        cleanupOldJvmstatFiles(perfDir, shortName);
+        Pointer mapAddress = Mman.mmap(WordFactory.nullPointer(), WordFactory.unsigned(size), Mman.PROT_READ() | Mman.PROT_WRITE(), Mman.MAP_SHARED(), fd, 0);
 
-        assert size > 0 : "unexpected PerfDataBuffer region size";
+        int result = Unistd.NoTransitions.close(fd);
+        assert result != -1;
 
-        MappedByteBuffer buffer = createSharedBuffer(perfDir, filename, user, size);
-        if (buffer == null) {
+        String filePath = dirName + "/" + fileName;
+        if (mapAddress == Mman.MAP_FAILED()) {
+            restartableUnlink(filePath);
             return null;
         }
-        // save the file name for use in teardown()
-        backingStoreFile = filename;
-        buffer.order(ByteOrder.nativeOrder());
-        return buffer;
+
+        backingFilePath = filePath;
+
+        /* Clear the shared memory region. */
+        LibC.memset(mapAddress, WordFactory.signed(0), WordFactory.unsigned(size));
+        return SubstrateUtil.cast(new Target_java_nio_DirectByteBuffer(mapAddress.rawValue(), size), ByteBuffer.class);
     }
 
-    private static void cleanupOldJvmstatFiles(Path perfDir, String selfName) {
-        File[] files = perfDir.toFile().listFiles();
+    private static String getUserName(int uid) {
+        /* Determine max. pwBuf size. */
+        long bufSize = Unistd.sysconf(_SC_GETPW_R_SIZE_MAX());
+        if (bufSize == -1) {
+            bufSize = 1024;
+        }
 
-        if (files == null) {
-            return;
+        /* Retrieve the username and copy it to a String object. */
+        CCharPointer pwBuf = ImageSingletons.lookup(UnmanagedMemorySupport.class).malloc(WordFactory.unsigned(bufSize));
+        if (pwBuf.isNull()) {
+            return null;
         }
-        for (File f : files) {
-            String name = f.getName();
-            int pid = getPidFromFile(f);
 
-            if (pid == 0) {
-                // attempt to remove all unexpected files
-                f.delete();
-                continue;
+        try {
+            passwd pwent = StackValue.get(passwd.class);
+            passwdPointer p = StackValue.get(passwdPointer.class);
+            int code = Pwd.getpwuid_r(uid, pwent, pwBuf, WordFactory.unsigned(bufSize), p);
+            if (code != 0) {
+                return null;
             }
 
-            /*
-             * We now have a file name that converts to a valid integer that could represent a
-             * process id. If this process id matches the current process id or the process is not
-             * running, then remove the stale file resources.
-             * 
-             * Process liveness is detected by sending signal number 0 to the process id (see
-             * kill(2)). If kill determines that the process does not exist, then the file resources
-             * are removed. If kill determines that we don't have permission to signal the process,
-             * then the file resources are assumed to be stale and are removed because the resources
-             * for such a process should be in a different user specific directory.
-             */
-            if (name.equals(selfName)) {
-                f.delete();
-            } else {
-                int ret = Signal.kill(pid, 0);
+            passwd result = p.read();
+            if (result.isNull()) {
+                return null;
+            }
+
+            CCharPointer pwName = result.pw_name();
+            if (pwName.isNull() || pwName.read() == '\0') {
+                return null;
+            }
+
+            return CTypeConversion.toJavaString(pwName);
+        } finally {
+            UnmanagedMemory.free(pwBuf);
+        }
+    }
+
+    private static String getUserTmpDir(String user, int vmId, int nsPid) {
+        String tmpDir = Target_jdk_internal_vm_VMSupport.getVMTemporaryDirectory();
+        if (Platform.includedIn(Platform.LINUX.class)) {
+            if (nsPid != -1) {
+                /* Use different directory if we have a containerized process on Linux. */
+                tmpDir = "/proc/" + vmId + "/root" + tmpDir;
+            }
+        }
+        return tmpDir + "/" + PERFDATA_NAME + "_" + user;
+    }
 
-                if (ret == -1) {
-                    int errno = LibC.errno();
-                    if (errno == Errno.ESRCH() || errno == Errno.EPERM()) {
-                        f.delete();
+    private static String getSharedMemFileName(int vmId, int nspid) {
+        int pid = vmId;
+        if (Platform.includedIn(Platform.LINUX.class) && nspid != -1) {
+            pid = nspid;
+        }
+        return Integer.toString(pid);
+    }
+
+    /**
+     * This method attempts to remove stale shared memory files in the user's temp directory. It
+     * scans for files matching the pattern ^$[0-9]*$. For each file found, the process id is
+     * extracted from the file name and a test is run to determine if the process is alive. If the
+     * process is not alive, any stale file resources are removed.
+     */
+    private static void cleanupSharedMemResources(CCharPointer directoryPath, int selfPid) {
+        try (SecureDirectory s = openDirectorySecure(directoryPath)) {
+            if (s == null) {
+                return;
+            }
+
+            dirent entry;
+            while ((entry = Dirent.readdir(s.dir)).isNonNull()) {
+                String name = CTypeConversion.toJavaString(entry.d_name());
+                int pid = filenameToPid(name);
+                if (pid == 0) {
+                    if (!".".equals(name) && !"..".equals(name)) {
+                        /* Attempt to remove all unexpected files. */
+                        Fcntl.NoTransitions.unlinkat(s.fd, entry.d_name(), 0);
                     }
+                    continue;
+                }
+
+                /*
+                 * We now have a file name that converts to a valid integer that could represent a
+                 * process id. If this process id matches the current process id or the process is
+                 * not running, then remove the stale file resources.
+                 */
+                if (pid == selfPid || canFileBeDeleted(pid)) {
+                    Fcntl.NoTransitions.unlinkat(s.fd, entry.d_name(), 0);
                 }
             }
         }
     }
 
+    private static int createSharedMemResources(CCharPointer directoryPath, CCharPointer filename, int size) {
+        if (!makeUserTmpDir(directoryPath)) {
+            return -1;
+        }
+
+        int fd = tryCreatePerfFile(directoryPath, filename);
+        if (fd == -1) {
+            return -1;
+        }
+
+        if (!isFileSecure(fd)) {
+            Unistd.NoTransitions.close(fd);
+            return -1;
+        }
+
+        /* Truncate the file to get rid of any existing data. */
+        int result = restartableFtruncate(fd, 0);
+        if (result == -1) {
+            Unistd.NoTransitions.close(fd);
+            return -1;
+        }
+
+        /* Set the file size. */
+        result = restartableFtruncate(fd, size);
+        if (result == -1) {
+            Unistd.NoTransitions.close(fd);
+            return -1;
+        }
+
+        /*
+         * Verify that we have enough disk space for this file. We'll get random SIGBUS crashes on
+         * memory accesses if we don't.
+         */
+        RawFileOperationSupport fs = RawFileOperationSupport.nativeByteOrder();
+        RawFileDescriptor rawFd = WordFactory.signed(fd);
+        int pageSize = NumUtil.safeToInt(VirtualMemoryProvider.get().getGranularity().rawValue());
+
+        boolean success = true;
+        for (int pos = 0; pos < size; pos += pageSize) {
+            success = fs.seek(rawFd, pos);
+            if (!success) {
+                break;
+            }
+            success = fs.writeInt(rawFd, 0);
+            if (!success) {
+                break;
+            }
+        }
+
+        if (!success) {
+            Unistd.NoTransitions.close(fd);
+            return -1;
+        }
+
+        return fd;
+    }
+
+    private static int tryCreatePerfFile(CCharPointer directoryPath, CCharPointer filename) {
+        try (SecureDirectory s = openDirectorySecure(directoryPath)) {
+            if (s == null) {
+                return -1;
+            }
+
+            /*
+             * Open the filename in the current directory. Cannot use O_TRUNC here; truncation of an
+             * existing file has to happen after the is_file_secure() check below.
+             */
+            return restartableOpenat(s.fd, filename, O_RDWR() | O_CREAT() | O_NOFOLLOW(), PosixStat.S_IRUSR() | PosixStat.S_IWUSR());
+        }
+    }
+
     /**
      * Convert the given file name into a process id. If the file does not meet the file naming
      * constraints, return 0.
      */
-    private static int getPidFromFile(File file) {
-        String name = file.getName();
-
+    private static int filenameToPid(String filename) {
         try {
-            return Integer.parseInt(name);
+            return Integer.parseInt(filename);
         } catch (NumberFormatException ex) {
             return 0;
         }
     }
 
-    /**
-     * Create the shared memory file resources.
-     *
-     * This method creates the shared memory file with the given size. This method also creates the
-     * user specific temporary directory, if it does not yet exist.
-     */
-    private static MappedByteBuffer createSharedBuffer(Path perfDir, Path file, String userName, int size) {
+    @Uninterruptible(reason = "LibC.errno() must not be overwritten accidentally.")
+    private static boolean makeUserTmpDir(CCharPointer directory) {
+        /*
+         * Create the directory with 0755 permissions. note that the directory will be owned by
+         * euid::egid, which may not be the same as uid::gid.
+         */
+        if (PosixStat.NoTransitions.mkdir(directory, S_IRWXU() | S_IRGRP() | S_IXGRP() | S_IROTH() | S_IXOTH()) == -1) {
+            if (LibC.errno() == EEXIST()) {
+                /*
+                 * The directory already exists and was probably created by another JVM instance.
+                 * However, this could also be the result of a deliberate symlink. Verify that the
+                 * existing directory is safe.
+                 */
+                return isDirectorySecure(directory);
+            } else {
+                return false;
+            }
+        }
+        return true;
+    }
 
-        // make the user temporary directory
-        if (!createUserDir(perfDir, userName)) {
-            // could not make/find the directory or the found directory
-            // was not secure
+    private static SecureDirectory openDirectorySecure(CCharPointer directory) {
+        int fd = restartableOpen(directory, O_RDONLY() | O_NOFOLLOW(), 0);
+        if (fd == -1) {
             return null;
         }
 
-        // create a file with a set of specified attributes
-        Set perms = EnumSet.of(OWNER_READ, OWNER_WRITE);
-        FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms);
-        EnumSet options = EnumSet.of(CREATE_NEW, WRITE, READ);
-        try {
-            FileChannel channel = FileChannel.open(file, options, attr);
-            return channel.map(FileChannel.MapMode.READ_WRITE, 0, size);
-        } catch (IOException ex) {
-            // log exception
+        if (!isDirFdSecure(fd)) {
+            Unistd.NoTransitions.close(fd);
             return null;
         }
-    }
 
-    private static boolean createUserDir(Path userDir, String userName) {
-        Path dir = userDir;
-        try {
-            Set perms = EnumSet.of(OWNER_READ, OWNER_WRITE, OWNER_EXECUTE,
-                            GROUP_READ, GROUP_EXECUTE,
-                            OTHERS_READ, OTHERS_EXECUTE);
-            FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms);
-            dir = Files.createDirectories(dir, attr);
-            return isDirSecure(dir, userName);
-        } catch (IOException ex) {
-            // log exception
-            return false;
+        DIR dir = Dirent.fdopendir(fd);
+        if (dir.isNull()) {
+            Unistd.NoTransitions.close(fd);
+            return null;
         }
+
+        return new SecureDirectory(fd, dir);
     }
 
     /**
@@ -238,40 +371,181 @@ private static boolean createUserDir(Path userDir, String userName) {
      * true if the directory is considered a secure location. Returns false if the dir is a symbolic
      * link or if an error occurred.
      */
-    private static boolean isDirSecure(Path dir, String userName) {
-        try {
-            if (!Files.isDirectory(dir, NOFOLLOW_LINKS)) {
-                // The path represents a link or some non-directory file type,
-                // which is not what we expected. Declare it insecure.
-                return false;
-            }
-            Set perms = Files.getPosixFilePermissions(dir, NOFOLLOW_LINKS);
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    private static boolean isDirectorySecure(CCharPointer directory) {
+        PosixStat.stat buf = StackValue.get(PosixStat.sizeOfStatStruct());
+        int result = restartableLstat(directory, buf);
+        if (result == -1) {
+            return false;
+        }
+        return isStatBufSecure(buf);
+    }
 
-            if (perms.contains(GROUP_WRITE) || perms.contains(OTHERS_WRITE)) {
-                // The directory is open for writing and could be subjected
-                // to a symlink or a hard link attack. Declare it insecure.
-                return false;
-            }
-            // If user is not root then see if the uid of the directory matches the effective uid of
-            // the process.
-            int euid = Unistd.getegid();
-            UserPrincipal euser = Files.getOwner(dir, NOFOLLOW_LINKS);
-            if ((euid != 0) && (!euser.getName().equals(userName))) {
-                // The directory was not created by this user. Declare it insecure.
-                return false;
-            }
-        } catch (IOException ex) {
-            // log exception
+    /**
+     * Check if the given directory file descriptor is considered a secure directory for the backing
+     * store files. Returns true if the directory exists and is considered a secure location.
+     * Returns false if the path is a symbolic link or if an error occurred.
+     */
+    private static boolean isDirFdSecure(int dirFd) {
+        PosixStat.stat buf = StackValue.get(PosixStat.sizeOfStatStruct());
+        int result = restartableFstat(dirFd, buf);
+        if (result == -1) {
+            return false;
+        }
+        return isStatBufSecure(buf);
+    }
+
+    private static boolean isFileSecure(int fd) {
+        PosixStat.stat buf = StackValue.get(PosixStat.sizeOfStatStruct());
+        int result = restartableFstat(fd, buf);
+        if (result == -1) {
+            return false;
+        }
+        if (PosixStat.st_nlink(buf) > 1) {
+            return false;
+        }
+        return true;
+    }
+
+    /**
+     * Check if the given statbuf is considered a secure directory for the backing store files.
+     * Returns true if the directory is considered a secure location. Returns false if the statbuf
+     * is a symbolic link or if an error occurred.
+     */
+    @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
+    private static boolean isStatBufSecure(PosixStat.stat statp) {
+        if (PosixStat.S_ISLNK(statp) || !PosixStat.S_ISDIR(statp)) {
+            /*
+             * The path represents a link or some non-directory file type, which is not what we
+             * expected. Declare it insecure.
+             */
+            return false;
+        }
+
+        if ((PosixStat.st_mode(statp) & (PosixStat.S_IWGRP() | PosixStat.S_IWOTH())) != 0) {
+            /*
+             * The directory is open for writing and could be subjected to a symlink or a hard link
+             * attack. Declare it insecure.
+             */
+            return false;
+        }
+
+        /*
+         * If user is not root then see if the uid of the directory matches the effective uid of the
+         * process.
+         */
+        int euid = Unistd.NoTransitions.geteuid();
+        if (euid != 0 && PosixStat.st_uid(statp) != euid) {
+            /* The directory was not created by this user, declare it insecure. */
             return false;
         }
         return true;
     }
 
+    /**
+     * Process liveness is detected by sending signal number 0 to the process id. If kill determines
+     * that the process does not exist, then the file resources are removed. If kill determines that
+     * we don't have permission to signal the process, then the file resources are assumed to be
+     * stale and are removed because the resources for such a process should be in a different user
+     * specific directory.
+     */
+    @Uninterruptible(reason = "LibC.errno() must not be overwritten accidentally.")
+    private static boolean canFileBeDeleted(int pid) {
+        int ret = Signal.NoTransitions.kill(pid, 0);
+        if (ret == -1) {
+            int errno = LibC.errno();
+            return errno == ESRCH() || errno == EPERM();
+        }
+        return false;
+    }
+
+    @Uninterruptible(reason = "LibC.errno() must not be overwritten accidentally.")
+    private static int restartableOpen(CCharPointer directory, int flags, int mode) {
+        int result;
+        do {
+            result = Fcntl.NoTransitions.open(directory, flags, mode);
+        } while (result == -1 && LibC.errno() == Errno.EINTR());
+
+        return result;
+    }
+
+    @Uninterruptible(reason = "LibC.errno() must not be overwritten accidentally.")
+    private static int restartableOpenat(int fd, CCharPointer filename, int flags, int mode) {
+        int result;
+        do {
+            result = Fcntl.NoTransitions.openat(fd, filename, flags, mode);
+        } while (result == -1 && LibC.errno() == Errno.EINTR());
+
+        return result;
+    }
+
+    private static int restartableUnlink(String pathname) {
+        try (CTypeConversion.CCharPointerHolder f = CTypeConversion.toCString(pathname)) {
+            return restartableUnlink(f.get());
+        }
+    }
+
+    @Uninterruptible(reason = "LibC.errno() must not be overwritten accidentally.")
+    private static int restartableUnlink(CCharPointer pathname) {
+        int result;
+        do {
+            result = Fcntl.NoTransitions.unlink(pathname);
+        } while (result == -1 && LibC.errno() == Errno.EINTR());
+
+        return result;
+    }
+
+    @Uninterruptible(reason = "LibC.errno() must not be overwritten accidentally.")
+    private static int restartableFtruncate(int fd, int size) {
+        int result;
+        do {
+            result = Unistd.NoTransitions.ftruncate(fd, WordFactory.signed(size));
+        } while (result == -1 && LibC.errno() == Errno.EINTR());
+
+        return result;
+    }
+
+    @Uninterruptible(reason = "LibC.errno() must not be overwritten accidentally.")
+    private static int restartableFstat(int fd, PosixStat.stat buf) {
+        int result;
+        do {
+            result = PosixStat.NoTransitions.fstat(fd, buf);
+        } while (result == -1 && LibC.errno() == Errno.EINTR());
+
+        return result;
+    }
+
+    @Uninterruptible(reason = "LibC.errno() must not be overwritten accidentally.")
+    private static int restartableLstat(CCharPointer directory, PosixStat.stat buf) {
+        int result;
+        do {
+            result = PosixStat.NoTransitions.lstat(directory, buf);
+        } while (result == -1 && LibC.errno() == Errno.EINTR());
+
+        return result;
+    }
+
     @Override
     public void teardown() {
-        if (backingStoreFile != null) {
-            backingStoreFile.toFile().delete();
-            backingStoreFile = null;
+        if (backingFilePath != null) {
+            restartableUnlink(backingFilePath);
+            backingFilePath = null;
+        }
+    }
+
+    private static class SecureDirectory implements AutoCloseable {
+        private final int fd;
+        private final DIR dir;
+
+        SecureDirectory(int fd, DIR dir) {
+            this.fd = fd;
+            this.dir = dir;
+        }
+
+        @Override
+        public void close() {
+            /* Close the directory (and implicitly the file descriptor). */
+            Dirent.closedir(dir);
         }
     }
 }
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/PerfMemory.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/PerfMemory.java
index bd864320046..787705f9880 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/PerfMemory.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/PerfMemory.java
@@ -111,6 +111,7 @@ private boolean createBuffer() {
         buffer = b;
         capacity = b.capacity();
         rawMemory = WordFactory.pointer(SubstrateUtil.cast(b, Target_java_nio_Buffer.class).address);
+
         assert verifyRawMemoryAccess();
 
         return true;
diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/SystemCounters.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/SystemCounters.java
index 42aa6395182..66ffbf021f4 100644
--- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/SystemCounters.java
+++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jvmstat/SystemCounters.java
@@ -123,15 +123,15 @@ public void allocate() {
         loadedClasses.allocate(numberOfLoadedClasses());
         processors.allocate(getAvailableProcessors());
 
-        tempDir.allocate(System.getProperty("java.io.tmpdir"));
-        javaVersion.allocate(System.getProperty("java.version"));
-        vmName.allocate(System.getProperty("java.vm.name"));
-        vmVendor.allocate(System.getProperty("java.vm.vendor"));
-        vmVersion.allocate(System.getProperty("java.vm.version"));
-        osArch.allocate(System.getProperty("os.arch"));
-        osName.allocate(System.getProperty("os.name"));
-        userDir.allocate(System.getProperty("user.dir"));
-        userName.allocate(System.getProperty("user.name"));
+        tempDir.allocate(getSystemProperty("java.io.tmpdir"));
+        javaVersion.allocate(getSystemProperty("java.version"));
+        vmName.allocate(getSystemProperty("java.vm.name"));
+        vmVendor.allocate(getSystemProperty("java.vm.vendor"));
+        vmVersion.allocate(getSystemProperty("java.vm.version"));
+        osArch.allocate(getSystemProperty("os.arch"));
+        osName.allocate(getSystemProperty("os.name"));
+        userDir.allocate(getSystemProperty("user.dir"));
+        userName.allocate(getSystemProperty("user.name"));
 
         gcInProgress.allocate();
 
@@ -145,6 +145,15 @@ public void allocate() {
         initDoneTime.allocate(System.currentTimeMillis());
     }
 
+    private static String getSystemProperty(String s) {
+        /* Certain system properties (e.g., "user.dir"), may throw an exception. */
+        try {
+            return System.getProperty(s);
+        } catch (Throwable e) {
+            return "";
+        }
+    }
+
     @Override
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public void vmOperationChanged(VMOperation operation) {

From 4bec3af23fe57294e8f07e05e7fcec84a5357e8c Mon Sep 17 00:00:00 2001
From: Christian Haeubl 
Date: Sun, 12 Mar 2023 10:14:40 +0100
Subject: [PATCH 40/43] Darwin and aarch64 fixes.

---
 .../src/com/oracle/svm/core/posix/PosixStat.java         | 9 +++++----
 .../oracle/svm/core/posix/headers/darwin/DarwinStat.java | 8 ++++++--
 .../oracle/svm/core/posix/headers/linux/LinuxStat.java   | 8 ++++++--
 .../svm/core/posix/jvmstat/PosixPerfMemoryProvider.java  | 4 ++--
 4 files changed, 19 insertions(+), 10 deletions(-)

diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixStat.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixStat.java
index 652b9655a71..fdf21d9afaf 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixStat.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/PosixStat.java
@@ -35,6 +35,7 @@
 import org.graalvm.nativeimage.c.struct.SizeOf;
 import org.graalvm.nativeimage.c.type.CCharPointer;
 import org.graalvm.nativeimage.c.type.CConst;
+import org.graalvm.word.UnsignedWord;
 import org.graalvm.word.WordBase;
 
 import com.oracle.svm.core.Uninterruptible;
@@ -123,16 +124,16 @@ public static int st_uid(stat buf) {
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean S_ISLNK(stat buf) {
-        return (st_mode(buf) & S_IFLNK()) == S_IFLNK();
+        return st_mode(buf).and(S_IFLNK()).equal(S_IFLNK());
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
     public static boolean S_ISDIR(stat buf) {
-        return (st_mode(buf) & S_IFDIR()) == S_IFDIR();
+        return st_mode(buf).and(S_IFDIR()).equal(S_IFDIR());
     }
 
     @Uninterruptible(reason = "Called from uninterruptible code.", mayBeInlined = true)
-    public static int st_mode(stat buf) {
+    public static UnsignedWord st_mode(stat buf) {
         if (Platform.includedIn(Platform.LINUX.class)) {
             return ((LinuxStat.stat64) buf).st_mode();
         } else if (Platform.includedIn(Platform.DARWIN.class)) {
@@ -142,7 +143,7 @@ public static int st_mode(stat buf) {
         }
     }
 
-    public static long st_nlink(stat buf) {
+    public static UnsignedWord st_nlink(stat buf) {
         if (Platform.includedIn(Platform.LINUX.class)) {
             return ((LinuxStat.stat64) buf).st_nlink();
         } else if (Platform.includedIn(Platform.DARWIN.class)) {
diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinStat.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinStat.java
index fca031dc296..94c5fc909df 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinStat.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinStat.java
@@ -28,11 +28,13 @@
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.nativeimage.c.CContext;
 import org.graalvm.nativeimage.c.function.CFunction;
+import org.graalvm.nativeimage.c.struct.AllowWideningCast;
 import org.graalvm.nativeimage.c.struct.CField;
 import org.graalvm.nativeimage.c.struct.CStruct;
 import org.graalvm.nativeimage.c.type.CCharPointer;
 import org.graalvm.nativeimage.c.type.CConst;
 import org.graalvm.word.PointerBase;
+import org.graalvm.word.UnsignedWord;
 
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.posix.headers.PosixDirectives;
@@ -58,7 +60,8 @@ public interface stat extends PointerBase {
         long st_ino();
 
         @CField
-        int st_mode();
+        @AllowWideningCast
+        UnsignedWord st_mode();
 
         @CField
         int st_uid();
@@ -67,7 +70,8 @@ public interface stat extends PointerBase {
         long st_size();
 
         @CField
-        long st_nlink();
+        @AllowWideningCast
+        UnsignedWord st_nlink();
     }
 
     @CFunction("fstat$INODE64")
diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/linux/LinuxStat.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/linux/LinuxStat.java
index 73d7a316e71..35164de9feb 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/linux/LinuxStat.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/linux/LinuxStat.java
@@ -28,11 +28,13 @@
 
 import org.graalvm.nativeimage.c.CContext;
 import org.graalvm.nativeimage.c.function.CFunction;
+import org.graalvm.nativeimage.c.struct.AllowWideningCast;
 import org.graalvm.nativeimage.c.struct.CField;
 import org.graalvm.nativeimage.c.struct.CStruct;
 import org.graalvm.nativeimage.c.type.CCharPointer;
 import org.graalvm.nativeimage.c.type.CConst;
 import org.graalvm.word.PointerBase;
+import org.graalvm.word.UnsignedWord;
 
 import com.oracle.svm.core.posix.headers.PosixDirectives;
 
@@ -50,7 +52,8 @@ public interface stat64 extends PointerBase {
         long st_ino();
 
         @CField
-        int st_mode();
+        @AllowWideningCast
+        UnsignedWord st_mode();
 
         @CField
         int st_uid();
@@ -59,7 +62,8 @@ public interface stat64 extends PointerBase {
         long st_size();
 
         @CField
-        long st_nlink();
+        @AllowWideningCast
+        UnsignedWord st_nlink();
     }
 
     public static class NoTransitions {
diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java
index 4766dd8baaf..59726fb4a10 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java
@@ -401,7 +401,7 @@ private static boolean isFileSecure(int fd) {
         if (result == -1) {
             return false;
         }
-        if (PosixStat.st_nlink(buf) > 1) {
+        if (PosixStat.st_nlink(buf).aboveThan(1)) {
             return false;
         }
         return true;
@@ -422,7 +422,7 @@ private static boolean isStatBufSecure(PosixStat.stat statp) {
             return false;
         }
 
-        if ((PosixStat.st_mode(statp) & (PosixStat.S_IWGRP() | PosixStat.S_IWOTH())) != 0) {
+        if (PosixStat.st_mode(statp).and(PosixStat.S_IWGRP() | PosixStat.S_IWOTH()).notEqual(0)) {
             /*
              * The directory is open for writing and could be subjected to a symlink or a hard link
              * attack. Declare it insecure.

From eaeb3ec90520deb14d3c9ce4e501713fa224eba3 Mon Sep 17 00:00:00 2001
From: Gilles Duboscq 
Date: Wed, 26 Apr 2023 18:39:15 +0200
Subject: [PATCH 41/43] Add error message in DarwinStat.NoTransitions#lstat

---
 .../com/oracle/svm/core/posix/headers/darwin/DarwinStat.java    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinStat.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinStat.java
index 94c5fc909df..fdf28ad9632 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinStat.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/headers/darwin/DarwinStat.java
@@ -130,7 +130,7 @@ public static int lstat(CCharPointer path, stat buf) {
             } else if (Platform.includedIn(Platform.AARCH64.class)) {
                 return lstat_aarch64(path, buf);
             } else {
-                throw VMError.shouldNotReachHere();
+                throw VMError.shouldNotReachHere("Unknown architecture");
             }
         }
 

From 3c79e4fd3a368231eb9fbca6cd73884510299477 Mon Sep 17 00:00:00 2001
From: Gilles Duboscq 
Date: Wed, 26 Apr 2023 19:58:34 +0200
Subject: [PATCH 42/43] PosixPerfMemoryProvider: use correct DirectByteBuffer
 constructor

---
 .../svm/core/posix/jvmstat/PosixPerfMemoryProvider.java    | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java
index 59726fb4a10..0900e586dff 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java
@@ -42,6 +42,7 @@
 import java.nio.ByteBuffer;
 
 import org.graalvm.compiler.core.common.NumUtil;
+import org.graalvm.compiler.serviceprovider.JavaVersionUtil;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
@@ -147,7 +148,11 @@ public ByteBuffer create() {
 
         /* Clear the shared memory region. */
         LibC.memset(mapAddress, WordFactory.signed(0), WordFactory.unsigned(size));
-        return SubstrateUtil.cast(new Target_java_nio_DirectByteBuffer(mapAddress.rawValue(), size), ByteBuffer.class);
+        if (JavaVersionUtil.JAVA_SPEC >= 21) {
+            return SubstrateUtil.cast(new Target_java_nio_DirectByteBuffer(mapAddress.rawValue(), (long) size), ByteBuffer.class);
+        } else {
+            return SubstrateUtil.cast(new Target_java_nio_DirectByteBuffer(mapAddress.rawValue(), size), ByteBuffer.class);
+        }
     }
 
     private static String getUserName(int uid) {

From 6a74d773287052cfb43d61d4affc6895c9092b5b Mon Sep 17 00:00:00 2001
From: Gilles Duboscq 
Date: Thu, 27 Apr 2023 09:27:22 +0200
Subject: [PATCH 43/43] Use DirectByteBufferUtil.allocate

---
 .../core/posix/jvmstat/PosixPerfMemoryProvider.java    | 10 ++--------
 1 file changed, 2 insertions(+), 8 deletions(-)

diff --git a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java
index 0900e586dff..bc216e4b52c 100644
--- a/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java
+++ b/substratevm/src/com.oracle.svm.core.posix/src/com/oracle/svm/core/posix/jvmstat/PosixPerfMemoryProvider.java
@@ -42,7 +42,6 @@
 import java.nio.ByteBuffer;
 
 import org.graalvm.compiler.core.common.NumUtil;
-import org.graalvm.compiler.serviceprovider.JavaVersionUtil;
 import org.graalvm.nativeimage.ImageSingletons;
 import org.graalvm.nativeimage.Platform;
 import org.graalvm.nativeimage.Platforms;
@@ -54,7 +53,6 @@
 import org.graalvm.word.Pointer;
 import org.graalvm.word.WordFactory;
 
-import com.oracle.svm.core.SubstrateUtil;
 import com.oracle.svm.core.Uninterruptible;
 import com.oracle.svm.core.VMInspectionOptions;
 import com.oracle.svm.core.annotate.Alias;
@@ -62,7 +60,7 @@
 import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature;
 import com.oracle.svm.core.feature.InternalFeature;
 import com.oracle.svm.core.headers.LibC;
-import com.oracle.svm.core.jdk.Target_java_nio_DirectByteBuffer;
+import com.oracle.svm.core.jdk.DirectByteBufferUtil;
 import com.oracle.svm.core.jvmstat.PerfManager;
 import com.oracle.svm.core.jvmstat.PerfMemoryPrologue;
 import com.oracle.svm.core.jvmstat.PerfMemoryProvider;
@@ -148,11 +146,7 @@ public ByteBuffer create() {
 
         /* Clear the shared memory region. */
         LibC.memset(mapAddress, WordFactory.signed(0), WordFactory.unsigned(size));
-        if (JavaVersionUtil.JAVA_SPEC >= 21) {
-            return SubstrateUtil.cast(new Target_java_nio_DirectByteBuffer(mapAddress.rawValue(), (long) size), ByteBuffer.class);
-        } else {
-            return SubstrateUtil.cast(new Target_java_nio_DirectByteBuffer(mapAddress.rawValue(), size), ByteBuffer.class);
-        }
+        return DirectByteBufferUtil.allocate(mapAddress.rawValue(), size);
     }
 
     private static String getUserName(int uid) {