diff --git a/ci.jsonnet b/ci.jsonnet index 1eb8fa7fcd..02b45a26a0 100644 --- a/ci.jsonnet +++ b/ci.jsonnet @@ -5,7 +5,7 @@ (import "ci/python-gate.libsonnet") + (import "ci/python-bench.libsonnet") + { - overlay: "40bd8048e1a6ba45494605955ffe748ae4db20be", + overlay: "30def35dbfc43256d57ad3b9b981d92718728e2a", specVersion: "8", // Until buildbot issues around CI tiers are resolved, we cannot use them // tierConfig: self.tierConfig, @@ -30,6 +30,9 @@ BISECT_EMAIL_FROM: "", npm_config_registry: "", RODINIA_DATASET_ZIP: "", + GRAALPY_GRAALOS_TOOLCHAIN_URL: "", + GRAALPY_GRAALOS_RUNTIME_URL: "", + GRAALPY_GRAALOS_ARTIFACT_BASE_URL: "", BUILDBOT_COMMIT_SERVICE: "", INTERNET_ACCESS_ENV: {}, }, @@ -330,6 +333,15 @@ "tox-example": gpgate_ee + require(GPYEE_NATIVE_STANDALONE) + platform_spec(no_jobs) + platform_spec({ "linux:amd64:jdk-latest" : tier3, }), + "python-svm-graalos-standalone-build": gpgate_ee + internet_access_env + platform_spec(no_jobs) + platform_spec({ + "linux:amd64:jdk-latest": tier3 + $.ol8 + task_spec({ + environment +: { + GRAALPY_GRAALOS_TOOLCHAIN_URL: $.overlay_imports.GRAALPY_GRAALOS_TOOLCHAIN_URL, + GRAALPY_GRAALOS_RUNTIME_URL: $.overlay_imports.GRAALPY_GRAALOS_RUNTIME_URL, + GRAALPY_GRAALOS_ARTIFACT_BASE_URL: $.overlay_imports.GRAALPY_GRAALOS_ARTIFACT_BASE_URL, + }, + }), + }), }, local need_pgo = task_spec({runAfter: ["python-pgo-profile-post_merge-linux-amd64-jdk-latest"]}), diff --git a/graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/PythonResource.java b/graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/PythonResource.java index 4b76ac03e2..cb769e9925 100644 --- a/graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/PythonResource.java +++ b/graalpython/com.oracle.graal.python.resources/src/com/oracle/graal/python/resources/PythonResource.java @@ -90,7 +90,8 @@ public final class PythonResource implements InternalResource { while ((ch = is.read()) != '\n' && ch != -1) { // skip ABI version } - PYTHON_ABIFLAGS = ch == -1 ? "" : new String(is.readAllBytes(), StandardCharsets.US_ASCII).strip(); + String[] abiParts = ch == -1 ? new String[0] : new String(is.readAllBytes(), StandardCharsets.US_ASCII).split("\\R", 4); + PYTHON_ABIFLAGS = abiParts.length > 0 ? abiParts[0].strip() : ""; } catch (IOException e) { throw new RuntimeException(e); } diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_graalos_standalone.py b/graalpython/com.oracle.graal.python.test/src/tests/test_graalos_standalone.py new file mode 100644 index 0000000000..49b73eddb8 --- /dev/null +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_graalos_standalone.py @@ -0,0 +1,59 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import sysconfig +import unittest + + +def test_graalos_sqlite3_native_extension_smoke(): + soabi = sysconfig.get_config_var("SOABI") or "" + if "graalos" not in soabi: + raise unittest.SkipTest(f"requires GraalOS SOABI, got {soabi!r}") + + import _sqlite3 + import sqlite3 + + assert _sqlite3.sqlite_version + conn = sqlite3.connect(":memory:") + try: + conn.execute("create table values_for_sum(value integer)") + conn.executemany("insert into values_for_sum(value) values (?)", [(1,), (2,), (3,)]) + assert conn.execute("select sum(value) from values_for_sum").fetchone()[0] == 6 + finally: + conn.close() diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java index 9e1ba1e502..c7a68d788b 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java @@ -203,6 +203,9 @@ public final class PythonLanguage extends TruffleLanguage { /** See {@code mx_graalpython.py:abi_version} */ public static final String GRAALPY_ABI_VERSION; public static final String GRAALPY_ABIFLAGS; + public static final String GRAALPY_SOABI; + public static final String GRAALPY_EXT_SUFFIX; + public static final String GRAALPY_MULTIARCH; /* Magic number used to mark pyc files */ public static final int MAGIC_NUMBER = 21000 + Compiler.BYTECODE_VERSION * 10; @@ -252,9 +255,12 @@ public final class PythonLanguage extends TruffleLanguage { default: RELEASE_LEVEL_STRING = tsLiteral("final"); } - String[] abiParts = new String(is.readAllBytes(), StandardCharsets.US_ASCII).split("\\R", 2); + String[] abiParts = new String(is.readAllBytes(), StandardCharsets.US_ASCII).split("\\R", 5); GRAALPY_ABI_VERSION = abiParts[0].strip(); GRAALPY_ABIFLAGS = abiParts.length > 1 ? abiParts[1].strip() : ""; + GRAALPY_SOABI = abiParts.length > 2 ? abiParts[2].strip() : ""; + GRAALPY_EXT_SUFFIX = abiParts.length > 3 ? abiParts[3].strip() : ""; + GRAALPY_MULTIARCH = abiParts.length > 4 ? abiParts[4].strip() : ""; } catch (IOException e) { throw new RuntimeException(e); } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java index f45ad58611..be8ad9dc07 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/ImpModuleBuiltins.java @@ -833,7 +833,7 @@ public abstract static class ExtensionSuffixesNode extends PythonBuiltinNode { @Specialization Object run( @Bind PythonLanguage language) { - return PFactory.createList(language, new Object[]{PythonContext.get(this).getSoAbi(), T_EXT_SO, T_EXT_PYD}); + return PFactory.createList(language, new Object[]{PythonContext.get(this).getExtensionSuffix(), T_EXT_SO, T_EXT_PYD}); } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java index 71d80cccf8..0d72d19d6b 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java @@ -60,7 +60,6 @@ import static com.oracle.graal.python.builtins.modules.io.IONodes.T_R; import static com.oracle.graal.python.builtins.modules.io.IONodes.T_W; import static com.oracle.graal.python.builtins.modules.io.IONodes.T_WRITE; -import static com.oracle.graal.python.builtins.objects.str.StringUtils.cat; import static com.oracle.graal.python.lib.PyTraceBackPrint.castToString; import static com.oracle.graal.python.lib.PyTraceBackPrint.classNameNoDot; import static com.oracle.graal.python.lib.PyTraceBackPrint.fileFlush; @@ -119,7 +118,6 @@ import static com.oracle.graal.python.nodes.StringLiterals.T_BASE_PREFIX; import static com.oracle.graal.python.nodes.StringLiterals.T_BIG; import static com.oracle.graal.python.nodes.StringLiterals.T_COMMA; -import static com.oracle.graal.python.nodes.StringLiterals.T_DASH; import static com.oracle.graal.python.nodes.StringLiterals.T_DOT; import static com.oracle.graal.python.nodes.StringLiterals.T_EMPTY_STRING; import static com.oracle.graal.python.nodes.StringLiterals.T_JAVA; @@ -561,7 +559,7 @@ public void initialize(Python3Core core) { if (os == PLATFORM_DARWIN) { addBuiltinConstant("_framework", FRAMEWORK); } - final TruffleString gmultiarch = cat(PythonUtils.getPythonArch(), T_DASH, osName); + final TruffleString gmultiarch = toTruffleStringUncached(PythonLanguage.GRAALPY_MULTIARCH); addBuiltinConstant("__gmultiarch", gmultiarch); // Initialized later in postInitialize diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java index 303358bcc1..9a95ce0a7e 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/capi/CApiContext.java @@ -1119,7 +1119,7 @@ public static Object loadCExtModule(Node location, PythonContext context, Module CApiContext cApiContext = CApiContext.ensureCapiWasLoaded(location, context, spec.name, spec.path); NativeLibrary library; - TruffleFile realPath = context.getPublicTruffleFileRelaxed(spec.path, context.getSoAbi()).getCanonicalFile(); + TruffleFile realPath = context.getPublicTruffleFileRelaxed(spec.path, context.getExtensionSuffix()).getCanonicalFile(); String loadPath = cApiContext.nativeLibraryLocator.resolve(context, realPath); getLogger(CApiContext.class).config(String.format("loading module %s (real path: %s) as native", spec.path, loadPath)); int dlopenFlags = context.getDlopenFlags(); diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryLocator.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryLocator.java index db4aa1d54f..7c3c772db5 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryLocator.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/cext/copying/NativeLibraryLocator.java @@ -149,7 +149,7 @@ public static void replicate(TruffleFile venvDirectory, PythonContext context, i "but we are preparing %d copies. The extra copies will only be used if a different value " + "of the system property %s is set.", MAX_CEXT_COPIES, count, J_MAX_CAPI_COPIES)); } - String suffix = context.getSoAbi().toJavaStringUncached(); + String suffix = context.getExtensionSuffix().toJavaStringUncached(); TruffleFile capiLibrary = context.getPublicTruffleFileRelaxed(context.getCAPIHome()).resolve(PythonContext.getSupportLibName("python-" + J_NATIVE)); for (int i = 0; i < count; i++) { // Relocate the C API library diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java index 548eaa1012..e85719887a 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java @@ -27,17 +27,12 @@ import static com.oracle.graal.python.PythonLanguage.getPythonOS; import static com.oracle.graal.python.PythonLanguage.throwIfUnsupported; -import static com.oracle.graal.python.annotations.PythonOS.PLATFORM_DARWIN; import static com.oracle.graal.python.annotations.PythonOS.PLATFORM_WIN32; import static com.oracle.graal.python.builtins.PythonBuiltinClassType.SystemError; -import static com.oracle.graal.python.builtins.modules.SysModuleBuiltins.T_ABIFLAGS; -import static com.oracle.graal.python.builtins.modules.SysModuleBuiltins.T_CACHE_TAG; -import static com.oracle.graal.python.builtins.modules.SysModuleBuiltins.T__MULTIARCH; import static com.oracle.graal.python.builtins.modules.io.IONodes.T_CLOSED; import static com.oracle.graal.python.builtins.modules.io.IONodes.T_FLUSH; import static com.oracle.graal.python.builtins.objects.PythonAbstractObject.NATIVE_POINTER_FREED; import static com.oracle.graal.python.builtins.objects.PythonAbstractObject.UNINITIALIZED; -import static com.oracle.graal.python.builtins.objects.str.StringUtils.cat; import static com.oracle.graal.python.builtins.objects.thread.PThread.GRAALPYTHON_THREADS; import static com.oracle.graal.python.nodes.BuiltinNames.T_PYEXPAT; import static com.oracle.graal.python.nodes.BuiltinNames.T_SHA3; @@ -55,11 +50,8 @@ import static com.oracle.graal.python.nodes.StringLiterals.J_EXT_DYLIB; import static com.oracle.graal.python.nodes.StringLiterals.J_EXT_SO; import static com.oracle.graal.python.nodes.StringLiterals.J_LIB_PREFIX; -import static com.oracle.graal.python.nodes.StringLiterals.T_DASH; import static com.oracle.graal.python.nodes.StringLiterals.T_DOT; import static com.oracle.graal.python.nodes.StringLiterals.T_EMPTY_STRING; -import static com.oracle.graal.python.nodes.StringLiterals.T_EXT_PYD; -import static com.oracle.graal.python.nodes.StringLiterals.T_EXT_SO; import static com.oracle.graal.python.nodes.StringLiterals.T_JAVA; import static com.oracle.graal.python.nodes.StringLiterals.T_NATIVE; import static com.oracle.graal.python.nodes.StringLiterals.T_PATH; @@ -151,7 +143,6 @@ import com.oracle.graal.python.nodes.SpecialAttributeNames; import com.oracle.graal.python.nodes.SpecialMethodNames; import com.oracle.graal.python.nodes.WriteUnraisableNode; -import com.oracle.graal.python.nodes.attributes.ReadAttributeFromModuleNode; import com.oracle.graal.python.nodes.attributes.ReadAttributeFromObjectNode; import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode; import com.oracle.graal.python.nodes.call.CallNode; @@ -807,7 +798,7 @@ public enum CApiState { @CompilationFinal private boolean nativeAccessAllowed; @CompilationFinal private NativeContext nativeContext; - private TruffleString soABI; + private TruffleString extensionSuffix; private static final class GlobalInterpreterLock extends ReentrantLock { private static final long serialVersionUID = 1L; @@ -2932,31 +2923,11 @@ public boolean isFinalizing() { } @TruffleBoundary - public TruffleString getSoAbi() { - if (soABI == null) { - PythonModule sysModule = this.lookupBuiltinModule(T_SYS); - Object implementationObj = ReadAttributeFromModuleNode.getUncached().execute(sysModule, T_IMPLEMENTATION); - // sys.implementation.cache_tag - TruffleString cacheTag = (TruffleString) PyObjectGetAttr.executeUncached(implementationObj, T_CACHE_TAG); - TruffleString abiFlags = (TruffleString) ReadAttributeFromModuleNode.getUncached().execute(sysModule, T_ABIFLAGS); - // sys.implementation._multiarch - TruffleString multiArch = (TruffleString) PyObjectGetAttr.executeUncached(implementationObj, T__MULTIARCH); - - // only use '.pyd' if we are on 'Win32-native' - TruffleString soExt; - if (getPythonOS() == PLATFORM_DARWIN) { - // not ".dylib", similar to CPython: - // https://github.com/python/cpython/issues/37510 - soExt = T_EXT_SO; - } else if (getPythonOS() == PLATFORM_WIN32) { - soExt = T_EXT_PYD; - } else { - soExt = T_EXT_SO; - } - - soABI = cat(T_DOT, cacheTag, abiFlags, T_DASH, T_NATIVE, T_DASH, multiArch, soExt); + public TruffleString getExtensionSuffix() { + if (extensionSuffix == null) { + extensionSuffix = toTruffleStringUncached(PythonLanguage.GRAALPY_EXT_SUFFIX); } - return soABI; + return extensionSuffix; } public Thread getMainThread() { diff --git a/graalpython/graalpy-versions/CMakeLists.txt b/graalpython/graalpy-versions/CMakeLists.txt index a2ba93cafe..1664e2baad 100644 --- a/graalpython/graalpy-versions/CMakeLists.txt +++ b/graalpython/graalpy-versions/CMakeLists.txt @@ -35,8 +35,17 @@ endif() if (NOT DEFINED GRAALPY_ABIFLAGS) set(GRAALPY_ABIFLAGS "") endif() +if (NOT DEFINED GRAALPY_EXT_SUFFIX) + message(FATAL_ERROR "GRAALPY_EXT_SUFFIX needs to be set") +endif() +if (NOT DEFINED GRAALPY_MULTIARCH) + message(FATAL_ERROR "GRAALPY_MULTIARCH needs to be set") +endif() +if (NOT DEFINED GRAALPY_SOABI) + message(FATAL_ERROR "GRAALPY_SOABI needs to be set") +endif() # Generates file 'graalpy_versions' with the given content. # The file will be created if it does not exist and will only be updated if the # content changes. -file(GENERATE OUTPUT "graalpy_versions" CONTENT "${GRAALPY_VER}\n${GRAALPY_ABIFLAGS}") +file(GENERATE OUTPUT "graalpy_versions" CONTENT "${GRAALPY_VER}\n${GRAALPY_ABIFLAGS}\n${GRAALPY_SOABI}\n${GRAALPY_EXT_SUFFIX}\n${GRAALPY_MULTIARCH}\n") diff --git a/graalpython/graalpy_graalos_standalone_payload/CMakeLists.txt b/graalpython/graalpy_graalos_standalone_payload/CMakeLists.txt new file mode 100644 index 0000000000..268e254b0d --- /dev/null +++ b/graalpython/graalpy_graalos_standalone_payload/CMakeLists.txt @@ -0,0 +1,195 @@ +# +# Copyright (c) 2026, Oracle and/or its affiliates. +# +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without modification, are +# permitted provided that the following conditions are met: +# +# 1. Redistributions of source code must retain the above copyright notice, this list of +# conditions and the following disclaimer. +# +# 2. Redistributions in binary form must reproduce the above copyright notice, this list of +# conditions and the following disclaimer in the documentation and/or other materials provided +# with the distribution. +# 3. Neither the name of the copyright holder nor the names of its contributors may be used to +# endorse or promote products derived from this software without specific prior written +# permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS +# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +# MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +# GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +# OF THE POSSIBILITY OF SUCH DAMAGE. +# +cmake_minimum_required(VERSION 3.22) +project(graalpy-graalos-standalone-payload LANGUAGES NONE) + +foreach(required_var + GRAALPY_LAUNCHER + PYTHON_LAUNCHER + PYTHON3_LAUNCHER + GRAALPY_CONFIG_LAUNCHER + GRAALPY_POLYGLOT_GET_LAUNCHER + GRAALPY_SUITE_PARENT) + if(NOT DEFINED ${required_var}) + message(FATAL_ERROR "${required_var} needs to be set") + endif() +endforeach() + +set(PAYLOAD_DIR "${CMAKE_BINARY_DIR}") +set(GRAALOS_DIR "${PAYLOAD_DIR}/lib/graalos") +set(SYSROOT_LIB_DIR "${GRAALOS_DIR}/sysroot/lib") + +file(REMOVE_RECURSE + "${PAYLOAD_DIR}/bin" + "${PAYLOAD_DIR}/libexec" + "${PAYLOAD_DIR}/lib" + "${PAYLOAD_DIR}/config.json") +file(MAKE_DIRECTORY + "${PAYLOAD_DIR}/bin" + "${PAYLOAD_DIR}/libexec" + "${GRAALOS_DIR}" + "${SYSROOT_LIB_DIR}") + +function(_append_env_path out_var env_name) + if(DEFINED ENV{${env_name}} AND NOT "$ENV{${env_name}}" STREQUAL "") + list(APPEND ${out_var} "$ENV{${env_name}}") + set(${out_var} "${${out_var}}" PARENT_SCOPE) + endif() +endfunction() + +function(_find_graalos_runtime out_var) + set(candidates) + _append_env_path(candidates GRAALOS_RUNTIME_HOME) + _append_env_path(candidates GRAALOS_HOME) + _append_env_path(candidates GRAALOS_BUILD_HOME) + list(APPEND candidates + "${GRAALPY_SUITE_PARENT}/graalos/cmake-build-clang-pkeysoff" + "${GRAALPY_SUITE_PARENT}/graalos/build") + + foreach(candidate IN LISTS candidates) + if(EXISTS "${candidate}/graalhost/graalhost" + AND EXISTS "${candidate}/graalhost/libc.so" + AND EXISTS "${candidate}/graalhost/libbinsweep.so") + set(${out_var} "${candidate}" PARENT_SCOPE) + return() + endif() + endforeach() + + string(REPLACE ";" "\n" candidate_text "${candidates}") + message(FATAL_ERROR + "Could not find GraalOS graalhost, safe libc.so, and libbinsweep.so. Checked:\n${candidate_text}") +endfunction() + +function(_find_graalos_sysroot_lib_dir out_var) + set(candidates) + if(DEFINED ENV{MUSL_TOOLCHAIN} AND NOT "$ENV{MUSL_TOOLCHAIN}" STREQUAL "") + list(APPEND candidates "$ENV{MUSL_TOOLCHAIN}/lib") + endif() + if(DEFINED ENV{GRAALOS_TOOLCHAIN_PATH} AND NOT "$ENV{GRAALOS_TOOLCHAIN_PATH}" STREQUAL "") + list(APPEND candidates "$ENV{GRAALOS_TOOLCHAIN_PATH}/lib") + endif() + if(DEFINED ENV{JAVA_HOME} AND NOT "$ENV{JAVA_HOME}" STREQUAL "") + list(APPEND candidates + "$ENV{JAVA_HOME}/lib/toolchains/musl-swcfi/lib" + "$ENV{JAVA_HOME}/lib/toolchains/sysroot/x86_64-unknown-linux-musl_swcfi/lib") + endif() + file(GLOB graalos_prebuilt_musl_lib_dirs + LIST_DIRECTORIES TRUE + "${GRAALPY_SUITE_PARENT}/graalos/build-tools/prebuilt/graalvm-graalos-*/lib/toolchains/musl-swcfi/lib") + file(GLOB graalos_prebuilt_sysroot_lib_dirs + LIST_DIRECTORIES TRUE + "${GRAALPY_SUITE_PARENT}/graalos/build-tools/prebuilt/graalvm-graalos-*/lib/toolchains/sysroot/x86_64-unknown-linux-musl_swcfi/lib") + list(APPEND candidates + ${graalos_prebuilt_musl_lib_dirs} + ${graalos_prebuilt_sysroot_lib_dirs}) + + set(required_libs + libc++.so + libc++.so.1 + libc++.so.1.0 + libc++abi.so + libc++abi.so.1 + libc++abi.so.1.0 + libunwind.so + libunwind.so.1 + libunwind.so.1.0) + + foreach(candidate IN LISTS candidates) + set(found TRUE) + foreach(lib IN LISTS required_libs) + if(NOT EXISTS "${candidate}/${lib}") + set(found FALSE) + break() + endif() + endforeach() + if(found) + set(${out_var} "${candidate}" PARENT_SCOPE) + return() + endif() + endforeach() + + string(REPLACE ";" "\n" candidate_text "${candidates}") + message(FATAL_ERROR "Could not find required GraalOS musl-swcfi runtime libraries. Checked:\n${candidate_text}") +endfunction() + +function(_copy_executable source target) + configure_file("${source}" "${target}" COPYONLY) + file(CHMOD "${target}" + PERMISSIONS + OWNER_READ OWNER_WRITE OWNER_EXECUTE + GROUP_READ GROUP_EXECUTE + WORLD_READ WORLD_EXECUTE) +endfunction() + +function(_copy_file source target) + configure_file("${source}" "${target}" COPYONLY) +endfunction() + +function(_write_launcher target virtual_executable) + file(READ "${CMAKE_CURRENT_LIST_DIR}/launcher-wrapper.sh" wrapper_content) + string(REPLACE "@GRAALPY_SANDBOX_VIRTUAL_EXECUTABLE@" "${virtual_executable}" wrapper_content "${wrapper_content}") + file(WRITE "${target}" "${wrapper_content}") + file(CHMOD "${target}" + PERMISSIONS + OWNER_READ OWNER_WRITE OWNER_EXECUTE + GROUP_READ GROUP_EXECUTE + WORLD_READ WORLD_EXECUTE) +endfunction() + +_write_launcher("${PAYLOAD_DIR}/bin/${GRAALPY_LAUNCHER}" "/bin/graalpy") +_write_launcher("${PAYLOAD_DIR}/bin/${PYTHON_LAUNCHER}" "/bin/python") +_write_launcher("${PAYLOAD_DIR}/bin/${PYTHON3_LAUNCHER}" "/bin/python3") +_write_launcher("${PAYLOAD_DIR}/bin/${GRAALPY_CONFIG_LAUNCHER}" "/bin/graalpy-config") +_write_launcher("${PAYLOAD_DIR}/libexec/${GRAALPY_POLYGLOT_GET_LAUNCHER}" "/libexec/graalpy-polyglot-get") + +_copy_file("${CMAKE_CURRENT_LIST_DIR}/config.json" "${PAYLOAD_DIR}/config.json") +_copy_executable("${CMAKE_CURRENT_LIST_DIR}/graalpy-sandbox-launcher.sh" "${GRAALOS_DIR}/graalpy-sandbox-launcher") +_copy_executable( + "${CMAKE_CURRENT_LIST_DIR}/graalpy-sandbox-expand-config.sh" + "${GRAALOS_DIR}/graalpy-sandbox-expand-config") +_copy_executable("${CMAKE_CURRENT_LIST_DIR}/graalpy-sandbox-fsmappings.sh" "${GRAALOS_DIR}/graalpy-sandbox-fsmappings") + +_find_graalos_runtime(graalos_runtime) +_copy_executable("${graalos_runtime}/graalhost/graalhost" "${GRAALOS_DIR}/graalhost") +_copy_file("${graalos_runtime}/graalhost/libc.so" "${GRAALOS_DIR}/libc.so") +_copy_file("${graalos_runtime}/graalhost/libbinsweep.so" "${GRAALOS_DIR}/libbinsweep.so") + +_find_graalos_sysroot_lib_dir(graalos_sysroot_lib_dir) +foreach(lib + libc++.so + libc++.so.1 + libc++.so.1.0 + libc++abi.so + libc++abi.so.1 + libc++abi.so.1.0 + libunwind.so + libunwind.so.1 + libunwind.so.1.0) + file(COPY "${graalos_sysroot_lib_dir}/${lib}" DESTINATION "${SYSROOT_LIB_DIR}") +endforeach() diff --git a/graalpython/graalpy_graalos_standalone_payload/config.json b/graalpython/graalpy_graalos_standalone_payload/config.json new file mode 100644 index 0000000000..20940f1998 --- /dev/null +++ b/graalpython/graalpy_graalos_standalone_payload/config.json @@ -0,0 +1,16 @@ +{ + "env": { + "HOME": "/home", + "PATH": "/bin", + "PYTHONHOME": "/" + }, + "working_dir": "/", + "testing_default_mappings": true, + "allowed_ports": [], + "graalhost": { + "seccomp": null, + "log_level": null, + "extra_args": [] + }, + "fsmappings": [] +} diff --git a/graalpython/graalpy_graalos_standalone_payload/graalpy-sandbox-expand-config.sh b/graalpython/graalpy_graalos_standalone_payload/graalpy-sandbox-expand-config.sh new file mode 100644 index 0000000000..a8e79d6eeb --- /dev/null +++ b/graalpython/graalpy_graalos_standalone_payload/graalpy-sandbox-expand-config.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +set -euo pipefail + +if [ "$#" -ne 3 ]; then + echo "usage: $0 STANDALONE_HOME CONFIG_JSON OUTFILE" >&2 + exit 2 +fi + +standalone_home="$(cd "$1" && pwd -P)" +config_json="$2" +outfile="$3" +helper_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +. "${helper_dir}/graalpy-sandbox-fsmappings" + +if [ ! -d "$standalone_home" ]; then + echo "missing standalone home: $standalone_home" >&2 + exit 126 +fi + +if [ ! -f "$config_json" ]; then + echo "missing sandbox config: $config_json" >&2 + exit 126 +fi + +tmp_prefix="${outfile}.tmp" +trap 'rm -f "${tmp_prefix}.head" "${tmp_prefix}.tail"' EXIT + +awk ' + BEGIN { skipping = 0; skipping_graalhost = 0; replaced = 0 } + /^[[:space:]]*"graalhost"[[:space:]]*:/ { + skipping_graalhost = 1 + if ($0 ~ /\}[[:space:]]*,?[[:space:]]*$/) { + skipping_graalhost = 0 + } + next + } + skipping_graalhost { + if ($0 ~ /^[[:space:]]*\}[[:space:]]*,?[[:space:]]*$/) { + skipping_graalhost = 0 + } + next + } + /^[[:space:]]*"fsmappings"[[:space:]]*:/ { + print " \"fsmappings\": [" + print "@@GRAALPY_SANDBOX_FSMAPPINGS@@" + skipping = 1 + replaced = 1 + if ($0 ~ /\]/) { + skipping = 0 + } + next + } + skipping { + if ($0 ~ /\]/) { + skipping = 0 + } + next + } + { print } + END { + if (!replaced) { + exit 42 + } + } +' "$config_json" > "${tmp_prefix}.head" || { + status=$? + if [ "$status" -eq 42 ]; then + echo "sandbox config must contain a top-level fsmappings array" >&2 + fi + exit "$status" +} + +before_marker="${tmp_prefix}.head" +sed -n '1,/@@GRAALPY_SANDBOX_FSMAPPINGS@@/p' "$before_marker" | sed '$d' > "$outfile" + +graalpy_sandbox_emit_fsmappings "$standalone_home" "$outfile" + +printf '\n ]\n' >> "$outfile" +sed -n '/@@GRAALPY_SANDBOX_FSMAPPINGS@@/,$p' "$before_marker" | sed '1d' >> "$outfile" diff --git a/graalpython/graalpy_graalos_standalone_payload/graalpy-sandbox-fsmappings.sh b/graalpython/graalpy_graalos_standalone_payload/graalpy-sandbox-fsmappings.sh new file mode 100644 index 0000000000..a418f208f1 --- /dev/null +++ b/graalpython/graalpy_graalos_standalone_payload/graalpy-sandbox-fsmappings.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +json_escape() { + printf '%s' "$1" | sed 's/\\/\\\\/g; s/"/\\"/g' +} + +emit_mapping() { + local outfile="$1" + local concrete="$2" + local virt="$3" + local extra="${4:-}" + if [ "$need_comma" = "true" ]; then + printf ',\n' >> "$outfile" + fi + need_comma=true + { + printf ' {\n' + printf ' "concrete": "%s",\n' "$(json_escape "$concrete")" + if [ -n "$extra" ]; then + printf '%s\n' "$extra" + fi + printf ' "virt": "%s"\n' "$(json_escape "$virt")" + printf ' }' + } >> "$outfile" +} + +emit_pseudo_mapping() { + local outfile="$1" + local concrete="$2" + local virt="$3" + local mutable="${4:-false}" + local extra + extra=' "using": {"handler": "pseudo_fs"},' + if [ "$mutable" = "true" ]; then + extra="${extra}"$'\n'' "mutable": true,' + fi + emit_mapping "$outfile" "$concrete" "$virt" "$extra" +} + +graalpy_sandbox_emit_fsmappings() { + local standalone_home="$1" + local outfile="$2" + local sysroot_lib="${GRAALPY_SANDBOX_SYSROOT_LIB_DIR:-${standalone_home}/lib/graalos/sysroot/lib}" + local safe_libc="${GRAALPY_SANDBOX_SAFE_LIBC:-${standalone_home}/lib/graalos/libc.so}" + local native_bin="${GRAALPY_SANDBOX_NATIVE_BIN_DIR:-${standalone_home}/lib/graalos/native-bin}" + + if [ ! -d "$standalone_home" ]; then + echo "missing standalone home: $standalone_home" >&2 + return 126 + fi + + if [ ! -f "$safe_libc" ]; then + echo "missing GraalHost safe libc: $safe_libc" >&2 + return 126 + fi + + need_comma=false + emit_mapping "$outfile" "$standalone_home" "/" ' "using": {"handler": "host_fs"}, + "mutable": true, + "allow_set_x_bit": true, + "verif": false,' + emit_pseudo_mapping "$outfile" "/etc/passwd" "/etc/passwd" + emit_pseudo_mapping "$outfile" "/proc" "/proc" + emit_mapping "$outfile" "/proc/sys/vm/max_map_count" "/proc/sys/vm/max_map_count" ' "verif": true,' + emit_pseudo_mapping "$outfile" "/dev" "/dev" true + + if [ -d "$native_bin" ]; then + while IFS= read -r file; do + virt="/bin/${file#"$native_bin"/}" + emit_mapping "$outfile" "$file" "$virt" ' "verif": true,' + done < <(find "$native_bin" -maxdepth 1 -type f -perm -111 | sort) + else + while IFS= read -r file; do + virt="/${file#"$standalone_home"/}" + emit_mapping "$outfile" "$file" "$virt" ' "verif": true,' + done < <(find "${standalone_home}/bin" -maxdepth 1 -type f -perm -111 | sort) + fi + + while IFS= read -r file; do + virt="/${file#"$standalone_home"/}" + emit_mapping "$outfile" "$file" "$virt" ' "verif": true,' + done < <(find "$standalone_home" -type f \( -name "*.so" -o -name "*.so.*" \) ! -path "${standalone_home}/lib/graalos/*" | sort) + + for lib in libc++.so libc++.so.1 libc++.so.1.0 libc++abi.so libc++abi.so.1 libc++abi.so.1.0 libunwind.so libunwind.so.1 libunwind.so.1.0; do + if [ -e "${sysroot_lib}/${lib}" ]; then + emit_mapping "$outfile" "${sysroot_lib}/${lib}" "/lib/${lib}" ' "verif": true,' + fi + done + + emit_mapping "$outfile" "$safe_libc" "/lib/libc.so" ' "verif": true,' + emit_mapping "$outfile" "$safe_libc" "/lib/ld-musl-x86_64.so.1" ' "verif": true,' +} diff --git a/scripts/generate-fsmappings.sh b/graalpython/graalpy_graalos_standalone_payload/graalpy-sandbox-launcher.sh old mode 100755 new mode 100644 similarity index 50% rename from scripts/generate-fsmappings.sh rename to graalpython/graalpy_graalos_standalone_payload/graalpy-sandbox-launcher.sh index 3299232066..2d30dc5295 --- a/scripts/generate-fsmappings.sh +++ b/graalpython/graalpy_graalos_standalone_payload/graalpy-sandbox-launcher.sh @@ -1,4 +1,5 @@ -# Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved. +#!/usr/bin/env bash +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. # # The Universal Permissive License (UPL), Version 1.0 @@ -37,74 +38,67 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. -#!/bin/bash -if [ -z "$MUSL_TOOLCHAIN" ]; then - echo "MUSL_TOOLCHAIN not set to point to the toolchain path." - exit 1 +set -euo pipefail + +if [ "$#" -lt 1 ]; then + echo "usage: $0 VIRTUAL_EXECUTABLE [ARGS...]" >&2 + exit 2 +fi + +virtual_executable="$1" +shift + +script="${BASH_SOURCE[0]}" +while [ -L "$script" ]; do + script_dir="$(cd "$(dirname "$script")" && pwd -P)" + script="$(readlink "$script")" + case "$script" in + /*) ;; + *) script="${script_dir}/${script}" ;; + esac +done + +script_dir="$(cd "$(dirname "$script")" && pwd -P)" +standalone_home="$(cd "${script_dir}/../.." && pwd -P)" +graalhost="${standalone_home}/lib/graalos/graalhost" +libc="${standalone_home}/lib/graalos/libc.so" +expand_config="${standalone_home}/lib/graalos/graalpy-sandbox-expand-config" +config="${standalone_home}/config.json" + +if [ ! -x "$graalhost" ]; then + echo "missing or non-executable GraalHost binary: $graalhost" >&2 + exit 126 +fi + +if [ ! -f "$libc" ]; then + echo "missing GraalHost libc: $libc" >&2 + exit 126 fi -if [ $# -ne 1 ]; then - echo "Usage: $0 [TARGET_CONFIG_FILE]" - exit 1 +if [ ! -x "$expand_config" ]; then + echo "missing or non-executable sandbox config expander: $expand_config" >&2 + exit 126 fi -PARENT_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}" | xargs dirname)") -OUTFILE=$(readlink -f "$1") +if [ ! -f "$config" ]; then + echo "missing sandbox config: $config" >&2 + exit 126 +fi -cat < $OUTFILE -{ - "allowed_ports": [], - "env": {}, - "working_dir": "/proc", - "testing_default_mappings": true, - "fsmappings": [ - { - "concrete": "${PARENT_DIR}/mxbuild/linux-amd64/GRAALPY_NATIVE_STANDALONE", - "mutable": false, - "verif": false, - "virt": "/proc" - }, - { - "concrete": "${PARENT_DIR}/mxbuild/linux-amd64/GRAALPY_NATIVE_STANDALONE/bin", - "mutable": false, - "verif": false, - "virt": "/proc/self" - }, - { - "concrete": "${PARENT_DIR}/mxbuild/linux-amd64/GRAALPY_NATIVE_STANDALONE/bin/graalpy", - "verif": true, - "virt": "/proc/self/exe" - }, -EOF +tmpdir="$(mktemp -d "${TMPDIR:-/tmp}/graalpy-sandbox.XXXXXXXXXX")" +trap 'rm -rf "$tmpdir"' EXIT +endpoint_config="${tmpdir}/config.json" +"$expand_config" "$standalone_home" "$config" "$endpoint_config" -pushd "${PARENT_DIR}/mxbuild/linux-amd64/GRAALPY_NATIVE_STANDALONE/" >/dev/null -for i in `find . -name "*.so"`; do - cat <> $OUTFILE - { - "concrete": "${PARENT_DIR}/mxbuild/linux-amd64/GRAALPY_NATIVE_STANDALONE/$i", - "verif": true, - "virt": "/proc/$i" - }, -EOF -done -popd >/dev/null +graalhost_args=() +log_level="$(sed -n 's/^[[:space:]]*"log_level"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$config" | head -n 1)" +if [ -n "$log_level" ]; then + graalhost_args+=(--log_level "$log_level") +fi -cat <> $OUTFILE - { - "concrete": "${MUSL_TOOLCHAIN}/lib/libc++.so.1.0", - "verif": true, - "virt": "/lib/libc++.so.1.0" - }, - { - "concrete": "${MUSL_TOOLCHAIN}/lib/libc++abi.so.1.0", - "verif": true, - "virt": "/lib/libc++abi.so.1.0" - }, - { - "concrete": "${MUSL_TOOLCHAIN}/lib/libunwind.so.1.0", - "verif": true, - "virt": "/lib/libunwind.so.1.0" - } - ] -} -EOF +exec "$graalhost" \ + ${graalhost_args[@]+"${graalhost_args[@]}"} \ + --musl_path "$libc" \ + --run_config=@"$endpoint_config" \ + --run_virtual "$virtual_executable" \ + "$@" diff --git a/graalpython/graalpy_graalos_standalone_payload/launcher-wrapper.sh b/graalpython/graalpy_graalos_standalone_payload/launcher-wrapper.sh new file mode 100644 index 0000000000..d73eab066c --- /dev/null +++ b/graalpython/graalpy_graalos_standalone_payload/launcher-wrapper.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +set -euo pipefail + +script="${BASH_SOURCE[0]}" +while [ -L "$script" ]; do + script_dir="$(cd "$(dirname "$script")" && pwd -P)" + script="$(readlink "$script")" + case "$script" in + /*) ;; + *) script="${script_dir}/${script}" ;; + esac +done + +script_dir="$(cd "$(dirname "$script")" && pwd -P)" +standalone_home="$(cd "${script_dir}/.." && pwd -P)" + +exec "${standalone_home}/lib/graalos/graalpy-sandbox-launcher" "@GRAALPY_SANDBOX_VIRTUAL_EXECUTABLE@" "$@" diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py index a46adbaa6b..c173468159 100644 --- a/mx.graalpython/mx_graalpython.py +++ b/mx.graalpython/mx_graalpython.py @@ -30,6 +30,7 @@ import glob import gzip import itertools +import json import os import pathlib import re @@ -62,6 +63,7 @@ import mx_subst import mx_truffle import mx_graalpython_bisect +import mx_graalpython_graalos import mx_graalpython_import import mx_pominit import mx_graalpython_python_benchmarks @@ -1064,6 +1066,7 @@ class GraalPythonTags(object): unittest_standalone = 'python-unittest-standalone' tagged = 'python-tagged-unittest' svmbuild = 'python-svm-build' + svm_graalos_standalone_build = 'python-svm-graalos-standalone-build' svmunit = 'python-svm-unittest' svmunit_sandboxed = 'python-svm-unittest-sandboxed' graalvm = 'python-graalvm' @@ -2013,6 +2016,10 @@ def graalpython_gate_runner(_, tasks): if task: graalpy_standalone_native() + with Task('GraalPython GraalOS standalone build on SVM', tasks, tags=[GraalPythonTags.svm_graalos_standalone_build]) as task: + if task: + mx_graalpython_graalos.graalpy_graalos_standalone_build_and_test(report=report()) + with Task('GraalPython tests on SVM', tasks, tags=[GraalPythonTags.svmunit]) as task: if task: run_python_unittests(graalpy_standalone_native(), parallel=8, report=report()) @@ -2347,17 +2354,12 @@ def graalpy_cmake_build_type(*_): return 'Debug' if 'GRAALPY_NATIVE_DEBUG_BUILD' in os.environ else 'Release' -def _graalpy_sysconfig_platform(os): +def _graalpy_sysconfig_platform(): + os = mx_subst.path_substitutions.substitute('') if os == 'darwin': return 'darwin' if os == 'windows': return 'win32' - if _is_graalos_build(): - libc = _libc() - if 'swcfi' in libc: - return 'graalos' - elif 'hwcfi' in libc: - return 'graalos_hwcfi' return 'linux' @@ -2371,27 +2373,33 @@ def _graalpy_sysconfig_arch(): def graalpy_soabi(*_): - os = mx_subst.path_substitutions.substitute('') - pyos = _graalpy_sysconfig_platform(os) + pyos = _graalpy_sysconfig_platform() return f'{abi_version()}{graalpy_abiflags()}-native-{graalpy_multiarch(os=pyos)}' def graalpy_ext(*_): - os = mx_subst.path_substitutions.substitute('') + os = _graalpy_sysconfig_platform() # on Windows we use '.pyd' else '.so' but never '.dylib' (similar to CPython): # https://github.com/python/cpython/issues/37510 - ext = 'pyd' if os == 'windows' else 'so' + ext = 'pyd' if os == 'win32' else 'so' return f'.{graalpy_soabi()}.{ext}' def graalpy_sysconfigdata(*_): - os = mx_subst.path_substitutions.substitute('') - pyos = _graalpy_sysconfig_platform(os) + pyos = _graalpy_sysconfig_platform() return f'_sysconfigdata_{graalpy_abiflags()}_{pyos}_{graalpy_multiarch(os=pyos)}' def graalpy_multiarch(*_, os=None): - pyos = os if os is not None else _graalpy_sysconfig_platform(mx_subst.path_substitutions.substitute('')) + pyos = os if os is not None else _graalpy_sysconfig_platform() + if pyos == 'linux' and _is_graalos_build(): + libc = _libc() + if 'swcfi' in libc: + pyos = 'graalos' + elif 'hwcfi' in libc: + pyos = 'graalos_hwcfi' + elif 'nocfi' in libc: + pyos = 'graalos_nocfi' return f'{_graalpy_sysconfig_arch()}-{pyos}' @@ -3498,7 +3506,6 @@ def run_downstream_test(args): def _get_github_unittest_tag_pr_commits(): import urllib - import json params = { 'q': "repo:oracle/graalpython is:pr in:title Weekly Retagger: Update tags", diff --git a/mx.graalpython/mx_graalpython_graalos.py b/mx.graalpython/mx_graalpython_graalos.py new file mode 100644 index 0000000000..85c0570e65 --- /dev/null +++ b/mx.graalpython/mx_graalpython_graalos.py @@ -0,0 +1,208 @@ +# Copyright (c) 2026, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# The Universal Permissive License (UPL), Version 1.0 +# +# Subject to the condition set forth below, permission is hereby granted to any +# person obtaining a copy of this software, associated documentation and/or +# data (collectively the "Software"), free of charge and under any and all +# copyright rights in the Software, and any and all patent rights owned or +# freely licensable by each licensor hereunder covering either (i) the +# unmodified Software as contributed to or provided by such licensor, or (ii) +# the Larger Works (as defined below), to deal in both +# +# (a) the Software, and +# +# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if +# one is included with the Software each a "Larger Work" to which the Software +# is contributed by such licensors), +# +# without restriction, including without limitation the rights to copy, create +# derivative works of, display, perform, and distribute the Software and make, +# use, sell, offer for sale, import, export, have made, and have sold the +# Software and the Larger Work(s), and to sublicense the foregoing rights on +# either these or other terms. +# +# This license is subject to the following condition: +# +# The above copyright notice and either this complete permission notice or at a +# minimum a reference to the UPL must be included in all copies or substantial +# portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from __future__ import annotations + +# pylint: disable=cyclic-import + +import base64 +import gzip +import json +import os +import shutil +import sys +import tarfile +import urllib.parse +import urllib.request + +import mx +import mx_urlrewrites +import mx_util + + +SUITE = mx.suite('graalpython') + + +def run(*args, **kwargs): + from mx_graalpython import run + return run(*args, **kwargs) + + +def _download_graalos_standalone_artifact(source, target): + source = mx_urlrewrites.rewriteurl(source) + if "artifact/latest" in source: + artifact_base_url = os.environ.get("GRAALPY_GRAALOS_ARTIFACT_BASE_URL") + if not artifact_base_url: + mx.abort("GRAALPY_GRAALOS_ARTIFACT_BASE_URL must be set to resolve GraalOS artifact metadata") + with urllib.request.urlopen(urllib.request.Request(source, headers={"Accept": "application/json"})) as response: + metadata = json.loads(response.read().decode("utf-8")) + artifact_name = metadata.get("artifactName") + if not artifact_name: + mx.abort(f"GraalOS artifact metadata does not contain artifactName: {source}") + if script := os.environ.get("ARTIFACT_DOWNLOAD_SCRIPT"): + run([sys.executable, script, artifact_name, target]) + return + source = urllib.parse.urljoin(artifact_base_url, artifact_name) + + mx.log(f"Downloading {source} to {target}") + with urllib.request.urlopen(source) as response, open(target, "wb") as fp: + shutil.copyfileobj(response, fp) + + +def _extract_tarball(tarball, destination, strip_components=0): + if os.path.isdir(destination): + shutil.rmtree(destination) + mx_util.ensure_dir_exists(destination) + if strip_components == 0: + mx.Extractor.create(tarball).extract(destination) + return + if strip_components < 0: + mx.abort(f"strip_components must not be negative: {strip_components}") + + with tarfile.open(tarball) as archive: + for member in archive: + original_name = member.name + stripped_name = "/".join(original_name.split("/")[strip_components:]) + if not stripped_name: + continue + if not mx.Extractor._is_sane_name(stripped_name): # pylint: disable=protected-access + mx.abort(f"Refusing to extract unsafe archive entry after stripping: {original_name}") + member.name = stripped_name + archive.extract(member, destination) + member.name = original_name + os.utime(destination, None) + + +def _find_graalos_runtime_home(runtime_root): + default = os.path.join(runtime_root, "opt", "graalos") + if os.path.isdir(default): + return default + for root, dirs, _ in os.walk(runtime_root): + if os.path.basename(root) == "opt" and "graalos" in dirs: + return os.path.join(root, "graalos") + mx.abort(f"Could not find opt/graalos in extracted GraalOS runtime artifact: {runtime_root}") + + +def _ensure_graalos_runtime_inputs(runtime_home): + graalhost_dir = os.path.join(runtime_home, "graalhost") + required = [ + os.path.join(graalhost_dir, "graalhost"), + os.path.join(graalhost_dir, "libc.so"), + ] + libbinsweep = os.path.join(graalhost_dir, "libbinsweep.so") + if not os.path.exists(libbinsweep): + optional_libbinsweep = os.path.join(graalhost_dir, "optional", "libbinsweep.so.gz") + if os.path.exists(optional_libbinsweep): + with gzip.open(optional_libbinsweep, "rb") as src, open(libbinsweep, "wb") as dst: + shutil.copyfileobj(src, dst) + required.append(libbinsweep) + missing = [path for path in required if not os.path.exists(path)] + if missing: + mx.abort("Extracted GraalOS runtime artifact is missing required files:\n" + "\n".join(missing)) + + +def graalpy_graalos_standalone_build_and_test(report=None): + del report # This gate executes an in-sandbox smoke test directly instead of using the source-tree test runner. + toolchain_url = os.environ.get("GRAALPY_GRAALOS_TOOLCHAIN_URL") + runtime_url = os.environ.get("GRAALPY_GRAALOS_RUNTIME_URL") + if not toolchain_url or not runtime_url: + mx.log("Skipping GRAALPY_NATIVE_GRAALOS_STANDALONE build: GraalOS artifact URLs are not configured") + return + + work_dir = os.path.join(SUITE.dir, "mxbuild", "graalos-standalone-ci") + if os.path.isdir(work_dir): + shutil.rmtree(work_dir) + mx_util.ensure_dir_exists(work_dir) + + graalvm_tarball = os.path.join(work_dir, "graalvm.tar.gz") + runtime_tarball = os.path.join(work_dir, "graalos-runtime.tar.gz") + graalvm_home = os.path.join(work_dir, "graalvm") + runtime_root = os.path.join(work_dir, "runtime") + + _download_graalos_standalone_artifact(toolchain_url, graalvm_tarball) + _extract_tarball(graalvm_tarball, graalvm_home, strip_components=1) + musl_toolchain = os.path.join(graalvm_home, "lib", "toolchains", "musl-swcfi") + if not os.path.exists(os.path.join(graalvm_home, "bin", mx.exe_suffix("java"))): + mx.abort(f"Extracted GraalOS toolchain artifact does not contain bin/java: {graalvm_home}") + if not os.path.isdir(musl_toolchain): + mx.abort(f"Extracted GraalOS toolchain artifact does not contain musl-swcfi toolchain: {musl_toolchain}") + + _download_graalos_standalone_artifact(runtime_url, runtime_tarball) + _extract_tarball(runtime_tarball, runtime_root) + graalos_runtime_home = _find_graalos_runtime_home(runtime_root) + _ensure_graalos_runtime_inputs(graalos_runtime_home) + + from mx_graalpython import extend_os_env, run_mx, _graalpy_launcher + env = extend_os_env( + JAVA_HOME=graalvm_home, + MUSL_TOOLCHAIN=musl_toolchain, + GRAALOS_TOOLCHAIN_PATH=musl_toolchain, + GRAALOS_RUNTIME_HOME=graalos_runtime_home, + NATIVE_IMAGE_EXPERIMENTAL_OPTIONS_ARE_FATAL="false", + ) + run_mx([ + "--multitarget=linux-amd64-musl-swcfi", + "build", + "--target", "GRAALPY_NATIVE_GRAALOS_STANDALONE", + ], env=env) + + standalone_home = os.path.join(SUITE.dir, "mxbuild", "linux-amd64", "GRAALPY_NATIVE_GRAALOS_STANDALONE") + launcher = os.path.join(standalone_home, "bin", _graalpy_launcher()) + if not os.path.exists(launcher): + mx.abort(f"GRAALPY_NATIVE_GRAALOS_STANDALONE launcher was not built: {launcher}") + + test_path = os.path.join( + SUITE.dir, + "graalpython", + "com.oracle.graal.python.test", + "src", + "tests", + "test_graalos_standalone.py", + ) + with open(test_path, "r", encoding="utf-8") as f: + smoke_test = f.read() + smoke_test += """ +try: + test_graalos_sqlite3_native_extension_smoke() +except unittest.SkipTest as e: + print(f"skipped: {e}") +""" + smoke_test_arg = base64.b64encode(smoke_test.encode("utf-8")).decode("ascii") + smoke_test_command = f"import base64; exec(base64.b64decode({smoke_test_arg!r}).decode('utf-8'))" + run([launcher, "-c", smoke_test_command], env=env) diff --git a/mx.graalpython/suite.py b/mx.graalpython/suite.py index 9baacc20a2..7fb7dba95d 100644 --- a/mx.graalpython/suite.py +++ b/mx.graalpython/suite.py @@ -647,11 +647,18 @@ "graalpy-versions": { "subDir": "graalpython", "class": "CMakeNinjaProject", + "multitarget": [ + {"libc": ["glibc", "default"]}, + {"libc": ["musl"], "variant": ["swcfi"]}, + ], "max_jobs": "1", "ninja_targets": ["all"], "cmakeConfig": { "GRAALPY_VER": "", "GRAALPY_ABIFLAGS": "", + "GRAALPY_EXT_SUFFIX": "", + "GRAALPY_MULTIARCH": "", + "GRAALPY_SOABI": "", }, "results": [ "graalpy_versions" @@ -871,6 +878,45 @@ "community_3rd_party_license_file": "THIRD_PARTY_LICENSE.txt", }, + "graalpy_graalos_standalone_payload": { + "subDir": "graalpython", + "class": "CMakeNinjaProject", + "defaultBuild": False, + "max_jobs": "1", + "ninja_targets": ["all"], + "results": [ + "bin/", + "bin/", + "bin/", + "bin/", + "libexec/", + "config.json", + "lib/graalos/graalpy-sandbox-launcher", + "lib/graalos/graalpy-sandbox-expand-config", + "lib/graalos/graalpy-sandbox-fsmappings", + "lib/graalos/graalhost", + "lib/graalos/libc.so", + "lib/graalos/libbinsweep.so", + "lib/graalos/sysroot/lib/libc++.so", + "lib/graalos/sysroot/lib/libc++.so.1", + "lib/graalos/sysroot/lib/libc++.so.1.0", + "lib/graalos/sysroot/lib/libc++abi.so", + "lib/graalos/sysroot/lib/libc++abi.so.1", + "lib/graalos/sysroot/lib/libc++abi.so.1.0", + "lib/graalos/sysroot/lib/libunwind.so", + "lib/graalos/sysroot/lib/libunwind.so.1", + "lib/graalos/sysroot/lib/libunwind.so.1.0", + ], + "cmakeConfig": { + "GRAALPY_LAUNCHER": "", + "PYTHON_LAUNCHER": "", + "PYTHON3_LAUNCHER": "", + "GRAALPY_CONFIG_LAUNCHER": "", + "GRAALPY_POLYGLOT_GET_LAUNCHER": "", + "GRAALPY_SUITE_PARENT": "", + }, + }, + "graalpy_thin_launcher": { "class": "ThinLauncherProject", "mainClass": "com.oracle.graal.python.shell.GraalPythonMain", @@ -950,13 +996,13 @@ "GRAALPYTHON_VERSIONS_RES": { "type": "dir", "layout": { - "./": "dependency:graalpy-versions/graalpy_versions", + "./": "dependency:graalpy-versions/-//graalpy_versions", }, }, "GRAALPYTHON_VERSIONS_MAIN": { "type": "dir", "layout": { - "./": "dependency:graalpy-versions/graalpy_versions", + "./": "dependency:graalpy-versions/-//graalpy_versions", }, }, @@ -1510,6 +1556,19 @@ }, }, + "GRAALPY_GRAALOS_STANDALONE_PAYLOAD": { + "description": "GraalOS runtime and generated support files for the GraalPy GraalOS standalone", + "type": "dir", + "defaultBuild": False, + "platformDependent": True, + "platforms": "local", + "layout": { + "./": [ + "dependency:graalpy_graalos_standalone_payload/*", + ], + }, + }, + "GRAALPY_NATIVE_STANDALONE": { "description": "GraalPy Native standalone", "type": "dir", @@ -1523,6 +1582,21 @@ }, }, + "GRAALPY_NATIVE_GRAALOS_STANDALONE": { + "description": "GraalPy GraalOS standalone", + "type": "dir", + "defaultBuild": False, + "platformDependent": True, + "platforms": "local", + "layout": { + "./": [ + "dependency:GRAALPY_NATIVE_STANDALONE/*", + "dependency:GRAALPY_GRAALOS_STANDALONE_PAYLOAD/*", + ], + "lib/graalos/native-bin/": "dependency:GRAALPY_NATIVE_STANDALONE/bin/*", + }, + }, + "GRAALPY_JVM_STANDALONE": { "description": "GraalPy JVM standalone", "type": "dir", @@ -1730,5 +1804,16 @@ "enterprise_archive_name": "graalpy-jvm", "language_id": "python", }, + + "GRAALPY_GRAALOS_STANDALONE_RELEASE_ARCHIVE": { + "class": "DeliverableStandaloneArchive", + "deploy": False, + "defaultBuild": False, + "platformDependent": True, + "standalone_dist": "GRAALPY_NATIVE_GRAALOS_STANDALONE", + "community_archive_name": "graalpy-community-graalos", + "enterprise_archive_name": "graalpy-graalos", + "language_id": "python", + }, }, }