diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_indirect_call.py b/graalpython/com.oracle.graal.python.test/src/tests/test_indirect_call.py index 052067ec3d..55d21382d5 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_indirect_call.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_indirect_call.py @@ -39,7 +39,6 @@ import sys import os -from tests.cpyext import CPyExtTestCase, CPyExtType from tests.util import has_capi, needs_capi # synchronize with Java implementation of __graalpython__.indirect_call_tester @@ -72,6 +71,8 @@ def was_stack_walk(new_value): if has_capi(): + from tests.cpyext import CPyExtType + IndirectCApiCallTester = CPyExtType( 'IndirectCApiCallTester', code=''' diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_startup.py b/graalpython/com.oracle.graal.python.test/src/tests/test_startup.py index c98e7057fc..d3dbbff6d3 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_startup.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_startup.py @@ -45,17 +45,22 @@ # Both lists should remain as small as possible to avoid adding overhead to startup IS_WINDOWS = platform.system() == 'Windows' -WINDOWS_CORE_MODULES = ['_nt', '_winapi', '_overlapped', 'winreg', '_winreg'] if IS_WINDOWS else [] -WINDOWS_FULL_STARTUP_MODULES = [ - '_datetime', - 'datetime', - '_ctypes', - '_struct', - 'struct', - 'ctypes._endian', - 'ctypes', - 'ctypes.wintypes', -] if IS_WINDOWS else [] + +if IS_WINDOWS and sys.implementation.name == 'graalpy' and __graalpython__.native_access_is_available(): + WINDOWS_CORE_MODULES = ['_nt', '_winapi', '_overlapped', 'winreg', '_winreg'] + WINDOWS_FULL_STARTUP_MODULES = [ + '_datetime', + 'datetime', + '_ctypes', + '_struct', + 'struct', + 'ctypes._endian', + 'ctypes', + 'ctypes.wintypes', + ] +else: + WINDOWS_CORE_MODULES = [] + WINDOWS_FULL_STARTUP_MODULES = ['_winapi'] if IS_WINDOWS else [] expected_nosite_startup_modules = [ '_frozen_importlib', @@ -90,11 +95,11 @@ def test_startup_nosite(self): result = subprocess.check_output([sys.executable, '--log.level=FINE', '-S', '-v', '-c', 'print("Hello")'], stderr=subprocess.STDOUT, text=True) assert 'Hello' in result imports = re.findall(r"import '(\S+)'", result) - self.assertEqual(expected_nosite_startup_modules, imports) + self.assertEqual(sorted(expected_nosite_startup_modules), sorted(imports)) @unittest.skipUnless(sys.implementation.name == 'graalpy', "GraalPy-specific test") def test_startup_full(self): result = subprocess.check_output([sys.executable, '--log.level=FINE', '-s', '-v', '-c', 'print("Hello")'], stderr=subprocess.STDOUT, text=True) assert 'Hello' in result imports = re.findall(r"import '(\S+)'", result) - self.assertEqual(expected_full_startup_modules, imports) + self.assertEqual(sorted(expected_full_startup_modules), sorted(imports)) diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_sys_settrace.py b/graalpython/com.oracle.graal.python.test/src/tests/test_sys_settrace.py index a4bb8236e7..3c9ab0d7ef 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_sys_settrace.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_sys_settrace.py @@ -43,7 +43,6 @@ import signal from tests import util import builtins -import asyncio GRAALPY_POSIX_BACKEND_IS_JAVA = ( @@ -520,6 +519,15 @@ def func(): self.assert_events(self.events, events) class AsyncTracingEventsUnitTest(TracingEventsUnitTest): + @staticmethod + def run_awaitable(awaitable): + iterator = awaitable.__await__() + try: + while True: + next(iterator) + except StopIteration as e: + return e.value + def async_trace(self, frame, event, arg): code = frame.f_code name = code.co_name @@ -539,7 +547,7 @@ async def async_wrapper(self, afunc, names): def trace_async_function(self, afunc, names): self.first_line = afunc.__code__.co_firstlineno self.events = [] - asyncio.run(self.async_wrapper(afunc, names)) + self.run_awaitable(self.async_wrapper(afunc, names)) @unittest.skipIf( sys.implementation.name == "graalpy", diff --git a/graalpython/com.oracle.graal.python.test/src/tests/test_wheel.py b/graalpython/com.oracle.graal.python.test/src/tests/test_wheel.py index f27271477a..2d68f5c9b6 100644 --- a/graalpython/com.oracle.graal.python.test/src/tests/test_wheel.py +++ b/graalpython/com.oracle.graal.python.test/src/tests/test_wheel.py @@ -52,6 +52,10 @@ # cannot be moved to tagged as it uses testlib_helper @unittest.skipIf(os.environ.get("GITHUB_CI"), "Skip on Github CI") +@unittest.skipIf( + sys.implementation.name == "graalpy" and not __graalpython__.native_access_is_available(), + "Skipped because importing the repaired wheel requires native access support", +) class TestWheelBuildAndRun(unittest.TestCase): def test_build_install_and_run(self): # Build a C library and a wheel that depends on it. Then run it through repair_wheel to vendor the library in and verify that it can be imported diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java index 72124281d5..64a0c471f3 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/Python3Core.java @@ -1051,12 +1051,14 @@ private void initializePython3Core(TruffleString coreHome) { } private void initializeWindowsCoreFiles(TruffleString coreHome) { - assert !ImageInfo.inImageBuildtimeCode(); - loadFile(toTruffleStringUncached("_nt"), coreHome); - loadFile(toTruffleStringUncached("_winapi"), toTruffleStringUncached("modules/_winapi"), coreHome); - loadFile(toTruffleStringUncached("_overlapped"), toTruffleStringUncached("modules/_overlapped"), coreHome); - loadFile(toTruffleStringUncached("winreg"), toTruffleStringUncached("modules/winreg"), coreHome); - loadFile(toTruffleStringUncached("_winreg"), toTruffleStringUncached("modules/_winreg"), coreHome); + if (PythonLanguage.getPythonOS() == PythonOS.PLATFORM_WIN32 && getContext().isNativeAccessAllowed()) { + assert !ImageInfo.inImageBuildtimeCode(); + loadFile(toTruffleStringUncached("_nt"), coreHome); + loadFile(toTruffleStringUncached("_winapi"), toTruffleStringUncached("modules/_winapi"), coreHome); + loadFile(toTruffleStringUncached("_overlapped"), toTruffleStringUncached("modules/_overlapped"), coreHome); + loadFile(toTruffleStringUncached("winreg"), toTruffleStringUncached("modules/winreg"), coreHome); + loadFile(toTruffleStringUncached("_winreg"), toTruffleStringUncached("modules/_winreg"), coreHome); + } } /** diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java index 559fea13d7..81bb52d4a2 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java @@ -928,6 +928,16 @@ private static int which() { } } + @Builtin(name = "native_access_is_available", minNumOfPositionalArgs = 0) + @GenerateNodeFactory + public abstract static class NativeAccessNode extends PythonBuiltinNode { + @TruffleBoundary + @Specialization + static boolean isAvailable(@Bind PythonContext context) { + return context.isNativeAccessAllowed(); + } + } + @Builtin(name = "posix_module_backend", minNumOfPositionalArgs = 0) @GenerateNodeFactory public abstract static class PosixModuleBackendNode extends PythonBuiltinNode { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java index 71d80cccf8..3ff2a50c5e 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/SysModuleBuiltins.java @@ -707,6 +707,10 @@ public void postInitialize0(Python3Core core) { sys.setAttribute(T___UNRAISABLEHOOK__, sys.getAttribute(T_UNRAISABLEHOOK)); sys.setAttribute(T___DISPLAYHOOK__, sys.getAttribute(T_DISPLAYHOOK)); sys.setAttribute(T___BREAKPOINTHOOK__, sys.getAttribute(T_BREAKPOINTHOOK)); + + if (!context.isNativeAccessAllowed()) { + sys.setAttribute(tsInternedLiteral("getrefcount"), PNone.NO_VALUE); + } } private static PFrozenSet createStdLibModulesSet(PythonLanguage language) { @@ -1054,8 +1058,7 @@ public abstract static class GetrefcountNode extends PythonUnaryBuiltinNode { static long doGeneric(Object object, @Bind PythonContext context) { if (!context.isNativeAccessAllowed()) { - CompilerDirectives.transferToInterpreterAndInvalidate(); - throw new RuntimeException(ErrorMessages.NATIVE_ACCESS_NOT_ALLOWED.toJavaStringUncached()); + throw CompilerDirectives.shouldNotReachHere(); } if (context.getCApiState() != CApiState.INITIALIZED) { if (object instanceof PythonAbstractObject pythonAbstractObject) { diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/WinapiModuleBuiltins.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/WinapiModuleBuiltins.java index 3e409be157..21b03b1f6b 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/WinapiModuleBuiltins.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/WinapiModuleBuiltins.java @@ -1,5 +1,5 @@ /* - * Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2022, 2026, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * The Universal Permissive License (UPL), Version 1.0 @@ -42,16 +42,62 @@ import java.util.List; +import com.oracle.graal.python.annotations.Builtin; +import com.oracle.graal.python.annotations.PythonOS; import com.oracle.graal.python.builtins.CoreFunctions; +import com.oracle.graal.python.builtins.Python3Core; import com.oracle.graal.python.builtins.PythonBuiltins; -import com.oracle.graal.python.annotations.PythonOS; +import com.oracle.graal.python.builtins.objects.PNone; +import com.oracle.graal.python.builtins.objects.str.StringUtils; import com.oracle.graal.python.nodes.function.PythonBuiltinBaseNode; +import com.oracle.graal.python.nodes.function.PythonBuiltinNode; +import com.oracle.graal.python.runtime.PythonContext; +import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; +import com.oracle.truffle.api.dsl.Bind; +import com.oracle.truffle.api.dsl.GenerateNodeFactory; import com.oracle.truffle.api.dsl.NodeFactory; +import com.oracle.truffle.api.dsl.Specialization; @CoreFunctions(defineModule = "_winapi", os = PythonOS.PLATFORM_WIN32) public final class WinapiModuleBuiltins extends PythonBuiltins { + @Override + public void initialize(Python3Core core) { + super.initialize(core); + addBuiltinConstant("NULL", 0); + addBuiltinConstant("INFINITE", 0xFFFFFFFFL); + } + @Override protected List> getNodeFactories() { - return List.of(); + return WinapiModuleBuiltinsFactory.getFactories(); + } + + // Managed fallback for native kernel32.GetACP call so encodings work sandboxed + @Builtin(name = "GetACP", minNumOfPositionalArgs = 0) + @GenerateNodeFactory + abstract static class GetfullpathnameNode extends PythonBuiltinNode { + @TruffleBoundary + @Specialization + int getacp(@Bind PythonContext context) { + var ts = SysModuleBuiltins.GetFileSystemEncodingNode.getFileSystemEncoding(); + var cp = StringUtils.toLowerCase(ts.toJavaStringUncached()); + if (cp.startsWith("cp")) { + try { + return Integer.valueOf(cp.substring(2)); + } catch (Exception e) { + // pass + } + } + return 0; + } + } + + @Builtin(name = "CloseHandle", minNumOfPositionalArgs = 1) + @GenerateNodeFactory + abstract static class CloseHandleNode extends PythonBuiltinNode { + @Specialization + static Object closeHandle(@SuppressWarnings("unused") Object handle) { + return PNone.NONE; + } } } diff --git a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java index 548eaa1012..69b4a90633 100644 --- a/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java +++ b/graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java @@ -1261,7 +1261,7 @@ public PythonContext(PythonLanguage language, TruffleLanguage.Env env) { this.in = env.in(); this.out = env.out(); this.err = env.err(); - this.nativeAccessAllowed = env.isNativeAccessAllowed() && !PythonOS.isUnsupported(); + this.nativeAccessAllowed = env.isNativeAccessAllowed() && !PythonOS.isUnsupported() && NativeAccessSupport.isAvailable(); } private static final ContextReference REFERENCE = ContextReference.create(PythonLanguage.class); @@ -1439,7 +1439,7 @@ public void setEnv(TruffleLanguage.Env newEnv) { err = env.err(); posixSupport.setEnv(env); optionValues = PythonOptions.createOptionValuesStorage(newEnv); - nativeAccessAllowed = newEnv.isNativeAccessAllowed() && !PythonOS.isUnsupported(); + nativeAccessAllowed = newEnv.isNativeAccessAllowed() && !PythonOS.isUnsupported() && NativeAccessSupport.isAvailable(); } /** diff --git a/graalpython/lib-graalpython/modules/_winapi.py b/graalpython/lib-graalpython/modules/_winapi.py index c5c8faef47..b1a3bb8997 100644 --- a/graalpython/lib-graalpython/modules/_winapi.py +++ b/graalpython/lib-graalpython/modules/_winapi.py @@ -44,6 +44,10 @@ raise ImportError("win32 only") +if not __graalpython__.native_access_is_available(): + raise ImportError("needs native access") + + _POINTER_BITS = 64 diff --git a/graalpython/lib-graalpython/modules/winreg.py b/graalpython/lib-graalpython/modules/winreg.py index b56c55928b..d8bf369297 100644 --- a/graalpython/lib-graalpython/modules/winreg.py +++ b/graalpython/lib-graalpython/modules/winreg.py @@ -44,6 +44,10 @@ raise ImportError("win32 only") +if not __graalpython__.native_access_is_available(): + raise ImportError("needs native access") + + error = OSError ERROR_SUCCESS = 0 diff --git a/graalpython/lib-python/3/test/test_xml_etree.py b/graalpython/lib-python/3/test/test_xml_etree.py index 72bd499aa5..b6f05af2a0 100644 --- a/graalpython/lib-python/3/test/test_xml_etree.py +++ b/graalpython/lib-python/3/test/test_xml_etree.py @@ -2144,6 +2144,8 @@ def test_bug_xmltoolkit62(self): 'A new cultivar of Begonia plant named \u2018BCT9801BEG\u2019.') @unittest.skipIf(sys.gettrace(), "Skips under coverage.") + @unittest.skipUnless(hasattr(sys, 'getrefcount'), + 'test needs sys.getrefcount()') def test_bug_xmltoolkit63(self): # Check reference leak. def xmltoolkit63(): diff --git a/mx.graalpython/mx_graalpython.py b/mx.graalpython/mx_graalpython.py index a46adbaa6b..14ab45529b 100644 --- a/mx.graalpython/mx_graalpython.py +++ b/mx.graalpython/mx_graalpython.py @@ -105,7 +105,7 @@ def get_boolean_env(name, default=False): LATEST_JAVA_HOME = {"JAVA_HOME": os.environ.get("LATEST_JAVA_HOME", mx.get_jdk().home)} RUNNING_ON_LATEST_JAVA = os.environ.get("LATEST_JAVA_HOME", os.environ.get("JAVA_HOME")) == mx.get_jdk().home - +HAS_JEP_454 = mx.get_jdk().version >= mx.VersionSpec("22.0.0") # this environment variable is used by some of our maven projects to build against the unreleased master version during development os.environ["GRAALPY_VERSION"] = GRAAL_VERSION @@ -892,7 +892,7 @@ def get_path_with_patchelf(graalpy=None): subprocess.check_call([sys.executable, "-m", "venv", str(venv)]) subprocess.check_call([str(venv / "bin" / "pip"), "install", "patchelf"]) mx.log(f"{time.strftime('[%H:%M:%S] ')} Building patchelf-venv with {sys.executable}... [duration: {time.time() - t0}]") - if mx.is_windows() and not shutil.which("delvewheel"): + if mx.is_windows() and HAS_JEP_454 and not shutil.which("delvewheel"): venv = Path(SUITE.get_output_root()).absolute() / "delvewheel-venv" path += os.pathsep + str(venv / "Scripts") if not shutil.which("delvewheel", path=path): @@ -976,14 +976,13 @@ def __post_init__(self): # test leaks with Python code only run_leak_launcher(["--code", "pass", ]) run_leak_launcher(["--repeat-and-check-size", "250", "--null-stdout", "--code", "print('hello')"]) - has_jep_454 = mx.get_jdk().version >= mx.VersionSpec("22.0.0") c_api_leak_test = ( 'import _testcapi; ' 't = _testcapi.tuple_pack(2, "a", "b"); ' 'assert _testcapi.tuple_get_item(t, 1) == "b"' ) # test leaks when some C module code is involved - if has_jep_454: + if HAS_JEP_454: run_leak_launcher([ "--forbid-capi-residue", "--code", c_api_leak_test, @@ -992,7 +991,7 @@ def __post_init__(self): run_leak_launcher(["--shared-engine", "--code", "pass"]) run_leak_launcher(["--shared-engine", "--repeat-and-check-size", "250", "--null-stdout", "--code", "print('hello')"]) # test leaks with shared engine when some C module code is involved - if has_jep_454: + if HAS_JEP_454: run_leak_launcher([ "--shared-engine", "--forbid-capi-residue", "--code", c_api_leak_test, @@ -1178,7 +1177,7 @@ def graalpy_standalone_home(standalone_type, enterprise=False, dev=False, build= # Build if standalone_type == 'jvm': - if dev or jdk_version < mx.VersionSpec("22.0.0"): + if dev or not HAS_JEP_454: env_file = 'jvm' else: env_file = 'jvm-ee-libgraal' if enterprise else 'jvm-ce-libgraal' @@ -1683,8 +1682,7 @@ def run_python_unittests(python_binary, args=None, paths=None, exclude=None, env def run_sandboxed_tests(python_binary, report, args=None, **kwargs): args = SANDBOXED_OPTIONS + (args or []) exclude = list(kwargs.pop("exclude", []) or []) - if mx.get_jdk().version < mx.VersionSpec("22.0.0"): - # C API doesn't work on JDK 21 + if not HAS_JEP_454: exclude += [ "cpyext", "test_ctypes", @@ -1835,7 +1833,7 @@ def graalpython_gate_runner(_, tasks): env['PATH'] = get_path_with_patchelf() default_runtime_arg = [] - if mx.get_jdk().version < mx.VersionSpec("22.0.0"): + if not HAS_JEP_454: # Bytecode DSL does not work on JDK21 with optimizing runtime (GR-72424) default_runtime_arg = ['-Dtruffle.UseFallbackRuntime=true'] graalvm_jdk_path = graalvm_jdk()