Add basic support for GraalPy (#1645)

* add basic support for graalpy * Fix cargo build and format python code --------- Co-authored-by: messense <messense@icloud.com>
PyO3 · Jun 4, 2023 · dcbb65f · dcbb65f
1 parent 6fe618b
commit dcbb65f
Show file tree

Hide file tree

Showing 10 changed files with 83 additions and 18 deletions.
diff --git a/README.md b/README.md
@@ -12,7 +12,7 @@ _formerly pyo3-pack_
 Build and publish crates with pyo3, rust-cpython, cffi and uniffi bindings as well as rust binaries as python packages.
 
 This project is meant as a zero configuration replacement for [setuptools-rust](https://github.com/PyO3/setuptools-rust) and [milksnake](https://github.com/getsentry/milksnake).
-It supports building wheels for python 3.5+ on windows, linux, mac and freebsd, can upload them to [pypi](https://pypi.org/) and has basic pypy support.
+It supports building wheels for python 3.5+ on windows, linux, mac and freebsd, can upload them to [pypi](https://pypi.org/) and has basic pypy and graalpy support.
 
 Check out the [User Guide](https://maturin.rs/)!
 

diff --git a/guide/src/bindings.md b/guide/src/bindings.md
@@ -8,7 +8,7 @@ specify which bindings to use.
 
 [pyo3](https://github.com/PyO3/pyo3) is Rust bindings for Python,
 including tools for creating native Python extension modules.
-It supports both CPython and PyPy.
+It supports CPython, PyPy, and GraalPy.
 
 maturin automatically detects pyo3 bindings when it's added as a dependency in `Cargo.toml`.
 

diff --git a/guide/src/platform_support.md b/guide/src/platform_support.md
@@ -36,7 +36,7 @@ supported by [manylinux](https://github.com/pypa/manylinux).
 CPython 3.7 to 3.10 are supported and tested on CI, though the entire 3.x series should work.
 This will be changed as new python versions are released and others have their end of life.
 
-PyPy 3.6 and later also works.
+PyPy 3.6 and later also works, as does GraalPy 23.0 and later.
 
 ## Manylinux/Musllinux
 

diff --git a/src/build_context.rs b/src/build_context.rs
@@ -42,7 +42,7 @@ pub enum BridgeModel {
     /// providing crate, e.g. pyo3, the number is the minimum minor python version
     Bindings(String, usize),
     /// `Bindings`, but specifically for pyo3 with feature flags that allow building a single wheel
-    /// for all cpython versions (pypy still needs multiple versions).
+    /// for all cpython versions (pypy & graalpy still need multiple versions).
     /// The numbers are the minimum major and minor version
     BindingsAbi3(u8, u8),
     /// A native module with c bindings, i.e. `#[no_mangle] extern "C" <some item>`
@@ -234,7 +234,9 @@ impl BuildContext {
                     let interp_names: HashSet<_> = non_abi3_interps
                         .iter()
                         .map(|interp| match interp.interpreter_kind {
-                            InterpreterKind::CPython => interp.implementation_name.to_string(),
+                            InterpreterKind::CPython | InterpreterKind::GraalPy => {
+                                interp.implementation_name.to_string()
+                            }
                             InterpreterKind::PyPy => "PyPy".to_string(),
                         })
                         .collect();

diff --git a/src/build_options.rs b/src/build_options.rs
@@ -269,6 +269,8 @@ impl BuildOptions {
                                     Some(InterpreterKind::PyPy)
                                 } else if tag.starts_with("cpython") {
                                     Some(InterpreterKind::CPython)
+                                } else if tag.starts_with("graalpy") {
+                                    Some(InterpreterKind::GraalPy)
                                 } else {
                                     None
                                 }
@@ -1133,6 +1135,11 @@ fn find_interpreter_in_sysconfig(
         let python = interp.display().to_string();
         let (python_impl, python_ver) = if let Some(ver) = python.strip_prefix("pypy") {
             (InterpreterKind::PyPy, ver.strip_prefix('-').unwrap_or(ver))
+        } else if let Some(ver) = python.strip_prefix("graalpy") {
+            (
+                InterpreterKind::GraalPy,
+                ver.strip_prefix('-').unwrap_or(ver),
+            )
         } else if let Some(ver) = python.strip_prefix("python") {
             (
                 InterpreterKind::CPython,

diff --git a/src/compile.rs b/src/compile.rs
@@ -350,10 +350,10 @@ fn compile_target(
     }
 
     if let BridgeModel::BindingsAbi3(_, _) = bridge_model {
-        let is_pypy = python_interpreter
-            .map(|p| p.interpreter_kind.is_pypy())
+        let is_pypy_or_graalpy = python_interpreter
+            .map(|p| p.interpreter_kind.is_pypy() || p.interpreter_kind.is_graalpy())
             .unwrap_or(false);
-        if !is_pypy && !target.is_windows() {
+        if !is_pypy_or_graalpy && !target.is_windows() {
             let pyo3_ver = pyo3_version(&context.cargo_metadata)
                 .context("Failed to get pyo3 version from cargo metadata")?;
             if pyo3_ver < PYO3_ABI3_NO_PYTHON_VERSION {
@@ -389,7 +389,8 @@ fn compile_target(
         } else if (bridge_model.is_bindings("pyo3")
             || bridge_model.is_bindings("pyo3-ffi")
             || (matches!(bridge_model, BridgeModel::BindingsAbi3(_, _))
-                && interpreter.interpreter_kind.is_pypy()))
+                && (interpreter.interpreter_kind.is_pypy()
+                    || interpreter.interpreter_kind.is_graalpy())))
             && env::var_os("PYO3_CONFIG_FILE").is_none()
         {
             let pyo3_config = interpreter.pyo3_config_file();

diff --git a/src/python_interpreter/config.rs b/src/python_interpreter/config.rs
@@ -9,6 +9,7 @@ use std::io::{BufRead, BufReader};
 use std::path::Path;
 
 const PYPY_ABI_TAG: &str = "pp73";
+const GRAALPY_ABI_TAG: &str = "graalpy230_310_native";
 
 /// Some of the sysconfigdata of Python interpreter we care about
 #[derive(Debug, Clone, Deserialize, Eq, PartialEq)]
@@ -17,7 +18,7 @@ pub struct InterpreterConfig {
     pub major: usize,
     /// Python's minor version
     pub minor: usize,
-    /// cpython or pypy
+    /// cpython, pypy, or graalpy
     #[serde(rename = "interpreter")]
     pub interpreter_kind: InterpreterKind,
     /// For linux and mac, this contains the value of the abiflags, e.g. "m"
@@ -68,7 +69,7 @@ impl InterpreterConfig {
                     target.target_env().to_string().replace("musl", "gnu")
                 }
             }
-            PyPy => "gnu".to_string(),
+            PyPy | GraalPy => "gnu".to_string(),
         };
         match (target.target_os(), python_impl) {
             (Os::Linux, CPython) => {
@@ -316,6 +317,7 @@ impl InterpreterConfig {
                 }
             }
             InterpreterKind::PyPy => abi_tag.unwrap_or_else(|| PYPY_ABI_TAG.to_string()),
+            InterpreterKind::GraalPy => abi_tag.unwrap_or_else(|| GRAALPY_ABI_TAG.to_string()),
         };
         let file_ext = if target.is_windows() { "pyd" } else { "so" };
         let ext_suffix = if target.is_linux() || target.is_macos() {
@@ -350,6 +352,16 @@ impl InterpreterConfig {
                         file_ext,
                     )
                 }),
+                InterpreterKind::GraalPy => ext_suffix.unwrap_or_else(|| {
+                    // e.g. .graalpy230-310-native-x86_64-linux.so
+                    format!(
+                        ".{}-{}-{}.{}",
+                        abi_tag.replace('_', "-"),
+                        target.get_python_arch(),
+                        target.get_python_os(),
+                        file_ext,
+                    )
+                }),
             }
         } else if target.is_emscripten() && matches!(interpreter_kind, InterpreterKind::CPython) {
             ext_suffix.unwrap_or_else(|| {

diff --git a/src/python_interpreter/get_interpreter_metadata.py b/src/python_interpreter/get_interpreter_metadata.py
@@ -19,6 +19,23 @@
 else:
     ext_suffix = sysconfig.get_config_var("EXT_SUFFIX")
 
+
+def get_abi_tag():
+    # This should probably return the ABI tag based on EXT_SUFFIX in the same
+    # way as pypa/packaging. See https://github.com/pypa/packaging/pull/607.
+    # For simplicity, we just fix it up for GraalPy for now and leave the logic
+    # for the other interpreters untouched, but this should be fixed properly
+    # down the road.
+    if platform.python_implementation() == "GraalVM":
+        ext_suffix = sysconfig.get_config_var("EXT_SUFFIX")
+        parts = ext_suffix.split(".")
+        soabi = parts[1]
+        abi = "-".join(soabi.split("-")[:3])
+        return abi.replace(".", "_").replace("-", "_")
+    else:
+        return (sysconfig.get_config_var("SOABI") or "-").split("-")[1]
+
+
 metadata = {
     # sys.implementation.name can differ from platform.python_implementation(), for example
     # Pyston has sys.implementation.name == "pyston" while platform.python_implementation() == cpython
@@ -30,7 +47,7 @@
     "interpreter": platform.python_implementation().lower(),
     "ext_suffix": ext_suffix,
     "soabi": sysconfig.get_config_var("SOABI") or None,
-    "abi_tag": (sysconfig.get_config_var("SOABI") or "-").split("-")[1] or None,
+    "abi_tag": get_abi_tag() or None,
     "platform": sysconfig.get_platform(),
     # This one isn't technically necessary, but still very useful for sanity checks
     "system": platform.system().lower(),

diff --git a/src/python_interpreter/mod.rs b/src/python_interpreter/mod.rs
@@ -291,6 +291,7 @@ fn windows_python_info(executable: &Path) -> Result<Option<InterpreterConfig>> {
 pub enum InterpreterKind {
     CPython,
     PyPy,
+    GraalPy,
 }
 
 impl InterpreterKind {
@@ -303,13 +304,19 @@ impl InterpreterKind {
     pub fn is_pypy(&self) -> bool {
         matches!(self, InterpreterKind::PyPy)
     }
+
+    /// Is this a GraalPy interpreter?
+    pub fn is_graalpy(&self) -> bool {
+        matches!(self, InterpreterKind::GraalPy)
+    }
 }
 
 impl fmt::Display for InterpreterKind {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match *self {
             InterpreterKind::CPython => write!(f, "CPython"),
             InterpreterKind::PyPy => write!(f, "PyPy"),
+            InterpreterKind::GraalPy => write!(f, "GraalPy"),
         }
     }
 }
@@ -321,6 +328,7 @@ impl FromStr for InterpreterKind {
         match s.to_ascii_lowercase().as_str() {
             "cpython" => Ok(InterpreterKind::CPython),
             "pypy" => Ok(InterpreterKind::PyPy),
+            "graalvm" | "graalpy" => Ok(InterpreterKind::GraalPy),
             unknown => Err(format!("Unknown interpreter kind '{unknown}'")),
         }
     }
@@ -407,8 +415,8 @@ fn fun_with_abiflags(
         );
     }
 
-    if message.interpreter == "pypy" {
-        // pypy does not specify abi flags
+    if message.interpreter == "pypy" || message.interpreter == "graalvm" {
+        // pypy and graalpy do not specify abi flags
         Ok("".to_string())
     } else if message.system == "windows" {
         if matches!(message.abiflags.as_deref(), Some("") | None) {
@@ -438,7 +446,7 @@ impl PythonInterpreter {
         } else {
             match self.interpreter_kind {
                 InterpreterKind::CPython => true,
-                InterpreterKind::PyPy => false,
+                InterpreterKind::PyPy | InterpreterKind::GraalPy => false,
             }
         }
     }
@@ -516,6 +524,23 @@ impl PythonInterpreter {
                         platform = platform,
                     )
                 }
+                InterpreterKind::GraalPy => {
+                    // GraalPy suffers from pypa/packaging#614, where
+                    // packaging misdetects it as a 32-bit implementation,
+                    // so GraalPy adds the correct platform itself, e.g.
+                    // graalpy 3.10 23.1 => numpy-1.23.5-graalpy310-graalpy231_310_native_x86_64_linux-linux_i686.whl
+                    format!(
+                        "graalpy{major}{minor}-{abi_tag}_{arch}_{os}-{os}_i686",
+                        major = self.major,
+                        minor = self.minor,
+                        abi_tag = self
+                            .abi_tag
+                            .clone()
+                            .expect("GraalPy's syconfig didn't define an `EXT_SUFFIX` ಠ_ಠ"),
+                        os = target.get_python_os(),
+                        arch = target.get_python_arch(),
+                    )
+                }
             }
         };
         Ok(tag)
@@ -635,6 +660,7 @@ impl PythonInterpreter {
         let interpreter = match message.interpreter.as_str() {
             "cpython" => InterpreterKind::CPython,
             "pypy" => InterpreterKind::PyPy,
+            "graalvm" | "graalpy" => InterpreterKind::GraalPy,
             other => {
                 bail!("Unsupported interpreter {}", other);
             }
@@ -888,7 +914,7 @@ impl PythonInterpreter {
     pub fn get_venv_site_package(&self, venv_base: impl AsRef<Path>, target: &Target) -> PathBuf {
         if target.is_unix() {
             match self.interpreter_kind {
-                InterpreterKind::CPython => {
+                InterpreterKind::CPython | InterpreterKind::GraalPy => {
                     let python_dir = format!("python{}.{}", self.major, self.minor);
                     venv_base
                         .as_ref()

diff --git a/tests/run.rs b/tests/run.rs
@@ -158,8 +158,8 @@ fn integration_pyo3_bin() {
         target.get_python()
     });
     let python_implementation = get_python_implementation(&python).unwrap();
-    if python_implementation == "pypy" {
-        // PyPy doesn't support the 'auto-initialize' feature of pyo3
+    if python_implementation == "pypy" || python_implementation == "graalpy" {
+        // PyPy & GraalPy do not support the 'auto-initialize' feature of pyo3
         return;
     }