Update pyo3 to version 0.23.2 (#2732)

* update pyo3 to latest version

* add python bindings to workspace

* make pyo3 stuff dependent of workspace again

* adapt implementation for the newest version of pyo3

Cargo.toml

@@ -25,6 +25,7 @@ members = [
   "utils/gramatron/construct_automata",
   "utils/libafl_benches",
   "utils/libafl_jumper",
+  "bindings/pylibafl",
 ]
 default-members = [
   "libafl",
@@ -35,7 +36,6 @@ default-members = [
 ]
 
 exclude = [
-  "bindings",
   "fuzzers",
   "libafl_libfuzzer_runtime",
   "utils/noaslr",
@@ -102,6 +102,8 @@ paste = "1.0.15"
 postcard = { version = "1.0.10", features = [
   "alloc",
 ], default-features = false } # no_std compatible serde serialization format
+pyo3 = "0.23.2"
+pyo3-build-config = "0.23.2"
 rangemap = "1.5.1"
 regex = "1.10.6"
 rustversion = "1.0.17"

bindings/pylibafl/Cargo.toml

@@ -9,8 +9,8 @@ edition = "2021"
 categories = ["development-tools::testing", "emulators", "embedded", "os"]
 
 [dependencies]
-pyo3 = { version = "0.22.3", features = ["extension-module"] }
+pyo3 = { workspace = true, features = ["extension-module"] }
-pyo3-log = { version = "0.11.0" }
+pyo3-log = { version = "0.12.0" }
 libafl_sugar = { path = "../../libafl_sugar", version = "0.14.0", features = [
   "python",
 ] }
@@ -24,7 +24,7 @@ libafl_qemu = { path = "../../libafl_qemu", version = "0.14.0", features = [
 ] }
 
 [build-dependencies]
-pyo3-build-config = { version = "0.22.3" }
+pyo3-build-config = { workspace = true }
 
 [lib]
 name = "pylibafl"

bindings/pylibafl/src/lib.rs

@@ -9,22 +9,22 @@ use pyo3::prelude::*;
 pub fn python_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
     pyo3_log::init();
 
-    let modules = m.py().import_bound("sys")?.getattr("modules")?;
+    let modules = m.py().import("sys")?.getattr("modules")?;
 
-    let sugar_module = PyModule::new_bound(m.py(), "sugar")?;
+    let sugar_module = PyModule::new(m.py(), "sugar")?;
     libafl_sugar::python_module(&sugar_module)?;
     m.add_submodule(&sugar_module)?;
     modules.set_item("pylibafl.sugar", sugar_module)?;
 
     #[cfg(target_os = "linux")]
     {
-        let qemu_module = PyModule::new_bound(m.py(), "qemu")?;
+        let qemu_module = PyModule::new(m.py(), "qemu")?;
         libafl_qemu::python_module(&qemu_module)?;
         m.add_submodule(&qemu_module)?;
         modules.set_item("pylibafl.qemu", qemu_module)?;
     }
 
-    let bolts_module = PyModule::new_bound(m.py(), "libafl_bolts")?;
+    let bolts_module = PyModule::new(m.py(), "libafl_bolts")?;
     libafl_bolts::pybind::python_module(&bolts_module)?;
     m.add_submodule(&bolts_module)?;
     modules.set_item("pylibafl.libafl_bolts", bolts_module)?;

libafl/Cargo.toml

@@ -275,8 +275,8 @@ arrayvec = { version = "0.7.6", optional = true, default-features = false } # us
 const_format = "0.2.33" # used for providing helpful compiler output
 const_panic = "0.2.9"   # similarly, for formatting const panic output
 
-pyo3 = { version = "0.22.3", features = ["gil-refs"], optional = true }
+pyo3 = { workspace = true, optional = true }
-regex-syntax = { version = "0.8.4", optional = true }                   # For nautilus
+regex-syntax = { version = "0.8.4", optional = true } # For nautilus
 
 # optional-dev deps (change when target.'cfg(accessible(::std))'.test-dependencies will be stable)
 serial_test = { workspace = true, optional = true, default-features = false, features = [

libafl_bolts/Cargo.toml

@@ -156,7 +156,7 @@ clap = { workspace = true, features = [
   "wrap_help",
 ], optional = true } # CLI parsing, for libafl_bolts::cli / the `cli` feature
 log = { workspace = true }
-pyo3 = { version = "0.22.3", optional = true, features = ["serde", "macros"] }
+pyo3 = { workspace = true, optional = true, features = ["serde", "macros"] }
 
 # optional-dev deps (change when target.'cfg(accessible(::std))'.test-dependencies will be stable)
 serial_test = { workspace = true, optional = true, default-features = false, features = [

libafl_bolts/src/lib.rs

@@ -635,10 +635,13 @@ impl From<windows_result::Error> for Error {
 impl From<pyo3::PyErr> for Error {
     fn from(err: pyo3::PyErr) -> Self {
         pyo3::Python::with_gil(|py| {
-            if err.matches(
+            if err
-                py,
+                .matches(
-                pyo3::types::PyType::new_bound::<pyo3::exceptions::PyKeyboardInterrupt>(py),
+                    py,
-            ) {
+                    pyo3::types::PyType::new::<pyo3::exceptions::PyKeyboardInterrupt>(py),
+                )
+                .unwrap()
+            {
                 Self::shutting_down()
             } else {
                 Self::illegal_state(format!("Python exception: {err:?}"))

libafl_qemu/Cargo.toml

@@ -120,9 +120,7 @@ paste = { workspace = true }
 enum-map = "2.7.3"
 serde_yaml = { workspace = true, optional = true } # For parsing the injections yaml file
 toml = { workspace = true, optional = true } # For parsing the injections toml file
-pyo3 = { version = "0.22.3", optional = true, features = [
+pyo3 = { workspace = true, optional = true, features = ["multiple-pymethods"] }
-  "multiple-pymethods",
-] }
 bytes-utils = "0.1.4"
 typed-builder = { workspace = true }
 memmap2 = "0.9.5"
@@ -132,7 +130,7 @@ document-features = { workspace = true, optional = true }
 
 [build-dependencies]
 libafl_qemu_build = { workspace = true, default-features = true, version = "0.14.0" }
-pyo3-build-config = { version = "0.23.1", optional = true }
+pyo3-build-config = { workspace = true, optional = true }
 rustversion = { workspace = true }
 bindgen = { workspace = true }
 cc = { workspace = true }

libafl_qemu/libafl_qemu_sys/Cargo.toml

@@ -63,11 +63,11 @@ num_enum = { workspace = true, default-features = true }
 libc = { workspace = true }
 strum = { workspace = true }
 strum_macros = { workspace = true }
-pyo3 = { version = "0.22.3", optional = true }
+pyo3 = { workspace = true, optional = true }
 
 [build-dependencies]
 libafl_qemu_build = { workspace = true, default-features = true }
-pyo3-build-config = { version = "0.23.1", optional = true }
+pyo3-build-config = { workspace = true, optional = true }
 rustversion = { workspace = true }
 
 [lints]

libafl_qemu/libafl_qemu_sys/src/usermode.rs

@@ -1,12 +1,18 @@
+#[cfg(target_os = "linux")]
 use core::{slice::from_raw_parts, str::from_utf8_unchecked};
+#[cfg(feature = "python")]
+use std::convert::Infallible;
 
+#[cfg(target_os = "linux")]
 use libc::{c_char, strlen};
 use num_enum::{IntoPrimitive, TryFromPrimitive};
 #[cfg(feature = "python")]
-use pyo3::{pyclass, pymethods, IntoPy, PyObject, Python};
+use pyo3::{pyclass, pymethods, types::PyInt, Bound, IntoPyObject, Python};
 use strum_macros::EnumIter;
 
-use crate::{libafl_mapinfo, GuestAddr, MmapPerms};
+use crate::MmapPerms;
+#[cfg(target_os = "linux")]
+use crate::{libafl_mapinfo, GuestAddr};
 
 #[derive(IntoPrimitive, TryFromPrimitive, Debug, Clone, Copy, EnumIter, PartialEq, Eq)]
 #[repr(i32)]
@@ -98,10 +104,14 @@ impl MmapPerms {
 }
 
 #[cfg(feature = "python")]
-impl IntoPy<PyObject> for MmapPerms {
+impl<'py> IntoPyObject<'py> for MmapPerms {
-    fn into_py(self, py: Python) -> PyObject {
+    type Target = PyInt;
+    type Output = Bound<'py, Self::Target>;
+    type Error = Infallible;
+
+    fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
         let n: i32 = self.into();
-        n.into_py(py)
+        n.into_pyobject(py)
     }
 }
 

libafl_qemu/src/arch/aarch64.rs

@@ -73,14 +73,6 @@ impl Regs {
     pub const Lr: Regs = Regs::X30;
 }
 
-#[cfg(feature = "python")]
-impl IntoPy<PyObject> for Regs {
-    fn into_py(self, py: Python) -> PyObject {
-        let n: i32 = self.into();
-        n.into_py(py)
-    }
-}
-
 /// Return an ARM64 ArchCapstoneBuilder
 pub fn capstone() -> capstone::arch::arm64::ArchCapstoneBuilder {
     capstone::Capstone::new()

libafl_qemu/src/arch/arm.rs

@@ -63,14 +63,6 @@ impl Regs {
     pub const Cpsr: Regs = Regs::R25;
 }
 
-#[cfg(feature = "python")]
-impl IntoPy<PyObject> for Regs {
-    fn into_py(self, py: Python) -> PyObject {
-        let n: i32 = self.into();
-        n.into_py(py)
-    }
-}
-
 /// Return an ARM ArchCapstoneBuilder
 pub fn capstone() -> capstone::arch::arm::ArchCapstoneBuilder {
     capstone::Capstone::new()

libafl_qemu/src/arch/i386.rs

@@ -49,14 +49,6 @@ impl Regs {
     pub const Pc: Regs = Regs::Eip;
 }
 
-#[cfg(feature = "python")]
-impl IntoPy<PyObject> for Regs {
-    fn into_py(self, py: Python) -> PyObject {
-        let n: i32 = self.into();
-        n.into_py(py)
-    }
-}
-
 /// Return an X86 ArchCapstoneBuilder
 pub fn capstone() -> capstone::arch::x86::ArchCapstoneBuilder {
     capstone::Capstone::new()

libafl_qemu/src/arch/mips.rs

@@ -72,14 +72,6 @@ impl Regs {
     pub const Zero: Regs = Regs::R0;
 }
 
-#[cfg(feature = "python")]
-impl IntoPy<PyObject> for Regs {
-    fn into_py(self, py: Python) -> PyObject {
-        let n: i32 = self.into();
-        n.into_py(py)
-    }
-}
-
 /// Return an MIPS ArchCapstoneBuilder
 pub fn capstone() -> capstone::arch::mips::ArchCapstoneBuilder {
     capstone::Capstone::new().mips()

libafl_qemu/src/arch/mod.rs

@@ -1,5 +1,12 @@
+#[cfg(feature = "python")]
+use std::convert::Infallible;
+
+#[cfg(feature = "python")]
+use pyo3::{prelude::*, types::PyInt};
+
 #[cfg(cpu_target = "aarch64")]
 pub mod aarch64;
+
 #[cfg(all(cpu_target = "aarch64", not(feature = "clippy")))]
 pub use aarch64::*;
 
@@ -37,3 +44,15 @@ pub use hexagon::*;
 pub mod riscv;
 #[cfg(any(cpu_target = "riscv32", cpu_target = "riscv64"))]
 pub use riscv::*;
+
+#[cfg(feature = "python")]
+impl<'py> IntoPyObject<'py> for Regs {
+    type Target = PyInt;
+    type Output = Bound<'py, Self::Target>;
+    type Error = Infallible;
+
+    fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
+        let n: i32 = self.into();
+        n.into_pyobject(py)
+    }
+}

libafl_qemu/src/arch/ppc.rs

@@ -112,14 +112,6 @@ impl Regs {
     pub const Sp: Regs = Regs::R1;
 }
 
-#[cfg(feature = "python")]
-impl IntoPy<PyObject> for Regs {
-    fn into_py(self, py: Python) -> PyObject {
-        let n: i32 = self.into();
-        n.into_py(py)
-    }
-}
-
 /// Return an MIPS ArchCapstoneBuilder
 pub fn capstone() -> capstone::arch::ppc::ArchCapstoneBuilder {
     capstone::Capstone::new().ppc()

libafl_qemu/src/arch/x86_64.rs

@@ -3,8 +3,6 @@ use std::{mem::size_of, ops::Range, sync::OnceLock};
 use capstone::arch::BuildsCapstone;
 use enum_map::{enum_map, EnumMap};
 use num_enum::{IntoPrimitive, TryFromPrimitive};
-#[cfg(feature = "python")]
-use pyo3::prelude::*;
 pub use strum_macros::EnumIter;
 pub use syscall_numbers::x86_64::*;
 
@@ -57,14 +55,6 @@ impl Regs {
     pub const Pc: Regs = Regs::Rip;
 }
 
-#[cfg(feature = "python")]
-impl IntoPy<PyObject> for Regs {
-    fn into_py(self, py: Python) -> PyObject {
-        let n: i32 = self.into();
-        n.into_py(py)
-    }
-}
-
 /// Return an X86 `ArchCapstoneBuilder`
 #[must_use]
 pub fn capstone() -> capstone::arch::x86::ArchCapstoneBuilder {

libafl_qemu/src/lib.rs

@@ -75,17 +75,17 @@ use pyo3::prelude::*;
 pub fn python_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
     use pyo3::types::PyString;
 
-    let regsm = PyModule::new_bound(m.py(), "regs")?;
+    let regsm = PyModule::new(m.py(), "regs")?;
     for r in Regs::iter() {
         let v: i32 = r.into();
-        regsm.add(PyString::new_bound(m.py(), &format!("{r:?}")), v)?;
+        regsm.add(PyString::new(m.py(), &format!("{r:?}")), v)?;
     }
     m.add_submodule(&regsm)?;
 
-    let mmapm = PyModule::new_bound(m.py(), "mmap")?;
+    let mmapm = PyModule::new(m.py(), "mmap")?;
     for r in MmapPerms::iter() {
         let v: i32 = r.into();
-        mmapm.add(PyString::new_bound(m.py(), &format!("{r:?}")), v)?;
+        mmapm.add(PyString::new(m.py(), &format!("{r:?}")), v)?;
     }
     m.add_submodule(&mmapm)?;
 

libafl_qemu/src/qemu/mod.rs

@@ -1103,8 +1103,8 @@ pub mod pybind {
 
     extern "C" fn py_generic_hook_wrapper(idx: u64, _pc: GuestAddr) {
         let obj = unsafe {
-            let hooks = &mut *&raw mut PY_GENERIC_HOOKS;
+            let hooks = &raw mut PY_GENERIC_HOOKS;
-            &hooks[idx as usize].1
+            &(*hooks)[idx as usize].1
         };
         Python::with_gil(|py| {
             obj.call0(py).expect("Error in the hook");
@@ -1183,9 +1183,9 @@ pub mod pybind {
         /// Removes a hooke from `PY_GENERIC_HOOKS` -> may not be called concurrently!
         unsafe fn set_hook(&self, addr: GuestAddr, hook: PyObject) {
             unsafe {
-                let hooks = &mut *&raw mut PY_GENERIC_HOOKS;
+                let hooks = &raw mut PY_GENERIC_HOOKS;
-                let idx = hooks.len();
+                let idx = (*hooks).len();
-                hooks.push((addr, hook));
+                (*hooks).push((addr, hook));
                 self.qemu.hooks().add_instruction_hooks(
                     idx as u64,
                     addr,
@@ -1199,8 +1199,8 @@ pub mod pybind {
         /// Removes a hooke from `PY_GENERIC_HOOKS` -> may not be called concurrently!
         unsafe fn remove_hooks_at(&self, addr: GuestAddr) -> usize {
             unsafe {
-                let hooks = &mut *&raw mut PY_GENERIC_HOOKS;
+                let hooks = &raw mut PY_GENERIC_HOOKS;
-                hooks.retain(|(a, _)| *a != addr);
+                (*hooks).retain(|(a, _)| *a != addr);
             }
             self.qemu.hooks().remove_instruction_hooks_at(addr, true)
         }

libafl_qemu/src/qemu/usermode.rs

@@ -11,7 +11,7 @@ use libafl_qemu_sys::{
 };
 use libc::{c_int, c_uchar, strlen};
 #[cfg(feature = "python")]
-use pyo3::{pyclass, pymethods, IntoPy, PyObject, PyRef, PyRefMut, Python};
+use pyo3::{pyclass, pymethods, IntoPyObject, Py, PyRef, PyRefMut, Python};
 
 use crate::{Qemu, CPU};
 
@@ -65,8 +65,9 @@ impl GuestMaps {
     fn __iter__(slf: PyRef<Self>) -> PyRef<Self> {
         slf
     }
-    fn __next__(mut slf: PyRefMut<Self>) -> Option<PyObject> {
+
-        Python::with_gil(|py| slf.next().map(|x| x.into_py(py)))
+    fn __next__(mut slf: PyRefMut<Self>) -> Option<Py<MapInfo>> {
+        Python::with_gil(|py| slf.next().map(|x| x.into_pyobject(py).unwrap().into()))
     }
 }
 
@@ -281,7 +282,7 @@ pub mod pybind {
         a6: u64,
         a7: u64,
     ) -> SyscallHookResult {
-        unsafe { (*&raw const PY_SYSCALL_HOOK).as_ref() }.map_or_else(
+        unsafe { PY_SYSCALL_HOOK.as_ref() }.map_or_else(
             || SyscallHookResult::new(None),
             |obj| {
                 let args = (sys_num, a0, a1, a2, a3, a4, a5, a6, a7);
@@ -362,7 +363,7 @@ pub mod pybind {
         /// Accesses the global `PY_SYSCALL_HOOK` and may not be called concurrently.
         unsafe fn set_syscall_hook(&self, hook: PyObject) {
             unsafe {
-                (*&raw mut (PY_SYSCALL_HOOK)) = Some(hook);
+                PY_SYSCALL_HOOK = Some(hook);
             }
             self.qemu
                 .hooks()

libafl_sugar/Cargo.toml

@@ -57,7 +57,7 @@ riscv32 = ["libafl_qemu/riscv32"]
 riscv64 = ["libafl_qemu/riscv64"]
 
 [build-dependencies]
-pyo3-build-config = { version = "0.23.1", optional = true }
+pyo3-build-config = { workspace = true, optional = true }
 
 [dependencies]
 libafl = { workspace = true, default-features = true }
@@ -67,8 +67,8 @@ libafl_targets = { workspace = true, default-features = true }
 # Document all features of this crate (for `cargo doc`)
 document-features = { workspace = true, optional = true }
 
-typed-builder = { workspace = true }           # Implement the builder pattern at compiletime
+typed-builder = { workspace = true }         # Implement the builder pattern at compiletime
-pyo3 = { version = "0.22.3", optional = true }
+pyo3 = { workspace = true, optional = true }
 log = { workspace = true }
 
 [target.'cfg(target_os = "linux")'.dependencies]

libafl_sugar/src/inmemory.rs

@@ -436,7 +436,7 @@ pub mod pybind {
                 .cores(&self.cores)
                 .harness(|buf| {
                     Python::with_gil(|py| -> PyResult<()> {
-                        let args = (PyBytes::new_bound(py, buf),); // TODO avoid copy
+                        let args = (PyBytes::new(py, buf),); // TODO avoid copy
                         harness.call1(py, args)?;
                         Ok(())
                     })

libafl_sugar/src/qemu.rs

@@ -541,7 +541,7 @@ pub mod pybind {
                 .cores(&self.cores)
                 .harness(|buf| {
                     Python::with_gil(|py| -> PyResult<()> {
-                        let args = (PyBytes::new_bound(py, buf),); // TODO avoid copy
+                        let args = (PyBytes::new(py, buf),); // TODO avoid copy
                         harness.call1(py, args)?;
                         Ok(())
                     })