From 94fa4014acaa6176e0b8c1e13d65ab366653c589 Mon Sep 17 00:00:00 2001 From: Romain Malmain Date: Wed, 27 Nov 2024 19:01:31 +0100 Subject: [PATCH] Update pyo3 to version 0.23.2 (#2732) * update pyo3 to latest version * add python bindings to workspace * make pyo3 stuff dependent of workspace again * adapt implementation for the newest version of pyo3 --- Cargo.toml | 4 +++- bindings/pylibafl/Cargo.toml | 6 +++--- bindings/pylibafl/src/lib.rs | 8 ++++---- libafl/Cargo.toml | 4 ++-- libafl_bolts/Cargo.toml | 2 +- libafl_bolts/src/lib.rs | 11 +++++++---- libafl_qemu/Cargo.toml | 6 ++---- libafl_qemu/libafl_qemu_sys/Cargo.toml | 4 ++-- libafl_qemu/libafl_qemu_sys/src/usermode.rs | 20 +++++++++++++++----- libafl_qemu/src/arch/aarch64.rs | 8 -------- libafl_qemu/src/arch/arm.rs | 8 -------- libafl_qemu/src/arch/i386.rs | 8 -------- libafl_qemu/src/arch/mips.rs | 8 -------- libafl_qemu/src/arch/mod.rs | 19 +++++++++++++++++++ libafl_qemu/src/arch/ppc.rs | 8 -------- libafl_qemu/src/arch/x86_64.rs | 10 ---------- libafl_qemu/src/lib.rs | 8 ++++---- libafl_qemu/src/qemu/mod.rs | 14 +++++++------- libafl_qemu/src/qemu/usermode.rs | 11 ++++++----- libafl_sugar/Cargo.toml | 6 +++--- libafl_sugar/src/inmemory.rs | 2 +- libafl_sugar/src/qemu.rs | 2 +- 22 files changed, 80 insertions(+), 97 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ef625ea260..159ff0c783 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -25,6 +25,7 @@ members = [ "utils/gramatron/construct_automata", "utils/libafl_benches", "utils/libafl_jumper", + "bindings/pylibafl", ] default-members = [ "libafl", @@ -35,7 +36,6 @@ default-members = [ ] exclude = [ - "bindings", "fuzzers", "libafl_libfuzzer_runtime", "utils/noaslr", @@ -102,6 +102,8 @@ paste = "1.0.15" postcard = { version = "1.0.10", features = [ "alloc", ], default-features = false } # no_std compatible serde serialization format +pyo3 = "0.23.2" +pyo3-build-config = "0.23.2" rangemap = "1.5.1" regex = "1.10.6" rustversion = "1.0.17" diff --git a/bindings/pylibafl/Cargo.toml b/bindings/pylibafl/Cargo.toml index cf3c5ea928..9ad77cc799 100644 --- a/bindings/pylibafl/Cargo.toml +++ b/bindings/pylibafl/Cargo.toml @@ -9,8 +9,8 @@ edition = "2021" categories = ["development-tools::testing", "emulators", "embedded", "os"] [dependencies] -pyo3 = { version = "0.22.3", features = ["extension-module"] } -pyo3-log = { version = "0.11.0" } +pyo3 = { workspace = true, features = ["extension-module"] } +pyo3-log = { version = "0.12.0" } libafl_sugar = { path = "../../libafl_sugar", version = "0.14.0", features = [ "python", ] } @@ -24,7 +24,7 @@ libafl_qemu = { path = "../../libafl_qemu", version = "0.14.0", features = [ ] } [build-dependencies] -pyo3-build-config = { version = "0.22.3" } +pyo3-build-config = { workspace = true } [lib] name = "pylibafl" diff --git a/bindings/pylibafl/src/lib.rs b/bindings/pylibafl/src/lib.rs index c4371f89d2..537c820dbd 100644 --- a/bindings/pylibafl/src/lib.rs +++ b/bindings/pylibafl/src/lib.rs @@ -9,22 +9,22 @@ use pyo3::prelude::*; pub fn python_module(m: &Bound<'_, PyModule>) -> PyResult<()> { pyo3_log::init(); - let modules = m.py().import_bound("sys")?.getattr("modules")?; + let modules = m.py().import("sys")?.getattr("modules")?; - let sugar_module = PyModule::new_bound(m.py(), "sugar")?; + let sugar_module = PyModule::new(m.py(), "sugar")?; libafl_sugar::python_module(&sugar_module)?; m.add_submodule(&sugar_module)?; modules.set_item("pylibafl.sugar", sugar_module)?; #[cfg(target_os = "linux")] { - let qemu_module = PyModule::new_bound(m.py(), "qemu")?; + let qemu_module = PyModule::new(m.py(), "qemu")?; libafl_qemu::python_module(&qemu_module)?; m.add_submodule(&qemu_module)?; modules.set_item("pylibafl.qemu", qemu_module)?; } - let bolts_module = PyModule::new_bound(m.py(), "libafl_bolts")?; + let bolts_module = PyModule::new(m.py(), "libafl_bolts")?; libafl_bolts::pybind::python_module(&bolts_module)?; m.add_submodule(&bolts_module)?; modules.set_item("pylibafl.libafl_bolts", bolts_module)?; diff --git a/libafl/Cargo.toml b/libafl/Cargo.toml index 9c3aa8fabd..d7c1823a43 100644 --- a/libafl/Cargo.toml +++ b/libafl/Cargo.toml @@ -275,8 +275,8 @@ arrayvec = { version = "0.7.6", optional = true, default-features = false } # us const_format = "0.2.33" # used for providing helpful compiler output const_panic = "0.2.9" # similarly, for formatting const panic output -pyo3 = { version = "0.22.3", features = ["gil-refs"], optional = true } -regex-syntax = { version = "0.8.4", optional = true } # For nautilus +pyo3 = { workspace = true, optional = true } +regex-syntax = { version = "0.8.4", optional = true } # For nautilus # optional-dev deps (change when target.'cfg(accessible(::std))'.test-dependencies will be stable) serial_test = { workspace = true, optional = true, default-features = false, features = [ diff --git a/libafl_bolts/Cargo.toml b/libafl_bolts/Cargo.toml index d0e0a897dd..897c6f1394 100644 --- a/libafl_bolts/Cargo.toml +++ b/libafl_bolts/Cargo.toml @@ -156,7 +156,7 @@ clap = { workspace = true, features = [ "wrap_help", ], optional = true } # CLI parsing, for libafl_bolts::cli / the `cli` feature log = { workspace = true } -pyo3 = { version = "0.22.3", optional = true, features = ["serde", "macros"] } +pyo3 = { workspace = true, optional = true, features = ["serde", "macros"] } # optional-dev deps (change when target.'cfg(accessible(::std))'.test-dependencies will be stable) serial_test = { workspace = true, optional = true, default-features = false, features = [ diff --git a/libafl_bolts/src/lib.rs b/libafl_bolts/src/lib.rs index 0ea64b09c9..0c6af328db 100644 --- a/libafl_bolts/src/lib.rs +++ b/libafl_bolts/src/lib.rs @@ -635,10 +635,13 @@ impl From for Error { impl From for Error { fn from(err: pyo3::PyErr) -> Self { pyo3::Python::with_gil(|py| { - if err.matches( - py, - pyo3::types::PyType::new_bound::(py), - ) { + if err + .matches( + py, + pyo3::types::PyType::new::(py), + ) + .unwrap() + { Self::shutting_down() } else { Self::illegal_state(format!("Python exception: {err:?}")) diff --git a/libafl_qemu/Cargo.toml b/libafl_qemu/Cargo.toml index b699533e0e..25cb7af3b8 100644 --- a/libafl_qemu/Cargo.toml +++ b/libafl_qemu/Cargo.toml @@ -120,9 +120,7 @@ paste = { workspace = true } enum-map = "2.7.3" serde_yaml = { workspace = true, optional = true } # For parsing the injections yaml file toml = { workspace = true, optional = true } # For parsing the injections toml file -pyo3 = { version = "0.22.3", optional = true, features = [ - "multiple-pymethods", -] } +pyo3 = { workspace = true, optional = true, features = ["multiple-pymethods"] } bytes-utils = "0.1.4" typed-builder = { workspace = true } memmap2 = "0.9.5" @@ -132,7 +130,7 @@ document-features = { workspace = true, optional = true } [build-dependencies] libafl_qemu_build = { workspace = true, default-features = true, version = "0.14.0" } -pyo3-build-config = { version = "0.23.1", optional = true } +pyo3-build-config = { workspace = true, optional = true } rustversion = { workspace = true } bindgen = { workspace = true } cc = { workspace = true } diff --git a/libafl_qemu/libafl_qemu_sys/Cargo.toml b/libafl_qemu/libafl_qemu_sys/Cargo.toml index 833ef0d4cb..0ad98885bd 100644 --- a/libafl_qemu/libafl_qemu_sys/Cargo.toml +++ b/libafl_qemu/libafl_qemu_sys/Cargo.toml @@ -63,11 +63,11 @@ num_enum = { workspace = true, default-features = true } libc = { workspace = true } strum = { workspace = true } strum_macros = { workspace = true } -pyo3 = { version = "0.22.3", optional = true } +pyo3 = { workspace = true, optional = true } [build-dependencies] libafl_qemu_build = { workspace = true, default-features = true } -pyo3-build-config = { version = "0.23.1", optional = true } +pyo3-build-config = { workspace = true, optional = true } rustversion = { workspace = true } [lints] diff --git a/libafl_qemu/libafl_qemu_sys/src/usermode.rs b/libafl_qemu/libafl_qemu_sys/src/usermode.rs index 94c6e64859..50b526c5b1 100644 --- a/libafl_qemu/libafl_qemu_sys/src/usermode.rs +++ b/libafl_qemu/libafl_qemu_sys/src/usermode.rs @@ -1,12 +1,18 @@ +#[cfg(target_os = "linux")] use core::{slice::from_raw_parts, str::from_utf8_unchecked}; +#[cfg(feature = "python")] +use std::convert::Infallible; +#[cfg(target_os = "linux")] use libc::{c_char, strlen}; use num_enum::{IntoPrimitive, TryFromPrimitive}; #[cfg(feature = "python")] -use pyo3::{pyclass, pymethods, IntoPy, PyObject, Python}; +use pyo3::{pyclass, pymethods, types::PyInt, Bound, IntoPyObject, Python}; use strum_macros::EnumIter; -use crate::{libafl_mapinfo, GuestAddr, MmapPerms}; +use crate::MmapPerms; +#[cfg(target_os = "linux")] +use crate::{libafl_mapinfo, GuestAddr}; #[derive(IntoPrimitive, TryFromPrimitive, Debug, Clone, Copy, EnumIter, PartialEq, Eq)] #[repr(i32)] @@ -98,10 +104,14 @@ impl MmapPerms { } #[cfg(feature = "python")] -impl IntoPy for MmapPerms { - fn into_py(self, py: Python) -> PyObject { +impl<'py> IntoPyObject<'py> for MmapPerms { + type Target = PyInt; + type Output = Bound<'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { let n: i32 = self.into(); - n.into_py(py) + n.into_pyobject(py) } } diff --git a/libafl_qemu/src/arch/aarch64.rs b/libafl_qemu/src/arch/aarch64.rs index bb1b49d0ed..f68fe27b35 100644 --- a/libafl_qemu/src/arch/aarch64.rs +++ b/libafl_qemu/src/arch/aarch64.rs @@ -73,14 +73,6 @@ impl Regs { pub const Lr: Regs = Regs::X30; } -#[cfg(feature = "python")] -impl IntoPy for Regs { - fn into_py(self, py: Python) -> PyObject { - let n: i32 = self.into(); - n.into_py(py) - } -} - /// Return an ARM64 ArchCapstoneBuilder pub fn capstone() -> capstone::arch::arm64::ArchCapstoneBuilder { capstone::Capstone::new() diff --git a/libafl_qemu/src/arch/arm.rs b/libafl_qemu/src/arch/arm.rs index 952d08ae6a..18955bb964 100644 --- a/libafl_qemu/src/arch/arm.rs +++ b/libafl_qemu/src/arch/arm.rs @@ -63,14 +63,6 @@ impl Regs { pub const Cpsr: Regs = Regs::R25; } -#[cfg(feature = "python")] -impl IntoPy for Regs { - fn into_py(self, py: Python) -> PyObject { - let n: i32 = self.into(); - n.into_py(py) - } -} - /// Return an ARM ArchCapstoneBuilder pub fn capstone() -> capstone::arch::arm::ArchCapstoneBuilder { capstone::Capstone::new() diff --git a/libafl_qemu/src/arch/i386.rs b/libafl_qemu/src/arch/i386.rs index 435f990b34..56bbd28562 100644 --- a/libafl_qemu/src/arch/i386.rs +++ b/libafl_qemu/src/arch/i386.rs @@ -49,14 +49,6 @@ impl Regs { pub const Pc: Regs = Regs::Eip; } -#[cfg(feature = "python")] -impl IntoPy for Regs { - fn into_py(self, py: Python) -> PyObject { - let n: i32 = self.into(); - n.into_py(py) - } -} - /// Return an X86 ArchCapstoneBuilder pub fn capstone() -> capstone::arch::x86::ArchCapstoneBuilder { capstone::Capstone::new() diff --git a/libafl_qemu/src/arch/mips.rs b/libafl_qemu/src/arch/mips.rs index 0e5e1f18fb..f770b96817 100644 --- a/libafl_qemu/src/arch/mips.rs +++ b/libafl_qemu/src/arch/mips.rs @@ -72,14 +72,6 @@ impl Regs { pub const Zero: Regs = Regs::R0; } -#[cfg(feature = "python")] -impl IntoPy for Regs { - fn into_py(self, py: Python) -> PyObject { - let n: i32 = self.into(); - n.into_py(py) - } -} - /// Return an MIPS ArchCapstoneBuilder pub fn capstone() -> capstone::arch::mips::ArchCapstoneBuilder { capstone::Capstone::new().mips() diff --git a/libafl_qemu/src/arch/mod.rs b/libafl_qemu/src/arch/mod.rs index f4a03b63b8..d4ed7b978f 100644 --- a/libafl_qemu/src/arch/mod.rs +++ b/libafl_qemu/src/arch/mod.rs @@ -1,5 +1,12 @@ +#[cfg(feature = "python")] +use std::convert::Infallible; + +#[cfg(feature = "python")] +use pyo3::{prelude::*, types::PyInt}; + #[cfg(cpu_target = "aarch64")] pub mod aarch64; + #[cfg(all(cpu_target = "aarch64", not(feature = "clippy")))] pub use aarch64::*; @@ -37,3 +44,15 @@ pub use hexagon::*; pub mod riscv; #[cfg(any(cpu_target = "riscv32", cpu_target = "riscv64"))] pub use riscv::*; + +#[cfg(feature = "python")] +impl<'py> IntoPyObject<'py> for Regs { + type Target = PyInt; + type Output = Bound<'py, Self::Target>; + type Error = Infallible; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let n: i32 = self.into(); + n.into_pyobject(py) + } +} diff --git a/libafl_qemu/src/arch/ppc.rs b/libafl_qemu/src/arch/ppc.rs index d8838d7d68..4bc7a64cb8 100644 --- a/libafl_qemu/src/arch/ppc.rs +++ b/libafl_qemu/src/arch/ppc.rs @@ -112,14 +112,6 @@ impl Regs { pub const Sp: Regs = Regs::R1; } -#[cfg(feature = "python")] -impl IntoPy for Regs { - fn into_py(self, py: Python) -> PyObject { - let n: i32 = self.into(); - n.into_py(py) - } -} - /// Return an MIPS ArchCapstoneBuilder pub fn capstone() -> capstone::arch::ppc::ArchCapstoneBuilder { capstone::Capstone::new().ppc() diff --git a/libafl_qemu/src/arch/x86_64.rs b/libafl_qemu/src/arch/x86_64.rs index d48568813d..119c5a9c62 100644 --- a/libafl_qemu/src/arch/x86_64.rs +++ b/libafl_qemu/src/arch/x86_64.rs @@ -3,8 +3,6 @@ use std::{mem::size_of, ops::Range, sync::OnceLock}; use capstone::arch::BuildsCapstone; use enum_map::{enum_map, EnumMap}; use num_enum::{IntoPrimitive, TryFromPrimitive}; -#[cfg(feature = "python")] -use pyo3::prelude::*; pub use strum_macros::EnumIter; pub use syscall_numbers::x86_64::*; @@ -57,14 +55,6 @@ impl Regs { pub const Pc: Regs = Regs::Rip; } -#[cfg(feature = "python")] -impl IntoPy for Regs { - fn into_py(self, py: Python) -> PyObject { - let n: i32 = self.into(); - n.into_py(py) - } -} - /// Return an X86 `ArchCapstoneBuilder` #[must_use] pub fn capstone() -> capstone::arch::x86::ArchCapstoneBuilder { diff --git a/libafl_qemu/src/lib.rs b/libafl_qemu/src/lib.rs index 13401b3a77..6f27560e56 100644 --- a/libafl_qemu/src/lib.rs +++ b/libafl_qemu/src/lib.rs @@ -75,17 +75,17 @@ use pyo3::prelude::*; pub fn python_module(m: &Bound<'_, PyModule>) -> PyResult<()> { use pyo3::types::PyString; - let regsm = PyModule::new_bound(m.py(), "regs")?; + let regsm = PyModule::new(m.py(), "regs")?; for r in Regs::iter() { let v: i32 = r.into(); - regsm.add(PyString::new_bound(m.py(), &format!("{r:?}")), v)?; + regsm.add(PyString::new(m.py(), &format!("{r:?}")), v)?; } m.add_submodule(®sm)?; - let mmapm = PyModule::new_bound(m.py(), "mmap")?; + let mmapm = PyModule::new(m.py(), "mmap")?; for r in MmapPerms::iter() { let v: i32 = r.into(); - mmapm.add(PyString::new_bound(m.py(), &format!("{r:?}")), v)?; + mmapm.add(PyString::new(m.py(), &format!("{r:?}")), v)?; } m.add_submodule(&mmapm)?; diff --git a/libafl_qemu/src/qemu/mod.rs b/libafl_qemu/src/qemu/mod.rs index e742dfbda1..d9fc7ce657 100644 --- a/libafl_qemu/src/qemu/mod.rs +++ b/libafl_qemu/src/qemu/mod.rs @@ -1103,8 +1103,8 @@ pub mod pybind { extern "C" fn py_generic_hook_wrapper(idx: u64, _pc: GuestAddr) { let obj = unsafe { - let hooks = &mut *&raw mut PY_GENERIC_HOOKS; - &hooks[idx as usize].1 + let hooks = &raw mut PY_GENERIC_HOOKS; + &(*hooks)[idx as usize].1 }; Python::with_gil(|py| { obj.call0(py).expect("Error in the hook"); @@ -1183,9 +1183,9 @@ pub mod pybind { /// Removes a hooke from `PY_GENERIC_HOOKS` -> may not be called concurrently! unsafe fn set_hook(&self, addr: GuestAddr, hook: PyObject) { unsafe { - let hooks = &mut *&raw mut PY_GENERIC_HOOKS; - let idx = hooks.len(); - hooks.push((addr, hook)); + let hooks = &raw mut PY_GENERIC_HOOKS; + let idx = (*hooks).len(); + (*hooks).push((addr, hook)); self.qemu.hooks().add_instruction_hooks( idx as u64, addr, @@ -1199,8 +1199,8 @@ pub mod pybind { /// Removes a hooke from `PY_GENERIC_HOOKS` -> may not be called concurrently! unsafe fn remove_hooks_at(&self, addr: GuestAddr) -> usize { unsafe { - let hooks = &mut *&raw mut PY_GENERIC_HOOKS; - hooks.retain(|(a, _)| *a != addr); + let hooks = &raw mut PY_GENERIC_HOOKS; + (*hooks).retain(|(a, _)| *a != addr); } self.qemu.hooks().remove_instruction_hooks_at(addr, true) } diff --git a/libafl_qemu/src/qemu/usermode.rs b/libafl_qemu/src/qemu/usermode.rs index 955de9536d..d4d3b89efa 100644 --- a/libafl_qemu/src/qemu/usermode.rs +++ b/libafl_qemu/src/qemu/usermode.rs @@ -11,7 +11,7 @@ use libafl_qemu_sys::{ }; use libc::{c_int, c_uchar, strlen}; #[cfg(feature = "python")] -use pyo3::{pyclass, pymethods, IntoPy, PyObject, PyRef, PyRefMut, Python}; +use pyo3::{pyclass, pymethods, IntoPyObject, Py, PyRef, PyRefMut, Python}; use crate::{Qemu, CPU}; @@ -65,8 +65,9 @@ impl GuestMaps { fn __iter__(slf: PyRef) -> PyRef { slf } - fn __next__(mut slf: PyRefMut) -> Option { - Python::with_gil(|py| slf.next().map(|x| x.into_py(py))) + + fn __next__(mut slf: PyRefMut) -> Option> { + Python::with_gil(|py| slf.next().map(|x| x.into_pyobject(py).unwrap().into())) } } @@ -281,7 +282,7 @@ pub mod pybind { a6: u64, a7: u64, ) -> SyscallHookResult { - unsafe { (*&raw const PY_SYSCALL_HOOK).as_ref() }.map_or_else( + unsafe { PY_SYSCALL_HOOK.as_ref() }.map_or_else( || SyscallHookResult::new(None), |obj| { let args = (sys_num, a0, a1, a2, a3, a4, a5, a6, a7); @@ -362,7 +363,7 @@ pub mod pybind { /// Accesses the global `PY_SYSCALL_HOOK` and may not be called concurrently. unsafe fn set_syscall_hook(&self, hook: PyObject) { unsafe { - (*&raw mut (PY_SYSCALL_HOOK)) = Some(hook); + PY_SYSCALL_HOOK = Some(hook); } self.qemu .hooks() diff --git a/libafl_sugar/Cargo.toml b/libafl_sugar/Cargo.toml index a3cbb06114..b32a3ff708 100644 --- a/libafl_sugar/Cargo.toml +++ b/libafl_sugar/Cargo.toml @@ -57,7 +57,7 @@ riscv32 = ["libafl_qemu/riscv32"] riscv64 = ["libafl_qemu/riscv64"] [build-dependencies] -pyo3-build-config = { version = "0.23.1", optional = true } +pyo3-build-config = { workspace = true, optional = true } [dependencies] libafl = { workspace = true, default-features = true } @@ -67,8 +67,8 @@ libafl_targets = { workspace = true, default-features = true } # Document all features of this crate (for `cargo doc`) document-features = { workspace = true, optional = true } -typed-builder = { workspace = true } # Implement the builder pattern at compiletime -pyo3 = { version = "0.22.3", optional = true } +typed-builder = { workspace = true } # Implement the builder pattern at compiletime +pyo3 = { workspace = true, optional = true } log = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] diff --git a/libafl_sugar/src/inmemory.rs b/libafl_sugar/src/inmemory.rs index d971a5f70f..dfac4dd2d6 100644 --- a/libafl_sugar/src/inmemory.rs +++ b/libafl_sugar/src/inmemory.rs @@ -436,7 +436,7 @@ pub mod pybind { .cores(&self.cores) .harness(|buf| { Python::with_gil(|py| -> PyResult<()> { - let args = (PyBytes::new_bound(py, buf),); // TODO avoid copy + let args = (PyBytes::new(py, buf),); // TODO avoid copy harness.call1(py, args)?; Ok(()) }) diff --git a/libafl_sugar/src/qemu.rs b/libafl_sugar/src/qemu.rs index f9d3958f57..e0126c3b01 100644 --- a/libafl_sugar/src/qemu.rs +++ b/libafl_sugar/src/qemu.rs @@ -541,7 +541,7 @@ pub mod pybind { .cores(&self.cores) .harness(|buf| { Python::with_gil(|py| -> PyResult<()> { - let args = (PyBytes::new_bound(py, buf),); // TODO avoid copy + let args = (PyBytes::new(py, buf),); // TODO avoid copy harness.call1(py, args)?; Ok(()) })