From c4c0fb6750cf3a0d407e853f34fddf64acfa7e0c Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Mon, 29 Jul 2024 22:55:28 +0200 Subject: [PATCH] Update pyo3 (#2459) * Update pyo3 * Update pyo3 to 0.22 * Fix qemu python * clippy * fmt * nautilus python * More pyo3 * Make signature more legible --- bindings/pylibafl/Cargo.toml | 6 ++--- bindings/pylibafl/src/lib.rs | 22 +++++++++---------- libafl/Cargo.toml | 2 +- libafl/src/common/nautilus/grammartec/tree.rs | 15 +++++++------ libafl_bolts/Cargo.toml | 2 +- libafl_bolts/src/lib.rs | 8 +++---- libafl_bolts/src/rands/mod.rs | 6 ++--- libafl_qemu/Cargo.toml | 4 ++-- libafl_qemu/libafl_qemu_sys/Cargo.toml | 4 ++-- libafl_qemu/libafl_qemu_sys/src/lib.rs | 2 ++ libafl_qemu/src/lib.rs | 16 ++++++++------ libafl_qemu/src/qemu/hooks.rs | 21 +++++++++--------- libafl_qemu/src/qemu/usermode.rs | 12 ++++++---- libafl_sugar/Cargo.toml | 4 ++-- libafl_sugar/src/forkserver.rs | 12 +++++++++- libafl_sugar/src/inmemory.rs | 14 ++++++++++-- libafl_sugar/src/lib.rs | 8 +++---- libafl_sugar/src/qemu.rs | 14 ++++++++++-- 18 files changed, 105 insertions(+), 67 deletions(-) diff --git a/bindings/pylibafl/Cargo.toml b/bindings/pylibafl/Cargo.toml index d723c35839..93a1e33bb5 100644 --- a/bindings/pylibafl/Cargo.toml +++ b/bindings/pylibafl/Cargo.toml @@ -4,8 +4,8 @@ version = "0.13.2" edition = "2021" [dependencies] -pyo3 = { version = "0.18.3", features = ["extension-module"] } -pyo3-log = "0.8.1" +pyo3 = { version = "0.22", features = ["extension-module"] } +pyo3-log = "0.11" libafl_sugar = { path = "../../libafl_sugar", version = "0.13.2", features = [ "python", ] } @@ -19,7 +19,7 @@ libafl_qemu = { path = "../../libafl_qemu", version = "0.13.2", features = [ ] } [build-dependencies] -pyo3-build-config = { version = "0.17" } +pyo3-build-config = { version = "0.22" } [lib] name = "pylibafl" diff --git a/bindings/pylibafl/src/lib.rs b/bindings/pylibafl/src/lib.rs index 017dae5211..c4371f89d2 100644 --- a/bindings/pylibafl/src/lib.rs +++ b/bindings/pylibafl/src/lib.rs @@ -6,27 +6,27 @@ use pyo3::prelude::*; /// Returns error if python libafl setup failed. #[pymodule] #[pyo3(name = "pylibafl")] -pub fn python_module(py: Python, m: &PyModule) -> PyResult<()> { +pub fn python_module(m: &Bound<'_, PyModule>) -> PyResult<()> { pyo3_log::init(); - let modules = py.import("sys")?.getattr("modules")?; + let modules = m.py().import_bound("sys")?.getattr("modules")?; - let sugar_module = PyModule::new(py, "sugar")?; - libafl_sugar::python_module(py, sugar_module)?; - m.add_submodule(sugar_module)?; + let sugar_module = PyModule::new_bound(m.py(), "sugar")?; + libafl_sugar::python_module(&sugar_module)?; + m.add_submodule(&sugar_module)?; modules.set_item("pylibafl.sugar", sugar_module)?; #[cfg(target_os = "linux")] { - let qemu_module = PyModule::new(py, "qemu")?; - libafl_qemu::python_module(py, qemu_module)?; - m.add_submodule(qemu_module)?; + let qemu_module = PyModule::new_bound(m.py(), "qemu")?; + libafl_qemu::python_module(&qemu_module)?; + m.add_submodule(&qemu_module)?; modules.set_item("pylibafl.qemu", qemu_module)?; } - let bolts_module = PyModule::new(py, "libafl_bolts")?; - libafl_bolts::pybind::python_module(py, bolts_module)?; - m.add_submodule(bolts_module)?; + let bolts_module = PyModule::new_bound(m.py(), "libafl_bolts")?; + libafl_bolts::pybind::python_module(&bolts_module)?; + m.add_submodule(&bolts_module)?; modules.set_item("pylibafl.libafl_bolts", bolts_module)?; Ok(()) diff --git a/libafl/Cargo.toml b/libafl/Cargo.toml index 09e0a5f20d..88753194fd 100644 --- a/libafl/Cargo.toml +++ b/libafl/Cargo.toml @@ -264,7 +264,7 @@ arrayvec = { version = "0.7.4", optional = true, default-features = false } # us const_format = "0.2.32" # used for providing helpful compiler output const_panic = "0.2.8" # similarly, for formatting const panic output -pyo3 = { version = "0.18.3", optional = true } # For nautilus +pyo3 = { version = "0.22", optional = true } # For nautilus regex-syntax = { version = "0.8.3", optional = true } # For nautilus # optional-dev deps (change when target.'cfg(accessible(::std))'.test-dependencies will be stable) diff --git a/libafl/src/common/nautilus/grammartec/tree.rs b/libafl/src/common/nautilus/grammartec/tree.rs index e56e27b003..71ecc53cd4 100644 --- a/libafl/src/common/nautilus/grammartec/tree.rs +++ b/libafl/src/common/nautilus/grammartec/tree.rs @@ -5,7 +5,7 @@ use hashbrown::HashSet; use libafl_bolts::rands::Rand; use pyo3::{ prelude::{PyObject, PyResult, Python}, - types::{PyBytes, PyString, PyTuple}, + types::{PyAnyMethods, PyBytes, PyBytesMethods, PyString, PyStringMethods, PyTuple}, FromPyObject, PyTypeInfo, }; use serde::{Deserialize, Serialize}; @@ -85,13 +85,14 @@ impl<'data, 'tree: 'data, 'ctx: 'data, W: Write, T: TreeLike> Unparser<'data, 't .into_iter() .map(io::Cursor::into_inner) .collect::>(); - let byte_arrays = bufs.iter().map(|b| PyBytes::new(py, b)); - let res = expr.call1(py, PyTuple::new(py, byte_arrays))?; - if PyString::is_type_of(res.as_ref(py)) { - let pystr = <&PyString>::extract(res.as_ref(py))?; + let byte_arrays = bufs.iter().map(|b| PyBytes::new_bound(py, b)); + let res = expr.call1(py, PyTuple::new_bound(py, byte_arrays))?; + let bound = res.bind(py); + if PyString::is_type_of_bound(bound) { + let pystr = bound.downcast::()?; self.write(pystr.to_string_lossy().as_bytes()); - } else if PyBytes::is_type_of(res.as_ref(py)) { - let pybytes = <&PyBytes>::extract(res.as_ref(py))?; + } else if PyBytes::is_type_of_bound(bound) { + let pybytes = bound.downcast::()?; self.write(pybytes.as_bytes()); } else { return Err(pyo3::exceptions::PyValueError::new_err( diff --git a/libafl_bolts/Cargo.toml b/libafl_bolts/Cargo.toml index d738d9665a..92b794b030 100644 --- a/libafl_bolts/Cargo.toml +++ b/libafl_bolts/Cargo.toml @@ -164,7 +164,7 @@ clap = { version = "4.5", features = [ ], optional = true } # CLI parsing, for libafl_bolts::cli / the `cli` feature log = { version = "0.4" } -pyo3 = { version = "0.18", optional = true, features = ["serde", "macros"] } +pyo3 = { version = "0.22", optional = true, features = ["serde", "macros"] } # optional-dev deps (change when target.'cfg(accessible(::std))'.test-dependencies will be stable) serial_test = { version = "3", optional = true, default-features = false, features = [ diff --git a/libafl_bolts/src/lib.rs b/libafl_bolts/src/lib.rs index f8b4ea0bea..12c6b380ce 100644 --- a/libafl_bolts/src/lib.rs +++ b/libafl_bolts/src/lib.rs @@ -623,7 +623,7 @@ impl From for Error { pyo3::Python::with_gil(|py| { if err.matches( py, - pyo3::types::PyType::new::(py), + pyo3::types::PyType::new_bound::(py), ) { Self::shutting_down() } else { @@ -1056,7 +1056,7 @@ pub unsafe fn set_error_print_panic_hook(new_stderr: RawFd) { #[allow(missing_docs)] pub mod pybind { - use pyo3::{pymodule, types::PyModule, PyResult, Python}; + use pyo3::{pymodule, types::PyModule, Bound, PyResult}; #[macro_export] macro_rules! unwrap_me_body { @@ -1188,8 +1188,8 @@ pub mod pybind { #[pymodule] #[pyo3(name = "libafl_bolts")] /// Register the classes to the python module - pub fn python_module(py: Python, m: &PyModule) -> PyResult<()> { - crate::rands::pybind::register(py, m)?; + pub fn python_module(m: &Bound<'_, PyModule>) -> PyResult<()> { + crate::rands::pybind::register(m)?; Ok(()) } } diff --git a/libafl_bolts/src/rands/mod.rs b/libafl_bolts/src/rands/mod.rs index e7516e8358..308fb243e1 100644 --- a/libafl_bolts/src/rands/mod.rs +++ b/libafl_bolts/src/rands/mod.rs @@ -689,7 +689,7 @@ pub mod pybind { } } - #[derive(Serialize, Deserialize, Debug, Clone)] + #[derive(Serialize, Deserialize, Debug)] enum PythonRandWrapper { Std(Py), } @@ -697,7 +697,7 @@ pub mod pybind { /// Rand Trait binding #[pyclass(unsendable, name = "Rand")] #[allow(clippy::unsafe_derive_deserialize)] - #[derive(Serialize, Deserialize, Debug, Clone)] + #[derive(Serialize, Deserialize, Debug)] pub struct PythonRand { wrapper: PythonRandWrapper, } @@ -730,7 +730,7 @@ pub mod pybind { } /// Register the classes to the python module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { + pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; Ok(()) diff --git a/libafl_qemu/Cargo.toml b/libafl_qemu/Cargo.toml index 6f5fafdd0c..f634a47410 100644 --- a/libafl_qemu/Cargo.toml +++ b/libafl_qemu/Cargo.toml @@ -123,7 +123,7 @@ paste = "1" enum-map = "2.7" serde_yaml = { version = "0.9", optional = true } # For parsing the injections yaml file toml = { version = "0.8.13", optional = true } # For parsing the injections toml file -pyo3 = { version = "0.18", optional = true, features = ["multiple-pymethods"] } +pyo3 = { version = "0.22", optional = true, features = ["multiple-pymethods"] } bytes-utils = "0.1" typed-builder = "0.18" memmap2 = "0.9" @@ -132,7 +132,7 @@ document-features = { version = "0.2", optional = true } [build-dependencies] libafl_qemu_build = { path = "./libafl_qemu_build", version = "0.13.2" } -pyo3-build-config = { version = "0.21", optional = true } +pyo3-build-config = { version = "0.22", optional = true } rustversion = "1.0" bindgen = "0.69" cc = "1.0" diff --git a/libafl_qemu/libafl_qemu_sys/Cargo.toml b/libafl_qemu/libafl_qemu_sys/Cargo.toml index 6a337d5459..c0a493e094 100644 --- a/libafl_qemu/libafl_qemu_sys/Cargo.toml +++ b/libafl_qemu/libafl_qemu_sys/Cargo.toml @@ -59,9 +59,9 @@ num_enum = "0.7" libc = "0.2" strum = "0.26" strum_macros = "0.26" -pyo3 = { version = "0.18", optional = true } +pyo3 = { version = "0.22", optional = true } [build-dependencies] libafl_qemu_build = { path = "../libafl_qemu_build", version = "0.13.2" } -pyo3-build-config = { version = "0.21", optional = true } +pyo3-build-config = { version = "0.22", optional = true } rustversion = "1.0" diff --git a/libafl_qemu/libafl_qemu_sys/src/lib.rs b/libafl_qemu/libafl_qemu_sys/src/lib.rs index 069eaff8fd..11020d04f2 100644 --- a/libafl_qemu/libafl_qemu_sys/src/lib.rs +++ b/libafl_qemu/libafl_qemu_sys/src/lib.rs @@ -83,6 +83,7 @@ macro_rules! extern_c_checked { unsafe impl Sync for [<__ $c_var:upper _STRUCT__>] {} #[cfg_attr(nightly, used(linker))] + #[allow(unused_unsafe)] static [<__ $c_var:upper __>]: [<__ $c_var:upper _STRUCT__>] = unsafe { [<__ $c_var:upper _STRUCT__>] { member: core::ptr::addr_of!($c_var) } }; } @@ -102,6 +103,7 @@ macro_rules! extern_c_checked { unsafe impl Sync for [<__ $c_var:upper _STRUCT__>] {} #[cfg_attr(nightly, used(linker))] + #[allow(unused_unsafe)] static mut [<__ $c_var:upper __>]: [<__ $c_var:upper _STRUCT__>] = unsafe { [<__ $c_var:upper _STRUCT__>] { member: core::ptr::addr_of!($c_var) } }; } diff --git a/libafl_qemu/src/lib.rs b/libafl_qemu/src/lib.rs index 67dbbf0796..68efb0ff02 100644 --- a/libafl_qemu/src/lib.rs +++ b/libafl_qemu/src/lib.rs @@ -79,20 +79,22 @@ use pyo3::prelude::*; #[pymodule] #[pyo3(name = "libafl_qemu")] #[allow(clippy::items_after_statements, clippy::too_many_lines)] -pub fn python_module(py: Python, m: &PyModule) -> PyResult<()> { - let regsm = PyModule::new(py, "regs")?; +pub fn python_module(m: &Bound<'_, PyModule>) -> PyResult<()> { + use pyo3::types::PyString; + + let regsm = PyModule::new_bound(m.py(), "regs")?; for r in Regs::iter() { let v: i32 = r.into(); - regsm.add(&format!("{r:?}"), v)?; + regsm.add(PyString::new_bound(m.py(), &format!("{r:?}")), v)?; } - m.add_submodule(regsm)?; + m.add_submodule(®sm)?; - let mmapm = PyModule::new(py, "mmap")?; + let mmapm = PyModule::new_bound(m.py(), "mmap")?; for r in sys::MmapPerms::iter() { let v: i32 = r.into(); - mmapm.add(&format!("{r:?}"), v)?; + mmapm.add(PyString::new_bound(m.py(), &format!("{r:?}")), v)?; } - m.add_submodule(mmapm)?; + m.add_submodule(&mmapm)?; m.add_class::()?; diff --git a/libafl_qemu/src/qemu/hooks.rs b/libafl_qemu/src/qemu/hooks.rs index 8053270ec3..e4f7cb31e2 100644 --- a/libafl_qemu/src/qemu/hooks.rs +++ b/libafl_qemu/src/qemu/hooks.rs @@ -934,25 +934,24 @@ impl QemuHooks { #[pymethods] impl SyscallHookResult { #[new] + #[pyo3(signature = ( + value=None + ))] #[must_use] pub fn new(value: Option) -> Self { - value.map_or( - Self { - retval: 0, - skip_syscall: false, - }, - |v| Self { - retval: v, - skip_syscall: true, - }, - ) + Self::new_internal(value) } } -#[cfg(not(feature = "python"))] impl SyscallHookResult { + #[cfg(not(feature = "python"))] #[must_use] pub fn new(value: Option) -> Self { + Self::new_internal(value) + } + + #[must_use] + fn new_internal(value: Option) -> Self { value.map_or( Self { retval: 0, diff --git a/libafl_qemu/src/qemu/usermode.rs b/libafl_qemu/src/qemu/usermode.rs index 763b3a9420..c754973594 100644 --- a/libafl_qemu/src/qemu/usermode.rs +++ b/libafl_qemu/src/qemu/usermode.rs @@ -233,7 +233,11 @@ impl Qemu { pub mod pybind { use libafl_qemu_sys::{GuestAddr, MmapPerms}; use pyo3::{ - exceptions::PyValueError, pymethods, types::PyInt, FromPyObject, PyObject, PyResult, Python, + conversion::FromPyObject, + exceptions::PyValueError, + pymethods, + types::{PyAnyMethods, PyInt}, + Bound, PyObject, PyResult, Python, }; use crate::{pybind::Qemu, qemu::hooks::SyscallHookResult}; @@ -258,17 +262,17 @@ pub mod pybind { let args = (sys_num, a0, a1, a2, a3, a4, a5, a6, a7); Python::with_gil(|py| { let ret = obj.call1(py, args).expect("Error in the syscall hook"); - let any = ret.as_ref(py); + let any = ret.bind(py); if any.is_none() { SyscallHookResult::new(None) } else { - let a: Result<&PyInt, _> = any.downcast(); + let a: Result<&Bound<'_, PyInt>, _> = any.downcast_exact(); if let Ok(i) = a { SyscallHookResult::new(Some( i.extract().expect("Invalid syscall hook return value"), )) } else { - SyscallHookResult::extract(any) + SyscallHookResult::extract_bound(ret.bind(py)) .expect("The syscall hook must return a SyscallHookResult") } } diff --git a/libafl_sugar/Cargo.toml b/libafl_sugar/Cargo.toml index ecc5f232a8..3d0f67dec9 100644 --- a/libafl_sugar/Cargo.toml +++ b/libafl_sugar/Cargo.toml @@ -53,7 +53,7 @@ ppc = ["libafl_qemu/ppc"] hexagon = ["libafl_qemu/hexagon"] [build-dependencies] -pyo3-build-config = { version = "0.21", optional = true } +pyo3-build-config = { version = "0.22", optional = true } [dependencies] libafl = { path = "../libafl", version = "0.13.2" } @@ -64,7 +64,7 @@ libafl_targets = { path = "../libafl_targets", version = "0.13.2" } document-features = { version = "0.2", optional = true } typed-builder = "0.18" # Implement the builder pattern at compiletime -pyo3 = { version = "0.18", optional = true } +pyo3 = { version = "0.22", optional = true } log = "0.4.20" [target.'cfg(target_os = "linux")'.dependencies] diff --git a/libafl_sugar/src/forkserver.rs b/libafl_sugar/src/forkserver.rs index 9ac5574670..8949af3588 100644 --- a/libafl_sugar/src/forkserver.rs +++ b/libafl_sugar/src/forkserver.rs @@ -335,6 +335,16 @@ pub mod pybind { /// Create a new [`ForkserverBytesCoverageSugar`] #[new] #[allow(clippy::too_many_arguments)] + #[pyo3(signature = ( + input_dirs, + output_dir, + broker_port, + cores, + use_cmplog=None, + iterations=None, + tokens_file=None, + timeout=None + ))] fn new( input_dirs: Vec, output_dir: PathBuf, @@ -377,7 +387,7 @@ pub mod pybind { } /// Register the module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { + pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } diff --git a/libafl_sugar/src/inmemory.rs b/libafl_sugar/src/inmemory.rs index cc05399f18..8e4b6675d3 100644 --- a/libafl_sugar/src/inmemory.rs +++ b/libafl_sugar/src/inmemory.rs @@ -392,6 +392,16 @@ pub mod pybind { /// Create a new [`InMemoryBytesCoverageSugar`] #[new] #[allow(clippy::too_many_arguments)] + #[pyo3(signature = ( + input_dirs, + output_dir, + broker_port, + cores, + use_cmplog=None, + iterations=None, + tokens_file=None, + timeout=None + ))] fn new( input_dirs: Vec, output_dir: PathBuf, @@ -424,7 +434,7 @@ pub mod pybind { .cores(&self.cores) .harness(|buf| { Python::with_gil(|py| -> PyResult<()> { - let args = (PyBytes::new(py, buf),); // TODO avoid copy + let args = (PyBytes::new_bound(py, buf),); // TODO avoid copy harness.call1(py, args)?; Ok(()) }) @@ -440,7 +450,7 @@ pub mod pybind { } /// Register the module - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { + pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } diff --git a/libafl_sugar/src/lib.rs b/libafl_sugar/src/lib.rs index 459562a437..fbd1bbcec8 100644 --- a/libafl_sugar/src/lib.rs +++ b/libafl_sugar/src/lib.rs @@ -86,15 +86,15 @@ use pyo3::prelude::*; #[cfg(feature = "python")] #[pymodule] #[pyo3(name = "libafl_sugar")] -pub fn python_module(py: Python, m: &PyModule) -> PyResult<()> { - inmemory::pybind::register(py, m)?; +pub fn python_module(m: &Bound<'_, PyModule>) -> PyResult<()> { + inmemory::pybind::register(m)?; #[cfg(target_os = "linux")] { - qemu::pybind::register(py, m)?; + qemu::pybind::register(m)?; } #[cfg(unix)] { - forkserver::pybind::register(py, m)?; + forkserver::pybind::register(m)?; } Ok(()) } diff --git a/libafl_sugar/src/qemu.rs b/libafl_sugar/src/qemu.rs index 4fd22bac33..203f5208fe 100644 --- a/libafl_sugar/src/qemu.rs +++ b/libafl_sugar/src/qemu.rs @@ -494,6 +494,16 @@ pub mod pybind { /// Create a new [`QemuBytesCoverageSugar`] #[new] #[allow(clippy::too_many_arguments)] + #[pyo3(signature = ( + input_dirs, + output_dir, + broker_port, + cores, + use_cmplog=None, + iterations=None, + tokens_file=None, + timeout=None + ))] fn new( input_dirs: Vec, output_dir: PathBuf, @@ -526,7 +536,7 @@ pub mod pybind { .cores(&self.cores) .harness(|buf| { Python::with_gil(|py| -> PyResult<()> { - let args = (PyBytes::new(py, buf),); // TODO avoid copy + let args = (PyBytes::new_bound(py, buf),); // TODO avoid copy harness.call1(py, args)?; Ok(()) }) @@ -542,7 +552,7 @@ pub mod pybind { } /// Register this class - pub fn register(_py: Python, m: &PyModule) -> PyResult<()> { + pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) }