SYS-OSS/FRET-LibAFL
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update pyo3 to version 0.23.2 (#2732)

Browse Source 
* update pyo3 to latest version

* add python bindings to workspace

* make pyo3 stuff dependent of workspace again

* adapt implementation for the newest version of pyo3
This commit is contained in:
Romain Malmain 2024-11-27 19:01:31 +01:00 committed by GitHub
parent f30cd2a8ab
commit 94fa4014ac
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 80 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.toml
View File

@ -25,6 +25,7 @@ members = [
"utils/gramatron/construct_automata",
"utils/libafl_benches",
"utils/libafl_jumper",
"bindings/pylibafl",
]
default-members = [
"libafl",
@ -35,7 +36,6 @@ default-members = [
]
exclude = [
"bindings",
"fuzzers",
"libafl_libfuzzer_runtime",
"utils/noaslr",
@ -102,6 +102,8 @@ paste = "1.0.15"
postcard = { version = "1.0.10", features = [
"alloc",
], default-features = false } # no_std compatible serde serialization format
pyo3 = "0.23.2"
pyo3-build-config = "0.23.2"
rangemap = "1.5.1"
regex = "1.10.6"
rustversion = "1.0.17"

6
bindings/pylibafl/Cargo.toml
View File

@ -9,8 +9,8 @@ edition = "2021"
categories = ["development-tools::testing", "emulators", "embedded", "os"]
[dependencies]
pyo3 = { version = "0.22.3", features = ["extension-module"] }
pyo3-log = { version = "0.11.0" }
pyo3 = { workspace = true, features = ["extension-module"] }
pyo3-log = { version = "0.12.0" }
libafl_sugar = { path = "../../libafl_sugar", version = "0.14.0", features = [
"python",
] }
@ -24,7 +24,7 @@ libafl_qemu = { path = "../../libafl_qemu", version = "0.14.0", features = [
] }
[build-dependencies]
pyo3-build-config = { version = "0.22.3" }
pyo3-build-config = { workspace = true }
[lib]
name = "pylibafl"

8
bindings/pylibafl/src/lib.rs
View File

@ -9,22 +9,22 @@ use pyo3::prelude::*;
pub fn python_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
pyo3_log::init();
let modules = m.py().import_bound("sys")?.getattr("modules")?;
let modules = m.py().import("sys")?.getattr("modules")?;
let sugar_module = PyModule::new_bound(m.py(), "sugar")?;
let sugar_module = PyModule::new(m.py(), "sugar")?;
libafl_sugar::python_module(&sugar_module)?;
m.add_submodule(&sugar_module)?;
modules.set_item("pylibafl.sugar", sugar_module)?;
#[cfg(target_os = "linux")]
{
let qemu_module = PyModule::new_bound(m.py(), "qemu")?;
let qemu_module = PyModule::new(m.py(), "qemu")?;
libafl_qemu::python_module(&qemu_module)?;
m.add_submodule(&qemu_module)?;
modules.set_item("pylibafl.qemu", qemu_module)?;
}
let bolts_module = PyModule::new_bound(m.py(), "libafl_bolts")?;
let bolts_module = PyModule::new(m.py(), "libafl_bolts")?;
libafl_bolts::pybind::python_module(&bolts_module)?;
m.add_submodule(&bolts_module)?;
modules.set_item("pylibafl.libafl_bolts", bolts_module)?;

4
libafl/Cargo.toml
View File

@ -275,8 +275,8 @@ arrayvec = { version = "0.7.6", optional = true, default-features = false } # us
const_format = "0.2.33" # used for providing helpful compiler output
const_panic = "0.2.9" # similarly, for formatting const panic output
pyo3 = { version = "0.22.3", features = ["gil-refs"], optional = true }
regex-syntax = { version = "0.8.4", optional = true } # For nautilus
pyo3 = { workspace = true, optional = true }
regex-syntax = { version = "0.8.4", optional = true } # For nautilus
# optional-dev deps (change when target.'cfg(accessible(::std))'.test-dependencies will be stable)
serial_test = { workspace = true, optional = true, default-features = false, features = [

2
libafl_bolts/Cargo.toml
View File

@ -156,7 +156,7 @@ clap = { workspace = true, features = [
"wrap_help",
], optional = true } # CLI parsing, for libafl_bolts::cli / the `cli` feature
log = { workspace = true }
pyo3 = { version = "0.22.3", optional = true, features = ["serde", "macros"] }
pyo3 = { workspace = true, optional = true, features = ["serde", "macros"] }
# optional-dev deps (change when target.'cfg(accessible(::std))'.test-dependencies will be stable)
serial_test = { workspace = true, optional = true, default-features = false, features = [

11
libafl_bolts/src/lib.rs
View File

@ -635,10 +635,13 @@ impl From<windows_result::Error> for Error {
impl From<pyo3::PyErr> for Error {
fn from(err: pyo3::PyErr) -> Self {
pyo3::Python::with_gil(|py| {
if err.matches(
py,
pyo3::types::PyType::new_bound::<pyo3::exceptions::PyKeyboardInterrupt>(py),
) {
if err
.matches(
py,
pyo3::types::PyType::new::<pyo3::exceptions::PyKeyboardInterrupt>(py),
)
.unwrap()
{
Self::shutting_down()
} else {
Self::illegal_state(format!("Python exception: {err:?}"))

6
libafl_qemu/Cargo.toml
View File

@ -120,9 +120,7 @@ paste = { workspace = true }
enum-map = "2.7.3"
serde_yaml = { workspace = true, optional = true } # For parsing the injections yaml file
toml = { workspace = true, optional = true } # For parsing the injections toml file
pyo3 = { version = "0.22.3", optional = true, features = [
"multiple-pymethods",
] }
pyo3 = { workspace = true, optional = true, features = ["multiple-pymethods"] }
bytes-utils = "0.1.4"
typed-builder = { workspace = true }
memmap2 = "0.9.5"
@ -132,7 +130,7 @@ document-features = { workspace = true, optional = true }
[build-dependencies]
libafl_qemu_build = { workspace = true, default-features = true, version = "0.14.0" }
pyo3-build-config = { version = "0.23.1", optional = true }
pyo3-build-config = { workspace = true, optional = true }
rustversion = { workspace = true }
bindgen = { workspace = true }
cc = { workspace = true }

4
libafl_qemu/libafl_qemu_sys/Cargo.toml
View File

@ -63,11 +63,11 @@ num_enum = { workspace = true, default-features = true }
libc = { workspace = true }
strum = { workspace = true }
strum_macros = { workspace = true }
pyo3 = { version = "0.22.3", optional = true }
pyo3 = { workspace = true, optional = true }
[build-dependencies]
libafl_qemu_build = { workspace = true, default-features = true }
pyo3-build-config = { version = "0.23.1", optional = true }
pyo3-build-config = { workspace = true, optional = true }
rustversion = { workspace = true }
[lints]

20
libafl_qemu/libafl_qemu_sys/src/usermode.rs
View File

@ -1,12 +1,18 @@
#[cfg(target_os = "linux")]
use core::{slice::from_raw_parts, str::from_utf8_unchecked};
#[cfg(feature = "python")]
use std::convert::Infallible;
#[cfg(target_os = "linux")]
use libc::{c_char, strlen};
use num_enum::{IntoPrimitive, TryFromPrimitive};
#[cfg(feature = "python")]
use pyo3::{pyclass, pymethods, IntoPy, PyObject, Python};
use pyo3::{pyclass, pymethods, types::PyInt, Bound, IntoPyObject, Python};
use strum_macros::EnumIter;
use crate::{libafl_mapinfo, GuestAddr, MmapPerms};
use crate::MmapPerms;
#[cfg(target_os = "linux")]
use crate::{libafl_mapinfo, GuestAddr};
#[derive(IntoPrimitive, TryFromPrimitive, Debug, Clone, Copy, EnumIter, PartialEq, Eq)]
#[repr(i32)]
@ -98,10 +104,14 @@ impl MmapPerms {
}
#[cfg(feature = "python")]
impl IntoPy<PyObject> for MmapPerms {
fn into_py(self, py: Python) -> PyObject {
impl<'py> IntoPyObject<'py> for MmapPerms {
type Target = PyInt;
type Output = Bound<'py, Self::Target>;
type Error = Infallible;
fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
let n: i32 = self.into();
n.into_py(py)
n.into_pyobject(py)
}
}

8
libafl_qemu/src/arch/aarch64.rs
View File

@ -73,14 +73,6 @@ impl Regs {
pub const Lr: Regs = Regs::X30;
}
#[cfg(feature = "python")]
impl IntoPy<PyObject> for Regs {
fn into_py(self, py: Python) -> PyObject {
let n: i32 = self.into();
n.into_py(py)
}
}
/// Return an ARM64 ArchCapstoneBuilder
pub fn capstone() -> capstone::arch::arm64::ArchCapstoneBuilder {
capstone::Capstone::new()

8
libafl_qemu/src/arch/arm.rs
View File

@ -63,14 +63,6 @@ impl Regs {
pub const Cpsr: Regs = Regs::R25;
}
#[cfg(feature = "python")]
impl IntoPy<PyObject> for Regs {
fn into_py(self, py: Python) -> PyObject {
let n: i32 = self.into();
n.into_py(py)
}
}
/// Return an ARM ArchCapstoneBuilder
pub fn capstone() -> capstone::arch::arm::ArchCapstoneBuilder {
capstone::Capstone::new()

8
libafl_qemu/src/arch/i386.rs
View File

@ -49,14 +49,6 @@ impl Regs {
pub const Pc: Regs = Regs::Eip;
}
#[cfg(feature = "python")]
impl IntoPy<PyObject> for Regs {
fn into_py(self, py: Python) -> PyObject {
let n: i32 = self.into();
n.into_py(py)
}
}
/// Return an X86 ArchCapstoneBuilder
pub fn capstone() -> capstone::arch::x86::ArchCapstoneBuilder {
capstone::Capstone::new()

8
libafl_qemu/src/arch/mips.rs
View File

@ -72,14 +72,6 @@ impl Regs {
pub const Zero: Regs = Regs::R0;
}
#[cfg(feature = "python")]
impl IntoPy<PyObject> for Regs {
fn into_py(self, py: Python) -> PyObject {
let n: i32 = self.into();
n.into_py(py)
}
}
/// Return an MIPS ArchCapstoneBuilder
pub fn capstone() -> capstone::arch::mips::ArchCapstoneBuilder {
capstone::Capstone::new().mips()

19
libafl_qemu/src/arch/mod.rs
View File

@ -1,5 +1,12 @@
#[cfg(feature = "python")]
use std::convert::Infallible;
#[cfg(feature = "python")]
use pyo3::{prelude::*, types::PyInt};
#[cfg(cpu_target = "aarch64")]
pub mod aarch64;
#[cfg(all(cpu_target = "aarch64", not(feature = "clippy")))]
pub use aarch64::*;
@ -37,3 +44,15 @@ pub use hexagon::*;
pub mod riscv;
#[cfg(any(cpu_target = "riscv32", cpu_target = "riscv64"))]
pub use riscv::*;
#[cfg(feature = "python")]
impl<'py> IntoPyObject<'py> for Regs {
type Target = PyInt;
type Output = Bound<'py, Self::Target>;
type Error = Infallible;
fn into_pyobject(self, py: Python<'py>) -> Result<Self::Output, Self::Error> {
let n: i32 = self.into();
n.into_pyobject(py)
}
}

8
libafl_qemu/src/arch/ppc.rs
View File

@ -112,14 +112,6 @@ impl Regs {
pub const Sp: Regs = Regs::R1;
}
#[cfg(feature = "python")]
impl IntoPy<PyObject> for Regs {
fn into_py(self, py: Python) -> PyObject {
let n: i32 = self.into();
n.into_py(py)
}
}
/// Return an MIPS ArchCapstoneBuilder
pub fn capstone() -> capstone::arch::ppc::ArchCapstoneBuilder {
capstone::Capstone::new().ppc()

10
libafl_qemu/src/arch/x86_64.rs
View File

@ -3,8 +3,6 @@ use std::{mem::size_of, ops::Range, sync::OnceLock};
use capstone::arch::BuildsCapstone;
use enum_map::{enum_map, EnumMap};
use num_enum::{IntoPrimitive, TryFromPrimitive};
#[cfg(feature = "python")]
use pyo3::prelude::*;
pub use strum_macros::EnumIter;
pub use syscall_numbers::x86_64::*;
@ -57,14 +55,6 @@ impl Regs {
pub const Pc: Regs = Regs::Rip;
}
#[cfg(feature = "python")]
impl IntoPy<PyObject> for Regs {
fn into_py(self, py: Python) -> PyObject {
let n: i32 = self.into();
n.into_py(py)
}
}
/// Return an X86 `ArchCapstoneBuilder`
#[must_use]
pub fn capstone() -> capstone::arch::x86::ArchCapstoneBuilder {

8
libafl_qemu/src/lib.rs
View File

@ -75,17 +75,17 @@ use pyo3::prelude::*;
pub fn python_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
use pyo3::types::PyString;
let regsm = PyModule::new_bound(m.py(), "regs")?;
let regsm = PyModule::new(m.py(), "regs")?;
for r in Regs::iter() {
let v: i32 = r.into();
regsm.add(PyString::new_bound(m.py(), &format!("{r:?}")), v)?;
regsm.add(PyString::new(m.py(), &format!("{r:?}")), v)?;
}
m.add_submodule(&regsm)?;
let mmapm = PyModule::new_bound(m.py(), "mmap")?;
let mmapm = PyModule::new(m.py(), "mmap")?;
for r in MmapPerms::iter() {
let v: i32 = r.into();
mmapm.add(PyString::new_bound(m.py(), &format!("{r:?}")), v)?;
mmapm.add(PyString::new(m.py(), &format!("{r:?}")), v)?;
}
m.add_submodule(&mmapm)?;

14
libafl_qemu/src/qemu/mod.rs
View File

@ -1103,8 +1103,8 @@ pub mod pybind {
extern "C" fn py_generic_hook_wrapper(idx: u64, _pc: GuestAddr) {
let obj = unsafe {
let hooks = &mut *&raw mut PY_GENERIC_HOOKS;
&hooks[idx as usize].1
let hooks = &raw mut PY_GENERIC_HOOKS;
&(*hooks)[idx as usize].1
};
Python::with_gil(|py| {
obj.call0(py).expect("Error in the hook");
@ -1183,9 +1183,9 @@ pub mod pybind {
/// Removes a hooke from `PY_GENERIC_HOOKS` -> may not be called concurrently!
unsafe fn set_hook(&self, addr: GuestAddr, hook: PyObject) {
unsafe {
let hooks = &mut *&raw mut PY_GENERIC_HOOKS;
let idx = hooks.len();
hooks.push((addr, hook));
let hooks = &raw mut PY_GENERIC_HOOKS;
let idx = (*hooks).len();
(*hooks).push((addr, hook));
self.qemu.hooks().add_instruction_hooks(
idx as u64,
addr,
@ -1199,8 +1199,8 @@ pub mod pybind {
/// Removes a hooke from `PY_GENERIC_HOOKS` -> may not be called concurrently!
unsafe fn remove_hooks_at(&self, addr: GuestAddr) -> usize {
unsafe {
let hooks = &mut *&raw mut PY_GENERIC_HOOKS;
hooks.retain(|(a, _)| *a != addr);
let hooks = &raw mut PY_GENERIC_HOOKS;
(*hooks).retain(|(a, _)| *a != addr);
}
self.qemu.hooks().remove_instruction_hooks_at(addr, true)
}

11
libafl_qemu/src/qemu/usermode.rs
View File

@ -11,7 +11,7 @@ use libafl_qemu_sys::{
};
use libc::{c_int, c_uchar, strlen};
#[cfg(feature = "python")]
use pyo3::{pyclass, pymethods, IntoPy, PyObject, PyRef, PyRefMut, Python};
use pyo3::{pyclass, pymethods, IntoPyObject, Py, PyRef, PyRefMut, Python};
use crate::{Qemu, CPU};
@ -65,8 +65,9 @@ impl GuestMaps {
fn __iter__(slf: PyRef<Self>) -> PyRef<Self> {
slf
}
fn __next__(mut slf: PyRefMut<Self>) -> Option<PyObject> {
Python::with_gil(|py| slf.next().map(|x| x.into_py(py)))
fn __next__(mut slf: PyRefMut<Self>) -> Option<Py<MapInfo>> {
Python::with_gil(|py| slf.next().map(|x| x.into_pyobject(py).unwrap().into()))
}
}
@ -281,7 +282,7 @@ pub mod pybind {
a6: u64,
a7: u64,
) -> SyscallHookResult {
unsafe { (*&raw const PY_SYSCALL_HOOK).as_ref() }.map_or_else(
unsafe { PY_SYSCALL_HOOK.as_ref() }.map_or_else(
|| SyscallHookResult::new(None),
|obj| {
let args = (sys_num, a0, a1, a2, a3, a4, a5, a6, a7);
@ -362,7 +363,7 @@ pub mod pybind {
/// Accesses the global `PY_SYSCALL_HOOK` and may not be called concurrently.
unsafe fn set_syscall_hook(&self, hook: PyObject) {
unsafe {
(*&raw mut (PY_SYSCALL_HOOK)) = Some(hook);
PY_SYSCALL_HOOK = Some(hook);
}
self.qemu
.hooks()

6
libafl_sugar/Cargo.toml
View File

@ -57,7 +57,7 @@ riscv32 = ["libafl_qemu/riscv32"]
riscv64 = ["libafl_qemu/riscv64"]
[build-dependencies]
pyo3-build-config = { version = "0.23.1", optional = true }
pyo3-build-config = { workspace = true, optional = true }
[dependencies]
libafl = { workspace = true, default-features = true }
@ -67,8 +67,8 @@ libafl_targets = { workspace = true, default-features = true }
# Document all features of this crate (for `cargo doc`)
document-features = { workspace = true, optional = true }
typed-builder = { workspace = true } # Implement the builder pattern at compiletime
pyo3 = { version = "0.22.3", optional = true }
typed-builder = { workspace = true } # Implement the builder pattern at compiletime
pyo3 = { workspace = true, optional = true }
log = { workspace = true }
[target.'cfg(target_os = "linux")'.dependencies]

2
libafl_sugar/src/inmemory.rs
View File

@ -436,7 +436,7 @@ pub mod pybind {
.cores(&self.cores)
.harness(|buf| {
Python::with_gil(|py| -> PyResult<()> {
let args = (PyBytes::new_bound(py, buf),); // TODO avoid copy
let args = (PyBytes::new(py, buf),); // TODO avoid copy
harness.call1(py, args)?;
Ok(())
})

2
libafl_sugar/src/qemu.rs
View File

@ -541,7 +541,7 @@ pub mod pybind {
.cores(&self.cores)
.harness(|buf| {
Python::with_gil(|py| -> PyResult<()> {
let args = (PyBytes::new_bound(py, buf),); // TODO avoid copy
let args = (PyBytes::new(py, buf),); // TODO avoid copy
harness.call1(py, args)?;
Ok(())
})