SYS-OSS/FRET-LibAFL
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update pyo3 (#2459)

Browse Source 
* Update pyo3

* Update pyo3 to 0.22

* Fix qemu python

* clippy

* fmt

* nautilus python

* More pyo3

* Make signature more legible
This commit is contained in:
Dominik Maier 2024-07-29 22:55:28 +02:00 committed by GitHub
parent 651ea027b9
commit c4c0fb6750
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 105 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bindings/pylibafl/Cargo.toml
View File

@ -4,8 +4,8 @@ version = "0.13.2"
edition = "2021"
[dependencies]
pyo3 = { version = "0.18.3", features = ["extension-module"] }
pyo3-log = "0.8.1"
pyo3 = { version = "0.22", features = ["extension-module"] }
pyo3-log = "0.11"
libafl_sugar = { path = "../../libafl_sugar", version = "0.13.2", features = [
"python",
] }
@ -19,7 +19,7 @@ libafl_qemu = { path = "../../libafl_qemu", version = "0.13.2", features = [
] }
[build-dependencies]
pyo3-build-config = { version = "0.17" }
pyo3-build-config = { version = "0.22" }
[lib]
name = "pylibafl"

22
bindings/pylibafl/src/lib.rs
View File

@ -6,27 +6,27 @@ use pyo3::prelude::*;
/// Returns error if python libafl setup failed.
#[pymodule]
#[pyo3(name = "pylibafl")]
pub fn python_module(py: Python, m: &PyModule) -> PyResult<()> {
pub fn python_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
pyo3_log::init();
let modules = py.import("sys")?.getattr("modules")?;
let modules = m.py().import_bound("sys")?.getattr("modules")?;
let sugar_module = PyModule::new(py, "sugar")?;
libafl_sugar::python_module(py, sugar_module)?;
m.add_submodule(sugar_module)?;
let sugar_module = PyModule::new_bound(m.py(), "sugar")?;
libafl_sugar::python_module(&sugar_module)?;
m.add_submodule(&sugar_module)?;
modules.set_item("pylibafl.sugar", sugar_module)?;
#[cfg(target_os = "linux")]
{
let qemu_module = PyModule::new(py, "qemu")?;
libafl_qemu::python_module(py, qemu_module)?;
m.add_submodule(qemu_module)?;
let qemu_module = PyModule::new_bound(m.py(), "qemu")?;
libafl_qemu::python_module(&qemu_module)?;
m.add_submodule(&qemu_module)?;
modules.set_item("pylibafl.qemu", qemu_module)?;
}
let bolts_module = PyModule::new(py, "libafl_bolts")?;
libafl_bolts::pybind::python_module(py, bolts_module)?;
m.add_submodule(bolts_module)?;
let bolts_module = PyModule::new_bound(m.py(), "libafl_bolts")?;
libafl_bolts::pybind::python_module(&bolts_module)?;
m.add_submodule(&bolts_module)?;
modules.set_item("pylibafl.libafl_bolts", bolts_module)?;
Ok(())

2
libafl/Cargo.toml
View File

@ -264,7 +264,7 @@ arrayvec = { version = "0.7.4", optional = true, default-features = false } # us
const_format = "0.2.32" # used for providing helpful compiler output
const_panic = "0.2.8" # similarly, for formatting const panic output
pyo3 = { version = "0.18.3", optional = true } # For nautilus
pyo3 = { version = "0.22", optional = true } # For nautilus
regex-syntax = { version = "0.8.3", optional = true } # For nautilus
# optional-dev deps (change when target.'cfg(accessible(::std))'.test-dependencies will be stable)

15
libafl/src/common/nautilus/grammartec/tree.rs
View File

@ -5,7 +5,7 @@ use hashbrown::HashSet;
use libafl_bolts::rands::Rand;
use pyo3::{
prelude::{PyObject, PyResult, Python},
types::{PyBytes, PyString, PyTuple},
types::{PyAnyMethods, PyBytes, PyBytesMethods, PyString, PyStringMethods, PyTuple},
FromPyObject, PyTypeInfo,
};
use serde::{Deserialize, Serialize};
@ -85,13 +85,14 @@ impl<'data, 'tree: 'data, 'ctx: 'data, W: Write, T: TreeLike> Unparser<'data, 't
.into_iter()
.map(io::Cursor::into_inner)
.collect::<Vec<_>>();
let byte_arrays = bufs.iter().map(|b| PyBytes::new(py, b));
let res = expr.call1(py, PyTuple::new(py, byte_arrays))?;
if PyString::is_type_of(res.as_ref(py)) {
let pystr = <&PyString>::extract(res.as_ref(py))?;
let byte_arrays = bufs.iter().map(|b| PyBytes::new_bound(py, b));
let res = expr.call1(py, PyTuple::new_bound(py, byte_arrays))?;
let bound = res.bind(py);
if PyString::is_type_of_bound(bound) {
let pystr = bound.downcast::<PyString>()?;
self.write(pystr.to_string_lossy().as_bytes());
} else if PyBytes::is_type_of(res.as_ref(py)) {
let pybytes = <&PyBytes>::extract(res.as_ref(py))?;
} else if PyBytes::is_type_of_bound(bound) {
let pybytes = bound.downcast::<PyBytes>()?;
self.write(pybytes.as_bytes());
} else {
return Err(pyo3::exceptions::PyValueError::new_err(

2
libafl_bolts/Cargo.toml
View File

@ -164,7 +164,7 @@ clap = { version = "4.5", features = [
], optional = true } # CLI parsing, for libafl_bolts::cli / the `cli` feature
log = { version = "0.4" }
pyo3 = { version = "0.18", optional = true, features = ["serde", "macros"] }
pyo3 = { version = "0.22", optional = true, features = ["serde", "macros"] }
# optional-dev deps (change when target.'cfg(accessible(::std))'.test-dependencies will be stable)
serial_test = { version = "3", optional = true, default-features = false, features = [

8
libafl_bolts/src/lib.rs
View File

@ -623,7 +623,7 @@ impl From<pyo3::PyErr> for Error {
pyo3::Python::with_gil(|py| {
if err.matches(
py,
pyo3::types::PyType::new::<pyo3::exceptions::PyKeyboardInterrupt>(py),
pyo3::types::PyType::new_bound::<pyo3::exceptions::PyKeyboardInterrupt>(py),
) {
Self::shutting_down()
} else {
@ -1056,7 +1056,7 @@ pub unsafe fn set_error_print_panic_hook(new_stderr: RawFd) {
#[allow(missing_docs)]
pub mod pybind {
use pyo3::{pymodule, types::PyModule, PyResult, Python};
use pyo3::{pymodule, types::PyModule, Bound, PyResult};
#[macro_export]
macro_rules! unwrap_me_body {
@ -1188,8 +1188,8 @@ pub mod pybind {
#[pymodule]
#[pyo3(name = "libafl_bolts")]
/// Register the classes to the python module
pub fn python_module(py: Python, m: &PyModule) -> PyResult<()> {
crate::rands::pybind::register(py, m)?;
pub fn python_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
crate::rands::pybind::register(m)?;
Ok(())
}
}

6
libafl_bolts/src/rands/mod.rs
View File

@ -689,7 +689,7 @@ pub mod pybind {
}
}
#[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Serialize, Deserialize, Debug)]
enum PythonRandWrapper {
Std(Py<PythonStdRand>),
}
@ -697,7 +697,7 @@ pub mod pybind {
/// Rand Trait binding
#[pyclass(unsendable, name = "Rand")]
#[allow(clippy::unsafe_derive_deserialize)]
#[derive(Serialize, Deserialize, Debug, Clone)]
#[derive(Serialize, Deserialize, Debug)]
pub struct PythonRand {
wrapper: PythonRandWrapper,
}
@ -730,7 +730,7 @@ pub mod pybind {
}
/// Register the classes to the python module
pub fn register(_py: Python, m: &PyModule) -> PyResult<()> {
pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<PythonStdRand>()?;
m.add_class::<PythonRand>()?;
Ok(())

4
libafl_qemu/Cargo.toml
View File

@ -123,7 +123,7 @@ paste = "1"
enum-map = "2.7"
serde_yaml = { version = "0.9", optional = true } # For parsing the injections yaml file
toml = { version = "0.8.13", optional = true } # For parsing the injections toml file
pyo3 = { version = "0.18", optional = true, features = ["multiple-pymethods"] }
pyo3 = { version = "0.22", optional = true, features = ["multiple-pymethods"] }
bytes-utils = "0.1"
typed-builder = "0.18"
memmap2 = "0.9"
@ -132,7 +132,7 @@ document-features = { version = "0.2", optional = true }
[build-dependencies]
libafl_qemu_build = { path = "./libafl_qemu_build", version = "0.13.2" }
pyo3-build-config = { version = "0.21", optional = true }
pyo3-build-config = { version = "0.22", optional = true }
rustversion = "1.0"
bindgen = "0.69"
cc = "1.0"

4
libafl_qemu/libafl_qemu_sys/Cargo.toml
View File

@ -59,9 +59,9 @@ num_enum = "0.7"
libc = "0.2"
strum = "0.26"
strum_macros = "0.26"
pyo3 = { version = "0.18", optional = true }
pyo3 = { version = "0.22", optional = true }
[build-dependencies]
libafl_qemu_build = { path = "../libafl_qemu_build", version = "0.13.2" }
pyo3-build-config = { version = "0.21", optional = true }
pyo3-build-config = { version = "0.22", optional = true }
rustversion = "1.0"

2
libafl_qemu/libafl_qemu_sys/src/lib.rs
View File

@ -83,6 +83,7 @@ macro_rules! extern_c_checked {
unsafe impl Sync for [<__ $c_var:upper _STRUCT__>] {}
#[cfg_attr(nightly, used(linker))]
#[allow(unused_unsafe)]
static [<__ $c_var:upper __>]: [<__ $c_var:upper _STRUCT__>] = unsafe { [<__ $c_var:upper _STRUCT__>] { member: core::ptr::addr_of!($c_var) } };
}
@ -102,6 +103,7 @@ macro_rules! extern_c_checked {
unsafe impl Sync for [<__ $c_var:upper _STRUCT__>] {}
#[cfg_attr(nightly, used(linker))]
#[allow(unused_unsafe)]
static mut [<__ $c_var:upper __>]: [<__ $c_var:upper _STRUCT__>] = unsafe { [<__ $c_var:upper _STRUCT__>] { member: core::ptr::addr_of!($c_var) } };
}

16
libafl_qemu/src/lib.rs
View File

@ -79,20 +79,22 @@ use pyo3::prelude::*;
#[pymodule]
#[pyo3(name = "libafl_qemu")]
#[allow(clippy::items_after_statements, clippy::too_many_lines)]
pub fn python_module(py: Python, m: &PyModule) -> PyResult<()> {
let regsm = PyModule::new(py, "regs")?;
pub fn python_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
use pyo3::types::PyString;
let regsm = PyModule::new_bound(m.py(), "regs")?;
for r in Regs::iter() {
let v: i32 = r.into();
regsm.add(&format!("{r:?}"), v)?;
regsm.add(PyString::new_bound(m.py(), &format!("{r:?}")), v)?;
}
m.add_submodule(regsm)?;
m.add_submodule(&regsm)?;
let mmapm = PyModule::new(py, "mmap")?;
let mmapm = PyModule::new_bound(m.py(), "mmap")?;
for r in sys::MmapPerms::iter() {
let v: i32 = r.into();
mmapm.add(&format!("{r:?}"), v)?;
mmapm.add(PyString::new_bound(m.py(), &format!("{r:?}")), v)?;
}
m.add_submodule(mmapm)?;
m.add_submodule(&mmapm)?;
m.add_class::<sys::MapInfo>()?;

21
libafl_qemu/src/qemu/hooks.rs
View File

@ -934,25 +934,24 @@ impl QemuHooks {
#[pymethods]
impl SyscallHookResult {
#[new]
#[pyo3(signature = (
value=None
))]
#[must_use]
pub fn new(value: Option<GuestAddr>) -> Self {
value.map_or(
Self {
retval: 0,
skip_syscall: false,
},
|v| Self {
retval: v,
skip_syscall: true,
},
)
Self::new_internal(value)
}
}
#[cfg(not(feature = "python"))]
impl SyscallHookResult {
#[cfg(not(feature = "python"))]
#[must_use]
pub fn new(value: Option<GuestAddr>) -> Self {
Self::new_internal(value)
}
#[must_use]
fn new_internal(value: Option<GuestAddr>) -> Self {
value.map_or(
Self {
retval: 0,

12
libafl_qemu/src/qemu/usermode.rs
View File

@ -233,7 +233,11 @@ impl Qemu {
pub mod pybind {
use libafl_qemu_sys::{GuestAddr, MmapPerms};
use pyo3::{
exceptions::PyValueError, pymethods, types::PyInt, FromPyObject, PyObject, PyResult, Python,
conversion::FromPyObject,
exceptions::PyValueError,
pymethods,
types::{PyAnyMethods, PyInt},
Bound, PyObject, PyResult, Python,
};
use crate::{pybind::Qemu, qemu::hooks::SyscallHookResult};
@ -258,17 +262,17 @@ pub mod pybind {
let args = (sys_num, a0, a1, a2, a3, a4, a5, a6, a7);
Python::with_gil(|py| {
let ret = obj.call1(py, args).expect("Error in the syscall hook");
let any = ret.as_ref(py);
let any = ret.bind(py);
if any.is_none() {
SyscallHookResult::new(None)
} else {
let a: Result<&PyInt, _> = any.downcast();
let a: Result<&Bound<'_, PyInt>, _> = any.downcast_exact();
if let Ok(i) = a {
SyscallHookResult::new(Some(
i.extract().expect("Invalid syscall hook return value"),
))
} else {
SyscallHookResult::extract(any)
SyscallHookResult::extract_bound(ret.bind(py))
.expect("The syscall hook must return a SyscallHookResult")
}
}

4
libafl_sugar/Cargo.toml
View File

@ -53,7 +53,7 @@ ppc = ["libafl_qemu/ppc"]
hexagon = ["libafl_qemu/hexagon"]
[build-dependencies]
pyo3-build-config = { version = "0.21", optional = true }
pyo3-build-config = { version = "0.22", optional = true }
[dependencies]
libafl = { path = "../libafl", version = "0.13.2" }
@ -64,7 +64,7 @@ libafl_targets = { path = "../libafl_targets", version = "0.13.2" }
document-features = { version = "0.2", optional = true }
typed-builder = "0.18" # Implement the builder pattern at compiletime
pyo3 = { version = "0.18", optional = true }
pyo3 = { version = "0.22", optional = true }
log = "0.4.20"
[target.'cfg(target_os = "linux")'.dependencies]

12
libafl_sugar/src/forkserver.rs
View File

@ -335,6 +335,16 @@ pub mod pybind {
/// Create a new [`ForkserverBytesCoverageSugar`]
#[new]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (
input_dirs,
output_dir,
broker_port,
cores,
use_cmplog=None,
iterations=None,
tokens_file=None,
timeout=None
))]
fn new(
input_dirs: Vec<PathBuf>,
output_dir: PathBuf,
@ -377,7 +387,7 @@ pub mod pybind {
}
/// Register the module
pub fn register(_py: Python, m: &PyModule) -> PyResult<()> {
pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<ForkserverBytesCoverageSugar>()?;
Ok(())
}

14
libafl_sugar/src/inmemory.rs
View File

@ -392,6 +392,16 @@ pub mod pybind {
/// Create a new [`InMemoryBytesCoverageSugar`]
#[new]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (
input_dirs,
output_dir,
broker_port,
cores,
use_cmplog=None,
iterations=None,
tokens_file=None,
timeout=None
))]
fn new(
input_dirs: Vec<PathBuf>,
output_dir: PathBuf,
@ -424,7 +434,7 @@ pub mod pybind {
.cores(&self.cores)
.harness(|buf| {
Python::with_gil(|py| -> PyResult<()> {
let args = (PyBytes::new(py, buf),); // TODO avoid copy
let args = (PyBytes::new_bound(py, buf),); // TODO avoid copy
harness.call1(py, args)?;
Ok(())
})
@ -440,7 +450,7 @@ pub mod pybind {
}
/// Register the module
pub fn register(_py: Python, m: &PyModule) -> PyResult<()> {
pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<InMemoryBytesCoverageSugar>()?;
Ok(())
}

8
libafl_sugar/src/lib.rs
View File

@ -86,15 +86,15 @@ use pyo3::prelude::*;
#[cfg(feature = "python")]
#[pymodule]
#[pyo3(name = "libafl_sugar")]
pub fn python_module(py: Python, m: &PyModule) -> PyResult<()> {
inmemory::pybind::register(py, m)?;
pub fn python_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
inmemory::pybind::register(m)?;
#[cfg(target_os = "linux")]
{
qemu::pybind::register(py, m)?;
qemu::pybind::register(m)?;
}
#[cfg(unix)]
{
forkserver::pybind::register(py, m)?;
forkserver::pybind::register(m)?;
}
Ok(())
}

14
libafl_sugar/src/qemu.rs
View File

@ -494,6 +494,16 @@ pub mod pybind {
/// Create a new [`QemuBytesCoverageSugar`]
#[new]
#[allow(clippy::too_many_arguments)]
#[pyo3(signature = (
input_dirs,
output_dir,
broker_port,
cores,
use_cmplog=None,
iterations=None,
tokens_file=None,
timeout=None
))]
fn new(
input_dirs: Vec<PathBuf>,
output_dir: PathBuf,
@ -526,7 +536,7 @@ pub mod pybind {
.cores(&self.cores)
.harness(|buf| {
Python::with_gil(|py| -> PyResult<()> {
let args = (PyBytes::new(py, buf),); // TODO avoid copy
let args = (PyBytes::new_bound(py, buf),); // TODO avoid copy
harness.call1(py, args)?;
Ok(())
})
@ -542,7 +552,7 @@ pub mod pybind {
}
/// Register this class
pub fn register(_py: Python, m: &PyModule) -> PyResult<()> {
pub fn register(m: &Bound<'_, PyModule>) -> PyResult<()> {
m.add_class::<QemuBytesCoverageSugar>()?;
Ok(())
}