Improve map feedback/observer (#665)

* improve

* a

* fix it back

* more

* NO

* try

* fix python

* more

* specialize map map feedback with u8

* more

* fmt

* usable_count = len

* clp

* restore iterator based map feedback

* simd specialization

* optimize hitcounts

* fix hitcounts

* no_std

* moar unsafe

* fix

* clippy

* clippy

* opt non-specialized is_interesting

* fmt

* op post_exec

* cleanup

* even more

* allow

Co-authored-by: Andrea Fioraldi <andreafioraldi@gmail.com>
This commit is contained in:
Dongjia Zhang 2022-06-10 17:14:12 +09:00 committed by GitHub
parent 986030732a
commit c9f802a3b8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 492 additions and 115 deletions

View File

@ -197,7 +197,8 @@ pub fn main() {
state.add_metadata(tokens); state.add_metadata(tokens);
// Setup a mutational stage with a basic bytes mutator // Setup a mutational stage with a basic bytes mutator
let mutator = StdScheduledMutator::new(havoc_mutations().merge(tokens_mutations())); let mutator =
StdScheduledMutator::with_max_stack_pow(havoc_mutations().merge(tokens_mutations()), 6);
let mut stages = tuple_list!(StdMutationalStage::new(mutator)); let mut stages = tuple_list!(StdMutationalStage::new(mutator));
fuzzer fuzzer

View File

@ -32,4 +32,4 @@ int main(int argc, char **argv) {
vuln(buf); vuln(buf);
return 0; return 0;
} }

View File

@ -12,7 +12,7 @@ edition = "2021"
categories = ["development-tools::testing", "emulators", "embedded", "os", "no-std"] categories = ["development-tools::testing", "emulators", "embedded", "os", "no-std"]
[features] [features]
default = ["std", "derive", "llmp_compression", "rand_trait", "fork"] default = ["std", "derive", "llmp_compression", "rand_trait", "fork", "introspection"]
std = ["serde_json", "serde_json/std", "hostname", "nix", "serde/std", "bincode", "wait-timeout", "regex", "byteorder", "once_cell", "uuid", "tui_monitor", "ctor", "backtrace", "num_cpus"] # print, env, launcher ... support std = ["serde_json", "serde_json/std", "hostname", "nix", "serde/std", "bincode", "wait-timeout", "regex", "byteorder", "once_cell", "uuid", "tui_monitor", "ctor", "backtrace", "num_cpus"] # print, env, launcher ... support
derive = ["libafl_derive"] # provide derive(SerdeAny) macro. derive = ["libafl_derive"] # provide derive(SerdeAny) macro.
fork = [] # uses the fork() syscall to spawn children, instead of launching a new command, if supported by the OS (has no effect on Windows, no_std). fork = [] # uses the fork() syscall to spawn children, instead of launching a new command, if supported by the OS (has no effect on Windows, no_std).

View File

@ -308,7 +308,10 @@ where
/// Reset the map /// Reset the map
pub fn reset(&mut self) -> Result<(), Error> { pub fn reset(&mut self) -> Result<(), Error> {
self.history_map.iter_mut().for_each(|x| *x = T::default()); let cnt = self.history_map.len();
for i in 0..cnt {
self.history_map[i] = T::default();
}
Ok(()) Ok(())
} }
} }
@ -353,79 +356,36 @@ where
Ok(()) Ok(())
} }
#[allow(clippy::wrong_self_convention)] #[rustversion::nightly]
fn is_interesting<EM, OT>( default fn is_interesting<EM, OT>(
&mut self, &mut self,
state: &mut S, state: &mut S,
manager: &mut EM, manager: &mut EM,
_input: &I, input: &I,
observers: &OT, observers: &OT,
_exit_kind: &ExitKind, exit_kind: &ExitKind,
) -> Result<bool, Error> ) -> Result<bool, Error>
where where
EM: EventFirer<I>, EM: EventFirer<I>,
OT: ObserversTuple<I, S>, OT: ObserversTuple<I, S>,
{ {
let mut interesting = false; self.is_interesting_default(state, manager, input, observers, exit_kind)
// TODO Replace with match_name_type when stable }
let observer = observers.match_name::<O>(&self.observer_name).unwrap();
let size = observer.usable_count();
let initial = observer.initial();
let map_state = state #[rustversion::not(nightly)]
.named_metadata_mut() fn is_interesting<EM, OT>(
.get_mut::<MapFeedbackMetadata<T>>(&self.name) &mut self,
.unwrap(); state: &mut S,
if map_state.history_map.len() < observer.len() { manager: &mut EM,
map_state.history_map.resize(observer.len(), T::default()); input: &I,
} observers: &OT,
exit_kind: &ExitKind,
// assert!(size <= map_state.history_map.len(), "The size of the associated map observer cannot exceed the size of the history map of the feedback. If you are running multiple instances of slightly different fuzzers (e.g. one with ASan and another without) synchronized using LLMP please check the `configuration` field of the LLMP manager."); ) -> Result<bool, Error>
where
assert!(size <= observer.len()); EM: EventFirer<I>,
OT: ObserversTuple<I, S>,
if self.novelties.is_some() { {
for (i, &item) in observer.as_ref_iter().enumerate() { self.is_interesting_default(state, manager, input, observers, exit_kind)
let history = map_state.history_map[i];
let reduced = R::reduce(history, item);
if N::is_novel(history, reduced) {
map_state.history_map[i] = reduced;
interesting = true;
self.novelties.as_mut().unwrap().push(i);
}
}
} else {
for (i, &item) in observer.as_ref_iter().enumerate() {
let history = map_state.history_map[i];
let reduced = R::reduce(history, item);
if N::is_novel(history, reduced) {
map_state.history_map[i] = reduced;
interesting = true;
}
}
}
if interesting {
let mut filled = 0;
for i in 0..size {
if map_state.history_map[i] != initial {
filled += 1;
if self.indexes.is_some() {
self.indexes.as_mut().unwrap().push(i);
}
}
}
manager.fire(
state,
Event::UpdateUserStats {
name: self.stats_name.to_string(),
value: UserStats::Ratio(filled, size as u64),
phantom: PhantomData,
},
)?;
}
Ok(interesting)
} }
fn append_metadata(&mut self, _state: &mut S, testcase: &mut Testcase<I>) -> Result<(), Error> { fn append_metadata(&mut self, _state: &mut S, testcase: &mut Testcase<I>) -> Result<(), Error> {
@ -452,6 +412,127 @@ where
} }
} }
/// Specialize for the common coverage map size, maximization of u8s
#[rustversion::nightly]
impl<I, O, S> Feedback<I, S> for MapFeedback<I, DifferentIsNovel, O, MaxReducer, S, u8>
where
O: MapObserver<Entry = u8> + AsSlice<u8>,
for<'it> O: AsRefIterator<'it, Item = u8>,
I: Input,
S: HasNamedMetadata + HasClientPerfMonitor + Debug,
{
#[allow(clippy::wrong_self_convention)]
#[allow(clippy::needless_range_loop)]
fn is_interesting<EM, OT>(
&mut self,
state: &mut S,
manager: &mut EM,
_input: &I,
observers: &OT,
_exit_kind: &ExitKind,
) -> Result<bool, Error>
where
EM: EventFirer<I>,
OT: ObserversTuple<I, S>,
{
// 128 bits vectors
type VectorType = core::simd::u8x16;
let mut interesting = false;
// TODO Replace with match_name_type when stable
let observer = observers.match_name::<O>(&self.observer_name).unwrap();
let map_state = state
.named_metadata_mut()
.get_mut::<MapFeedbackMetadata<u8>>(&self.name)
.unwrap();
let size = observer.usable_count();
if map_state.history_map.len() < size {
map_state.history_map.resize(size, u8::default());
}
let map = observer.as_slice();
debug_assert!(map.len() >= size);
let history_map = map_state.history_map.as_mut_slice();
// Non vector implementation for reference
/*for (i, history) in history_map.iter_mut().enumerate() {
let item = map[i];
let reduced = MaxReducer::reduce(*history, item);
if DifferentIsNovel::is_novel(*history, reduced) {
*history = reduced;
interesting = true;
if self.novelties.is_some() {
self.novelties.as_mut().unwrap().push(i);
}
}
}*/
let steps = size / VectorType::LANES;
let left = size % VectorType::LANES;
for step in 0..steps {
let i = step * VectorType::LANES;
let history = VectorType::from_slice(&history_map[i..]);
let items = VectorType::from_slice(&map[i..]);
if items.max(history) != history {
interesting = true;
unsafe {
for j in i..(i + VectorType::LANES) {
let item = *map.get_unchecked(j);
if item > *history_map.get_unchecked(j) {
*history_map.get_unchecked_mut(j) = item;
if self.novelties.is_some() {
self.novelties.as_mut().unwrap().push(j);
}
}
}
}
}
}
for j in (size - left)..size {
unsafe {
let item = *map.get_unchecked(j);
if item > *history_map.get_unchecked(j) {
interesting = true;
*history_map.get_unchecked_mut(j) = item;
if self.novelties.is_some() {
self.novelties.as_mut().unwrap().push(j);
}
}
}
}
let initial = observer.initial();
if interesting {
let len = history_map.len();
let mut filled = 0;
for i in 0..len {
if history_map[i] != initial {
filled += 1;
if self.indexes.is_some() {
self.indexes.as_mut().unwrap().push(i);
}
}
}
manager.fire(
state,
Event::UpdateUserStats {
name: self.stats_name.to_string(),
value: UserStats::Ratio(filled, len as u64),
phantom: PhantomData,
},
)?;
}
Ok(interesting)
}
}
impl<I, N, O, R, S, T> Named for MapFeedback<I, N, O, R, S, T> impl<I, N, O, R, S, T> Named for MapFeedback<I, N, O, R, S, T>
where where
T: PartialEq + Default + Copy + 'static + Serialize + DeserializeOwned + Debug, T: PartialEq + Default + Copy + 'static + Serialize + DeserializeOwned + Debug,
@ -490,10 +571,11 @@ impl<I, N, O, R, S, T> MapFeedback<I, N, O, R, S, T>
where where
T: PartialEq + Default + Copy + 'static + Serialize + DeserializeOwned + Debug, T: PartialEq + Default + Copy + 'static + Serialize + DeserializeOwned + Debug,
R: Reducer<T>, R: Reducer<T>,
N: IsNovel<T>,
O: MapObserver<Entry = T>, O: MapObserver<Entry = T>,
for<'it> O: AsRefIterator<'it, Item = T>, for<'it> O: AsRefIterator<'it, Item = T>,
S: HasNamedMetadata, N: IsNovel<T>,
I: Input,
S: HasNamedMetadata + HasClientPerfMonitor + Debug,
{ {
/// Create new `MapFeedback` /// Create new `MapFeedback`
#[must_use] #[must_use]
@ -551,6 +633,76 @@ where
phantom: PhantomData, phantom: PhantomData,
} }
} }
#[allow(clippy::wrong_self_convention)]
#[allow(clippy::needless_range_loop)]
#[allow(clippy::trivially_copy_pass_by_ref)]
fn is_interesting_default<EM, OT>(
&mut self,
state: &mut S,
manager: &mut EM,
_input: &I,
observers: &OT,
_exit_kind: &ExitKind,
) -> Result<bool, Error>
where
EM: EventFirer<I>,
OT: ObserversTuple<I, S>,
{
let mut interesting = false;
// TODO Replace with match_name_type when stable
let observer = observers.match_name::<O>(&self.observer_name).unwrap();
let map_state = state
.named_metadata_mut()
.get_mut::<MapFeedbackMetadata<T>>(&self.name)
.unwrap();
let size = observer.usable_count();
if map_state.history_map.len() < size {
map_state.history_map.resize(size, T::default());
}
let history_map = map_state.history_map.as_mut_slice();
for (i, (item, history)) in observer
.as_ref_iter()
.zip(history_map.iter_mut())
.enumerate()
{
let reduced = R::reduce(*history, *item);
if N::is_novel(*history, reduced) {
*history = reduced;
interesting = true;
if self.novelties.is_some() {
self.novelties.as_mut().unwrap().push(i);
}
}
}
let initial = observer.initial();
if interesting {
let len = history_map.len();
let mut filled = 0;
for i in 0..len {
if history_map[i] != initial {
filled += 1;
if self.indexes.is_some() {
self.indexes.as_mut().unwrap().push(i);
}
}
}
manager.fire(
state,
Event::UpdateUserStats {
name: self.stats_name.to_string(),
value: UserStats::Ratio(filled, len as u64),
phantom: PhantomData,
},
)?;
}
Ok(interesting)
}
} }
/// A [`ReachabilityFeedback`] reports if a target has been reached. /// A [`ReachabilityFeedback`] reports if a target has been reached.

View File

@ -8,6 +8,8 @@ Welcome to `LibAFL`
#![cfg_attr(unstable_feature, feature(specialization))] #![cfg_attr(unstable_feature, feature(specialization))]
// For `type_id` and owned things // For `type_id` and owned things
#![cfg_attr(unstable_feature, feature(intrinsics))] #![cfg_attr(unstable_feature, feature(intrinsics))]
// For `std::simd`
#![cfg_attr(unstable_feature, feature(portable_simd))]
#![warn(clippy::cargo)] #![warn(clippy::cargo)]
#![deny(clippy::cargo_common_metadata)] #![deny(clippy::cargo_common_metadata)]
#![deny(rustdoc::broken_intra_doc_links)] #![deny(rustdoc::broken_intra_doc_links)]
@ -68,6 +70,8 @@ Welcome to `LibAFL`
while_true while_true
) )
)] )]
// Till they fix this buggy lint in clippy
#![allow(clippy::borrow_deref_ref)]
#[cfg(feature = "std")] #[cfg(feature = "std")]
#[macro_use] #[macro_use]

View File

@ -58,17 +58,7 @@ pub trait MapObserver: HasLen + Named + Serialize + serde::de::DeserializeOwned
fn usable_count(&self) -> usize; fn usable_count(&self) -> usize;
/// Count the set bytes in the map /// Count the set bytes in the map
fn count_bytes(&self) -> u64 { fn count_bytes(&self) -> u64;
let initial = self.initial();
let cnt = self.usable_count();
let mut res = 0;
for i in 0..cnt {
if *self.get(i) != initial {
res += 1;
}
}
res
}
/// Compute the hash of the map /// Compute the hash of the map
fn hash(&self) -> u64; fn hash(&self) -> u64;
@ -85,39 +75,13 @@ pub trait MapObserver: HasLen + Named + Serialize + serde::de::DeserializeOwned
} }
/// Reset the map /// Reset the map
#[inline] fn reset_map(&mut self) -> Result<(), Error>;
fn reset_map(&mut self) -> Result<(), Error> {
// Normal memset, see https://rust.godbolt.org/z/Trs5hv
let initial = self.initial();
let cnt = self.usable_count();
for i in 0..cnt {
*self.get_mut(i) = initial;
}
Ok(())
}
/// Get these observer's contents as [`Vec`] /// Get these observer's contents as [`Vec`]
fn to_vec(&self) -> Vec<Self::Entry> { fn to_vec(&self) -> Vec<Self::Entry>;
let cnt = self.usable_count();
let mut res = Vec::with_capacity(cnt);
for i in 0..cnt {
res.push(*self.get(i));
}
res
}
/// Get the number of set entries with the specified indexes /// Get the number of set entries with the specified indexes
fn how_many_set(&self, indexes: &[usize]) -> usize { fn how_many_set(&self, indexes: &[usize]) -> usize;
let initial = self.initial();
let cnt = self.usable_count();
let mut res = 0;
for i in indexes {
if *i < cnt && *self.get(*i) != initial {
res += 1;
}
}
res
}
} }
/// A Simple iterator calling `MapObserver::get` /// A Simple iterator calling `MapObserver::get`
@ -333,6 +297,20 @@ where
&mut self.as_mut_slice()[idx] &mut self.as_mut_slice()[idx]
} }
/// Count the set bytes in the map
fn count_bytes(&self) -> u64 {
let initial = self.initial();
let cnt = self.usable_count();
let map = self.as_slice();
let mut res = 0;
for x in map[0..cnt].iter() {
if *x != initial {
res += 1;
}
}
res
}
#[inline] #[inline]
fn usable_count(&self) -> usize { fn usable_count(&self) -> usize {
self.as_slice().len() self.as_slice().len()
@ -355,6 +333,32 @@ where
fn to_vec(&self) -> Vec<T> { fn to_vec(&self) -> Vec<T> {
self.as_slice().to_vec() self.as_slice().to_vec()
} }
/// Reset the map
#[inline]
fn reset_map(&mut self) -> Result<(), Error> {
// Normal memset, see https://rust.godbolt.org/z/Trs5hv
let initial = self.initial();
let cnt = self.usable_count();
let map = self.as_mut_slice();
for x in map[0..cnt].iter_mut() {
*x = initial;
}
Ok(())
}
fn how_many_set(&self, indexes: &[usize]) -> usize {
let initial = self.initial();
let cnt = self.usable_count();
let map = self.as_slice();
let mut res = 0;
for i in indexes {
if *i < cnt && map[*i] != initial {
res += 1;
}
}
res
}
} }
impl<'a, T> AsSlice<T> for StdMapObserver<'a, T> impl<'a, T> AsSlice<T> for StdMapObserver<'a, T>
@ -598,6 +602,20 @@ where
&mut self.as_mut_slice()[idx] &mut self.as_mut_slice()[idx]
} }
/// Count the set bytes in the map
fn count_bytes(&self) -> u64 {
let initial = self.initial();
let cnt = self.usable_count();
let map = self.as_slice();
let mut res = 0;
for x in map[0..cnt].iter() {
if *x != initial {
res += 1;
}
}
res
}
fn usable_count(&self) -> usize { fn usable_count(&self) -> usize {
self.as_slice().len() self.as_slice().len()
} }
@ -606,9 +624,36 @@ where
hash_slice(self.as_slice()) hash_slice(self.as_slice())
} }
/// Reset the map
#[inline]
fn reset_map(&mut self) -> Result<(), Error> {
// Normal memset, see https://rust.godbolt.org/z/Trs5hv
let initial = self.initial();
let cnt = self.usable_count();
let map = self.as_mut_slice();
for x in map[0..cnt].iter_mut() {
*x = initial;
}
Ok(())
}
fn to_vec(&self) -> Vec<T> { fn to_vec(&self) -> Vec<T> {
self.as_slice().to_vec() self.as_slice().to_vec()
} }
/// Get the number of set entries with the specified indexes
fn how_many_set(&self, indexes: &[usize]) -> usize {
let initial = self.initial();
let cnt = self.usable_count();
let map = self.as_slice();
let mut res = 0;
for i in indexes {
if *i < cnt && map[*i] != initial {
res += 1;
}
}
res
}
} }
impl<'a, T, const N: usize> AsSlice<T> for ConstMapObserver<'a, T, N> impl<'a, T, const N: usize> AsSlice<T> for ConstMapObserver<'a, T, N>
@ -831,12 +876,52 @@ where
&mut self.map.as_mut_slice()[idx] &mut self.map.as_mut_slice()[idx]
} }
/// Count the set bytes in the map
fn count_bytes(&self) -> u64 {
let initial = self.initial();
let cnt = self.usable_count();
let map = self.as_slice();
let mut res = 0;
for x in map[0..cnt].iter() {
if *x != initial {
res += 1;
}
}
res
}
fn hash(&self) -> u64 { fn hash(&self) -> u64 {
hash_slice(self.as_slice()) hash_slice(self.as_slice())
} }
/// Reset the map
#[inline]
fn reset_map(&mut self) -> Result<(), Error> {
// Normal memset, see https://rust.godbolt.org/z/Trs5hv
let initial = self.initial();
let cnt = self.usable_count();
let map = self.as_mut_slice();
for x in map[0..cnt].iter_mut() {
*x = initial;
}
Ok(())
}
fn to_vec(&self) -> Vec<T> { fn to_vec(&self) -> Vec<T> {
self.as_slice().to_vec() self.as_slice().to_vec()
} }
fn how_many_set(&self, indexes: &[usize]) -> usize {
let initial = self.initial();
let cnt = self.usable_count();
let map = self.as_slice();
let mut res = 0;
for i in indexes {
if *i < cnt && map[*i] != initial {
res += 1;
}
}
res
}
} }
impl<'a, T> AsSlice<T> for VariableMapObserver<'a, T> impl<'a, T> AsSlice<T> for VariableMapObserver<'a, T>
@ -924,9 +1009,27 @@ static COUNT_CLASS_LOOKUP: [u8; 256] = [
128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
]; ];
static mut COUNT_CLASS_LOOKUP_16: Vec<u16> = vec![];
fn init_count_class_16() {
unsafe {
if !COUNT_CLASS_LOOKUP_16.is_empty() {
return;
}
COUNT_CLASS_LOOKUP_16 = vec![0; 65536];
for i in 0..256 {
for j in 0..256 {
COUNT_CLASS_LOOKUP_16[(i << 8) + j] =
(u16::from(COUNT_CLASS_LOOKUP[i]) << 8) | u16::from(COUNT_CLASS_LOOKUP[j]);
}
}
}
}
impl<I, S, M> Observer<I, S> for HitcountsMapObserver<M> impl<I, S, M> Observer<I, S> for HitcountsMapObserver<M>
where where
M: MapObserver<Entry = u8> + Observer<I, S>, M: MapObserver<Entry = u8> + Observer<I, S> + AsMutSlice<u8>,
for<'it> M: AsMutIterator<'it, Item = u8>, for<'it> M: AsMutIterator<'it, Item = u8>,
{ {
#[inline] #[inline]
@ -935,9 +1038,23 @@ where
} }
#[inline] #[inline]
#[allow(clippy::cast_ptr_alignment)]
fn post_exec(&mut self, state: &mut S, input: &I, exit_kind: &ExitKind) -> Result<(), Error> { fn post_exec(&mut self, state: &mut S, input: &I, exit_kind: &ExitKind) -> Result<(), Error> {
for elem in self.as_mut_iter() { let map = self.as_mut_slice();
*elem = COUNT_CLASS_LOOKUP[*elem as usize]; let len = map.len();
if (len & 1) != 0 {
unsafe {
*map.get_unchecked_mut(len - 1) =
*COUNT_CLASS_LOOKUP.get_unchecked(*map.get_unchecked(len - 1) as usize);
}
}
let cnt = len / 2;
let map16 = unsafe { core::slice::from_raw_parts_mut(map.as_mut_ptr() as *mut u16, cnt) };
for (_i, item) in map16[0..cnt].iter_mut().enumerate() {
unsafe {
*item = *COUNT_CLASS_LOOKUP_16.get_unchecked(*item as usize);
}
} }
self.base.post_exec(state, input, exit_kind) self.base.post_exec(state, input, exit_kind)
} }
@ -995,12 +1112,27 @@ where
self.base.get_mut(idx) self.base.get_mut(idx)
} }
/// Count the set bytes in the map
fn count_bytes(&self) -> u64 {
self.base.count_bytes()
}
/// Reset the map
#[inline]
fn reset_map(&mut self) -> Result<(), Error> {
self.base.reset_map()
}
fn hash(&self) -> u64 { fn hash(&self) -> u64 {
self.base.hash() self.base.hash()
} }
fn to_vec(&self) -> Vec<u8> { fn to_vec(&self) -> Vec<u8> {
self.base.to_vec() self.base.to_vec()
} }
fn how_many_set(&self, indexes: &[usize]) -> usize {
self.base.how_many_set(indexes)
}
} }
impl<M> AsSlice<u8> for HitcountsMapObserver<M> impl<M> AsSlice<u8> for HitcountsMapObserver<M>
@ -1028,6 +1160,7 @@ where
{ {
/// Creates a new [`MapObserver`] /// Creates a new [`MapObserver`]
pub fn new(base: M) -> Self { pub fn new(base: M) -> Self {
init_count_class_16();
Self { base } Self { base }
} }
} }
@ -1207,6 +1340,28 @@ where
fn usable_count(&self) -> usize { fn usable_count(&self) -> usize {
self.len() self.len()
} }
fn to_vec(&self) -> Vec<Self::Entry> {
let cnt = self.usable_count();
let mut res = Vec::with_capacity(cnt);
for i in 0..cnt {
res.push(*self.get(i));
}
res
}
/// Get the number of set entries with the specified indexes
fn how_many_set(&self, indexes: &[usize]) -> usize {
let initial = self.initial();
let cnt = self.usable_count();
let mut res = 0;
for i in indexes {
if *i < cnt && *self.get(*i) != initial {
res += 1;
}
}
res
}
} }
impl<'a, T> MultiMapObserver<'a, T> impl<'a, T> MultiMapObserver<'a, T>
@ -1434,6 +1589,20 @@ where
&mut self.as_mut_slice()[idx] &mut self.as_mut_slice()[idx]
} }
/// Count the set bytes in the map
fn count_bytes(&self) -> u64 {
let initial = self.initial();
let cnt = self.usable_count();
let map = self.as_slice();
let mut res = 0;
for x in map[0..cnt].iter() {
if *x != initial {
res += 1;
}
}
res
}
#[inline] #[inline]
fn usable_count(&self) -> usize { fn usable_count(&self) -> usize {
self.as_slice().len() self.as_slice().len()
@ -1458,9 +1627,34 @@ where
self.initial = initial; self.initial = initial;
} }
/// Reset the map
#[inline]
fn reset_map(&mut self) -> Result<(), Error> {
// Normal memset, see https://rust.godbolt.org/z/Trs5hv
let initial = self.initial();
let cnt = self.usable_count();
let map = self.as_mut_slice();
for x in map[0..cnt].iter_mut() {
*x = initial;
}
Ok(())
}
fn to_vec(&self) -> Vec<T> { fn to_vec(&self) -> Vec<T> {
self.as_slice().to_vec() self.as_slice().to_vec()
} }
fn how_many_set(&self, indexes: &[usize]) -> usize {
let initial = self.initial();
let cnt = self.usable_count();
let map = self.as_slice();
let mut res = 0;
for i in indexes {
if *i < cnt && map[*i] != initial {
res += 1;
}
}
res
}
} }
impl<T> AsSlice<T> for OwnedMapObserver<T> impl<T> AsSlice<T> for OwnedMapObserver<T>
@ -1505,8 +1699,8 @@ where
#[allow(missing_docs)] #[allow(missing_docs)]
pub mod pybind { pub mod pybind {
use super::{ use super::{
AsMutIterator, AsRefIterator, Debug, Error, HasLen, Iter, IterMut, MapObserver, Named, AsMutIterator, AsMutSlice, AsRefIterator, AsSlice, Debug, Error, HasLen, Iter, IterMut,
Observer, OwnedMapObserver, StdMapObserver, String, Vec, MapObserver, Named, Observer, OwnedMapObserver, StdMapObserver, String, Vec,
}; };
use crate::observers::pybind::PythonObserver; use crate::observers::pybind::PythonObserver;
use concat_idents::concat_idents; use concat_idents::concat_idents;
@ -1758,6 +1952,18 @@ pub mod pybind {
} }
} }
impl AsSlice<$datatype> for $struct_name_trait {
fn as_slice(&self) -> &[$datatype] {
mapob_unwrap_me!($wrapper_name, self.wrapper, m, { unsafe { std::mem::transmute(m.as_slice()) }} )
}
}
impl AsMutSlice<$datatype> for $struct_name_trait {
fn as_mut_slice(&mut self) -> &mut [$datatype] {
mapob_unwrap_me_mut!($wrapper_name, self.wrapper, m, { unsafe { std::mem::transmute(m.as_mut_slice()) }} )
}
}
impl MapObserver for $struct_name_trait { impl MapObserver for $struct_name_trait {
type Entry = $datatype; type Entry = $datatype;
@ -1773,6 +1979,10 @@ pub mod pybind {
unsafe { ptr.as_mut().unwrap() } unsafe { ptr.as_mut().unwrap() }
} }
#[inline]
fn count_bytes(&self) -> u64 {
mapob_unwrap_me!($wrapper_name, self.wrapper, m, { m.count_bytes() })
}
#[inline] #[inline]
fn usable_count(&self) -> usize { fn usable_count(&self) -> usize {
mapob_unwrap_me!($wrapper_name, self.wrapper, m, { m.usable_count() }) mapob_unwrap_me!($wrapper_name, self.wrapper, m, { m.usable_count() })
@ -1798,9 +2008,20 @@ pub mod pybind {
mapob_unwrap_me_mut!($wrapper_name, self.wrapper, m, { m.set_initial(initial) }); mapob_unwrap_me_mut!($wrapper_name, self.wrapper, m, { m.set_initial(initial) });
} }
#[inline]
fn reset_map(&mut self) -> Result<(), Error> {
mapob_unwrap_me_mut!($wrapper_name, self.wrapper, m, { m.reset_map() })
}
#[inline]
fn to_vec(&self) -> Vec<$datatype> { fn to_vec(&self) -> Vec<$datatype> {
mapob_unwrap_me!($wrapper_name, self.wrapper, m, { m.to_vec() }) mapob_unwrap_me!($wrapper_name, self.wrapper, m, { m.to_vec() })
} }
#[inline]
fn how_many_set(&self, indexes: &[usize]) -> usize {
mapob_unwrap_me!($wrapper_name, self.wrapper, m, { m.how_many_set(indexes) })
}
} }
impl Named for $struct_name_trait { impl Named for $struct_name_trait {

View File

@ -813,7 +813,6 @@ pub mod pybind {
} }
#[pyo3(name = "match_name")] #[pyo3(name = "match_name")]
#[allow(clippy::all)]
fn pymatch_name(&self, name: &str) -> Option<PythonObserver> { fn pymatch_name(&self, name: &str) -> Option<PythonObserver> {
for ob in &self.list { for ob in &self.list {
if *ob.name() == *name { if *ob.name() == *name {