add alt_bitmap for use in trace mode, truncate trace file on new exec

libxdc does not create a bitmap in trace mode
This patch lets qemu create the bitmap instead

Note that the bitmap not compatible with libxdc bitmap since the trace
callback behavior is different.

nyx/auxiliary_buffer.h

@@ -74,7 +74,7 @@ typedef struct auxilary_buffer_config_s{
 
   /* trigger to enable / disable different QEMU-PT modes */
   uint8_t redqueen_mode; 
-  uint8_t trace_mode; 
+  uint8_t trace_mode;  /* dump decoded edge transitions to file */
   uint8_t reload_mode;
 
   uint8_t verbose_level;

nyx/pt.c

@@ -34,9 +34,12 @@ along with QEMU-PT.  If not, see <http://www.gnu.org/licenses/>.
 #include "nyx/memory_access.h"
 #include "nyx/interface.h"
 #include "nyx/debug.h"
+#include "nyx/file_helper.h"
+#ifdef CONFIG_REDQUEEN
 #include "nyx/redqueen.h"
 #include "nyx/redqueen_patch.h"
 #include "nyx/patcher.h"
+#endif
 #include "nyx/page_cache.h"
 #include "nyx/state/state.h"
 #include <libxdc.h>
@@ -47,8 +50,11 @@ along with QEMU-PT.  If not, see <http://www.gnu.org/licenses/>.
 uint32_t state_byte = 0;
 uint32_t last = 0;
 
+uint32_t alt_bitmap_size = 0;
+uint8_t* alt_bitmap = NULL;
+
 int pt_trace_dump_fd = 0;
-bool should_dump_pt_trace= false;
+bool should_dump_pt_trace= false; /* dump PT trace as returned from HW */
 
 void pt_open_pt_trace_file(char* filename){
   printf("using pt trace at %s",filename);
@@ -98,12 +104,41 @@ static inline int pt_ioctl(int fd, unsigned long request, unsigned long arg){
 	return ioctl(fd, request, arg);
 }
 
+void alt_bitmap_init(void* ptr, uint32_t size)
+{
+	alt_bitmap = (uint8_t*)ptr;
+	alt_bitmap_size = size;
+}
+
+void alt_bitmap_reset(void)
+{
+	if(alt_bitmap) {
+		memset(alt_bitmap, 0x00, alt_bitmap_size);
+	}
+}
+
 static inline uint64_t mix_bits(uint64_t v) {
   v ^= (v >> 31);
   v *= 0x7fb5d329728ea185;
   return v;
 }
 
+/*
+ * quick+dirty bitmap based on libxdc trace callback
+ * similar but not itentical to libxdc bitmap.
+ */
+void alt_bitmap_add(uint64_t from, uint64_t to)
+{
+	uint64_t transition_value;
+
+	if (GET_GLOBAL_STATE()->redqueen_state->trace_mode) {
+		if(alt_bitmap) {
+			transition_value = mix_bits(to)^(mix_bits(from)>>1);
+			alt_bitmap[transition_value & (alt_bitmap_size-1)]++;
+		}
+	}
+}
+
 #ifdef DUMP_AND_DEBUG_PT
 void dump_pt_trace(void* buffer, int bytes){
 	static FILE* f = NULL;
@@ -165,8 +200,11 @@ int pt_enable(CPUState *cpu, bool hmp_mode){
 	if(!fast_reload_set_bitmap(get_fast_reload_snapshot())){
 		coverage_bitmap_reset();
 	}
-	//pt_reset_bitmap();
-  pt_trucate_pt_trace_file();
+	if (GET_GLOBAL_STATE()->redqueen_state->trace_mode) {
+		delete_trace_files();
+		alt_bitmap_reset();
+	}
+	pt_trucate_pt_trace_file();
 	return pt_cmd(cpu, KVM_VMX_PT_ENABLE, hmp_mode);
 }
 	
@@ -248,6 +286,10 @@ void pt_init_decoder(CPUState *cpu){
 	GET_GLOBAL_STATE()->decoder = libxdc_init(filters, (void* (*)(void*, uint64_t, bool*))page_cache_fetch2, GET_GLOBAL_STATE()->page_cache, GET_GLOBAL_STATE()->shared_bitmap_ptr, GET_GLOBAL_STATE()->shared_bitmap_size);
 
 	libxdc_register_bb_callback(GET_GLOBAL_STATE()->decoder, (void (*)(void*, disassembler_mode_t, uint64_t, uint64_t))redqueen_callback, GET_GLOBAL_STATE()->redqueen_state);
+
+	alt_bitmap_init(
+			GET_GLOBAL_STATE()->shared_bitmap_ptr,
+			GET_GLOBAL_STATE()->shared_bitmap_size);
 }
 
 int pt_disable_ip_filtering(CPUState *cpu, uint8_t addrn, bool hmp_mode){

nyx/pt.h

@@ -27,6 +27,10 @@ void pt_init_decoder(CPUState *cpu);
 void pt_reset_bitmap(void);
 void pt_setup_bitmap(void* ptr);
 
+void alt_bitmap_reset(void);
+void alt_bitmap_init(void* ptr, uint32_t size);
+void alt_bitmap_add(uint64_t from, uint64_t to);
+
 int pt_enable(CPUState *cpu, bool hmp_mode);
 int pt_disable(CPUState *cpu, bool hmp_mode);
 int pt_enable_ip_filtering(CPUState *cpu, uint8_t addrn, bool redqueen, bool hmp_mode);
@@ -39,7 +43,6 @@ void pt_post_kvm_run(CPUState *cpu);
 
 void pt_handle_overflow(CPUState *cpu);
 void pt_dump(CPUState *cpu, int bytes);
-void pt_bitmap(uint64_t from, uint64_t to);
 
 void pt_open_pt_trace_file(char* filename);
 void pt_trucate_pt_trace_file(void);

nyx/redqueen.c

@@ -250,7 +250,6 @@ static void redqueen_trace_disabled(redqueen_t* self){
 }
 
 void redqueen_set_trace_mode(redqueen_t* self){
-	delete_trace_files();
 	self->trace_mode = true;
   redqueen_trace_enabled(self);
 }

nyx/redqueen_trace.c

@@ -4,11 +4,14 @@
 #include <assert.h>
 #include "redqueen_trace.h"
 
+void alt_bitmap_add(uint64_t from, uint64_t to);
+
 /* write full trace of edge transitions rather than sorted list? */
 //#define KAFL_FULL_TRACES
 
 #ifdef KAFL_FULL_TRACES
 #include "redqueen.h"
+extern int trace_fd;
 #endif
 
 redqueen_trace_t* redqueen_trace_new(void){
@@ -35,6 +38,10 @@ void redqueen_trace_free(redqueen_trace_t* self){
 void redqueen_trace_register_transition(redqueen_trace_t* self, disassembler_mode_t mode, uint64_t from, uint64_t to){
 	khiter_t k;
 	int ret;
+	uint64_t exit_ip = 0xffffffffffffffff;
+
+	if (from != exit_ip && to != exit_ip)
+		alt_bitmap_add(from, to);
 #ifdef KAFL_FULL_TRACES
 	extern int trace_fd;
 	if (!trace_fd)