redqueen_trace: disable unless 'edge_cb_trace' option is provided

Both, the legacy 'redqueen' trace via libxdc callback as well as new
dump_pt trace option are now toggled with aux-buffer trace_mode option.

This new qemu cmdline option allows to re-enable the old trace method,
or even use both trace methods at the same time.

nyx/auxiliary_buffer.c

@@ -104,6 +104,7 @@ void check_auxiliary_config_buffer(auxilary_buffer_t* auxilary_buffer, auxilary_
 #ifdef SUPPORT_COMPILE_TIME_REDQUEEN
         GET_GLOBAL_STATE()->pt_trace_mode_force = true;		  
 #endif
+		GET_GLOBAL_STATE()->trace_mode = true;
         redqueen_set_trace_mode();
         pt_trace_dump_enable(true);
       }
@@ -114,6 +115,7 @@ void check_auxiliary_config_buffer(auxilary_buffer_t* auxilary_buffer, auxilary_
 #ifdef SUPPORT_COMPILE_TIME_REDQUEEN
         GET_GLOBAL_STATE()->pt_trace_mode_force = false;
 #endif
+		GET_GLOBAL_STATE()->trace_mode = false;
         redqueen_unset_trace_mode();
         pt_trace_dump_enable(false);
       }

nyx/interface.c

@@ -90,6 +90,7 @@ typedef struct nyx_interface_state {
 	uint32_t input_buffer_size;
 
 	bool dump_pt_trace;
+	bool edge_cb_trace;
 
 	bool redqueen;
 	
@@ -283,6 +284,10 @@ static bool verify_workdir_state(nyx_interface_state *s, Error **errp){
 	free(tmp);
   }
 
+  if(s->edge_cb_trace){
+	redqueen_trace_init();
+  }
+
 
 	assert(asprintf(&tmp, "%s/aux_buffer_%d", workdir, id) != -1);
 	/*
@@ -427,6 +432,7 @@ static Property nyx_interface_properties[] = {
 	DEFINE_PROP_UINT32("bitmap_size", nyx_interface_state, bitmap_size, DEFAULT_NYX_BITMAP_SIZE),
 	DEFINE_PROP_UINT32("input_buffer_size", nyx_interface_state, input_buffer_size, DEFAULT_NYX_BITMAP_SIZE),
 	DEFINE_PROP_BOOL("dump_pt_trace", nyx_interface_state, dump_pt_trace, false),
+	DEFINE_PROP_BOOL("edge_cb_trace", nyx_interface_state, edge_cb_trace, false),
 
 
 	DEFINE_PROP_END_OF_LIST(),

nyx/redqueen_trace.c

@@ -15,6 +15,8 @@ void alt_bitmap_add(uint64_t from, uint64_t to);
 
 int trace_fd = 0;
 
+int redqueen_trace_enabled = false;
+
 static int reset_trace_fd(void) {
 	if (trace_fd)
 		close(trace_fd);
@@ -26,6 +28,10 @@ static int reset_trace_fd(void) {
 	return trace_fd;
 }
 
+void redqueen_trace_init(void) {
+	redqueen_trace_enabled = true;
+}
+
 redqueen_trace_t* redqueen_trace_new(void){
 	redqueen_trace_t* self = malloc(sizeof(redqueen_trace_t));
 	self->lookup = kh_init(RQ_TRACE);
@@ -35,13 +41,6 @@ redqueen_trace_t* redqueen_trace_new(void){
 	return self;
 }
 
-static void redqueen_state_reset(void){
-	redqueen_trace_t *self = GET_GLOBAL_STATE()->redqueen_state->trace_state;
-	kh_destroy(RQ_TRACE, self->lookup);
-	self->lookup = kh_init(RQ_TRACE);
-	self->num_ordered_transitions = 0;
-}
-
 void redqueen_trace_free(redqueen_trace_t* self){
 	kh_destroy(RQ_TRACE, self->lookup);
 	free(self->ordered_transitions);
@@ -88,28 +87,42 @@ static void redqueen_trace_write(void){
 	}
 }
 
+static void redqueen_state_reset(void){
+	redqueen_trace_t *self = GET_GLOBAL_STATE()->redqueen_state->trace_state;
+	kh_destroy(RQ_TRACE, self->lookup);
+	self->lookup = kh_init(RQ_TRACE);
+	self->num_ordered_transitions = 0;
+}
+
+
 void redqueen_trace_reset(void){
-	redqueen_state_reset();
+	if (redqueen_trace_enabled) {
-	reset_trace_fd();
+		redqueen_state_reset();
+		reset_trace_fd();
+	}
 }
 
 void redqueen_trace_flush(void){
-	redqueen_trace_write();
+	if (redqueen_trace_enabled) {
-	if (trace_fd)
+		redqueen_trace_write();
-		fsync(trace_fd);
+		if (trace_fd)
+			fsync(trace_fd);
+	}
 }
 
 void redqueen_set_trace_mode(void){
-	GET_GLOBAL_STATE()->trace_mode = true;
+	if (redqueen_trace_enabled) {
-	libxdc_enable_tracing(GET_GLOBAL_STATE()->decoder);
+		libxdc_enable_tracing(GET_GLOBAL_STATE()->decoder);
-	libxdc_register_edge_callback(GET_GLOBAL_STATE()->decoder,
+		libxdc_register_edge_callback(GET_GLOBAL_STATE()->decoder,
-			(void (*)(void*, disassembler_mode_t, uint64_t, uint64_t))&redqueen_trace_register_transition,
+				(void (*)(void*, disassembler_mode_t, uint64_t, uint64_t))&redqueen_trace_register_transition,
-			GET_GLOBAL_STATE()->redqueen_state->trace_state);
+				GET_GLOBAL_STATE()->redqueen_state->trace_state);
+	}
 }
 
 void redqueen_unset_trace_mode(void){
-    libxdc_disable_tracing(GET_GLOBAL_STATE()->decoder);
+	if (redqueen_trace_enabled) {
-	GET_GLOBAL_STATE()->trace_mode = false;
+		libxdc_disable_tracing(GET_GLOBAL_STATE()->decoder);
+	}
 }
 
 #ifdef DEBUG_MAIN
@@ -126,7 +139,7 @@ int main(int argc, char** argv){
 			redqueen_trace_register_transition(rq_obj, 0xBADBEEF, 0xC0FFEE);
 		}
 		redqueen_trace_write(rq_obj, STDOUT_FILENO);
-		redqueen_state_reset();
+		redqueen_trace_reset();
 	}
 
 	redqueen_trace_free(rq_obj);

nyx/redqueen_trace.h

@@ -1,3 +1,10 @@
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <stdbool.h>
+
+#include "qemu/osdep.h"
+
 #pragma once
 #include "khash.h"
 #include <libxdc.h>
@@ -40,6 +47,7 @@ redqueen_trace_t* redqueen_trace_new(void);
 void redqueen_trace_free(redqueen_trace_t* self);
 void redqueen_trace_register_transition(redqueen_trace_t* self, disassembler_mode_t mode, uint64_t from, uint64_t to);
 
+void redqueen_trace_init(void);
 void redqueen_set_trace_mode(void);
 void redqueen_unset_trace_mode(void);