From 35c4f356ab6e134833e52c8dfdc6a5ebda12396f Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Tue, 8 Feb 2022 18:45:03 +0100 Subject: [PATCH 01/35] patch KASAN hypercall back in --- nyx/auxiliary_buffer.c | 6 +++++- nyx/auxiliary_buffer.h | 4 +++- nyx/hypercall/hypercall.c | 36 ++++++++++++++++++++++++++++++++++-- nyx/synchronization.c | 19 +++++++++++++++++++ nyx/synchronization.h | 3 ++- 5 files changed, 63 insertions(+), 5 deletions(-) diff --git a/nyx/auxiliary_buffer.c b/nyx/auxiliary_buffer.c index a4d5928994..34df9c4d67 100644 --- a/nyx/auxiliary_buffer.c +++ b/nyx/auxiliary_buffer.c @@ -166,6 +166,10 @@ void set_crash_auxiliary_result_buffer(auxilary_buffer_t* auxilary_buffer){ VOLATILE_WRITE_8(auxilary_buffer->result.exec_result_code, rc_crash); } +void set_asan_auxiliary_result_buffer(auxilary_buffer_t* auxilary_buffer){ + VOLATILE_WRITE_8(auxilary_buffer->result.exec_result_code, rc_sanitizer); +} + void set_timeout_auxiliary_result_buffer(auxilary_buffer_t* auxilary_buffer){ VOLATILE_WRITE_8(auxilary_buffer->result.exec_result_code, rc_timeout); } @@ -259,4 +263,4 @@ void set_result_bb_coverage(auxilary_buffer_t* auxilary_buffer, uint32_t value){ if (value != auxilary_buffer->result.bb_coverage){ VOLATILE_WRITE_32(auxilary_buffer->result.bb_coverage, value); } -} \ No newline at end of file +} diff --git a/nyx/auxiliary_buffer.h b/nyx/auxiliary_buffer.h index 53edb09b2a..566f4aca3c 100644 --- a/nyx/auxiliary_buffer.h +++ b/nyx/auxiliary_buffer.h @@ -44,6 +44,7 @@ enum nyx_result_codes { rc_timeout = 3, rc_input_buffer_write = 4, rc_aborted = 5, + rc_sanitizer = 6, }; typedef struct auxilary_buffer_header_s{ @@ -149,6 +150,7 @@ void init_auxiliary_buffer(auxilary_buffer_t* auxilary_buffer); void check_auxiliary_config_buffer(auxilary_buffer_t* auxilary_buffer, auxilary_buffer_config_t* shadow_config); void set_crash_auxiliary_result_buffer(auxilary_buffer_t* auxilary_buffer); +void set_asan_auxiliary_result_buffer(auxilary_buffer_t* auxilary_buffer); void set_timeout_auxiliary_result_buffer(auxilary_buffer_t* auxilary_buffer); void set_reload_auxiliary_result_buffer(auxilary_buffer_t* auxilary_buffer); void set_pt_overflow_auxiliary_result_buffer(auxilary_buffer_t* auxilary_buffer); @@ -173,4 +175,4 @@ void set_result_pt_trace_size(auxilary_buffer_t* auxilary_buffer, uint32_t value void set_result_bb_coverage(auxilary_buffer_t* auxilary_buffer, uint32_t value); -void set_payload_buffer_write_reason_auxiliary_buffer(auxilary_buffer_t* auxilary_buffer, char* msg, uint32_t len); \ No newline at end of file +void set_payload_buffer_write_reason_auxiliary_buffer(auxilary_buffer_t* auxilary_buffer, char* msg, uint32_t len); diff --git a/nyx/hypercall/hypercall.c b/nyx/hypercall/hypercall.c index 64703657e2..21f478f93c 100644 --- a/nyx/hypercall/hypercall.c +++ b/nyx/hypercall/hypercall.c @@ -439,6 +439,13 @@ static void handle_hypercall_kafl_submit_panic(struct kvm_run *run, CPUState *cp } } +static void handle_hypercall_kafl_submit_kasan(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg){ + if(hypercall_enabled){ + QEMU_PT_PRINTF(CORE_PREFIX, "kASAN address:\t%lx", hypercall_arg); + write_virtual_memory(hypercall_arg, (uint8_t*)KASAN_PAYLOAD, PAYLOAD_BUFFER_SIZE, cpu); + } +} + //#define PANIC_DEBUG static void handle_hypercall_kafl_panic(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg){ @@ -550,6 +557,27 @@ static void handle_hypercall_kafl_panic_extended(struct kvm_run *run, CPUState * } } +static void handle_hypercall_kafl_kasan(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg){ + if(hypercall_enabled){ +#ifdef PANIC_DEBUG + if(hypercall_arg){ + QEMU_PT_PRINTF(CORE_PREFIX, "ASan notification in user mode!"); + } else{ + QEMU_PT_PRINTF(CORE_PREFIX, "ASan notification in kernel mode!"); + } +#endif + if(fast_reload_snapshot_exists(get_fast_reload_snapshot())){ + synchronization_lock_asan_found(); + //synchronization_stop_vm_kasan(cpu); + } else{ + QEMU_PT_PRINTF(CORE_PREFIX, "KASAN detected during initialization of stage 1 or stage 2 loader"); + //hypercall_snd_char(KAFL_PROTO_KASAN); + QEMU_PT_PRINTF_DEBUG("Protocol - SEND: KAFL_PROTO_KASAN"); + + } + } +} + static void handle_hypercall_kafl_lock(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg){ if(is_called_in_fuzzing_mode("KVM_EXIT_KAFL_LOCK")){ @@ -804,7 +832,9 @@ int handle_kafl_hypercall(struct kvm_run *run, CPUState *cpu, uint64_t hypercall ret = 0; break; case KVM_EXIT_KAFL_SUBMIT_KASAN: - nyx_abort((char*)"Deprecated hypercall called (HYPERCALL_SUBMIT_KASAN)..."); + //timeout_reload_pending = false; + //fprintf(stderr, "KVM_EXIT_KAFL_SUBMIT_KASAN\n"); + handle_hypercall_kafl_submit_kasan(run, cpu, arg); ret = 0; break; case KVM_EXIT_KAFL_PANIC: @@ -814,7 +844,9 @@ int handle_kafl_hypercall(struct kvm_run *run, CPUState *cpu, uint64_t hypercall ret = 0; break; case KVM_EXIT_KAFL_KASAN: - nyx_abort((char*)"Deprecated hypercall called (HYPERCALL_KAFL_KASAN)..."); + //timeout_reload_pending = false; + //fprintf(stderr, "KVM_EXIT_KAFL_KASAN\n"); + handle_hypercall_kafl_kasan(run, cpu, arg); ret = 0; break; case KVM_EXIT_KAFL_LOCK: diff --git a/nyx/synchronization.c b/nyx/synchronization.c index ad279dbbbe..fe0707a989 100644 --- a/nyx/synchronization.c +++ b/nyx/synchronization.c @@ -335,6 +335,25 @@ void synchronization_lock_crash_found(void){ in_fuzzing_loop = false; } +void synchronization_lock_asan_found(void){ + if(!in_fuzzing_loop){ + fprintf(stderr, "<%d-%ld>\t%s [NOT IN FUZZING LOOP]\n", getpid(), run_counter, __func__); + set_success_auxiliary_result_buffer(GET_GLOBAL_STATE()->auxilary_buffer, 0); + } + + pt_disable(qemu_get_cpu(0), false); + + handle_tmp_snapshot_state(); + + set_asan_auxiliary_result_buffer(GET_GLOBAL_STATE()->auxilary_buffer); + + perform_reload(); + + //synchronization_lock(); + + in_fuzzing_loop = false; +} + void synchronization_lock_timeout_found(void){ //fprintf(stderr, "<%d>\t%s\n", getpid(), __func__); diff --git a/nyx/synchronization.h b/nyx/synchronization.h index 7cd7ef902f..802d6f080c 100644 --- a/nyx/synchronization.h +++ b/nyx/synchronization.h @@ -37,6 +37,7 @@ void synchronization_lock_hprintf(void); void synchronization_lock(void); void synchronization_lock_crash_found(void); +void synchronization_lock_asan_found(void); void synchronization_lock_timeout_found(void); void synchronization_lock_shutdown_detected(void); void synchronization_cow_full_detected(void); @@ -45,4 +46,4 @@ void synchronization_enter_fuzzing_loop(CPUState *cpu); void synchronization_payload_buffer_write_detected(void); void enable_timeout_detector(timeout_detector_t* timeout_detector); -void reset_timeout_detector_timeout(timeout_detector_t* timeout_detector); \ No newline at end of file +void reset_timeout_detector_timeout(timeout_detector_t* timeout_detector); From 31b8c05afe57be63dd7b747a23a311aab003845b Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Tue, 11 Jan 2022 14:35:24 +0100 Subject: [PATCH 02/35] checkout specific libxdc commit --- compile_qemu_nyx.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compile_qemu_nyx.sh b/compile_qemu_nyx.sh index e36fe285f8..408a384be0 100755 --- a/compile_qemu_nyx.sh +++ b/compile_qemu_nyx.sh @@ -45,7 +45,8 @@ compile_libraries (){ echo "[!] compiling libxdc..." cd libxdc - CFLAGS="-I../capstone_v4/include/" V=1 make libxdc.a + git checkout 641de7539e99f7faf5c8e8f1c8a4b37a9df52a5f + sudo make install cd .. echo "[!] libxdc is ready!" } From 95742719f5d9a1b35aeb83453584b4c2d524e293 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Tue, 24 Nov 2020 17:30:29 +0000 Subject: [PATCH 03/35] use 32bit kasan/panic notifier payload when on 32bit --- nyx/hypercall/hypercall.c | 12 ++++++++++-- nyx/hypercall/hypercall.h | 28 +++++++++++++++++++++++++--- 2 files changed, 35 insertions(+), 5 deletions(-) diff --git a/nyx/hypercall/hypercall.c b/nyx/hypercall/hypercall.c index 21f478f93c..52a54db88e 100644 --- a/nyx/hypercall/hypercall.c +++ b/nyx/hypercall/hypercall.c @@ -435,14 +435,22 @@ static void handle_hypercall_kafl_submit_panic(struct kvm_run *run, CPUState *cp if(hypercall_enabled){ QEMU_PT_PRINTF(CORE_PREFIX, "Panic address:\t%lx", hypercall_arg); - write_virtual_memory(hypercall_arg, (uint8_t*)PANIC_PAYLOAD, PAYLOAD_BUFFER_SIZE, cpu); + if (run->hypercall.longmode) { + write_virtual_memory(hypercall_arg, (uint8_t*)PANIC_PAYLOAD_64, PAYLOAD_BUFFER_SIZE, cpu); + } else { + write_virtual_memory(hypercall_arg, (uint8_t*)PANIC_PAYLOAD_32, PAYLOAD_BUFFER_SIZE, cpu); + } } } static void handle_hypercall_kafl_submit_kasan(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg){ if(hypercall_enabled){ QEMU_PT_PRINTF(CORE_PREFIX, "kASAN address:\t%lx", hypercall_arg); - write_virtual_memory(hypercall_arg, (uint8_t*)KASAN_PAYLOAD, PAYLOAD_BUFFER_SIZE, cpu); + if (run->hypercall.longmode){ + write_virtual_memory(hypercall_arg, (uint8_t*)KASAN_PAYLOAD_64, PAYLOAD_BUFFER_SIZE, cpu); + } else { + write_virtual_memory(hypercall_arg, (uint8_t*)KASAN_PAYLOAD_32, PAYLOAD_BUFFER_SIZE, cpu); + } } } diff --git a/nyx/hypercall/hypercall.h b/nyx/hypercall/hypercall.h index c9cb788846..fa43b2ec3b 100644 --- a/nyx/hypercall/hypercall.h +++ b/nyx/hypercall/hypercall.h @@ -46,18 +46,40 @@ bool check_bitmap_byte(uint32_t value); * 0f 01 c1 vmcall * f4 hlt */ -#define PANIC_PAYLOAD "\xFA\x48\xC7\xC0\x1F\x00\x00\x00\x48\xC7\xC3\x08\x00\x00\x00\x48\xC7\xC1\x00\x00\x00\x00\x0F\x01\xC1\xF4" +#define PANIC_PAYLOAD_64 "\xFA\x48\xC7\xC0\x1F\x00\x00\x00\x48\xC7\xC3\x08\x00\x00\x00\x48\xC7\xC1\x00\x00\x00\x00\x0F\x01\xC1\xF4" + +/* + * Panic Notifier Payload (x86-32) + * fa cli + * b8 1f 00 00 00 mov $0x1f,%eax + * bb 08 00 00 00 mov $0x8,%ebx + * b9 00 00 00 00 mov $0x0,%ecx + * 0f 01 c1 vmcall + * f4 hlt + */ +#define PANIC_PAYLOAD_32 "\xFA\xB8\x1F\x00\x00\x00\xBB\x08\x00\x00\x00\xB9\x00\x00\x00\x00\x0F\x01\xC1\xF4" /* * KASAN Notifier Payload (x86-64) * fa cli * 48 c7 c0 1f 00 00 00 mov rax,0x1f - * 48 c7 c3 08 00 00 00 mov rbx,0x9 + * 48 c7 c3 09 00 00 00 mov rbx,0x9 * 48 c7 c1 00 00 00 00 mov rcx,0x0 * 0f 01 c1 vmcall * f4 hlt */ -#define KASAN_PAYLOAD "\xFA\x48\xC7\xC0\x1F\x00\x00\x00\x48\xC7\xC3\x09\x00\x00\x00\x48\xC7\xC1\x00\x00\x00\x00\x0F\x01\xC1\xF4" +#define KASAN_PAYLOAD_64 "\xFA\x48\xC7\xC0\x1F\x00\x00\x00\x48\xC7\xC3\x09\x00\x00\x00\x48\xC7\xC1\x00\x00\x00\x00\x0F\x01\xC1\xF4" + +/* + * KASAN Notifier Payload (x86-32) + * fa cli + * b8 1f 00 00 00 mov $0x1f,%eax + * bb 09 00 00 00 mov $0x9,%ebx + * b9 00 00 00 00 mov $0x0,%ecx + * 0f 01 c1 vmcall + * f4 hlt + */ +#define KASAN_PAYLOAD_32 "\xFA\xB8\x1F\x00\x00\x00\xBB\x09\x00\x00\x00\xB9\x00\x00\x00\x00\x0F\x01\xC1\xF4" /* * printk Notifier Payload (x86-64) From c12c6bd70db29f59dcf1069b8a3064a90ec4288d Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Tue, 29 Dec 2020 01:13:43 +0000 Subject: [PATCH 04/35] starved: signal if guest was reading beyond end of payload --- nyx/auxiliary_buffer.c | 7 ++++++- nyx/auxiliary_buffer.h | 1 + nyx/hypercall/hypercall.c | 7 +++++++ nyx/state/state.c | 1 + nyx/state/state.h | 1 + nyx/synchronization.c | 7 ++++++- 6 files changed, 22 insertions(+), 2 deletions(-) diff --git a/nyx/auxiliary_buffer.c b/nyx/auxiliary_buffer.c index 34df9c4d67..8c0682ecd8 100644 --- a/nyx/auxiliary_buffer.c +++ b/nyx/auxiliary_buffer.c @@ -229,7 +229,12 @@ void reset_page_not_found_result_buffer(auxilary_buffer_t* auxilary_buffer){ } void set_success_auxiliary_result_buffer(auxilary_buffer_t* auxilary_buffer, uint8_t success){ - VOLATILE_WRITE_8(auxilary_buffer->result.exec_result_code, rc_success); + //should refactor to let caller directly set the result codes + if (success == 2) { + VOLATILE_WRITE_8(auxilary_buffer->result.exec_result_code, rc_starved); + } else { + VOLATILE_WRITE_8(auxilary_buffer->result.exec_result_code, rc_success); + } } void set_payload_buffer_write_reason_auxiliary_buffer(auxilary_buffer_t* auxilary_buffer, char* msg, uint32_t len){ diff --git a/nyx/auxiliary_buffer.h b/nyx/auxiliary_buffer.h index 566f4aca3c..4a0aa3bdf0 100644 --- a/nyx/auxiliary_buffer.h +++ b/nyx/auxiliary_buffer.h @@ -45,6 +45,7 @@ enum nyx_result_codes { rc_input_buffer_write = 4, rc_aborted = 5, rc_sanitizer = 6, + rc_starved = 7, }; typedef struct auxilary_buffer_header_s{ diff --git a/nyx/hypercall/hypercall.c b/nyx/hypercall/hypercall.c index 52a54db88e..5c08a69c73 100644 --- a/nyx/hypercall/hypercall.c +++ b/nyx/hypercall/hypercall.c @@ -349,6 +349,13 @@ void handle_hypercall_kafl_release(struct kvm_run *run, CPUState *cpu, uint64_t if (init_state){ init_state = false; } else { + //printf(CORE_PREFIX, "Got STARVED notification (num=%llu)\n", run->hypercall.args[0]); + if (run->hypercall.args[0] > 0) { + GET_GLOBAL_STATE()->starved = 1; + } else { + GET_GLOBAL_STATE()->starved = 0; + } + synchronization_disable_pt(cpu); release_print_once(cpu); } diff --git a/nyx/state/state.c b/nyx/state/state.c index 9dde7c97f1..3edcbe2618 100644 --- a/nyx/state/state.c +++ b/nyx/state/state.c @@ -89,6 +89,7 @@ void state_init_global(void){ global_state.in_fuzzing_mode = false; global_state.in_reload_mode = true; + global_state.starved = false; global_state.shutdown_requested = false; global_state.cow_cache_full = false; diff --git a/nyx/state/state.h b/nyx/state/state.h index 7a9367f481..6970119157 100644 --- a/nyx/state/state.h +++ b/nyx/state/state.h @@ -131,6 +131,7 @@ typedef struct qemu_nyx_state_s{ bool in_fuzzing_mode; bool in_reload_mode; + bool starved; bool shutdown_requested; bool cow_cache_full; diff --git a/nyx/synchronization.c b/nyx/synchronization.c index fe0707a989..f63c42a078 100644 --- a/nyx/synchronization.c +++ b/nyx/synchronization.c @@ -291,7 +291,12 @@ void synchronization_lock(void){ pthread_mutex_unlock(&synchronization_lock_mutex); check_auxiliary_config_buffer(GET_GLOBAL_STATE()->auxilary_buffer, &GET_GLOBAL_STATE()->shadow_config); - set_success_auxiliary_result_buffer(GET_GLOBAL_STATE()->auxilary_buffer, 1); + + //set_success_auxiliary_result_buffer(GET_GLOBAL_STATE()->auxilary_buffer, 1); + if (GET_GLOBAL_STATE()->starved == true) + set_success_auxiliary_result_buffer(GET_GLOBAL_STATE()->auxilary_buffer, 2); + else + set_success_auxiliary_result_buffer(GET_GLOBAL_STATE()->auxilary_buffer, 1); GET_GLOBAL_STATE()->pt_trace_size = 0; /* From 169b084df5e61d20532a94bc631a229683e8ebc8 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Tue, 29 Dec 2020 01:57:33 +0000 Subject: [PATCH 05/35] report KVM_EXIT_SHUTDOWN and UNKNOWN_ERROR as panic events --- accel/kvm/kvm-all.c | 25 +++++++++++++++++++------ nyx/hypercall/hypercall.c | 2 +- nyx/hypercall/hypercall.h | 3 +-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 05bf2ceec9..11f7c3ad93 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -2562,18 +2562,23 @@ int kvm_cpu_exec(CPUState *cpu) ret = EXCP_INTERRUPT; break; case KVM_EXIT_SHUTDOWN: - DPRINTF("shutdown\n"); #ifndef QEMU_NYX qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); ret = EXCP_INTERRUPT; #else - fprintf(stderr, "ATTEMPT TO SHUTDOWN MACHINE (KVM_EXIT_SHUTDOWN)!\n"); if(GET_GLOBAL_STATE()->in_fuzzing_mode){ +#define CONFIG_KVM_EXIT_SHUTODWN_IS_PANIC // consider triple-fault etc as crash? +#ifndef CONFIG_KVM_EXIT_SHUTODWN_IS_PANIC /* Fuzzing is enabled at this point -> don't exit */ + fprintf(stderr, "Got KVM_EXIT_SHUTDOWN while in fuzzing mode => reload\n",); handle_hypercall_kafl_release(run, cpu, (uint64_t)run->hypercall.args[0]); - ret = 0; - } - else{ + ret = 0; +#else + debug_fprintf(stderr "Got KVM_EXIT_SHUTDOWN while in fuzzing mode => panic\n",); + handle_hypercall_kafl_panic(run, cpu, (uint64_t)run->hypercall.args[0]); + ret = 0; +#endif + } else{ qemu_system_reset_request(SHUTDOWN_CAUSE_GUEST_RESET); ret = EXCP_INTERRUPT; } @@ -2684,8 +2689,16 @@ int kvm_cpu_exec(CPUState *cpu) #ifndef QEMU_NYX DPRINTF("kvm_arch_handle_exit\n"); #else - printf("kvm_arch_handle_exit => %d\n", run->exit_reason); +#define CONFIG_UNKNOWN_ERROR_IS_PANIC +#ifndef CONFIG_UNKNOWN_ERROR_IS_PANIC + fprintf(stderr, "Unknown exit code (%d) => ABORT\n", run->exit_reason); assert(false); + ret = kvm_arch_handle_exit(cpu, run); +#else + debug_fprintf("kvm_arch_handle_exit(%d) => panic\n", run->exit_reason); + handle_hypercall_kafl_panic(run, cpu, (uint64_t)run->hypercall.args[0]); + ret = 0; +#endif #endif ret = kvm_arch_handle_exit(cpu, run); break; diff --git a/nyx/hypercall/hypercall.c b/nyx/hypercall/hypercall.c index 5c08a69c73..5df5b513ff 100644 --- a/nyx/hypercall/hypercall.c +++ b/nyx/hypercall/hypercall.c @@ -463,7 +463,7 @@ static void handle_hypercall_kafl_submit_kasan(struct kvm_run *run, CPUState *cp //#define PANIC_DEBUG -static void handle_hypercall_kafl_panic(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg){ +void handle_hypercall_kafl_panic(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg){ static char reason[1024]; if(hypercall_enabled){ #ifdef PANIC_DEBUG diff --git a/nyx/hypercall/hypercall.h b/nyx/hypercall/hypercall.h index fa43b2ec3b..f4f3dc30c6 100644 --- a/nyx/hypercall/hypercall.h +++ b/nyx/hypercall/hypercall.h @@ -114,8 +114,7 @@ void hypercall_reload(void); void handle_hypercall_kafl_acquire(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg); void handle_hypercall_kafl_release(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg); - - +void handle_hypercall_kafl_panic(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg); void handle_hypercall_kafl_page_dump_bp(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg, uint64_t page); From 81dbc38d46915628a698ff30640c58189444d1c4 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Mon, 26 Apr 2021 20:14:39 +0000 Subject: [PATCH 06/35] print error on invalid hget(), minor bugfix for QEMU_PR_PRINTF enable --- nyx/page_cache.c | 4 ++++ nyx/sharedir.c | 1 + 2 files changed, 5 insertions(+) diff --git a/nyx/page_cache.c b/nyx/page_cache.c index fcb9aa2f3f..7d40dff85e 100644 --- a/nyx/page_cache.c +++ b/nyx/page_cache.c @@ -381,7 +381,11 @@ page_cache_t* page_cache_new(const char* cache_file, uint8_t disassembler_word_w self->last_page = 0xFFFFFFFFFFFFFFFF; self->last_addr = 0xFFFFFFFFFFFFFFFF; +#ifndef STANDALONE_DECODER QEMU_PT_PRINTF(PAGE_CACHE_PREFIX, "%s (%s - %s)", __func__, tmp1, tmp2); +#else + QEMU_PT_PRINTF(PAGE_CACHE_PREFIX, "%s (%s - %s) WORD_WIDTH: %d", __func__, tmp1, tmp2, disassembler_word_width); +#endif free(tmp3); free(tmp2); diff --git a/nyx/sharedir.c b/nyx/sharedir.c index 5647dc234c..9cedf104a7 100644 --- a/nyx/sharedir.c +++ b/nyx/sharedir.c @@ -167,6 +167,7 @@ uint64_t sharedir_request_file(sharedir_t* self, const char* file, uint8_t* page } } else{ + fprintf(stderr, "WARNING: No such file in sharedir: %s\n", file); return 0xFFFFFFFFFFFFFFFFUL; } } \ No newline at end of file From 7dbb64e7c2f757650bd0bc733d6aad4ee8de0975 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Wed, 21 Jul 2021 03:30:02 -0700 Subject: [PATCH 07/35] compile-time option to restore kAFL style full edge traces --- nyx/redqueen_trace.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/nyx/redqueen_trace.c b/nyx/redqueen_trace.c index bf70982fb1..15bf64f2e2 100644 --- a/nyx/redqueen_trace.c +++ b/nyx/redqueen_trace.c @@ -4,6 +4,13 @@ #include #include "redqueen_trace.h" +/* write full trace of edge transitions rather than sorted list? */ +//#define KAFL_FULL_TRACES + +#ifdef KAFL_FULL_TRACES +#include "redqueen.h" +#endif + redqueen_trace_t* redqueen_trace_new(void){ redqueen_trace_t* self = malloc(sizeof(redqueen_trace_t)); self->lookup = kh_init(RQ_TRACE); @@ -28,6 +35,13 @@ void redqueen_trace_free(redqueen_trace_t* self){ void redqueen_trace_register_transition(redqueen_trace_t* self, disassembler_mode_t mode, uint64_t from, uint64_t to){ khiter_t k; int ret; +#ifdef KAFL_FULL_TRACES + extern int trace_fd; + if (!trace_fd) + trace_fd = open(redqueen_workdir.pt_trace_results, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU); + dprintf(trace_fd, "%lx,%lx\n", from, to); + return; +#endif uint128_t key = (((uint128_t)from)<<64) | ((uint128_t)to); k = kh_get(RQ_TRACE, self->lookup, key); if(k != kh_end(self->lookup)){ @@ -42,6 +56,9 @@ void redqueen_trace_register_transition(redqueen_trace_t* self, disassembler_mod } void redqueen_trace_write_file(redqueen_trace_t* self, int fd){ +#ifdef KAFL_FULL_TRACES + return; +#endif for(size_t i = 0; i < self->num_ordered_transitions; i++){ khiter_t k; uint128_t key = self->ordered_transitions[i]; From 0b6ec2cf72ccba873455311b3da182e5c2537f02 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Wed, 21 Jul 2021 03:30:40 -0700 Subject: [PATCH 08/35] kafl_dump_file: cleanups + select random filename if none provided --- nyx/hypercall/hypercall.c | 118 ++++++++++++++++++++++---------------- 1 file changed, 70 insertions(+), 48 deletions(-) diff --git a/nyx/hypercall/hypercall.c b/nyx/hypercall/hypercall.c index 5df5b513ff..5653c6b14f 100644 --- a/nyx/hypercall/hypercall.c +++ b/nyx/hypercall/hypercall.c @@ -727,68 +727,90 @@ void pt_disable_rqi_trace(CPUState *cpu){ } } -static void handle_hypercall_kafl_dump_file(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg){ - - /* TODO: check via aux buffer if we should allow this hypercall during fuzzing */ - /* - if(GET_GLOBAL_STATE()->in_fuzzing_mode){ - return; - } - */ - +static void handle_hypercall_kafl_dump_file(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg) +{ + kafl_dump_file_t file_obj; char filename[256] = {0}; + char* host_path = NULL; + FILE* f = NULL; uint64_t vaddr = hypercall_arg; - kafl_dump_file_t file_obj; memset((void*)&file_obj, 0, sizeof(kafl_dump_file_t)); + if (!read_virtual_memory(vaddr, (uint8_t*)&file_obj, sizeof(kafl_dump_file_t), cpu)){ + fprintf(stderr, "Failed to read file_obj in %s. Skipping..\n", __func__); + goto err_out1; + } - if(read_virtual_memory(vaddr, (uint8_t*)&file_obj, sizeof(kafl_dump_file_t), cpu)){ - - void* page = malloc(0x1000); + if (!read_virtual_memory(file_obj.file_name_str_ptr, (uint8_t*)filename, 255, cpu)) { + fprintf(stderr, "Failed to read file_name_str_ptr in %s. Skipping..\n", __func__); + goto err_out1; + } + filename[255] = 0; - read_virtual_memory(file_obj.file_name_str_ptr, (uint8_t*)&filename, sizeof(char)*256, cpu); - filename[255] = 0; - - char* base_name = basename(filename); - char* host_path = NULL; + //fprintf(stderr, "%s: dump %lu fbytes from %s (append=%u)\n", + // __func__, file_obj.bytes, filename, file_obj.append); + if (strnlen(filename, sizeof(filename))) { + char *base_name = basename(filename); assert(asprintf(&host_path, "%s/dump/%s", GET_GLOBAL_STATE()->workdir_path , base_name) != -1); - //fprintf(stderr, "dumping file %s -> %s (bytes %ld) in append_mode=%d\n", base_name, host_path, file_obj.bytes, file_obj.append); - - FILE* f = NULL; if(file_obj.append){ - f = fopen(host_path, "a+"); + f = fopen(host_path, "a+"); + } else{ + f = fopen(host_path, "w+"); } - else{ - f = fopen(host_path, "w+"); + } else { // no filename given - create tempfile + if (file_obj.append) { + fprintf(stderr, "Error request to append but no filename given in %s\n", __func__); + goto err_out1; } - - int32_t bytes = file_obj.bytes; - uint32_t pos = 0; - - while(bytes > 0){ - - if(bytes >= 0x1000){ - read_virtual_memory(file_obj.data_ptr+pos, (uint8_t*)page, 0x1000, cpu); - fwrite(page, 1, 0x1000, f); - } - else{ - read_virtual_memory(file_obj.data_ptr+pos, (uint8_t*)page, bytes, cpu); - fwrite(page, 1, bytes, f); - } - - bytes -= 0x1000; - pos += 0x1000; - } - - - fclose(f); - free(host_path); - free(page); - + + assert(asprintf(&host_path, "%s/dump/tmp.XXXXXX", GET_GLOBAL_STATE()->workdir_path) != -1); + f = fdopen(mkstemp(host_path), "w+"); } + + if (!f) { + fprintf(stderr, "Error in %s(%s): %s\n", host_path, __func__, strerror(errno)); + goto err_out1; + } + + uint32_t pos = 0; + int32_t bytes = file_obj.bytes; + void* page = malloc(PAGE_SIZE); + uint32_t written = 0; + + QEMU_PT_PRINTF(CORE_PREFIX, "%s: dump %d bytes to %s (append=%u)\n", + __func__, bytes, host_path, file_obj.append); + + while (bytes > 0) { + + if (bytes >= PAGE_SIZE) { + read_virtual_memory(file_obj.data_ptr+pos, (uint8_t*)page, PAGE_SIZE, cpu); + written = fwrite(page, 1, PAGE_SIZE, f); + } + else { + read_virtual_memory(file_obj.data_ptr+pos, (uint8_t*)page, bytes, cpu); + written = fwrite(page, 1, bytes, f); + break; + } + + if (!written) { + fprintf(stderr, "Error in %s(%s): %s\n", host_path, __func__, strerror(errno)); + goto err_out2; + } + + bytes -= written; + pos += written; + + } + + +err_out2: + free(page); + fclose(f); +err_out1: + free(host_path); } static void handle_hypercall_kafl_persist_page_past_snapshot(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg){ From f32d1cb3b708241efa5b2db05e10d12e9b449377 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Thu, 22 Jul 2021 03:49:32 -0700 Subject: [PATCH 09/35] add alt_bitmap for use in trace mode, truncate trace file on new exec libxdc does not create a bitmap in trace mode This patch lets qemu create the bitmap instead Note that the bitmap not compatible with libxdc bitmap since the trace callback behavior is different. --- nyx/auxiliary_buffer.h | 2 +- nyx/pt.c | 48 +++++++++++++++++++++++++++++++++++++++--- nyx/pt.h | 5 ++++- nyx/redqueen.c | 1 - nyx/redqueen_trace.c | 7 ++++++ 5 files changed, 57 insertions(+), 6 deletions(-) diff --git a/nyx/auxiliary_buffer.h b/nyx/auxiliary_buffer.h index 4a0aa3bdf0..0ac87ce8c2 100644 --- a/nyx/auxiliary_buffer.h +++ b/nyx/auxiliary_buffer.h @@ -74,7 +74,7 @@ typedef struct auxilary_buffer_config_s{ /* trigger to enable / disable different QEMU-PT modes */ uint8_t redqueen_mode; - uint8_t trace_mode; + uint8_t trace_mode; /* dump decoded edge transitions to file */ uint8_t reload_mode; uint8_t verbose_level; diff --git a/nyx/pt.c b/nyx/pt.c index abaa8cb35c..f90e0de259 100644 --- a/nyx/pt.c +++ b/nyx/pt.c @@ -34,9 +34,12 @@ along with QEMU-PT. If not, see . #include "nyx/memory_access.h" #include "nyx/interface.h" #include "nyx/debug.h" +#include "nyx/file_helper.h" +#ifdef CONFIG_REDQUEEN #include "nyx/redqueen.h" #include "nyx/redqueen_patch.h" #include "nyx/patcher.h" +#endif #include "nyx/page_cache.h" #include "nyx/state/state.h" #include @@ -47,8 +50,11 @@ along with QEMU-PT. If not, see . uint32_t state_byte = 0; uint32_t last = 0; +uint32_t alt_bitmap_size = 0; +uint8_t* alt_bitmap = NULL; + int pt_trace_dump_fd = 0; -bool should_dump_pt_trace= false; +bool should_dump_pt_trace= false; /* dump PT trace as returned from HW */ void pt_open_pt_trace_file(char* filename){ printf("using pt trace at %s",filename); @@ -98,12 +104,41 @@ static inline int pt_ioctl(int fd, unsigned long request, unsigned long arg){ return ioctl(fd, request, arg); } +void alt_bitmap_init(void* ptr, uint32_t size) +{ + alt_bitmap = (uint8_t*)ptr; + alt_bitmap_size = size; +} + +void alt_bitmap_reset(void) +{ + if(alt_bitmap) { + memset(alt_bitmap, 0x00, alt_bitmap_size); + } +} + static inline uint64_t mix_bits(uint64_t v) { v ^= (v >> 31); v *= 0x7fb5d329728ea185; return v; } +/* + * quick+dirty bitmap based on libxdc trace callback + * similar but not itentical to libxdc bitmap. + */ +void alt_bitmap_add(uint64_t from, uint64_t to) +{ + uint64_t transition_value; + + if (GET_GLOBAL_STATE()->redqueen_state->trace_mode) { + if(alt_bitmap) { + transition_value = mix_bits(to)^(mix_bits(from)>>1); + alt_bitmap[transition_value & (alt_bitmap_size-1)]++; + } + } +} + #ifdef DUMP_AND_DEBUG_PT void dump_pt_trace(void* buffer, int bytes){ static FILE* f = NULL; @@ -165,8 +200,11 @@ int pt_enable(CPUState *cpu, bool hmp_mode){ if(!fast_reload_set_bitmap(get_fast_reload_snapshot())){ coverage_bitmap_reset(); } - //pt_reset_bitmap(); - pt_trucate_pt_trace_file(); + if (GET_GLOBAL_STATE()->redqueen_state->trace_mode) { + delete_trace_files(); + alt_bitmap_reset(); + } + pt_trucate_pt_trace_file(); return pt_cmd(cpu, KVM_VMX_PT_ENABLE, hmp_mode); } @@ -248,6 +286,10 @@ void pt_init_decoder(CPUState *cpu){ GET_GLOBAL_STATE()->decoder = libxdc_init(filters, (void* (*)(void*, uint64_t, bool*))page_cache_fetch2, GET_GLOBAL_STATE()->page_cache, GET_GLOBAL_STATE()->shared_bitmap_ptr, GET_GLOBAL_STATE()->shared_bitmap_size); libxdc_register_bb_callback(GET_GLOBAL_STATE()->decoder, (void (*)(void*, disassembler_mode_t, uint64_t, uint64_t))redqueen_callback, GET_GLOBAL_STATE()->redqueen_state); + + alt_bitmap_init( + GET_GLOBAL_STATE()->shared_bitmap_ptr, + GET_GLOBAL_STATE()->shared_bitmap_size); } int pt_disable_ip_filtering(CPUState *cpu, uint8_t addrn, bool hmp_mode){ diff --git a/nyx/pt.h b/nyx/pt.h index 51b360c45a..7b23719ec1 100644 --- a/nyx/pt.h +++ b/nyx/pt.h @@ -27,6 +27,10 @@ void pt_init_decoder(CPUState *cpu); void pt_reset_bitmap(void); void pt_setup_bitmap(void* ptr); +void alt_bitmap_reset(void); +void alt_bitmap_init(void* ptr, uint32_t size); +void alt_bitmap_add(uint64_t from, uint64_t to); + int pt_enable(CPUState *cpu, bool hmp_mode); int pt_disable(CPUState *cpu, bool hmp_mode); int pt_enable_ip_filtering(CPUState *cpu, uint8_t addrn, bool redqueen, bool hmp_mode); @@ -39,7 +43,6 @@ void pt_post_kvm_run(CPUState *cpu); void pt_handle_overflow(CPUState *cpu); void pt_dump(CPUState *cpu, int bytes); -void pt_bitmap(uint64_t from, uint64_t to); void pt_open_pt_trace_file(char* filename); void pt_trucate_pt_trace_file(void); diff --git a/nyx/redqueen.c b/nyx/redqueen.c index d878013687..32f8c91e30 100644 --- a/nyx/redqueen.c +++ b/nyx/redqueen.c @@ -250,7 +250,6 @@ static void redqueen_trace_disabled(redqueen_t* self){ } void redqueen_set_trace_mode(redqueen_t* self){ - delete_trace_files(); self->trace_mode = true; redqueen_trace_enabled(self); } diff --git a/nyx/redqueen_trace.c b/nyx/redqueen_trace.c index 15bf64f2e2..b2916900e2 100644 --- a/nyx/redqueen_trace.c +++ b/nyx/redqueen_trace.c @@ -4,11 +4,14 @@ #include #include "redqueen_trace.h" +void alt_bitmap_add(uint64_t from, uint64_t to); + /* write full trace of edge transitions rather than sorted list? */ //#define KAFL_FULL_TRACES #ifdef KAFL_FULL_TRACES #include "redqueen.h" +extern int trace_fd; #endif redqueen_trace_t* redqueen_trace_new(void){ @@ -35,6 +38,10 @@ void redqueen_trace_free(redqueen_trace_t* self){ void redqueen_trace_register_transition(redqueen_trace_t* self, disassembler_mode_t mode, uint64_t from, uint64_t to){ khiter_t k; int ret; + uint64_t exit_ip = 0xffffffffffffffff; + + if (from != exit_ip && to != exit_ip) + alt_bitmap_add(from, to); #ifdef KAFL_FULL_TRACES extern int trace_fd; if (!trace_fd) From 6b008a1be46b58b39df56077935892368e84668d Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Sun, 1 Aug 2021 00:40:19 +0000 Subject: [PATCH 10/35] error checking on payload remap + other --- nyx/memory_access.c | 11 +++++++++-- nyx/pt.c | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/nyx/memory_access.c b/nyx/memory_access.c index e5e4bcdf30..d2b3203467 100644 --- a/nyx/memory_access.c +++ b/nyx/memory_access.c @@ -216,8 +216,14 @@ bool remap_slot(uint64_t addr, uint32_t slot, CPUState *cpu, int fd, uint64_t sh QLIST_FOREACH_RCU(block, &ram_list.blocks, next) { if(!memcmp(block->idstr, "pc.ram", 6)){ /* TODO: put assert calls here */ - munmap((void*)(((uint64_t)block->host) + phys_addr), x86_64_PAGE_SIZE); - mmap((void*)(((uint64_t)block->host) + phys_addr), 0x1000, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, (i*x86_64_PAGE_SIZE)); + if (munmap((void*)(((uint64_t)block->host) + phys_addr), x86_64_PAGE_SIZE) == -1) { + fprintf(stderr, "%s: munmap failed!\n", __func__); + assert(false); + } + if (mmap((void*)(((uint64_t)block->host) + phys_addr), 0x1000, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, (i*x86_64_PAGE_SIZE)) == MAP_FAILED) { + fprintf(stderr, "%s: mmap failed!\n", __func__); + assert(false); + } //printf("MMUNMAP: %d\n", munmap((void*)(((uint64_t)block->host) + phys_addr), x86_64_PAGE_SIZE)); //printf("MMAP: %p\n", mmap((void*)(((uint64_t)block->host) + phys_addr), 0x1000, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, fd, (i*x86_64_PAGE_SIZE))); @@ -286,6 +292,7 @@ void resize_shared_memory(uint32_t new_size, uint32_t* shm_size, void** shm_ptr, bool remap_payload_buffer(uint64_t virt_guest_addr, CPUState *cpu){ assert(GET_GLOBAL_STATE()->shared_payload_buffer_fd && GET_GLOBAL_STATE()->shared_payload_buffer_size); + assert(GET_GLOBAL_STATE()->shared_payload_buffer_size % x86_64_PAGE_SIZE == 0); RAMBlock *block; refresh_kvm_non_dirty(cpu); diff --git a/nyx/pt.c b/nyx/pt.c index f90e0de259..80de789ed1 100644 --- a/nyx/pt.c +++ b/nyx/pt.c @@ -185,7 +185,7 @@ void pt_dump(CPUState *cpu, int bytes){ GET_GLOBAL_STATE()->decoder_page_fault_addr = libxdc_get_page_fault_addr(GET_GLOBAL_STATE()->decoder); break; case decoder_unkown_packet: - fprintf(stderr, "WARNING: libxdc_decode returned decoder_error\n"); + fprintf(stderr, "WARNING: libxdc_decode returned unknown_packet\n"); break; case decoder_error: fprintf(stderr, "WARNING: libxdc_decode returned decoder_error\n"); From b8995723775c5ee75355ef4e87922cec359245e5 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Fri, 6 Aug 2021 07:51:16 -0700 Subject: [PATCH 11/35] page_cache: auto-create workdir files or resume based on existing files - relieve frontend from having to create these files - perhaps add some checks for resuming from existing page_cache files --- nyx/interface.c | 24 ------------------------ nyx/page_cache.c | 6 +++--- nyx/pt.c | 2 +- 3 files changed, 4 insertions(+), 28 deletions(-) diff --git a/nyx/interface.c b/nyx/interface.c index 56ffc54d7c..eabbae9972 100644 --- a/nyx/interface.c +++ b/nyx/interface.c @@ -260,30 +260,6 @@ static bool verify_workdir_state(nyx_interface_state *s, Error **errp){ } free(tmp); - assert(asprintf(&tmp, "%s/page_cache.lock", workdir) != -1); - if (!file_exits(tmp)){ - fprintf(stderr, "%s does not exist...", tmp); - free(tmp); - return false; - } - free(tmp); - - assert(asprintf(&tmp, "%s/page_cache.addr", workdir) != -1); - if (!file_exits(tmp)){ - fprintf(stderr, "%s does not exist...\n", tmp); - free(tmp); - return false; - } - free(tmp); - - assert(asprintf(&tmp, "%s/page_cache.dump", workdir) != -1); - if (!file_exits(tmp)){ - fprintf(stderr, "%s does not exist...\n", tmp); - free(tmp); - return false; - } - free(tmp); - assert(asprintf(&tmp, "%s/page_cache", workdir) != -1); init_page_cache(tmp); diff --git a/nyx/page_cache.c b/nyx/page_cache.c index 7d40dff85e..55689316c8 100644 --- a/nyx/page_cache.c +++ b/nyx/page_cache.c @@ -359,12 +359,12 @@ page_cache_t* page_cache_new(const char* cache_file, uint8_t disassembler_word_w self->lookup = kh_init(PC_CACHE); - self->fd_page_file = open(tmp1, O_CLOEXEC | O_RDWR, S_IRWXU); - self->fd_address_file = open(tmp2, O_CLOEXEC | O_RDWR, S_IRWXU); + self->fd_page_file = open(tmp1, O_CLOEXEC | O_CREAT | O_RDWR, 0644); + self->fd_address_file = open(tmp2, O_CLOEXEC | O_CREAT | O_RDWR, 0644); #ifndef STANDALONE_DECODER self->cpu = cpu; - self->fd_lock = open(tmp3, O_CLOEXEC); + self->fd_lock = open(tmp3, O_CLOEXEC | O_CREAT, 0644); assert(self->fd_lock > 0); #else if(self->fd_page_file == -1 || self->fd_address_file == -1){ diff --git a/nyx/pt.c b/nyx/pt.c index 80de789ed1..193ef8ea09 100644 --- a/nyx/pt.c +++ b/nyx/pt.c @@ -58,7 +58,7 @@ bool should_dump_pt_trace= false; /* dump PT trace as returned from HW */ void pt_open_pt_trace_file(char* filename){ printf("using pt trace at %s",filename); - pt_trace_dump_fd = open(filename, O_WRONLY); + pt_trace_dump_fd = open(filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); should_dump_pt_trace = true; assert(pt_trace_dump_fd >= 0); } From 5c24050a645b0db69a806cb100f4e263ca191838 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Fri, 6 Aug 2021 07:52:51 -0700 Subject: [PATCH 12/35] page_cache: use file lock also for read access Without this there may be a risk of reading partially written files...doesn't seem to happen in practice though? --- nyx/page_cache.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/nyx/page_cache.c b/nyx/page_cache.c index 55689316c8..1827c186a9 100644 --- a/nyx/page_cache.c +++ b/nyx/page_cache.c @@ -25,6 +25,9 @@ #define UNMAPPED_PAGE 0xFFFFFFFFFFFFFFFFULL +static void page_cache_unlock(page_cache_t* self); +static void page_cache_lock(page_cache_t* self); + #ifndef STANDALONE_DECODER static bool reload_addresses(page_cache_t* self){ #else @@ -40,6 +43,8 @@ bool reload_addresses(page_cache_t* self){ if(self_offset != self->num_pages*PAGE_CACHE_ADDR_LINE_SIZE){ //fprintf(stderr, "Reloading files ...\n"); + page_cache_lock(self); // don't read while someone else is writing? + lseek(self->fd_address_file, self->num_pages*PAGE_CACHE_ADDR_LINE_SIZE, SEEK_SET); offset = self->num_pages; while(read(self->fd_address_file, &value, PAGE_CACHE_ADDR_LINE_SIZE)){ @@ -79,6 +84,8 @@ bool reload_addresses(page_cache_t* self){ munmap(self->page_data, self->num_pages*PAGE_SIZE); self->num_pages = self_offset/PAGE_CACHE_ADDR_LINE_SIZE; self->page_data = mmap(NULL, (self->num_pages)*PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, self->fd_page_file, 0); + + page_cache_unlock(self); return true; } From 56bc5571beb7429c47175856823a1be29d3f2d6d Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Mon, 9 Aug 2021 16:24:27 -0700 Subject: [PATCH 13/35] dump_pt: create-open & truncate output file on each execution Previous implementation only opened the file once. --- nyx/interface.c | 6 +++--- nyx/pt.c | 40 ++++++++++++++++++++++++++-------------- nyx/pt.h | 4 ++-- 3 files changed, 31 insertions(+), 19 deletions(-) diff --git a/nyx/interface.c b/nyx/interface.c index eabbae9972..af4b431212 100644 --- a/nyx/interface.c +++ b/nyx/interface.c @@ -277,9 +277,9 @@ static bool verify_workdir_state(nyx_interface_state *s, Error **errp){ init_redqueen_state(); if(s->dump_pt_trace){ - assert(asprintf(&tmp, "%s/pt_trace_dump_%d", workdir, id) != -1); - pt_open_pt_trace_file(tmp); - free(tmp); + assert(asprintf(&tmp, "%s/pt_trace_dump_%d", workdir, id) != -1); + pt_trace_dump_enable(tmp); + free(tmp); } diff --git a/nyx/pt.c b/nyx/pt.c index 193ef8ea09..89b4093c97 100644 --- a/nyx/pt.c +++ b/nyx/pt.c @@ -54,20 +54,34 @@ uint32_t alt_bitmap_size = 0; uint8_t* alt_bitmap = NULL; int pt_trace_dump_fd = 0; +char *pt_trace_dump_filename; bool should_dump_pt_trace= false; /* dump PT trace as returned from HW */ -void pt_open_pt_trace_file(char* filename){ - printf("using pt trace at %s",filename); - pt_trace_dump_fd = open(filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); - should_dump_pt_trace = true; - assert(pt_trace_dump_fd >= 0); +void pt_trace_dump_enable(char* filename) +{ + int test_fd; + + printf("Enable pt trace dump at %s", filename); + pt_trace_dump_filename = filename; + should_dump_pt_trace = true; + + test_fd = open(filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); + if (test_fd < 0) + fprintf(stderr, "Error accessing pt_dump output path: %s", strerror(errno)); + assert(test_fd >= 0); } -void pt_trucate_pt_trace_file(void){ - if(should_dump_pt_trace){ - assert(lseek(pt_trace_dump_fd, 0, SEEK_SET) == 0); - assert(ftruncate(pt_trace_dump_fd, 0)==0); - } +void pt_write_pt_dump_file(uint8_t *data, size_t bytes) +{ + int fd; + + if (!should_dump_pt_trace) + return; + + fd = open(pt_trace_dump_filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); + assert(fd >= 0); + + assert(bytes == write(fd, data, bytes)); } static void pt_set(CPUState *cpu, run_on_cpu_data arg){ @@ -166,9 +180,8 @@ void dump_pt_trace(void* buffer, int bytes){ #endif void pt_dump(CPUState *cpu, int bytes){ - if(should_dump_pt_trace){ - assert(bytes == write(pt_trace_dump_fd, cpu->pt_mmap, bytes)); - } + pt_write_pt_dump_file(cpu->pt_mmap, bytes); + if(!(GET_GLOBAL_STATE()->redqueen_state && GET_GLOBAL_STATE()->redqueen_state->intercept_mode)){ if (GET_GLOBAL_STATE()->in_fuzzing_mode && GET_GLOBAL_STATE()->decoder_page_fault == false && GET_GLOBAL_STATE()->decoder && !GET_GLOBAL_STATE()->dump_page){ GET_GLOBAL_STATE()->pt_trace_size += bytes; @@ -204,7 +217,6 @@ int pt_enable(CPUState *cpu, bool hmp_mode){ delete_trace_files(); alt_bitmap_reset(); } - pt_trucate_pt_trace_file(); return pt_cmd(cpu, KVM_VMX_PT_ENABLE, hmp_mode); } diff --git a/nyx/pt.h b/nyx/pt.h index 7b23719ec1..22790eb85b 100644 --- a/nyx/pt.h +++ b/nyx/pt.h @@ -44,7 +44,7 @@ void pt_post_kvm_run(CPUState *cpu); void pt_handle_overflow(CPUState *cpu); void pt_dump(CPUState *cpu, int bytes); -void pt_open_pt_trace_file(char* filename); -void pt_trucate_pt_trace_file(void); +void pt_trace_dump_enable(char* filename); +void pt_write_pt_dump_file(uint8_t *data, size_t bytes); #endif From 24e6f39e1cd7d59a88bcf6a496227dd1cd7c7aa3 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Mon, 16 Aug 2021 17:29:58 -0700 Subject: [PATCH 14/35] fix pt_dump feature (append on VMexit, truncate on new execution) --- nyx/pt.c | 29 +++++++++++++++++++++++++---- nyx/pt.h | 1 - 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/nyx/pt.c b/nyx/pt.c index 89b4093c97..f0b96e4b95 100644 --- a/nyx/pt.c +++ b/nyx/pt.c @@ -71,17 +71,35 @@ void pt_trace_dump_enable(char* filename) assert(test_fd >= 0); } -void pt_write_pt_dump_file(uint8_t *data, size_t bytes) -{ +static void pt_truncate_pt_dump_file(void) { int fd; if (!should_dump_pt_trace) return; fd = open(pt_trace_dump_filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); - assert(fd >= 0); + if (fd < 0) { + fprintf(stderr, "Error truncating pt_trace_dump: %s\n", strerror(errno)); + assert(0); + } + close(fd); +} +static void pt_write_pt_dump_file(uint8_t *data, size_t bytes) +{ + int fd; + + if (!should_dump_pt_trace) + return; + + fd = open(pt_trace_dump_filename, O_APPEND|O_WRONLY, 0644); + //fd = open(pt_trace_dump_filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); + if (fd < 0) { + fprintf(stderr, "Error writing pt_trace_dump: %s\n", strerror(errno)); + assert(0); + } assert(bytes == write(fd, data, bytes)); + close(fd); } static void pt_set(CPUState *cpu, run_on_cpu_data arg){ @@ -180,11 +198,13 @@ void dump_pt_trace(void* buffer, int bytes){ #endif void pt_dump(CPUState *cpu, int bytes){ - pt_write_pt_dump_file(cpu->pt_mmap, bytes); + //pt_write_pt_dump_file(cpu->pt_mmap, bytes); if(!(GET_GLOBAL_STATE()->redqueen_state && GET_GLOBAL_STATE()->redqueen_state->intercept_mode)){ if (GET_GLOBAL_STATE()->in_fuzzing_mode && GET_GLOBAL_STATE()->decoder_page_fault == false && GET_GLOBAL_STATE()->decoder && !GET_GLOBAL_STATE()->dump_page){ GET_GLOBAL_STATE()->pt_trace_size += bytes; + //dump_pt_trace(cpu->pt_mmap, bytes); + pt_write_pt_dump_file(cpu->pt_mmap, bytes); decoder_result_t result = libxdc_decode(GET_GLOBAL_STATE()->decoder, cpu->pt_mmap, bytes); switch(result){ case decoder_success: @@ -217,6 +237,7 @@ int pt_enable(CPUState *cpu, bool hmp_mode){ delete_trace_files(); alt_bitmap_reset(); } + pt_truncate_pt_dump_file(); return pt_cmd(cpu, KVM_VMX_PT_ENABLE, hmp_mode); } diff --git a/nyx/pt.h b/nyx/pt.h index 22790eb85b..174b463229 100644 --- a/nyx/pt.h +++ b/nyx/pt.h @@ -45,6 +45,5 @@ void pt_handle_overflow(CPUState *cpu); void pt_dump(CPUState *cpu, int bytes); void pt_trace_dump_enable(char* filename); -void pt_write_pt_dump_file(uint8_t *data, size_t bytes); #endif From 68f74353b2f4bcecb4f8c6975f3e2375ba6b2c76 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Wed, 18 Aug 2021 10:31:43 -0700 Subject: [PATCH 15/35] record worker_id in state and report via KAFL_HYPERCALL_GET_HOST_CONFIG Modifies elements of host_config_t - update guest agent struct! --- nyx/hypercall/configuration.c | 3 ++- nyx/hypercall/configuration.h | 1 + nyx/interface.c | 1 + nyx/state/state.c | 3 ++- nyx/state/state.h | 1 + 5 files changed, 7 insertions(+), 2 deletions(-) diff --git a/nyx/hypercall/configuration.c b/nyx/hypercall/configuration.c index 3b54415653..bd04f27a42 100644 --- a/nyx/hypercall/configuration.c +++ b/nyx/hypercall/configuration.c @@ -24,6 +24,7 @@ void handle_hypercall_kafl_get_host_config(struct kvm_run *run, CPUState *cpu, u config.bitmap_size = GET_GLOBAL_STATE()->shared_bitmap_size; config.ijon_bitmap_size = GET_GLOBAL_STATE()->shared_ijon_bitmap_size; config.payload_buffer_size = GET_GLOBAL_STATE()->shared_payload_buffer_size; + config.worker_id = GET_GLOBAL_STATE()->worker_id; write_virtual_memory(vaddr, (uint8_t*)&config, sizeof(host_config_t), cpu); GET_GLOBAL_STATE()->get_host_config_done = true; @@ -101,4 +102,4 @@ void handle_hypercall_kafl_set_agent_config(struct kvm_run *run, CPUState *cpu, exit(1); } GET_GLOBAL_STATE()->set_agent_config_done = true; -} \ No newline at end of file +} diff --git a/nyx/hypercall/configuration.h b/nyx/hypercall/configuration.h index 4bb0440be2..3c4e9ea0f0 100644 --- a/nyx/hypercall/configuration.h +++ b/nyx/hypercall/configuration.h @@ -19,6 +19,7 @@ typedef struct host_config_s{ uint32_t bitmap_size; uint32_t ijon_bitmap_size; uint32_t payload_buffer_size; + uint32_t worker_id; /* more to come */ } __attribute__((packed)) host_config_t; diff --git a/nyx/interface.c b/nyx/interface.c index af4b431212..c1e6a8d137 100644 --- a/nyx/interface.c +++ b/nyx/interface.c @@ -374,6 +374,7 @@ static void nyx_realize(DeviceState *dev, Error **errp){ if(s->cow_primary_size){ set_global_cow_cache_primary_size(s->cow_primary_size); } + GET_GLOBAL_STATE()->worker_id = s->worker_id; if (!s->workdir || !verify_workdir_state(s, errp)){ fprintf(stderr, "[QEMU-Nyx] Error: work dir...\n"); diff --git a/nyx/state/state.c b/nyx/state/state.c index 3edcbe2618..1389fe5a66 100644 --- a/nyx/state/state.c +++ b/nyx/state/state.c @@ -45,6 +45,7 @@ void state_init_global(void){ global_state.nyx_fdl = false; global_state.workdir_path = NULL; + global_state.worker_id = 0xffff; global_state.fast_reload_enabled = false; global_state.fast_reload_mode = false; @@ -234,4 +235,4 @@ void set_payload_pages(uint64_t* payload_pages, uint32_t pages){ void set_workdir_path(char* workdir){ assert(workdir && !global_state.workdir_path); assert(asprintf(&global_state.workdir_path, "%s", workdir) != -1); -} \ No newline at end of file +} diff --git a/nyx/state/state.h b/nyx/state/state.h index 6970119157..fabc05f72b 100644 --- a/nyx/state/state.h +++ b/nyx/state/state.h @@ -49,6 +49,7 @@ typedef struct qemu_nyx_state_s{ bool nyx_fdl; char* workdir_path; + uint32_t worker_id; /* FAST VM RELOAD */ bool fast_reload_enabled; From d81b846608071591fad277317358b30a6ed41055 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Thu, 19 Aug 2021 05:08:00 -0700 Subject: [PATCH 16/35] dump_file: check for NULL filename, support mkstemp() template --- nyx/hypercall/hypercall.c | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/nyx/hypercall/hypercall.c b/nyx/hypercall/hypercall.c index 5653c6b14f..b3b5031404 100644 --- a/nyx/hypercall/hypercall.c +++ b/nyx/hypercall/hypercall.c @@ -742,32 +742,38 @@ static void handle_hypercall_kafl_dump_file(struct kvm_run *run, CPUState *cpu, goto err_out1; } - if (!read_virtual_memory(file_obj.file_name_str_ptr, (uint8_t*)filename, 255, cpu)) { - fprintf(stderr, "Failed to read file_name_str_ptr in %s. Skipping..\n", __func__); - goto err_out1; + if (file_obj.file_name_str_ptr != 0) { + if (!read_virtual_memory(file_obj.file_name_str_ptr, (uint8_t*)filename, sizeof(filename)-1, cpu)) { + fprintf(stderr, "Failed to read file_name_str_ptr in %s. Skipping..\n", __func__); + goto err_out1; + } + filename[sizeof(filename)-1] = 0; } - filename[255] = 0; //fprintf(stderr, "%s: dump %lu fbytes from %s (append=%u)\n", // __func__, file_obj.bytes, filename, file_obj.append); - if (strnlen(filename, sizeof(filename))) { - char *base_name = basename(filename); - assert(asprintf(&host_path, "%s/dump/%s", GET_GLOBAL_STATE()->workdir_path , base_name) != -1); + // use a tempfile if file_name_ptr == NULL or points to empty string + if (0 == strnlen(filename, sizeof(filename))) { + strncpy(filename, "tmp.XXXXXX", sizeof(filename)-1); + } - if(file_obj.append){ - f = fopen(host_path, "a+"); - } else{ - f = fopen(host_path, "w+"); - } - } else { // no filename given - create tempfile + char *base_name = basename(filename); // clobbers the filename buffer! + assert(asprintf(&host_path, "%s/dump/%s", GET_GLOBAL_STATE()->workdir_path , base_name) != -1); + + // check if base_name is mkstemp() pattern, otherwise write/append to exact name + if (strlen(base_name) > 6 && 0 == strcmp(base_name+strlen(base_name)-6, "XXXXXX")) { if (file_obj.append) { fprintf(stderr, "Error request to append but no filename given in %s\n", __func__); goto err_out1; } - - assert(asprintf(&host_path, "%s/dump/tmp.XXXXXX", GET_GLOBAL_STATE()->workdir_path) != -1); f = fdopen(mkstemp(host_path), "w+"); + } else { + if (file_obj.append){ + f = fopen(host_path, "a+"); + } else{ + f = fopen(host_path, "w+"); + } } if (!f) { From 7b9bd18dc3d90cd4bf3ed95a0ae7656f8fb5de21 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Mon, 23 Aug 2021 13:29:31 -0700 Subject: [PATCH 17/35] refactor 'redqueen trace' to separate redqueen_trace.c --- nyx/auxiliary_buffer.c | 4 +-- nyx/file_helper.c | 23 +-------------- nyx/file_helper.h | 10 ++----- nyx/hypercall/hypercall.c | 13 -------- nyx/hypercall/hypercall.h | 2 -- nyx/pt.c | 6 ++-- nyx/redqueen.c | 38 ------------------------ nyx/redqueen.h | 5 ---- nyx/redqueen_trace.c | 62 +++++++++++++++++++++++++++++++-------- nyx/redqueen_trace.h | 8 +++-- nyx/state/state.c | 1 + nyx/state/state.h | 3 +- nyx/synchronization.c | 12 ++++---- 13 files changed, 74 insertions(+), 113 deletions(-) diff --git a/nyx/auxiliary_buffer.c b/nyx/auxiliary_buffer.c index 8c0682ecd8..8680cf7685 100644 --- a/nyx/auxiliary_buffer.c +++ b/nyx/auxiliary_buffer.c @@ -103,7 +103,7 @@ void check_auxiliary_config_buffer(auxilary_buffer_t* auxilary_buffer, auxilary_ #ifdef SUPPORT_COMPILE_TIME_REDQUEEN GET_GLOBAL_STATE()->pt_trace_mode_force = true; #endif - redqueen_set_trace_mode(GET_GLOBAL_STATE()->redqueen_state); + redqueen_set_trace_mode(); } } else { @@ -112,7 +112,7 @@ void check_auxiliary_config_buffer(auxilary_buffer_t* auxilary_buffer, auxilary_ #ifdef SUPPORT_COMPILE_TIME_REDQUEEN GET_GLOBAL_STATE()->pt_trace_mode_force = false; #endif - redqueen_unset_trace_mode(GET_GLOBAL_STATE()->redqueen_state); + redqueen_unset_trace_mode(); } } diff --git a/nyx/file_helper.c b/nyx/file_helper.c index 5ab7153b49..0bb484a7b8 100644 --- a/nyx/file_helper.c +++ b/nyx/file_helper.c @@ -56,7 +56,6 @@ void parse_address_file(char* path, size_t* num_addrs, uint64_t** addrs){ int re_fd = 0; int se_fd = 0; -int trace_fd = 0; void write_re_result(char* buf){ int unused __attribute__((unused)); @@ -65,20 +64,7 @@ void write_re_result(char* buf){ unused = write(re_fd, buf, strlen(buf)); } -void write_trace_result(redqueen_trace_t* trace_state){ - //int fd; - int unused __attribute__((unused)); - if (!trace_fd) - trace_fd = open(redqueen_workdir.pt_trace_results, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU); - redqueen_trace_write_file(trace_state, trace_fd); - //unused = write(trace_fd, buf, strlen(buf)); - //close(fd); -} - -void fsync_all_traces(void){ - if (!trace_fd){ - fsync(trace_fd); - } +void fsync_redqueen_files(void){ if (!se_fd){ fsync(se_fd); } @@ -96,13 +82,6 @@ void write_se_result(char* buf){ //close(fd); } -void delete_trace_files(void){ - int unused __attribute__((unused)); - if (!trace_fd) - trace_fd = open(redqueen_workdir.pt_trace_results, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU); - unused = ftruncate(trace_fd, 0); -} - void delete_redqueen_files(void){ int unused __attribute__((unused)); if (!re_fd) diff --git a/nyx/file_helper.h b/nyx/file_helper.h index 485863e8b8..020ebc83d1 100644 --- a/nyx/file_helper.h +++ b/nyx/file_helper.h @@ -1,7 +1,8 @@ #include #include #include -#include "redqueen_trace.h" + +#pragma once //doesn't take ownership of path, num_addrs or addrs void parse_address_file(char* path, size_t* num_addrs, uint64_t** addrs); @@ -12,14 +13,9 @@ void write_re_result(char* buf); //doesn't take ownership of buf void write_se_result(char* buf); -//doesn't take ownership of buf -void write_trace_result(redqueen_trace_t* trace_state); - //doesn' take ownership of buf void write_debug_result(char* buf); void delete_redqueen_files(void); -void delete_trace_files(void); - -void fsync_all_traces(void); +void fsync_redqueen_files(void); diff --git a/nyx/hypercall/hypercall.c b/nyx/hypercall/hypercall.c index b3b5031404..23b584f75b 100644 --- a/nyx/hypercall/hypercall.c +++ b/nyx/hypercall/hypercall.c @@ -714,19 +714,6 @@ void pt_set_disable_patches_pending(CPUState *cpu){ GET_GLOBAL_STATE()->patches_disable_pending = true; } -void pt_enable_rqi_trace(CPUState *cpu){ - if (GET_GLOBAL_STATE()->redqueen_state){ - redqueen_set_trace_mode(GET_GLOBAL_STATE()->redqueen_state); - } -} - -void pt_disable_rqi_trace(CPUState *cpu){ - if (GET_GLOBAL_STATE()->redqueen_state){ - redqueen_unset_trace_mode(GET_GLOBAL_STATE()->redqueen_state); - return; - } -} - static void handle_hypercall_kafl_dump_file(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg) { kafl_dump_file_t file_obj; diff --git a/nyx/hypercall/hypercall.h b/nyx/hypercall/hypercall.h index f4f3dc30c6..d48115758c 100644 --- a/nyx/hypercall/hypercall.h +++ b/nyx/hypercall/hypercall.h @@ -130,8 +130,6 @@ void pt_enable_rqo(CPUState *cpu); void pt_disable_rqo(CPUState *cpu); void pt_enable_rqi(CPUState *cpu); void pt_disable_rqi(CPUState *cpu); -void pt_enable_rqi_trace(CPUState *cpu); -void pt_disable_rqi_trace(CPUState *cpu); void pt_set_redqueen_instrumentation_mode(CPUState *cpu, int redqueen_instruction_mode); void pt_set_redqueen_update_blacklist(CPUState *cpu, bool newval); void pt_set_enable_patches_pending(CPUState *cpu); diff --git a/nyx/pt.c b/nyx/pt.c index f0b96e4b95..c6597e3f63 100644 --- a/nyx/pt.c +++ b/nyx/pt.c @@ -163,7 +163,7 @@ void alt_bitmap_add(uint64_t from, uint64_t to) { uint64_t transition_value; - if (GET_GLOBAL_STATE()->redqueen_state->trace_mode) { + if (GET_GLOBAL_STATE()->trace_mode) { if(alt_bitmap) { transition_value = mix_bits(to)^(mix_bits(from)>>1); alt_bitmap[transition_value & (alt_bitmap_size-1)]++; @@ -233,8 +233,8 @@ int pt_enable(CPUState *cpu, bool hmp_mode){ if(!fast_reload_set_bitmap(get_fast_reload_snapshot())){ coverage_bitmap_reset(); } - if (GET_GLOBAL_STATE()->redqueen_state->trace_mode) { - delete_trace_files(); + if (GET_GLOBAL_STATE()->trace_mode) { + redqueen_trace_reset(); alt_bitmap_reset(); } pt_truncate_pt_dump_file(); diff --git a/nyx/redqueen.c b/nyx/redqueen.c index 32f8c91e30..e4fc3a38f3 100644 --- a/nyx/redqueen.c +++ b/nyx/redqueen.c @@ -49,7 +49,6 @@ redqueen_t* new_rq_state(CPUState *cpu, page_cache_t* page_cache){ res->cpu = cpu; res->intercept_mode = false; - res->trace_mode = false; res->singlestep_enabled = false; res->hooks_applied = 0; res->page_cache = page_cache; @@ -225,43 +224,6 @@ void redqueen_callback(void* opaque, disassembler_mode_t mode, uint64_t start_ad } } - - -static void redqueen_trace_enabled(redqueen_t* self){ - int unused __attribute__((unused)); - if(self->trace_mode){ - - //libxdc_enable_tracing(GET_GLOBAL_STATE()->decoder); - libxdc_enable_tracing(GET_GLOBAL_STATE()->decoder); - libxdc_register_edge_callback(GET_GLOBAL_STATE()->decoder, (void (*)(void*, disassembler_mode_t, uint64_t, uint64_t))&redqueen_trace_register_transition, self->trace_state); - //redqueen_trace_register_transition(self->trace_state, INIT_TRACE_IP, ip); - //last_ip = ip; - } -} - -static void redqueen_trace_disabled(redqueen_t* self){ - int unused __attribute__((unused)); - if(self->trace_mode){ - libxdc_disable_tracing(GET_GLOBAL_STATE()->decoder); - - //redqueen_trace_register_transition(self->trace_state, last_ip, ip); - //edqueen_trace_register_transition(self->trace_state, ip, INIT_TRACE_IP); - } -} - -void redqueen_set_trace_mode(redqueen_t* self){ - self->trace_mode = true; - redqueen_trace_enabled(self); -} - -void redqueen_unset_trace_mode(redqueen_t* self){ - //write_trace_result(self->trace_state); - //redqueen_trace_reset(self->trace_state); - redqueen_trace_disabled(self); - - self->trace_mode = false; -} - void destroy_rq_state(redqueen_t* self){ redqueen_trace_free(self->trace_state); kh_destroy(RQ, self->lookup); diff --git a/nyx/redqueen.h b/nyx/redqueen.h index ce0d03a788..636b60fdc0 100644 --- a/nyx/redqueen.h +++ b/nyx/redqueen.h @@ -70,7 +70,6 @@ KHASH_MAP_INIT_INT64(RQ, uint32_t) typedef struct redqueen_s{ khash_t(RQ) *lookup; bool intercept_mode; - bool trace_mode; bool singlestep_enabled; int hooks_applied; CPUState *cpu; @@ -109,10 +108,6 @@ void enable_rq_intercept_mode(redqueen_t* self); void disable_rq_intercept_mode(redqueen_t* self); -void redqueen_register_transition(redqueen_t* self, uint64_t ip, uint64_t transition_val); -void redqueen_set_trace_mode(redqueen_t* self); -void redqueen_unset_trace_mode(redqueen_t* self); - void set_se_instruction(redqueen_t* self, uint64_t addr); void dump_se_registers(redqueen_t* self); diff --git a/nyx/redqueen_trace.c b/nyx/redqueen_trace.c index b2916900e2..ece2cf0e3d 100644 --- a/nyx/redqueen_trace.c +++ b/nyx/redqueen_trace.c @@ -2,17 +2,29 @@ #include #include #include + #include "redqueen_trace.h" +#include "redqueen.h" +#include "state/state.h" + void alt_bitmap_add(uint64_t from, uint64_t to); /* write full trace of edge transitions rather than sorted list? */ //#define KAFL_FULL_TRACES -#ifdef KAFL_FULL_TRACES -#include "redqueen.h" -extern int trace_fd; -#endif +int trace_fd = 0; + +static int reset_trace_fd(void) { + if (trace_fd) + close(trace_fd); + trace_fd = open(redqueen_workdir.pt_trace_results, O_WRONLY | O_CREAT | O_TRUNC, 0644); + if (trace_fd < 0) { + fprintf(stderr, "Failed to initiate trace output: %s\n", strerror(errno)); + assert(0); + } + return trace_fd; +} redqueen_trace_t* redqueen_trace_new(void){ redqueen_trace_t* self = malloc(sizeof(redqueen_trace_t)); @@ -23,7 +35,8 @@ redqueen_trace_t* redqueen_trace_new(void){ return self; } -void redqueen_trace_reset(redqueen_trace_t* self){ +static void redqueen_state_reset(void){ + redqueen_trace_t *self = GET_GLOBAL_STATE()->redqueen_state->trace_state; kh_destroy(RQ_TRACE, self->lookup); self->lookup = kh_init(RQ_TRACE); self->num_ordered_transitions = 0; @@ -43,9 +56,7 @@ void redqueen_trace_register_transition(redqueen_trace_t* self, disassembler_mod if (from != exit_ip && to != exit_ip) alt_bitmap_add(from, to); #ifdef KAFL_FULL_TRACES - extern int trace_fd; - if (!trace_fd) - trace_fd = open(redqueen_workdir.pt_trace_results, O_WRONLY | O_CREAT | O_APPEND, S_IRWXU); + assert(trace_fd >= 0); dprintf(trace_fd, "%lx,%lx\n", from, to); return; #endif @@ -62,33 +73,60 @@ void redqueen_trace_register_transition(redqueen_trace_t* self, disassembler_mod } } -void redqueen_trace_write_file(redqueen_trace_t* self, int fd){ +static void redqueen_trace_write(void){ #ifdef KAFL_FULL_TRACES return; #endif + redqueen_trace_t *self = GET_GLOBAL_STATE()->redqueen_state->trace_state; + assert(trace_fd >= 0); for(size_t i = 0; i < self->num_ordered_transitions; i++){ khiter_t k; uint128_t key = self->ordered_transitions[i]; k = kh_get(RQ_TRACE, self->lookup, key); assert(k != kh_end(self->lookup)); - dprintf(fd, "%lx,%lx,%lx\n", (uint64_t)(key>>64), (uint64_t)key, kh_value(self->lookup, k) ); + dprintf(trace_fd, "%lx,%lx,%lx\n", (uint64_t)(key>>64), (uint64_t)key, kh_value(self->lookup, k) ); } } +void redqueen_trace_reset(void){ + redqueen_state_reset(); + reset_trace_fd(); +} + +void redqueen_trace_flush(void){ + redqueen_trace_write(); + if (trace_fd) + fsync(trace_fd); +} + +void redqueen_set_trace_mode(void){ + GET_GLOBAL_STATE()->trace_mode = true; + libxdc_enable_tracing(GET_GLOBAL_STATE()->decoder); + libxdc_register_edge_callback(GET_GLOBAL_STATE()->decoder, + (void (*)(void*, disassembler_mode_t, uint64_t, uint64_t))&redqueen_trace_register_transition, + GET_GLOBAL_STATE()->redqueen_state->trace_state); +} + +void redqueen_unset_trace_mode(void){ + libxdc_disable_tracing(GET_GLOBAL_STATE()->decoder); + GET_GLOBAL_STATE()->trace_mode = false; +} #ifdef DEBUG_MAIN int main(int argc, char** argv){ redqueen_trace_t* rq_obj = redqueen_trace_new(); + reset_trace_fd(); + for (uint64_t j = 0; j < 0x5; j++){ redqueen_trace_register_transition(rq_obj, 0xBADF, 0xC0FFEE); redqueen_trace_register_transition(rq_obj, 0xBADBEEF, 0xC0FFEE); for (uint64_t i = 0; i < 0x10000; i++){ redqueen_trace_register_transition(rq_obj, 0xBADBEEF, 0xC0FFEE); } - redqueen_trace_write_file(rq_obj, STDOUT_FILENO); - redqueen_trace_reset(rq_obj); + redqueen_trace_write(rq_obj, STDOUT_FILENO); + redqueen_state_reset(); } redqueen_trace_free(rq_obj); diff --git a/nyx/redqueen_trace.h b/nyx/redqueen_trace.h index 95de032a69..5ec72dcc91 100644 --- a/nyx/redqueen_trace.h +++ b/nyx/redqueen_trace.h @@ -37,7 +37,11 @@ typedef struct redqueen_trace_s{ } redqueen_trace_t; redqueen_trace_t* redqueen_trace_new(void); -void redqueen_trace_reset(redqueen_trace_t* self); void redqueen_trace_free(redqueen_trace_t* self); void redqueen_trace_register_transition(redqueen_trace_t* self, disassembler_mode_t mode, uint64_t from, uint64_t to); -void redqueen_trace_write_file(redqueen_trace_t* self, int fd); + +void redqueen_set_trace_mode(void); +void redqueen_unset_trace_mode(void); + +void redqueen_trace_flush(void); +void redqueen_trace_reset(void); diff --git a/nyx/state/state.c b/nyx/state/state.c index 1389fe5a66..7a527f75d2 100644 --- a/nyx/state/state.c +++ b/nyx/state/state.c @@ -91,6 +91,7 @@ void state_init_global(void){ global_state.in_fuzzing_mode = false; global_state.in_reload_mode = true; global_state.starved = false; + global_state.trace_mode = false; global_state.shutdown_requested = false; global_state.cow_cache_full = false; diff --git a/nyx/state/state.h b/nyx/state/state.h index fabc05f72b..e634d7375e 100644 --- a/nyx/state/state.h +++ b/nyx/state/state.h @@ -132,7 +132,8 @@ typedef struct qemu_nyx_state_s{ bool in_fuzzing_mode; bool in_reload_mode; - bool starved; + bool starved; + bool trace_mode; bool shutdown_requested; bool cow_cache_full; diff --git a/nyx/synchronization.c b/nyx/synchronization.c index f63c42a078..c502f89138 100644 --- a/nyx/synchronization.c +++ b/nyx/synchronization.c @@ -277,12 +277,12 @@ void synchronization_lock(void){ //last_timeout = false; - if(unlikely(GET_GLOBAL_STATE()->in_redqueen_reload_mode || GET_GLOBAL_STATE()->redqueen_state->trace_mode)){ - if(GET_GLOBAL_STATE()->redqueen_state->trace_mode){ - write_trace_result(GET_GLOBAL_STATE()->redqueen_state->trace_state); - redqueen_trace_reset(GET_GLOBAL_STATE()->redqueen_state->trace_state); - } - fsync_all_traces(); + if(unlikely(GET_GLOBAL_STATE()->in_redqueen_reload_mode)) { + fsync_redqueen_files(); + } + + if (unlikely(GET_GLOBAL_STATE()->trace_mode)) { + redqueen_trace_flush(); } interface_send_char(NYX_INTERFACE_PING); From 84f1a1b67b46ca46eb0416a2617c0153dfcf2ff2 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Mon, 23 Aug 2021 14:15:50 -0700 Subject: [PATCH 18/35] move dump_pt logic to trace_dump.c, enable via aux_buffer --- nyx/Makefile.objs | 1 + nyx/auxiliary_buffer.c | 3 ++ nyx/interface.c | 3 +- nyx/pt.c | 50 +------------------------------- nyx/pt.h | 1 - nyx/trace_dump.c | 66 ++++++++++++++++++++++++++++++++++++++++++ nyx/trace_dump.h | 6 ++++ 7 files changed, 79 insertions(+), 51 deletions(-) create mode 100644 nyx/trace_dump.c create mode 100644 nyx/trace_dump.h diff --git a/nyx/Makefile.objs b/nyx/Makefile.objs index 5400116161..93db8e07bb 100644 --- a/nyx/Makefile.objs +++ b/nyx/Makefile.objs @@ -7,6 +7,7 @@ synchronization.o \ page_cache.o \ kvm_nested.o \ debug.o \ +trace_dump.o \ auxiliary_buffer.o \ mmh3.o \ nested_hypercalls.o \ diff --git a/nyx/auxiliary_buffer.c b/nyx/auxiliary_buffer.c index 8680cf7685..c6dddacf73 100644 --- a/nyx/auxiliary_buffer.c +++ b/nyx/auxiliary_buffer.c @@ -25,6 +25,7 @@ along with QEMU-PT. If not, see . #include #include "nyx/state/state.h" #include "nyx/debug.h" +#include "nyx/trace_dump.h" /* experimental feature (currently broken) * enabled via trace mode @@ -104,6 +105,7 @@ void check_auxiliary_config_buffer(auxilary_buffer_t* auxilary_buffer, auxilary_ GET_GLOBAL_STATE()->pt_trace_mode_force = true; #endif redqueen_set_trace_mode(); + pt_trace_dump_enable(true); } } else { @@ -113,6 +115,7 @@ void check_auxiliary_config_buffer(auxilary_buffer_t* auxilary_buffer, auxilary_ GET_GLOBAL_STATE()->pt_trace_mode_force = false; #endif redqueen_unset_trace_mode(); + pt_trace_dump_enable(false); } } diff --git a/nyx/interface.c b/nyx/interface.c index c1e6a8d137..d1c751d097 100644 --- a/nyx/interface.c +++ b/nyx/interface.c @@ -51,6 +51,7 @@ along with QEMU-PT. If not, see . #include "nyx/state/state.h" #include "nyx/sharedir.h" #include "nyx/helpers.h" +#include "nyx/trace_dump.h" #include @@ -278,7 +279,7 @@ static bool verify_workdir_state(nyx_interface_state *s, Error **errp){ if(s->dump_pt_trace){ assert(asprintf(&tmp, "%s/pt_trace_dump_%d", workdir, id) != -1); - pt_trace_dump_enable(tmp); + pt_trace_dump_init(tmp); free(tmp); } diff --git a/nyx/pt.c b/nyx/pt.c index c6597e3f63..e1826ae852 100644 --- a/nyx/pt.c +++ b/nyx/pt.c @@ -44,6 +44,7 @@ along with QEMU-PT. If not, see . #include "nyx/state/state.h" #include #include "nyx/helpers.h" +#include "nyx/trace_dump.h" #define PT_BUFFER_MMAP_ADDR 0x3ffff0000000 @@ -53,55 +54,6 @@ uint32_t last = 0; uint32_t alt_bitmap_size = 0; uint8_t* alt_bitmap = NULL; -int pt_trace_dump_fd = 0; -char *pt_trace_dump_filename; -bool should_dump_pt_trace= false; /* dump PT trace as returned from HW */ - -void pt_trace_dump_enable(char* filename) -{ - int test_fd; - - printf("Enable pt trace dump at %s", filename); - pt_trace_dump_filename = filename; - should_dump_pt_trace = true; - - test_fd = open(filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); - if (test_fd < 0) - fprintf(stderr, "Error accessing pt_dump output path: %s", strerror(errno)); - assert(test_fd >= 0); -} - -static void pt_truncate_pt_dump_file(void) { - int fd; - - if (!should_dump_pt_trace) - return; - - fd = open(pt_trace_dump_filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); - if (fd < 0) { - fprintf(stderr, "Error truncating pt_trace_dump: %s\n", strerror(errno)); - assert(0); - } - close(fd); -} - -static void pt_write_pt_dump_file(uint8_t *data, size_t bytes) -{ - int fd; - - if (!should_dump_pt_trace) - return; - - fd = open(pt_trace_dump_filename, O_APPEND|O_WRONLY, 0644); - //fd = open(pt_trace_dump_filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); - if (fd < 0) { - fprintf(stderr, "Error writing pt_trace_dump: %s\n", strerror(errno)); - assert(0); - } - assert(bytes == write(fd, data, bytes)); - close(fd); -} - static void pt_set(CPUState *cpu, run_on_cpu_data arg){ asm volatile("" ::: "memory"); } diff --git a/nyx/pt.h b/nyx/pt.h index 174b463229..ca49602272 100644 --- a/nyx/pt.h +++ b/nyx/pt.h @@ -44,6 +44,5 @@ void pt_post_kvm_run(CPUState *cpu); void pt_handle_overflow(CPUState *cpu); void pt_dump(CPUState *cpu, int bytes); -void pt_trace_dump_enable(char* filename); #endif diff --git a/nyx/trace_dump.c b/nyx/trace_dump.c new file mode 100644 index 0000000000..f325377441 --- /dev/null +++ b/nyx/trace_dump.c @@ -0,0 +1,66 @@ +#include +#include +#include +#include + +#include "state/state.h" +#include "trace_dump.h" + +/* dump PT trace as returned from HW */ + +char *pt_trace_dump_filename; +bool pt_dump_initialized = false; +bool pt_dump_enabled = false; + +void pt_trace_dump_enable(bool enable){ + if (pt_dump_initialized) + pt_dump_enabled = enable; +} + +void pt_trace_dump_init(char* filename) +{ + int test_fd; + + //fprintf(stderr, "Enable pt trace dump at %s", filename); + pt_dump_initialized = true; + + test_fd = open(filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); + if (test_fd < 0) + fprintf(stderr, "Error accessing pt_dump output path %s: %s", pt_trace_dump_filename, strerror(errno)); + assert(test_fd >= 0); + + pt_trace_dump_filename = strdup(filename); + assert(pt_trace_dump_filename); +} + +void pt_truncate_pt_dump_file(void) { + int fd; + + if (!pt_dump_enabled) + return; + + fd = open(pt_trace_dump_filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); + if (fd < 0) { + fprintf(stderr, "Error truncating %s: %s\n", pt_trace_dump_filename, strerror(errno)); + assert(0); + } + close(fd); +} + +void pt_write_pt_dump_file(uint8_t *data, size_t bytes) +{ + int fd; + + if (!pt_dump_enabled) + return; + + fd = open(pt_trace_dump_filename, O_APPEND|O_WRONLY, 0644); + //fd = open(pt_trace_dump_filename, O_CREAT|O_TRUNC|O_WRONLY, 0644); + if (fd < 0) { + fprintf(stderr, "Error writing pt_trace_dump to %s: %s\n", pt_trace_dump_filename, strerror(errno)); + assert(0); + } + assert(bytes == write(fd, data, bytes)); + close(fd); +} + diff --git a/nyx/trace_dump.h b/nyx/trace_dump.h new file mode 100644 index 0000000000..fb3235f34b --- /dev/null +++ b/nyx/trace_dump.h @@ -0,0 +1,6 @@ +#pragma once + +void pt_trace_dump_init(char* filename); +void pt_trace_dump_enable(bool enable); +void pt_write_pt_dump_file(uint8_t *data, size_t bytes); +void pt_truncate_pt_dump_file(void); From f348dcfc2379b3b56c9036f10371512f8f0aaf03 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Wed, 25 Aug 2021 14:16:39 -0700 Subject: [PATCH 19/35] redqueen_trace: disable unless 'edge_cb_trace' option is provided Both, the legacy 'redqueen' trace via libxdc callback as well as new dump_pt trace option are now toggled with aux-buffer trace_mode option. This new qemu cmdline option allows to re-enable the old trace method, or even use both trace methods at the same time. --- nyx/auxiliary_buffer.c | 2 ++ nyx/interface.c | 6 +++++ nyx/redqueen_trace.c | 53 ++++++++++++++++++++++++++---------------- nyx/redqueen_trace.h | 8 +++++++ 4 files changed, 49 insertions(+), 20 deletions(-) diff --git a/nyx/auxiliary_buffer.c b/nyx/auxiliary_buffer.c index c6dddacf73..7ba076de3e 100644 --- a/nyx/auxiliary_buffer.c +++ b/nyx/auxiliary_buffer.c @@ -104,6 +104,7 @@ void check_auxiliary_config_buffer(auxilary_buffer_t* auxilary_buffer, auxilary_ #ifdef SUPPORT_COMPILE_TIME_REDQUEEN GET_GLOBAL_STATE()->pt_trace_mode_force = true; #endif + GET_GLOBAL_STATE()->trace_mode = true; redqueen_set_trace_mode(); pt_trace_dump_enable(true); } @@ -114,6 +115,7 @@ void check_auxiliary_config_buffer(auxilary_buffer_t* auxilary_buffer, auxilary_ #ifdef SUPPORT_COMPILE_TIME_REDQUEEN GET_GLOBAL_STATE()->pt_trace_mode_force = false; #endif + GET_GLOBAL_STATE()->trace_mode = false; redqueen_unset_trace_mode(); pt_trace_dump_enable(false); } diff --git a/nyx/interface.c b/nyx/interface.c index d1c751d097..09e57c0209 100644 --- a/nyx/interface.c +++ b/nyx/interface.c @@ -90,6 +90,7 @@ typedef struct nyx_interface_state { uint32_t input_buffer_size; bool dump_pt_trace; + bool edge_cb_trace; bool redqueen; @@ -283,6 +284,10 @@ static bool verify_workdir_state(nyx_interface_state *s, Error **errp){ free(tmp); } + if(s->edge_cb_trace){ + redqueen_trace_init(); + } + assert(asprintf(&tmp, "%s/aux_buffer_%d", workdir, id) != -1); /* @@ -427,6 +432,7 @@ static Property nyx_interface_properties[] = { DEFINE_PROP_UINT32("bitmap_size", nyx_interface_state, bitmap_size, DEFAULT_NYX_BITMAP_SIZE), DEFINE_PROP_UINT32("input_buffer_size", nyx_interface_state, input_buffer_size, DEFAULT_NYX_BITMAP_SIZE), DEFINE_PROP_BOOL("dump_pt_trace", nyx_interface_state, dump_pt_trace, false), + DEFINE_PROP_BOOL("edge_cb_trace", nyx_interface_state, edge_cb_trace, false), DEFINE_PROP_END_OF_LIST(), diff --git a/nyx/redqueen_trace.c b/nyx/redqueen_trace.c index ece2cf0e3d..3f4d0acd9a 100644 --- a/nyx/redqueen_trace.c +++ b/nyx/redqueen_trace.c @@ -15,6 +15,8 @@ void alt_bitmap_add(uint64_t from, uint64_t to); int trace_fd = 0; +int redqueen_trace_enabled = false; + static int reset_trace_fd(void) { if (trace_fd) close(trace_fd); @@ -26,6 +28,10 @@ static int reset_trace_fd(void) { return trace_fd; } +void redqueen_trace_init(void) { + redqueen_trace_enabled = true; +} + redqueen_trace_t* redqueen_trace_new(void){ redqueen_trace_t* self = malloc(sizeof(redqueen_trace_t)); self->lookup = kh_init(RQ_TRACE); @@ -35,13 +41,6 @@ redqueen_trace_t* redqueen_trace_new(void){ return self; } -static void redqueen_state_reset(void){ - redqueen_trace_t *self = GET_GLOBAL_STATE()->redqueen_state->trace_state; - kh_destroy(RQ_TRACE, self->lookup); - self->lookup = kh_init(RQ_TRACE); - self->num_ordered_transitions = 0; -} - void redqueen_trace_free(redqueen_trace_t* self){ kh_destroy(RQ_TRACE, self->lookup); free(self->ordered_transitions); @@ -88,28 +87,42 @@ static void redqueen_trace_write(void){ } } +static void redqueen_state_reset(void){ + redqueen_trace_t *self = GET_GLOBAL_STATE()->redqueen_state->trace_state; + kh_destroy(RQ_TRACE, self->lookup); + self->lookup = kh_init(RQ_TRACE); + self->num_ordered_transitions = 0; +} + + void redqueen_trace_reset(void){ - redqueen_state_reset(); - reset_trace_fd(); + if (redqueen_trace_enabled) { + redqueen_state_reset(); + reset_trace_fd(); + } } void redqueen_trace_flush(void){ - redqueen_trace_write(); - if (trace_fd) - fsync(trace_fd); + if (redqueen_trace_enabled) { + redqueen_trace_write(); + if (trace_fd) + fsync(trace_fd); + } } void redqueen_set_trace_mode(void){ - GET_GLOBAL_STATE()->trace_mode = true; - libxdc_enable_tracing(GET_GLOBAL_STATE()->decoder); - libxdc_register_edge_callback(GET_GLOBAL_STATE()->decoder, - (void (*)(void*, disassembler_mode_t, uint64_t, uint64_t))&redqueen_trace_register_transition, - GET_GLOBAL_STATE()->redqueen_state->trace_state); + if (redqueen_trace_enabled) { + libxdc_enable_tracing(GET_GLOBAL_STATE()->decoder); + libxdc_register_edge_callback(GET_GLOBAL_STATE()->decoder, + (void (*)(void*, disassembler_mode_t, uint64_t, uint64_t))&redqueen_trace_register_transition, + GET_GLOBAL_STATE()->redqueen_state->trace_state); + } } void redqueen_unset_trace_mode(void){ - libxdc_disable_tracing(GET_GLOBAL_STATE()->decoder); - GET_GLOBAL_STATE()->trace_mode = false; + if (redqueen_trace_enabled) { + libxdc_disable_tracing(GET_GLOBAL_STATE()->decoder); + } } #ifdef DEBUG_MAIN @@ -126,7 +139,7 @@ int main(int argc, char** argv){ redqueen_trace_register_transition(rq_obj, 0xBADBEEF, 0xC0FFEE); } redqueen_trace_write(rq_obj, STDOUT_FILENO); - redqueen_state_reset(); + redqueen_trace_reset(); } redqueen_trace_free(rq_obj); diff --git a/nyx/redqueen_trace.h b/nyx/redqueen_trace.h index 5ec72dcc91..979287dc3d 100644 --- a/nyx/redqueen_trace.h +++ b/nyx/redqueen_trace.h @@ -1,3 +1,10 @@ +#include +#include +#include +#include + +#include "qemu/osdep.h" + #pragma once #include "khash.h" #include @@ -40,6 +47,7 @@ redqueen_trace_t* redqueen_trace_new(void); void redqueen_trace_free(redqueen_trace_t* self); void redqueen_trace_register_transition(redqueen_trace_t* self, disassembler_mode_t mode, uint64_t from, uint64_t to); +void redqueen_trace_init(void); void redqueen_set_trace_mode(void); void redqueen_unset_trace_mode(void); From 96aac23864a0725656e01f05946ac275774b512d Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Wed, 25 Aug 2021 14:21:23 -0700 Subject: [PATCH 20/35] move alt_bitmap implementation to redqueen_trace.c alt_bitmap is only relevant in redqueen_trace mode, when libxdc does not produce a bitmap on its own.. --- nyx/pt.c | 42 +----------------------------------------- nyx/pt.h | 7 ------- nyx/redqueen_trace.c | 44 +++++++++++++++++++++++++++++++++++++++++--- nyx/redqueen_trace.h | 4 ++++ 4 files changed, 46 insertions(+), 51 deletions(-) diff --git a/nyx/pt.c b/nyx/pt.c index e1826ae852..d806152dad 100644 --- a/nyx/pt.c +++ b/nyx/pt.c @@ -45,15 +45,10 @@ along with QEMU-PT. If not, see . #include #include "nyx/helpers.h" #include "nyx/trace_dump.h" +#include "nyx/redqueen_trace.h" #define PT_BUFFER_MMAP_ADDR 0x3ffff0000000 -uint32_t state_byte = 0; -uint32_t last = 0; - -uint32_t alt_bitmap_size = 0; -uint8_t* alt_bitmap = NULL; - static void pt_set(CPUState *cpu, run_on_cpu_data arg){ asm volatile("" ::: "memory"); } @@ -88,41 +83,6 @@ static inline int pt_ioctl(int fd, unsigned long request, unsigned long arg){ return ioctl(fd, request, arg); } -void alt_bitmap_init(void* ptr, uint32_t size) -{ - alt_bitmap = (uint8_t*)ptr; - alt_bitmap_size = size; -} - -void alt_bitmap_reset(void) -{ - if(alt_bitmap) { - memset(alt_bitmap, 0x00, alt_bitmap_size); - } -} - -static inline uint64_t mix_bits(uint64_t v) { - v ^= (v >> 31); - v *= 0x7fb5d329728ea185; - return v; -} - -/* - * quick+dirty bitmap based on libxdc trace callback - * similar but not itentical to libxdc bitmap. - */ -void alt_bitmap_add(uint64_t from, uint64_t to) -{ - uint64_t transition_value; - - if (GET_GLOBAL_STATE()->trace_mode) { - if(alt_bitmap) { - transition_value = mix_bits(to)^(mix_bits(from)>>1); - alt_bitmap[transition_value & (alt_bitmap_size-1)]++; - } - } -} - #ifdef DUMP_AND_DEBUG_PT void dump_pt_trace(void* buffer, int bytes){ static FILE* f = NULL; diff --git a/nyx/pt.h b/nyx/pt.h index ca49602272..66c1678e2b 100644 --- a/nyx/pt.h +++ b/nyx/pt.h @@ -24,13 +24,6 @@ along with QEMU-PT. If not, see . void pt_init_decoder(CPUState *cpu); -void pt_reset_bitmap(void); -void pt_setup_bitmap(void* ptr); - -void alt_bitmap_reset(void); -void alt_bitmap_init(void* ptr, uint32_t size); -void alt_bitmap_add(uint64_t from, uint64_t to); - int pt_enable(CPUState *cpu, bool hmp_mode); int pt_disable(CPUState *cpu, bool hmp_mode); int pt_enable_ip_filtering(CPUState *cpu, uint8_t addrn, bool redqueen, bool hmp_mode); diff --git a/nyx/redqueen_trace.c b/nyx/redqueen_trace.c index 3f4d0acd9a..4c9dacbef8 100644 --- a/nyx/redqueen_trace.c +++ b/nyx/redqueen_trace.c @@ -8,15 +8,53 @@ #include "state/state.h" -void alt_bitmap_add(uint64_t from, uint64_t to); - /* write full trace of edge transitions rather than sorted list? */ //#define KAFL_FULL_TRACES int trace_fd = 0; - int redqueen_trace_enabled = false; +uint32_t alt_bitmap_size = 0; +uint8_t* alt_bitmap = NULL; + +void alt_bitmap_init(void* ptr, uint32_t size) +{ + if (redqueen_trace_enabled) { + alt_bitmap = (uint8_t*)ptr; + alt_bitmap_size = size; + } +} + +void alt_bitmap_reset(void) +{ + if (alt_bitmap) { + memset(alt_bitmap, 0x00, alt_bitmap_size); + } +} + +static inline uint64_t mix_bits(uint64_t v) { + v ^= (v >> 31); + v *= 0x7fb5d329728ea185; + return v; +} + +/* + * quick+dirty bitmap based on libxdc trace callback + * similar but not itentical to libxdc bitmap. + */ +static void alt_bitmap_add(uint64_t from, uint64_t to) +{ + uint64_t transition_value; + + if (GET_GLOBAL_STATE()->trace_mode) { + if(alt_bitmap) { + transition_value = mix_bits(to)^(mix_bits(from)>>1); + alt_bitmap[transition_value & (alt_bitmap_size-1)]++; + } + } +} + + static int reset_trace_fd(void) { if (trace_fd) close(trace_fd); diff --git a/nyx/redqueen_trace.h b/nyx/redqueen_trace.h index 979287dc3d..a4fdc17b3e 100644 --- a/nyx/redqueen_trace.h +++ b/nyx/redqueen_trace.h @@ -43,6 +43,10 @@ typedef struct redqueen_trace_s{ uint128_t* ordered_transitions; } redqueen_trace_t; +/* libxdc outputs no bitmap in trace mode */ +void alt_bitmap_reset(void); +void alt_bitmap_init(void* ptr, uint32_t size); + redqueen_trace_t* redqueen_trace_new(void); void redqueen_trace_free(redqueen_trace_t* self); void redqueen_trace_register_transition(redqueen_trace_t* self, disassembler_mode_t mode, uint64_t from, uint64_t to); From 46119f1f2cf0c93d7ec92c1171ccb405d83658a5 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Thu, 2 Sep 2021 17:29:06 -0700 Subject: [PATCH 21/35] KVM unknown exit: only fail after default handler also fails Qemu default handler covers some corner cases and prints diagnostics. Failing only afterwards seems to fix a KVM_EXIT_ENTRY_ERROR crash (code 9) --- accel/kvm/kvm-all.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/accel/kvm/kvm-all.c b/accel/kvm/kvm-all.c index 11f7c3ad93..0254410cda 100644 --- a/accel/kvm/kvm-all.c +++ b/accel/kvm/kvm-all.c @@ -2692,12 +2692,13 @@ int kvm_cpu_exec(CPUState *cpu) #define CONFIG_UNKNOWN_ERROR_IS_PANIC #ifndef CONFIG_UNKNOWN_ERROR_IS_PANIC fprintf(stderr, "Unknown exit code (%d) => ABORT\n", run->exit_reason); - assert(false); ret = kvm_arch_handle_exit(cpu, run); + assert(ret == 0); #else debug_fprintf("kvm_arch_handle_exit(%d) => panic\n", run->exit_reason); - handle_hypercall_kafl_panic(run, cpu, (uint64_t)run->hypercall.args[0]); - ret = 0; + ret = kvm_arch_handle_exit(cpu, run); + if (ret != 0) + handle_hypercall_kafl_panic(run, cpu, (uint64_t)run->hypercall.args[0]); #endif #endif ret = kvm_arch_handle_exit(cpu, run); From a5729842890495976bce19fb4351296728e865b5 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Thu, 9 Sep 2021 22:20:15 +0000 Subject: [PATCH 22/35] virtio snapshot restore virtio-blk still fails for usermode fuzzing --- hw/virtio/virtio.c | 4 +- nyx/snapshot/devices/nyx_device_state.c | 6 +++ nyx/snapshot/devices/state_reallocation.c | 57 ++++++++++++++++++++--- nyx/snapshot/devices/state_reallocation.h | 1 + 4 files changed, 60 insertions(+), 8 deletions(-) diff --git a/hw/virtio/virtio.c b/hw/virtio/virtio.c index 04716b5f6c..b9a88b2826 100644 --- a/hw/virtio/virtio.c +++ b/hw/virtio/virtio.c @@ -2807,7 +2807,9 @@ static int virtio_device_put(QEMUFile *f, void *opaque, size_t size, } /* A wrapper for use as a VMState .get function */ -static int virtio_device_get(QEMUFile *f, void *opaque, size_t size, +int virtio_device_get(QEMUFile *f, void *opaque, size_t size, + const VMStateField *field); +int virtio_device_get(QEMUFile *f, void *opaque, size_t size, const VMStateField *field) { VirtIODevice *vdev = VIRTIO_DEVICE(opaque); diff --git a/nyx/snapshot/devices/nyx_device_state.c b/nyx/snapshot/devices/nyx_device_state.c index 3911deb12c..098d9b7af8 100644 --- a/nyx/snapshot/devices/nyx_device_state.c +++ b/nyx/snapshot/devices/nyx_device_state.c @@ -376,6 +376,12 @@ nyx_device_state_t* nyx_device_state_init_from_snapshot(const char* snapshot_fol return self; } +/* + * This is where QemuFile is created for later fast_snapshot creation + * we use fast_qemu_savevm_state() to create a regular snapshot to QEMUFile + * backed by RAM. state_reallocation_new() then uses this file to build an + * optimized sequence of snapshot restore operations. + */ nyx_device_state_t* nyx_device_state_init(void){ nyx_device_state_t* self = malloc(sizeof(nyx_device_state_t)); diff --git a/nyx/snapshot/devices/state_reallocation.c b/nyx/snapshot/devices/state_reallocation.c index 666a207160..ab83f58acc 100644 --- a/nyx/snapshot/devices/state_reallocation.c +++ b/nyx/snapshot/devices/state_reallocation.c @@ -283,13 +283,45 @@ static void add_post_fptr(state_reallocation_t* self, void* fptr, uint32_t versi extern void fast_get_pci_config_device(void* data, size_t size, void* opaque); void fast_get_pci_irq_state(void* data, size_t size, void* opaque); +//void fast_virtio_device_get(void* data, size_t size, void* opaque); +int virtio_device_get(QEMUFile *f, void *opaque, size_t size, const VMStateField *field); + +static int fast_loadvm_fclose(void *opaque){ + return 0; +} + +static ssize_t fast_loadvm_get_buffer(void *opaque, uint8_t *buf, int64_t pos, size_t size){ + assert(pos < ((struct fast_savevm_opaque_t*)(opaque))->buflen); + memcpy(buf, (void*)(((struct fast_savevm_opaque_t*)(opaque))->buf + pos), size); + return size; +} + +static const QEMUFileOps fast_loadvm_ops = { + .get_buffer = (QEMUFileGetBufferFunc*)fast_loadvm_get_buffer, + .close = (QEMUFileCloseFunc*)fast_loadvm_fclose +}; + +/* use opaque data to bootstrap virtio restore from QEMUFile */ +static void fast_virtio_device_get(void* data, size_t size, void* opaque) +{ + struct fast_savevm_opaque_t fast_loadvm_opaque = { + .buf = data, + .buflen = size, + .f = NULL, + .pos = 0, + }; + QEMUFile* f = qemu_fopen_ops(&fast_loadvm_opaque, &fast_loadvm_ops); + + virtio_device_get(f, opaque, size, NULL); +} + static void add_get(state_reallocation_t* self, void* fptr, void* opaque, size_t size, void* field, QEMUFile* f, const char* name){ if(!self){ return; } void (*handler)(void* , size_t, void*) = NULL; - void* data = NULL; + uint8_t* data = NULL; if(!strcmp(name, "timer")){ debug_fprintf(stderr, "SKPPING: %ld\n", size*-1); @@ -315,13 +347,21 @@ static void add_get(state_reallocation_t* self, void* fptr, void* opaque, size_t data = malloc(sizeof(uint8_t)*size); qemu_get_buffer(f, (uint8_t*)data, size); } - + else if(!strcmp(name, "virtio")){ + fprintf(stderr, "WARNING: ATTEMPTING FAST GET for %s\n", name); + qemu_file_skip(f, size * -1); + handler = fast_virtio_device_get; + data = malloc(sizeof(uint8_t)*size); + qemu_get_buffer(f, (uint8_t*)data, size); + } else{ fprintf(stderr, "WARNING: NOT IMPLEMENTED FAST GET ROUTINE for %s\n", name); abort(); return; } + // will be called by pre-/post-save or pre-post-load? + // will be processed by fdl_fast_reload() self->get_fptr[self->fast_state_get_fptr_pos] = handler; self->get_opaque[self->fast_state_get_fptr_pos] = opaque; self->get_size[self->fast_state_get_fptr_pos] = size; @@ -481,19 +521,21 @@ static inline int get_handler(state_reallocation_t* self, QEMUFile* f, void* cur add_get(self, (void*) field->info->get, curr_elem, size, (void*) field, f, field->info->name); } else if(!strcmp(field->info->name, "pci config")){ - //fprintf(stderr, "type: %s (size: %x)\n", field->info->name, size); + fprintf(stderr, "type: %s (size: %lx)\n", field->info->name, size); add_get(self, (void*) field->info->get, curr_elem, size, (void*) field, f, field->info->name); } else if(!strcmp(field->info->name, "pci irq state")){ - //fprintf(stderr, "type: %s (size: %x)\n", field->info->name, size); + fprintf(stderr, "type: %s (size: %lx)\n", field->info->name, size); add_get(self, (void*) field->info->get, curr_elem, size, (void*) field, f, field->info->name); } else if(!strcmp(field->info->name, "virtio")){ - fprintf(stderr, "type: %s (size: %lx)\n", field->info->name, size); - abort(); /* not yet implemented */ + add_get(self, (void*) field->info->get, curr_elem, size, (void*) field, f, field->info->name); + //fprintf(stderr, "[QEMU-PT] %s: WARNING no handler for %s, type %s, size %lx!\n", + // __func__, vmsd_name, field->info->name, size); } else{ - fprintf(stderr, "FAIL field->info->name: %s\n", field->info->name); + fprintf(stderr, "[QEMU-PT] %s: WARNING no handler for %s, type %s, size %lx!\n", + __func__, vmsd_name, field->info->name, size); assert(0); } @@ -918,6 +960,7 @@ state_reallocation_t* state_reallocation_new(QEMUFile *f){ self->tmp_snapshot.enabled = false; self->tmp_snapshot.fast_state_size = 0; + // actually enumerate the devices here fdl_enumerate_global_states(self, f); self->tmp_snapshot.copy = malloc(sizeof(void*) * self->fast_state_pos); diff --git a/nyx/snapshot/devices/state_reallocation.h b/nyx/snapshot/devices/state_reallocation.h index 4d594d1fe2..539ff49095 100644 --- a/nyx/snapshot/devices/state_reallocation.h +++ b/nyx/snapshot/devices/state_reallocation.h @@ -50,6 +50,7 @@ struct QEMUFile_tmp { struct fast_savevm_opaque_t{ FILE* f; uint8_t* buf; + size_t buflen; uint64_t pos; void* output_buffer; uint32_t* output_buffer_size; From 6b4661a7582f6d8bfbba1f8df4a791232bf299ab Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Mon, 20 Sep 2021 09:32:18 +0000 Subject: [PATCH 23/35] dump_file hypercall: support mkstemps() template with suffix --- nyx/hypercall/hypercall.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nyx/hypercall/hypercall.c b/nyx/hypercall/hypercall.c index 23b584f75b..ef63ddfbd4 100644 --- a/nyx/hypercall/hypercall.c +++ b/nyx/hypercall/hypercall.c @@ -749,12 +749,13 @@ static void handle_hypercall_kafl_dump_file(struct kvm_run *run, CPUState *cpu, assert(asprintf(&host_path, "%s/dump/%s", GET_GLOBAL_STATE()->workdir_path , base_name) != -1); // check if base_name is mkstemp() pattern, otherwise write/append to exact name - if (strlen(base_name) > 6 && 0 == strcmp(base_name+strlen(base_name)-6, "XXXXXX")) { + char *pattern = strstr(base_name, "XXXXXX"); + if (pattern) { + unsigned suffix = strlen(pattern) - strlen("XXXXXX"); + f = fdopen(mkstemps(host_path, suffix), "w+"); if (file_obj.append) { - fprintf(stderr, "Error request to append but no filename given in %s\n", __func__); - goto err_out1; + fprintf(stderr, "Warning in %s: Writing unique generated file in append mode?\n", __func__); } - f = fdopen(mkstemp(host_path), "w+"); } else { if (file_obj.append){ f = fopen(host_path, "a+"); From c1d29a2399e6549bde8517effcdb7e019290d954 Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Fri, 24 Sep 2021 15:56:07 +0000 Subject: [PATCH 24/35] sharedir: allow reading anything stored or linked from sharedir In particular, we want to allow symlinks to external resources.. --- nyx/sharedir.c | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/nyx/sharedir.c b/nyx/sharedir.c index 9cedf104a7..0920013355 100644 --- a/nyx/sharedir.c +++ b/nyx/sharedir.c @@ -42,19 +42,19 @@ static size_t get_file_size(const char* file){ static char* sharedir_scan(sharedir_t* self, const char* file){ - char* path = NULL; - assert(asprintf(&path, "%s/%s", self->dir, file) != -1); + /* + * Agent is not under our control, but lets roughly constrain + * it to anything stored in or linked from sharedir + */ + chdir(self->dir); + char* real_path = realpath(file, NULL); - char* real_path = realpath(path, NULL); - - free(path); - if(real_path && !strncmp(self->dir, real_path, strlen(self->dir)) && file_exits(real_path)){ + if (file[0] != '/' && !strstr(file, "/../") && + real_path && file_exits(real_path)) { return real_path; } - if(real_path){ - free(real_path); - } + free(real_path); return NULL; } @@ -115,15 +115,15 @@ static FILE* get_file_ptr(sharedir_t* self, sharedir_file_t* obj){ if(obj == self->last_file_obj_ptr && self->last_file_f){ return self->last_file_f; } - else{ - if(self->last_file_f){ - fclose(self->last_file_f); - } - FILE* f = fopen(obj->path, "r"); - self->last_file_f = f; - self->last_file_obj_ptr = obj; - return f; + + if(self->last_file_f){ + fclose(self->last_file_f); } + + FILE* f = fopen(obj->path, "r"); + self->last_file_f = f; + self->last_file_obj_ptr = obj; + return f; } uint64_t sharedir_request_file(sharedir_t* self, const char* file, uint8_t* page_buffer){ From dacb4d5126ef02521f7d611e54a67a3accbd4aea Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Thu, 3 Feb 2022 13:57:12 -0800 Subject: [PATCH 25/35] initial support for Q35 platform Add option for "-machine kAFL64-Q35" Co-authored-by: Benoit Morgan --- hw/i386/pc_q35.c | 17 +++++++++++++++++ hw/isa/lpc_ich9.c | 1 + include/hw/i386/ich9.h | 5 +++++ 3 files changed, 23 insertions(+) diff --git a/hw/i386/pc_q35.c b/hw/i386/pc_q35.c index 385e5cffb1..e2c5a9bc6a 100644 --- a/hw/i386/pc_q35.c +++ b/hw/i386/pc_q35.c @@ -533,3 +533,20 @@ static void pc_q35_2_4_machine_options(MachineClass *m) DEFINE_Q35_MACHINE(v2_4, "pc-q35-2.4", NULL, pc_q35_2_4_machine_options); + +#ifdef QEMU_NYX +static void pc_kAFL64_vmx_v1_0_machine_options(MachineClass *m) +{ + pc_q35_4_2_machine_options(m); + m->alias = "kAFL64"; + //m->is_default = 1; + m->desc = "kAFL64 PC (Q35 + ICH9, 2009)"; +} + +static void kAFL64_init(MachineState *machine) +{ + pc_q35_init(machine); +} + +DEFINE_PC_MACHINE(v1, "kAFL64-Q35", kAFL64_init, pc_kAFL64_vmx_v1_0_machine_options); +#endif diff --git a/hw/isa/lpc_ich9.c b/hw/isa/lpc_ich9.c index 17c292e306..9e2d30b032 100644 --- a/hw/isa/lpc_ich9.c +++ b/hw/isa/lpc_ich9.c @@ -479,6 +479,7 @@ static void ich9_lpc_rcba_update(ICH9LPCState *lpc, uint32_t rcba_old) if (rcba_old & ICH9_LPC_RCBA_EN) { memory_region_del_subregion(get_system_memory(), &lpc->rcrb_mem); } + // Nyx snapshot reload fails here if ICH9_LPC_RCBA_EN=1 if (rcba & ICH9_LPC_RCBA_EN) { memory_region_add_subregion_overlap(get_system_memory(), rcba & ICH9_LPC_RCBA_BA_MASK, diff --git a/include/hw/i386/ich9.h b/include/hw/i386/ich9.h index 72e803f6e2..df72577592 100644 --- a/include/hw/i386/ich9.h +++ b/include/hw/i386/ich9.h @@ -177,7 +177,12 @@ typedef struct ICH9LPCState { #define ICH9_LPC_RCBA 0xf0 #define ICH9_LPC_RCBA_BA_MASK Q35_MASK(32, 31, 14) +#ifdef QEMU_NYX +// Nyx snapshot restore fails on this +#define ICH9_LPC_RCBA_EN 0x0 +#else #define ICH9_LPC_RCBA_EN 0x1 +#endif #define ICH9_LPC_RCBA_DEFAULT 0x0 #define ICH9_LPC_PIC_NUM_PINS 16 From 29f06964a940adf3ffb6ecc5aec2a9a4ffecef4f Mon Sep 17 00:00:00 2001 From: Steffen Schulz Date: Sat, 5 Feb 2022 16:24:30 +0100 Subject: [PATCH 26/35] fix hprintf EOL handling All other uses of misc buffer do not include 0 byte in length.. --- nyx/hypercall/hypercall.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nyx/hypercall/hypercall.c b/nyx/hypercall/hypercall.c index ef63ddfbd4..282da6860f 100644 --- a/nyx/hypercall/hypercall.c +++ b/nyx/hypercall/hypercall.c @@ -615,7 +615,7 @@ static void handle_hypercall_kafl_printf(struct kvm_run *run, CPUState *cpu, uin #ifdef DEBUG_HPRINTF fprintf(stderr, "%s %s\n", __func__, hprintf_buffer); #endif - set_hprintf_auxiliary_buffer(GET_GLOBAL_STATE()->auxilary_buffer, hprintf_buffer, strnlen(hprintf_buffer, HPRINTF_SIZE)+1); + set_hprintf_auxiliary_buffer(GET_GLOBAL_STATE()->auxilary_buffer, hprintf_buffer, strnlen(hprintf_buffer, HPRINTF_SIZE)); synchronization_lock(); } From 954158c43a34acdacdccf21a0303b027d95d6688 Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Wed, 23 Feb 2022 08:39:36 +0100 Subject: [PATCH 27/35] Revert "checkout specific libxdc commit" This reverts commit d5a7011ad20ba5ba91f1371f9d40154035d5d768. --- compile_qemu_nyx.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/compile_qemu_nyx.sh b/compile_qemu_nyx.sh index 408a384be0..641c648f86 100755 --- a/compile_qemu_nyx.sh +++ b/compile_qemu_nyx.sh @@ -45,7 +45,6 @@ compile_libraries (){ echo "[!] compiling libxdc..." cd libxdc - git checkout 641de7539e99f7faf5c8e8f1c8a4b37a9df52a5f sudo make install cd .. echo "[!] libxdc is ready!" From b95d6b9236390f7d8f60ed361a44524d25d595c5 Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Wed, 23 Feb 2022 08:55:00 +0100 Subject: [PATCH 28/35] fix a global oob read Use an additional constant to specifiy the size of the crash notifier code in 32 bit mode (submit_panic / submit_kasan). --- nyx/hypercall/hypercall.c | 8 ++++---- nyx/hypercall/hypercall.h | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/nyx/hypercall/hypercall.c b/nyx/hypercall/hypercall.c index 282da6860f..7d26c74469 100644 --- a/nyx/hypercall/hypercall.c +++ b/nyx/hypercall/hypercall.c @@ -443,9 +443,9 @@ static void handle_hypercall_kafl_submit_panic(struct kvm_run *run, CPUState *cp if(hypercall_enabled){ QEMU_PT_PRINTF(CORE_PREFIX, "Panic address:\t%lx", hypercall_arg); if (run->hypercall.longmode) { - write_virtual_memory(hypercall_arg, (uint8_t*)PANIC_PAYLOAD_64, PAYLOAD_BUFFER_SIZE, cpu); + write_virtual_memory(hypercall_arg, (uint8_t*)PANIC_PAYLOAD_64, PAYLOAD_BUFFER_SIZE_64, cpu); } else { - write_virtual_memory(hypercall_arg, (uint8_t*)PANIC_PAYLOAD_32, PAYLOAD_BUFFER_SIZE, cpu); + write_virtual_memory(hypercall_arg, (uint8_t*)PANIC_PAYLOAD_32, PAYLOAD_BUFFER_SIZE_32, cpu); } } } @@ -454,9 +454,9 @@ static void handle_hypercall_kafl_submit_kasan(struct kvm_run *run, CPUState *cp if(hypercall_enabled){ QEMU_PT_PRINTF(CORE_PREFIX, "kASAN address:\t%lx", hypercall_arg); if (run->hypercall.longmode){ - write_virtual_memory(hypercall_arg, (uint8_t*)KASAN_PAYLOAD_64, PAYLOAD_BUFFER_SIZE, cpu); + write_virtual_memory(hypercall_arg, (uint8_t*)KASAN_PAYLOAD_64, PAYLOAD_BUFFER_SIZE_64, cpu); } else { - write_virtual_memory(hypercall_arg, (uint8_t*)KASAN_PAYLOAD_32, PAYLOAD_BUFFER_SIZE, cpu); + write_virtual_memory(hypercall_arg, (uint8_t*)KASAN_PAYLOAD_32, PAYLOAD_BUFFER_SIZE_32, cpu); } } } diff --git a/nyx/hypercall/hypercall.h b/nyx/hypercall/hypercall.h index d48115758c..544292f98f 100644 --- a/nyx/hypercall/hypercall.h +++ b/nyx/hypercall/hypercall.h @@ -21,7 +21,8 @@ along with QEMU-PT. If not, see . #pragma once -#define PAYLOAD_BUFFER_SIZE 26 +#define PAYLOAD_BUFFER_SIZE_64 26 +#define PAYLOAD_BUFFER_SIZE_32 20 #define KAFL_MODE_64 0 #define KAFL_MODE_32 1 From 1f675b053a86125945cdc1fa6985d76015a01520 Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Wed, 23 Feb 2022 10:26:02 +0100 Subject: [PATCH 29/35] fix crash notifier injection Decide which crash notifier (32bit or 64bit) to inject, based on the current memory mode instead of the current CPU mode. Otherwise, in the case of a 32bit loader running on a 64bit operating system, the wrong notifier code will be injected. --- nyx/hypercall/hypercall.c | 36 ++++++++++++++++++++++++++++-------- nyx/memory_access.c | 27 +++++++++++++-------------- nyx/memory_access.h | 3 +++ nyx/state/state.h | 12 ++---------- nyx/types.h | 12 ++++++++++++ 5 files changed, 58 insertions(+), 32 deletions(-) create mode 100644 nyx/types.h diff --git a/nyx/hypercall/hypercall.c b/nyx/hypercall/hypercall.c index 7d26c74469..f091a16981 100644 --- a/nyx/hypercall/hypercall.c +++ b/nyx/hypercall/hypercall.c @@ -442,10 +442,20 @@ static void handle_hypercall_kafl_submit_panic(struct kvm_run *run, CPUState *cp if(hypercall_enabled){ QEMU_PT_PRINTF(CORE_PREFIX, "Panic address:\t%lx", hypercall_arg); - if (run->hypercall.longmode) { - write_virtual_memory(hypercall_arg, (uint8_t*)PANIC_PAYLOAD_64, PAYLOAD_BUFFER_SIZE_64, cpu); - } else { - write_virtual_memory(hypercall_arg, (uint8_t*)PANIC_PAYLOAD_32, PAYLOAD_BUFFER_SIZE_32, cpu); + + switch (get_current_mem_mode(cpu)){ + case mm_32_protected: + case mm_32_paging: + case mm_32_pae: + write_virtual_memory(hypercall_arg, (uint8_t*)PANIC_PAYLOAD_32, PAYLOAD_BUFFER_SIZE_32, cpu); + break; + case mm_64_l4_paging: + case mm_64_l5_paging: + write_virtual_memory(hypercall_arg, (uint8_t*)PANIC_PAYLOAD_64, PAYLOAD_BUFFER_SIZE_64, cpu); + break; + default: + abort(); + break; } } } @@ -453,10 +463,20 @@ static void handle_hypercall_kafl_submit_panic(struct kvm_run *run, CPUState *cp static void handle_hypercall_kafl_submit_kasan(struct kvm_run *run, CPUState *cpu, uint64_t hypercall_arg){ if(hypercall_enabled){ QEMU_PT_PRINTF(CORE_PREFIX, "kASAN address:\t%lx", hypercall_arg); - if (run->hypercall.longmode){ - write_virtual_memory(hypercall_arg, (uint8_t*)KASAN_PAYLOAD_64, PAYLOAD_BUFFER_SIZE_64, cpu); - } else { - write_virtual_memory(hypercall_arg, (uint8_t*)KASAN_PAYLOAD_32, PAYLOAD_BUFFER_SIZE_32, cpu); + + switch (get_current_mem_mode(cpu)){ + case mm_32_protected: + case mm_32_paging: + case mm_32_pae: + write_virtual_memory(hypercall_arg, (uint8_t*)KASAN_PAYLOAD_32, PAYLOAD_BUFFER_SIZE_32, cpu); + break; + case mm_64_l4_paging: + case mm_64_l5_paging: + write_virtual_memory(hypercall_arg, (uint8_t*)KASAN_PAYLOAD_64, PAYLOAD_BUFFER_SIZE_64, cpu); + break; + default: + abort(); + break; } } } diff --git a/nyx/memory_access.c b/nyx/memory_access.c index 158f46a8ff..bf3ae92280 100644 --- a/nyx/memory_access.c +++ b/nyx/memory_access.c @@ -41,39 +41,38 @@ static uint64_t get_48_paging_phys_addr(uint64_t cr3, uint64_t addr, bool read_f #define x86_64_PAGE_SIZE 0x1000 #define x86_64_PAGE_MASK ~(x86_64_PAGE_SIZE - 1) -static void set_mem_mode(CPUState *cpu){ +mem_mode_t get_current_mem_mode(CPUState *cpu){ kvm_arch_get_registers(cpu); X86CPU *cpux86 = X86_CPU(cpu); CPUX86State *env = &cpux86->env; - - if (!(env->cr[0] & CR0_PG_MASK)) { - GET_GLOBAL_STATE()->mem_mode = mm_32_protected; - return; + + if (!(env->cr[0] & CR0_PG_MASK)) { + return mm_32_protected; } else{ if (env->cr[4] & CR4_PAE_MASK) { if (env->hflags & HF_LMA_MASK) { if (env->cr[4] & CR4_LA57_MASK) { - GET_GLOBAL_STATE()->mem_mode = mm_64_l5_paging; - return; + return mm_64_l5_paging; } else { - GET_GLOBAL_STATE()->mem_mode = mm_64_l4_paging; - return; + return mm_64_l4_paging; } } else{ - GET_GLOBAL_STATE()->mem_mode = mm_32_pae; - return; + return mm_32_pae; } } else { - GET_GLOBAL_STATE()->mem_mode = mm_32_paging; - return; + return mm_32_paging; } } - return; + return mm_unkown; +} + +static void set_mem_mode(CPUState *cpu){ + GET_GLOBAL_STATE()->mem_mode = get_current_mem_mode(cpu); } /* Warning: This might break memory handling for hypervisor fuzzing => FIXME LATER */ diff --git a/nyx/memory_access.h b/nyx/memory_access.h index 893f59982a..496acc1473 100644 --- a/nyx/memory_access.h +++ b/nyx/memory_access.h @@ -26,6 +26,7 @@ along with QEMU-PT. If not, see . #include #include "qemu-common.h" #include "sysemu/kvm_int.h" +#include "nyx/types.h" #define MEM_SPLIT_START 0x0C0000000 #define MEM_SPLIT_END 0x100000000 @@ -34,6 +35,8 @@ along with QEMU-PT. If not, see . #define address_to_ram_offset(offset) (offset >= MEM_SPLIT_END ? (offset - MEM_SPLIT_END) + MEM_SPLIT_START : offset) #define ram_offset_to_address(offset) (offset >= MEM_SPLIT_START ? (offset - MEM_SPLIT_START) + MEM_SPLIT_END : offset) +mem_mode_t get_current_mem_mode(CPUState *cpu); + uint64_t get_paging_phys_addr(CPUState *cpu, uint64_t cr3, uint64_t addr); bool read_physical_memory(uint64_t address, uint8_t* data, uint32_t size, CPUState *cpu); diff --git a/nyx/state/state.h b/nyx/state/state.h index e634d7375e..e434c30724 100644 --- a/nyx/state/state.h +++ b/nyx/state/state.h @@ -29,20 +29,12 @@ along with QEMU-PT. If not, see . #include "nyx/auxiliary_buffer.h" #include "nyx/sharedir.h" #include "nyx/fast_vm_reload_sync.h" +#include "nyx/types.h" #include #define INTEL_PT_MAX_RANGES 4 -enum mem_mode { - mm_unkown, - mm_32_protected, /* 32 Bit / No MMU */ - mm_32_paging, /* 32 Bit / L3 Paging */ - mm_32_pae, /* 32 Bit / PAE Paging */ - mm_64_l4_paging, /* 64 Bit / L4 Paging */ - mm_64_l5_paging, /* 32 Bit / L5 Paging */ -}; - typedef struct qemu_nyx_state_s{ /* set if FDL backend is used (required to perform some additional runtime tests) */ @@ -117,7 +109,7 @@ typedef struct qemu_nyx_state_s{ uint64_t* nested_payload_pages; bool protect_payload_buffer; bool discard_tmp_snapshot; - uint8_t mem_mode; + mem_mode_t mem_mode; uint32_t input_buffer_size; diff --git a/nyx/types.h b/nyx/types.h new file mode 100644 index 0000000000..7c15f8ee72 --- /dev/null +++ b/nyx/types.h @@ -0,0 +1,12 @@ +#pragma once + +enum mem_mode { + mm_unkown, + mm_32_protected, /* 32 Bit / No MMU */ + mm_32_paging, /* 32 Bit / L3 Paging */ + mm_32_pae, /* 32 Bit / PAE Paging */ + mm_64_l4_paging, /* 64 Bit / L4 Paging */ + mm_64_l5_paging, /* 32 Bit / L5 Paging */ +}; + +typedef uint8_t mem_mode_t; From 8e8f6e5b2b5204f12f7863ba8172e84c59682a78 Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Wed, 23 Feb 2022 10:28:44 +0100 Subject: [PATCH 30/35] uncomment several fprintfs in state_reallocation --- nyx/snapshot/devices/state_reallocation.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nyx/snapshot/devices/state_reallocation.c b/nyx/snapshot/devices/state_reallocation.c index ab83f58acc..c9edfa7433 100644 --- a/nyx/snapshot/devices/state_reallocation.c +++ b/nyx/snapshot/devices/state_reallocation.c @@ -521,11 +521,11 @@ static inline int get_handler(state_reallocation_t* self, QEMUFile* f, void* cur add_get(self, (void*) field->info->get, curr_elem, size, (void*) field, f, field->info->name); } else if(!strcmp(field->info->name, "pci config")){ - fprintf(stderr, "type: %s (size: %lx)\n", field->info->name, size); + //fprintf(stderr, "type: %s (size: %lx)\n", field->info->name, size); add_get(self, (void*) field->info->get, curr_elem, size, (void*) field, f, field->info->name); } else if(!strcmp(field->info->name, "pci irq state")){ - fprintf(stderr, "type: %s (size: %lx)\n", field->info->name, size); + //fprintf(stderr, "type: %s (size: %lx)\n", field->info->name, size); add_get(self, (void*) field->info->get, curr_elem, size, (void*) field, f, field->info->name); } else if(!strcmp(field->info->name, "virtio")){ From e7f63f4401b10f85134d256a9f4e4d817fc29a1d Mon Sep 17 00:00:00 2001 From: Mark Brand Date: Wed, 16 Mar 2022 17:32:04 +0100 Subject: [PATCH 31/35] Reimplemented x86_64 page table walking code. --- nyx/hypercall/hypercall.c | 2 +- nyx/memory_access.c | 419 ++++++++++++++++++++------------------ nyx/memory_access.h | 2 +- 3 files changed, 221 insertions(+), 202 deletions(-) diff --git a/nyx/hypercall/hypercall.c b/nyx/hypercall/hypercall.c index 64703657e2..a7dd7b90c5 100644 --- a/nyx/hypercall/hypercall.c +++ b/nyx/hypercall/hypercall.c @@ -210,7 +210,7 @@ static void handle_hypercall_get_payload(struct kvm_run *run, CPUState *cpu, uin CPUX86State *env = &(X86_CPU(cpu))->env; GET_GLOBAL_STATE()->parent_cr3 = env->cr[3] & 0xFFFFFFFFFFFFF000ULL; QEMU_PT_PRINTF(CORE_PREFIX, "Payload CR3:\t%lx", (uint64_t)GET_GLOBAL_STATE()->parent_cr3 ); - //print_48_paging2(GET_GLOBAL_STATE()->parent_cr3); + //print_48_pagetables(GET_GLOBAL_STATE()->parent_cr3); if(hypercall_arg&0xFFF){ fprintf(stderr, "[QEMU-Nyx] Error: Payload buffer is not page-aligned! (0x%lx)\n", hypercall_arg); diff --git a/nyx/memory_access.c b/nyx/memory_access.c index ad6dfec8d5..cb4560d293 100644 --- a/nyx/memory_access.c +++ b/nyx/memory_access.c @@ -563,221 +563,240 @@ void remove_all_breakpoints(CPUState *cpu){ } - - - - - - - - - - - -#define PPAGE_SIZE 0x1000 #define PENTRIES 0x200 -#define PLEVEL_4_SHIFT 12 -#define PLEVEL_3_SHIFT 21 -#define PLEVEL_2_SHIFT 30 -#define PLEVEL_1_SHIFT 39 -#define SIGN_EXTEND_TRESHOLD 0x100 -#define SIGN_EXTEND 0xFFFF000000000000ULL -#define PAGETABLE_MASK 0x1FFFFFFFFF000ULL -#define PML4_ENTRY_MASK 0x1FFFFFFFFF000ULL -#define PML3_ENTRY_MASK 0x1FFFFC0000000ULL -#define PML2_ENTRY_MASK 0x1FFFFFFE00000ULL +#define PPAGE_SIZE 0x1000 -#define CHECK_BIT(var,pos) !!(((var) & (1ULL<<(pos)))) - - -static void write_address(uint64_t address, uint64_t size, uint64_t prot){ - //fprintf(stderr, "%s %lx %lx %lx\n", __func__, address, size, prot); - static uint64_t next_address = PAGETABLE_MASK; - static uint64_t last_address = 0x0; - static uint64_t last_prot = 0; - - if((address != next_address || prot != last_prot)){ - /* do not print guard pages or empty pages without any permissions */ - if(last_address && prot && (last_address+size != next_address || prot != last_prot)){ - fprintf(stderr, "%016lx - %016lx %c%c%c\n", - last_address, next_address, - CHECK_BIT(last_prot, 1) ? 'W' : '-', - CHECK_BIT(last_prot, 2) ? 'U' : 'K', - !CHECK_BIT(last_prot, 63)? 'X' : '-'); - } - last_address = address; +static bool read_memory(uint64_t address, uint64_t* buffer, size_t size, bool read_from_snapshot) { + if (unlikely(address == INVALID_ADDRESS)) { + return false; } - next_address = address+size; - last_prot = prot; + + if (unlikely(read_from_snapshot)) { + return read_snapshot_memory( + get_fast_reload_snapshot(), + address, (uint8_t *)buffer, size); + } + // NB: This API exposed by exec.h doesn't signal failure, although it can + // fail. Figure out how to expose the address space object instead and then + // we can actually check the return value here. Until then, will clear the + // buffer contents first. + memset(buffer, 0, size); + cpu_physical_memory_rw(address, (uint8_t*)buffer, size, false); + return true; } -void print_48_paging2(uint64_t cr3){ - uint64_t paging_entries_level_1[PENTRIES]; - uint64_t paging_entries_level_2[PENTRIES]; - uint64_t paging_entries_level_3[PENTRIES]; - uint64_t paging_entries_level_4[PENTRIES]; - - uint64_t address_identifier_1, address_identifier_2, address_identifier_3, address_identifier_4; - uint32_t i1, i2, i3,i4; - - cpu_physical_memory_rw((cr3&PAGETABLE_MASK), (uint8_t *) paging_entries_level_1, PPAGE_SIZE, false); - for(i1 = 0; i1 < 512; i1++){ - if(paging_entries_level_1[i1]){ - address_identifier_1 = ((uint64_t)i1) << PLEVEL_1_SHIFT; - if (i1 & SIGN_EXTEND_TRESHOLD){ - address_identifier_1 |= SIGN_EXTEND; - } - if(CHECK_BIT(paging_entries_level_1[i1], 0)){ /* otherwise swapped out */ - cpu_physical_memory_rw((paging_entries_level_1[i1]&PAGETABLE_MASK), (uint8_t *) paging_entries_level_2, PPAGE_SIZE, false); - for(i2 = 0; i2 < PENTRIES; i2++){ - if(paging_entries_level_2[i2]){ - address_identifier_2 = (((uint64_t)i2) << PLEVEL_2_SHIFT) + address_identifier_1; - if (CHECK_BIT(paging_entries_level_2[i2], 0)){ /* otherwise swapped out */ - if((paging_entries_level_2[i2]&PAGETABLE_MASK) == (paging_entries_level_1[i1]&PAGETABLE_MASK)){ - /* loop */ - continue; - } - - if (CHECK_BIT(paging_entries_level_2[i2], 7)){ - write_address(address_identifier_2, 0x40000000, (uint64_t)paging_entries_level_2[i2] & ((1ULL<<63) | (1ULL<<2) | (1ULL<<1))); - } - else{ - /* otherwise this PDPE references a 1GB page */ - cpu_physical_memory_rw((paging_entries_level_2[i2]&PAGETABLE_MASK), (uint8_t *) paging_entries_level_3, PPAGE_SIZE, false); - for(i3 = 0; i3 < PENTRIES; i3++){ - if(paging_entries_level_3[i3]){ - address_identifier_3 = (((uint64_t)i3) << PLEVEL_3_SHIFT) + address_identifier_2; - if (CHECK_BIT(paging_entries_level_3[i3], 0)){ /* otherwise swapped out */ - if (CHECK_BIT(paging_entries_level_3[i3], 7)){ - write_address(address_identifier_3, 0x200000, (uint64_t)paging_entries_level_3[i3] & ((1ULL<<63) | (1ULL<<2) | (1ULL<<1))); - } - else{ - cpu_physical_memory_rw((paging_entries_level_3[i3]&PAGETABLE_MASK), (uint8_t *) paging_entries_level_4, PPAGE_SIZE, false); - for(i4 = 0; i4 < PENTRIES; i4++){ - if(paging_entries_level_4[i4]){ - address_identifier_4 = (((uint64_t)i4) << PLEVEL_4_SHIFT) + address_identifier_3; - if (CHECK_BIT(paging_entries_level_4[i4], 0)){ - write_address(address_identifier_4, 0x1000, (uint64_t)paging_entries_level_4[i4] & ((1ULL<<63) | (1ULL<<2) | (1ULL<<1))); - } - } - } - } - } - } - } - - } - } - } - } - } - } - } - write_address(0, 0x1000, 0); +__attribute__((always_inline)) +static bool bit(uint64_t value, uint8_t lsb) { + return (value >> lsb) & 1; } - -static uint64_t* load_page_table(uint64_t page_table_address, uint64_t* paging_entries_buffer, uint8_t level, bool read_from_snapshot, bool *success){ - if(page_table_address == INVALID_ADDRESS){ - *success = false; - } - - if (read_from_snapshot){ - *success = read_snapshot_memory(get_fast_reload_snapshot(), page_table_address, (uint8_t *) paging_entries_buffer, PPAGE_SIZE); - } - else{ - cpu_physical_memory_rw(page_table_address, (uint8_t *) paging_entries_buffer, PPAGE_SIZE, false); - *success = true; /* fix this */ - } - return paging_entries_buffer; +__attribute__((always_inline)) +static uint64_t bits(uint64_t value, uint8_t lsb, uint8_t msb) { + return (value & ((0xffffffffffffffffull >> (64 - (msb - lsb + 1))) << lsb)) >> lsb; } -static uint64_t get_48_paging_phys_addr(uint64_t cr3, uint64_t addr, bool read_from_snapshot){ - /* signedness broken af -> fix me! */ - uint16_t pml_4_index = (addr & 0xFF8000000000ULL) >> 39; - uint16_t pml_3_index = (addr & 0x0007FC0000000UL) >> 30; - uint16_t pml_2_index = (addr & 0x000003FE00000UL) >> 21; - uint16_t pml_1_index = (addr & 0x00000001FF000UL) >> 12; - - uint64_t address_identifier_4; - uint64_t paging_entries_buffer[PENTRIES]; - uint64_t* paging_entries_buffer_ptr = NULL; - uint64_t page_table_address = 0; - - bool success = false; - - page_table_address = (cr3&PAGETABLE_MASK); - paging_entries_buffer_ptr = load_page_table(page_table_address, paging_entries_buffer, 0, read_from_snapshot, &success); - - if (unlikely(success == false)){ - goto fail; +// Helper function to load an entire pagetable table. These are PENTRIES +// 64-bit entries, so entries must point to a sufficiently large buffer. +static bool load_table(uint64_t address, uint64_t* entries, bool read_from_snapshot) { + if (unlikely(!read_memory(address, entries, 512 * sizeof(*entries), read_from_snapshot))) { + return false; } - if(paging_entries_buffer_ptr[pml_4_index]){ - address_identifier_4 = ((uint64_t)pml_4_index) << PLEVEL_1_SHIFT; - if (pml_4_index & SIGN_EXTEND_TRESHOLD){ - address_identifier_4 |= SIGN_EXTEND; - } - if(CHECK_BIT(paging_entries_buffer_ptr[pml_4_index], 0)){ /* otherwise swapped out */ + return true; +} - page_table_address = (paging_entries_buffer_ptr[pml_4_index]&PAGETABLE_MASK); - paging_entries_buffer_ptr = load_page_table(page_table_address, paging_entries_buffer, 1, read_from_snapshot, &success); - - if (unlikely(success == false)){ - goto fail; - } - - if(paging_entries_buffer_ptr[pml_3_index]){ - - if (CHECK_BIT(paging_entries_buffer_ptr[pml_3_index], 0)){ /* otherwise swapped out */ - - if (CHECK_BIT(paging_entries_buffer_ptr[pml_3_index], 7)){ - /* 1GB PAGE */ - return (paging_entries_buffer_ptr[pml_3_index] & PML3_ENTRY_MASK) | (0x7FFFFFFF & addr); - } - else{ - - page_table_address = (paging_entries_buffer_ptr[pml_3_index]&PAGETABLE_MASK); - paging_entries_buffer_ptr = load_page_table(page_table_address, paging_entries_buffer, 2, read_from_snapshot, &success); - - if (unlikely(success == false)){ - goto fail; - } - - if(paging_entries_buffer_ptr[pml_2_index]){ - if (CHECK_BIT(paging_entries_buffer_ptr[pml_2_index], 0)){ /* otherwise swapped out */ - if (CHECK_BIT(paging_entries_buffer_ptr[pml_2_index], 7)){ - /* 2MB PAGE */ - return (paging_entries_buffer_ptr[pml_2_index] & PML2_ENTRY_MASK) | (0x3FFFFF & addr); - } - else{ - - page_table_address = (paging_entries_buffer_ptr[pml_2_index]&PAGETABLE_MASK); - paging_entries_buffer_ptr = load_page_table(page_table_address, paging_entries_buffer, 3, read_from_snapshot, &success); - - if (unlikely(success == false)){ - goto fail; - } - - if(paging_entries_buffer_ptr[pml_1_index]){ - if (CHECK_BIT(paging_entries_buffer_ptr[pml_1_index], 0)){ - /* 4 KB PAGE */ - return (paging_entries_buffer_ptr[pml_1_index] & PML4_ENTRY_MASK) | (0xFFF & addr); - } - } - } - } - } - } - } - } - } +// Helper function to load a single pagetable entry. We simplify things by +// returning the same invalid value (0) for both non-present entries and +// any other error conditions, since we don't need to handle these cases +// differently. +static uint64_t load_entry(uint64_t address, uint64_t index, + bool read_from_snapshot) { + uint64_t entry = 0; + if (unlikely(!read_memory(address + (index * sizeof(entry)), &entry, sizeof(entry), + read_from_snapshot))) { + return 0; } - fail: - - return INVALID_ADDRESS; + // Check that the entry is present. + if (unlikely(!bit(entry, 0))) { + return 0; + } + + return entry; +} + +static void print_page(uint64_t address, uint64_t entry, size_t size, bool s, bool w, bool x) { + fprintf(stderr, " %c%c%c %016llx %zx", + s ? 's' : 'u', w ? 'w' : 'r', x ? 'x' : '-', + (bits(entry, 12, 51) << 12) & ~(size - 1), size); +} + +static void print_48_pte(uint64_t address, uint64_t pde_entry, bool read_from_snapshot, + bool s, bool w, bool x) { + uint64_t pte_address = bits(pde_entry, 12, 51) << 12; + uint64_t pte_table[PENTRIES]; + + if (!load_table(pte_address, pte_table, read_from_snapshot)) { + return; + } + + for (size_t i = 0; i < PENTRIES; ++i) { + uint64_t entry = pte_table[i]; + + if (entry) { + fprintf(stderr, "\n 1 %016llx", address | i << 12, entry); + } + + if (!bit(entry, 0)) { + // Not present. + } else { + print_page(address | i << 12, entry, 0x1000, + s & !bit(entry, 2), w & bit(entry, 1), x & !bit(entry, 63)); + } + } +} + +static void print_48_pde(uint64_t address, uint64_t pdpte_entry, bool read_from_snapshot, + bool s, bool w, bool x) { + uint64_t pde_address = bits(pdpte_entry, 12, 51) << 12; + uint64_t pde_table[PENTRIES]; + + if (!load_table(pde_address, pde_table, read_from_snapshot)) { + return; + } + + for (size_t i = 0; i < PENTRIES; ++i) { + uint64_t entry = pde_table[i]; + + if (entry) { + fprintf(stderr, "\n 2 %016llx", address | i << 21, entry); + } + + if (!bit(entry, 0)) { + // Not present. + } else if (bit(entry, 7)) { + print_page(address | i << 21, entry, 0x200000, + s & !bit(entry, 2), w & bit(entry, 1), x & !bit(entry, 63)); + } else { + print_48_pte(address | i << 21, entry, read_from_snapshot, + s & !bit(entry, 2), w & bit(entry, 1), x & !bit(entry, 63)); + } + } +} + +static void print_48_pdpte(uint64_t address, uint64_t pml4_entry, bool read_from_snapshot, + bool s, bool w, bool x) { + uint64_t pdpte_address = bits(pml4_entry, 12, 51) << 12; + uint64_t pdpte_table[PENTRIES]; + + if (!load_table(pdpte_address, pdpte_table, read_from_snapshot)) { + return; + } + + for (size_t i = 0; i < PENTRIES; ++i) { + uint64_t entry = pdpte_table[i]; + + if (entry) { + fprintf(stderr, "\n 3 %016llx", address | i << 30, entry); + } + + if (!bit(entry, 0)) { + // Not present. + } else if (bit(entry, 7)) { + print_page(address | i << 30, entry, 0x40000000, + s & !bit(entry, 2), w & bit(entry, 1), x & !bit(entry, 63)); + } else { + print_48_pde(address | i << 30, entry, read_from_snapshot, + s & !bit(entry, 2), w & bit(entry, 1), x & !bit(entry, 63)); + } + } +} + +static void print_48_pagetables_(uint64_t cr3, bool read_from_snapshot) { + uint64_t pml4_address = bits(cr3, 12, 51) << 12; + uint64_t pml4_table[PENTRIES]; + + if (!load_table(pml4_address, pml4_table, read_from_snapshot)) { + return; + } + + for (size_t i = 0; i < PENTRIES; ++i) { + uint64_t entry = pml4_table[i]; + uint64_t address = i << 39; + // Ensure canonical virtual address + if (bit(address, 47)) { + address |= 0xffff000000000000ul; + } + + if (entry) { + fprintf(stderr, "\n4 %016llx", address, entry); + } + + if (bit(entry, 0)) { + print_48_pdpte(address, entry, read_from_snapshot, + !bit(entry, 2), bit(entry, 1), !bit(entry, 63)); + } + } +} + +void print_48_pagetables(uint64_t cr3) { + static bool printed = false; + if (!printed) { + fprintf(stderr, "pagetables for cr3 %lx", cr3); + print_48_pagetables_(cr3, false); + printed = true; + fprintf(stderr, "\n"); + } +} + +static uint64_t get_48_paging_phys_addr(uint64_t cr3, uint64_t addr, bool read_from_snapshot) { + uint64_t pml4_address = bits(cr3, 12, 51) << 12; + uint64_t pml4_offset = bits(addr, 39, 47); + uint64_t pml4_entry = load_entry(pml4_address, pml4_offset, read_from_snapshot); + if (unlikely(!pml4_entry)) { + return INVALID_ADDRESS; + } + + uint64_t pdpte_address = bits(pml4_entry, 12, 51) << 12; + uint64_t pdpte_offset = bits(addr, 30, 38); + uint64_t pdpte_entry = load_entry(pdpte_address, pdpte_offset, read_from_snapshot); + if (unlikely(!pdpte_entry)) { + return INVALID_ADDRESS; + } + + if (unlikely(bit(pdpte_entry, 7))) { + // 1GByte page translation. + uint64_t page_address = bits(pdpte_entry, 12, 51) << 12; + uint64_t page_offset = bits(addr, 0, 29); + return page_address + page_offset; + } + + uint64_t pde_address = bits(pdpte_entry, 12, 51) << 12; + uint64_t pde_offset = bits(addr, 21, 29); + uint64_t pde_entry = load_entry(pde_address, pde_offset, read_from_snapshot); + if (unlikely(!pde_entry)) { + return INVALID_ADDRESS; + } + + if (unlikely(bit(pde_entry, 7))) { + // 2MByte page translation. + uint64_t page_address = bits(pde_entry, 12, 51) << 12; + uint64_t page_offset = bits(addr, 0, 20); + return page_address + page_offset; + } + + uint64_t pte_address = bits(pde_entry, 12, 51) << 12; + uint64_t pte_offset = bits(addr, 12, 20); + uint64_t pte_entry = load_entry(pte_address, pte_offset, read_from_snapshot); + if (unlikely(!pte_entry)) { + return INVALID_ADDRESS; + } + + // 4Kbyte page translation. + uint64_t page_address = bits(pte_entry, 12, 51) << 12; + uint64_t page_offset = bits(addr, 0, 11); + return page_address + page_offset; } //#define DEBUG_48BIT_WALK diff --git a/nyx/memory_access.h b/nyx/memory_access.h index 893f59982a..2181d25542 100644 --- a/nyx/memory_access.h +++ b/nyx/memory_access.h @@ -63,7 +63,7 @@ bool dump_page_cr3_snapshot(uint64_t address, uint8_t* data, CPUState *cpu, uint bool dump_page_cr3_ht(uint64_t address, uint8_t* data, CPUState *cpu, uint64_t cr3); bool is_addr_mapped_cr3_snapshot(uint64_t address, CPUState *cpu, uint64_t cr3); -void print_48_paging2(uint64_t cr3); +void print_48_pagetables(uint64_t cr3); bool dump_page_ht(uint64_t address, uint8_t* data, CPUState *cpu); From d45d4da2772b79a1228ed8cf4e579247bb6e04db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Morgan?= Date: Fri, 25 Mar 2022 10:38:34 +0100 Subject: [PATCH 32/35] Optarg bug --- vl.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vl.c b/vl.c index 10149b9f26..02ee5a4dd7 100644 --- a/vl.c +++ b/vl.c @@ -2928,6 +2928,7 @@ int main(int argc, char **argv, char **envp) #ifdef QEMU_NYX bool fast_vm_reload = false; state_init_global(); + char *fast_vm_reload_opt_arg = NULL; #endif int i; @@ -3083,9 +3084,11 @@ int main(int argc, char **argv, char **envp) #ifdef QEMU_NYX case QEMU_OPTION_fast_vm_reload: opts = qemu_opts_parse_noisily(qemu_find_opts("fast_vm_reload-opts"), - optarg, true); if (!opts) { + optarg, true); + if (!opts) { exit(1); } + fast_vm_reload_opt_arg = optarg; fast_vm_reload = true; break; #endif @@ -4569,7 +4572,7 @@ int main(int argc, char **argv, char **envp) exit(1); } - QemuOpts *opts = qemu_opts_parse_noisily(qemu_find_opts("fast_vm_reload-opts"), optarg, true); + QemuOpts *opts = qemu_opts_parse_noisily(qemu_find_opts("fast_vm_reload-opts"), fast_vm_reload_opt_arg, true); const char* snapshot_path = qemu_opt_get(opts, "path"); const char* pre_snapshot_path = qemu_opt_get(opts, "pre_path"); From 164f449a02ac5203c8ddc276b408c604ac5822ea Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Thu, 7 Apr 2022 10:52:30 +0200 Subject: [PATCH 33/35] fix several compiler warnings --- nyx/memory_access.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/nyx/memory_access.c b/nyx/memory_access.c index a523f00c6c..c449f4e0e3 100644 --- a/nyx/memory_access.c +++ b/nyx/memory_access.c @@ -592,12 +592,12 @@ static bool read_memory(uint64_t address, uint64_t* buffer, size_t size, bool re return true; } -__attribute__((always_inline)) +__attribute__((always_inline)) inline static bool bit(uint64_t value, uint8_t lsb) { return (value >> lsb) & 1; } -__attribute__((always_inline)) +__attribute__((always_inline)) inline static uint64_t bits(uint64_t value, uint8_t lsb, uint8_t msb) { return (value & ((0xffffffffffffffffull >> (64 - (msb - lsb + 1))) << lsb)) >> lsb; } @@ -633,7 +633,7 @@ static uint64_t load_entry(uint64_t address, uint64_t index, } static void print_page(uint64_t address, uint64_t entry, size_t size, bool s, bool w, bool x) { - fprintf(stderr, " %c%c%c %016llx %zx", + fprintf(stderr, " %c%c%c %016lx %zx", s ? 's' : 'u', w ? 'w' : 'r', x ? 'x' : '-', (bits(entry, 12, 51) << 12) & ~(size - 1), size); } @@ -651,7 +651,7 @@ static void print_48_pte(uint64_t address, uint64_t pde_entry, bool read_from_sn uint64_t entry = pte_table[i]; if (entry) { - fprintf(stderr, "\n 1 %016llx", address | i << 12, entry); + fprintf(stderr, "\n 1 %016lx [%ld]", address | i << 12, entry); } if (!bit(entry, 0)) { @@ -676,7 +676,7 @@ static void print_48_pde(uint64_t address, uint64_t pdpte_entry, bool read_from_ uint64_t entry = pde_table[i]; if (entry) { - fprintf(stderr, "\n 2 %016llx", address | i << 21, entry); + fprintf(stderr, "\n 2 %016lx [%ld]", address | i << 21, entry); } if (!bit(entry, 0)) { @@ -704,7 +704,7 @@ static void print_48_pdpte(uint64_t address, uint64_t pml4_entry, bool read_from uint64_t entry = pdpte_table[i]; if (entry) { - fprintf(stderr, "\n 3 %016llx", address | i << 30, entry); + fprintf(stderr, "\n 3 %016lx [%ld]", address | i << 30, entry); } if (!bit(entry, 0)) { @@ -736,7 +736,7 @@ static void print_48_pagetables_(uint64_t cr3, bool read_from_snapshot) { } if (entry) { - fprintf(stderr, "\n4 %016llx", address, entry); + fprintf(stderr, "\n4 %016lx [%ld]", address, entry); } if (bit(entry, 0)) { From 0449772d10a5513a50fd36aea16aa9010a7ad895 Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Thu, 7 Apr 2022 10:52:48 +0200 Subject: [PATCH 34/35] fix compile script --- compile_qemu_nyx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compile_qemu_nyx.sh b/compile_qemu_nyx.sh index 641c648f86..e36fe285f8 100755 --- a/compile_qemu_nyx.sh +++ b/compile_qemu_nyx.sh @@ -45,7 +45,7 @@ compile_libraries (){ echo "[!] compiling libxdc..." cd libxdc - sudo make install + CFLAGS="-I../capstone_v4/include/" V=1 make libxdc.a cd .. echo "[!] libxdc is ready!" } From 758e65871b311121d75adf984c1efd083bd36f49 Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Thu, 7 Apr 2022 10:58:16 +0200 Subject: [PATCH 35/35] update NYX_HOST_VERSION --- nyx/hypercall/configuration.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nyx/hypercall/configuration.h b/nyx/hypercall/configuration.h index 3c4e9ea0f0..a94e92b3f6 100644 --- a/nyx/hypercall/configuration.h +++ b/nyx/hypercall/configuration.h @@ -10,7 +10,7 @@ void handle_hypercall_kafl_set_agent_config(struct kvm_run *run, CPUState *cpu, #define NYX_HOST_MAGIC 0x4878794e #define NYX_AGENT_MAGIC 0x4178794e -#define NYX_HOST_VERSION 1 +#define NYX_HOST_VERSION 2 #define NYX_AGENT_VERSION 1 typedef struct host_config_s{