From 9f9dcb96a46a60fcf95f7baebafa3ec5e2a1b5ce Mon Sep 17 00:00:00 2001 From: Stephen Michael Jothen Date: Wed, 25 May 2022 17:33:36 +0200 Subject: [PATCH 01/29] target/i386/tcg: Fix masking of real-mode addresses with A20 bit The correct A20 masking is done if paging is enabled (protected mode) but it seems to have been forgotten in real mode. For example from the AMD64 APM Vol. 2 section 1.2.4: > If the sum of the segment base and effective address carries over into bit 20, > that bit can be optionally truncated to mimic the 20-bit address wrapping of the > 8086 processor by using the A20M# input signal to mask the A20 address bit. Most BIOSes will enable the A20 line on boot, but I found by disabling the A20 line afterwards, the correct wrapping wasn't taking place. `handle_mmu_fault' in target/i386/tcg/sysemu/excp_helper.c seems to be the culprit. In real mode, it fills the TLB with the raw unmasked address. However, for the protected mode, the `mmu_translate' function does the correct A20 masking. The fix then should be to just apply the A20 mask in the first branch of the if statement. Signed-off-by: Stephen Michael Jothen Message-Id: Signed-off-by: Paolo Bonzini --- target/i386/tcg/sysemu/excp_helper.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/target/i386/tcg/sysemu/excp_helper.c b/target/i386/tcg/sysemu/excp_helper.c index e1b6d88683..48feba7e75 100644 --- a/target/i386/tcg/sysemu/excp_helper.c +++ b/target/i386/tcg/sysemu/excp_helper.c @@ -359,6 +359,7 @@ static int handle_mmu_fault(CPUState *cs, vaddr addr, int size, CPUX86State *env = &cpu->env; int error_code = PG_ERROR_OK; int pg_mode, prot, page_size; + int32_t a20_mask; hwaddr paddr; hwaddr vaddr; @@ -368,7 +369,8 @@ static int handle_mmu_fault(CPUState *cs, vaddr addr, int size, #endif if (!(env->cr[0] & CR0_PG_MASK)) { - paddr = addr; + a20_mask = x86_get_a20_mask(env); + paddr = addr & a20_mask; #ifdef TARGET_X86_64 if (!(env->hflags & HF_LMA_MASK)) { /* Without long mode we can only address 32bits in real mode */ From d8f3a60931a700c327aa2ee10d0beaeea2cb97da Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 30 May 2022 14:24:34 +0200 Subject: [PATCH 02/29] build: add a "make modules" target Signed-off-by: Paolo Bonzini --- meson.build | 3 +++ 1 file changed, 3 insertions(+) diff --git a/meson.build b/meson.build index bc6234c85e..9b3fd4e52b 100644 --- a/meson.build +++ b/meson.build @@ -3285,6 +3285,9 @@ foreach m : block_mods + softmmu_mods install: true, install_dir: qemu_moddir) endforeach +if emulator_modules.length() > 0 + alias_target('modules', emulator_modules) +endif softmmu_ss.add(authz, blockdev, chardev, crypto, io, qmp) common_ss.add(qom, qemuutil) From b0f3184e82817b700ea81bc76ebc8d999de35278 Mon Sep 17 00:00:00 2001 From: Yang Zhong Date: Mon, 30 May 2022 09:18:34 -0400 Subject: [PATCH 03/29] target/i386: Fix wrong count setting The previous patch used wrong count setting with index value, which got wrong value from CPUID(EAX=12,ECX=0):EAX. So the SGX1 instruction can't be exposed to VM and the SGX decice can't work in VM. Fixes: d19d6ffa0710 ("target/i386: introduce helper to access supported CPUID") Signed-off-by: Yang Zhong Message-Id: <20220530131834.1222801-1-yang.zhong@intel.com> Signed-off-by: Paolo Bonzini --- target/i386/cpu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index bb6a5dd498..9fdfec9d8b 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -5559,7 +5559,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count, * supports. Features can be further restricted by userspace, but not * made more permissive. */ - x86_cpu_get_supported_cpuid(0x12, index, eax, ebx, ecx, edx); + x86_cpu_get_supported_cpuid(0x12, count, eax, ebx, ecx, edx); if (count == 0) { *eax &= env->features[FEAT_SGX_12_0_EAX]; From 24ce7aa77dd1a3095d994bf53d60cce6e672cf4d Mon Sep 17 00:00:00 2001 From: "Jose R. Ziviani" Date: Sat, 28 May 2022 00:20:23 +0200 Subject: [PATCH 04/29] modules: introduces module_kconfig directive module_kconfig is a new directive that should be used with module_obj whenever that module depends on the Kconfig to be enabled. When the module is enabled in Kconfig we are sure that its dependencies will be enabled as well, thus the module will be loaded without any problem. The correct way to use module_kconfig is by passing the Kconfig option to module_kconfig (or the *config-devices.mak without CONFIG_). Signed-off-by: Jose R. Ziviani Signed-off-by: Dario Faggioli Message-Id: <165369002370.5857.12150544416563557322.stgit@work> Signed-off-by: Paolo Bonzini --- hw/display/qxl.c | 1 + hw/display/vhost-user-gpu-pci.c | 1 + hw/display/vhost-user-gpu.c | 1 + hw/display/vhost-user-vga.c | 1 + hw/display/virtio-gpu-base.c | 1 + hw/display/virtio-gpu-gl.c | 1 + hw/display/virtio-gpu-pci-gl.c | 1 + hw/display/virtio-gpu-pci.c | 1 + hw/display/virtio-gpu.c | 1 + hw/display/virtio-vga-gl.c | 1 + hw/display/virtio-vga.c | 1 + hw/s390x/virtio-ccw-gpu.c | 1 + hw/usb/ccid-card-emulated.c | 1 + hw/usb/ccid-card-passthru.c | 1 + hw/usb/host-libusb.c | 1 + hw/usb/redirect.c | 1 + include/qemu/module.h | 10 ++++++++++ scripts/modinfo-generate.py | 2 ++ 18 files changed, 28 insertions(+) diff --git a/hw/display/qxl.c b/hw/display/qxl.c index 2db34714fb..5b10f697f1 100644 --- a/hw/display/qxl.c +++ b/hw/display/qxl.c @@ -2515,6 +2515,7 @@ static const TypeInfo qxl_primary_info = { .class_init = qxl_primary_class_init, }; module_obj("qxl-vga"); +module_kconfig(QXL); static void qxl_secondary_class_init(ObjectClass *klass, void *data) { diff --git a/hw/display/vhost-user-gpu-pci.c b/hw/display/vhost-user-gpu-pci.c index daefcf7101..d119bcae45 100644 --- a/hw/display/vhost-user-gpu-pci.c +++ b/hw/display/vhost-user-gpu-pci.c @@ -44,6 +44,7 @@ static const VirtioPCIDeviceTypeInfo vhost_user_gpu_pci_info = { .instance_init = vhost_user_gpu_pci_initfn, }; module_obj(TYPE_VHOST_USER_GPU_PCI); +module_kconfig(VHOST_USER_GPU); static void vhost_user_gpu_pci_register_types(void) { diff --git a/hw/display/vhost-user-gpu.c b/hw/display/vhost-user-gpu.c index 96e56c4467..3340ef9e5f 100644 --- a/hw/display/vhost-user-gpu.c +++ b/hw/display/vhost-user-gpu.c @@ -606,6 +606,7 @@ static const TypeInfo vhost_user_gpu_info = { .class_init = vhost_user_gpu_class_init, }; module_obj(TYPE_VHOST_USER_GPU); +module_kconfig(VHOST_USER_GPU); static void vhost_user_gpu_register_types(void) { diff --git a/hw/display/vhost-user-vga.c b/hw/display/vhost-user-vga.c index 072c9c65bc..0c146080fd 100644 --- a/hw/display/vhost-user-vga.c +++ b/hw/display/vhost-user-vga.c @@ -45,6 +45,7 @@ static const VirtioPCIDeviceTypeInfo vhost_user_vga_info = { .instance_init = vhost_user_vga_inst_initfn, }; module_obj(TYPE_VHOST_USER_VGA); +module_kconfig(VHOST_USER_VGA); static void vhost_user_vga_register_types(void) { diff --git a/hw/display/virtio-gpu-base.c b/hw/display/virtio-gpu-base.c index 8ba5da4312..790cec333c 100644 --- a/hw/display/virtio-gpu-base.c +++ b/hw/display/virtio-gpu-base.c @@ -260,6 +260,7 @@ static const TypeInfo virtio_gpu_base_info = { .abstract = true }; module_obj(TYPE_VIRTIO_GPU_BASE); +module_kconfig(VIRTIO_GPU); static void virtio_register_types(void) diff --git a/hw/display/virtio-gpu-gl.c b/hw/display/virtio-gpu-gl.c index 0bca887703..e06be60dfb 100644 --- a/hw/display/virtio-gpu-gl.c +++ b/hw/display/virtio-gpu-gl.c @@ -160,6 +160,7 @@ static const TypeInfo virtio_gpu_gl_info = { .class_init = virtio_gpu_gl_class_init, }; module_obj(TYPE_VIRTIO_GPU_GL); +module_kconfig(VIRTIO_GPU); static void virtio_register_types(void) { diff --git a/hw/display/virtio-gpu-pci-gl.c b/hw/display/virtio-gpu-pci-gl.c index 99b14a0718..a2819e1ca9 100644 --- a/hw/display/virtio-gpu-pci-gl.c +++ b/hw/display/virtio-gpu-pci-gl.c @@ -47,6 +47,7 @@ static const VirtioPCIDeviceTypeInfo virtio_gpu_gl_pci_info = { .instance_init = virtio_gpu_gl_initfn, }; module_obj(TYPE_VIRTIO_GPU_GL_PCI); +module_kconfig(VIRTIO_PCI); static void virtio_gpu_gl_pci_register_types(void) { diff --git a/hw/display/virtio-gpu-pci.c b/hw/display/virtio-gpu-pci.c index e36eee0c40..93f214ff58 100644 --- a/hw/display/virtio-gpu-pci.c +++ b/hw/display/virtio-gpu-pci.c @@ -65,6 +65,7 @@ static const TypeInfo virtio_gpu_pci_base_info = { .abstract = true }; module_obj(TYPE_VIRTIO_GPU_PCI_BASE); +module_kconfig(VIRTIO_PCI); #define TYPE_VIRTIO_GPU_PCI "virtio-gpu-pci" typedef struct VirtIOGPUPCI VirtIOGPUPCI; diff --git a/hw/display/virtio-gpu.c b/hw/display/virtio-gpu.c index 529b5246b2..cd4a56056f 100644 --- a/hw/display/virtio-gpu.c +++ b/hw/display/virtio-gpu.c @@ -1452,6 +1452,7 @@ static const TypeInfo virtio_gpu_info = { .class_init = virtio_gpu_class_init, }; module_obj(TYPE_VIRTIO_GPU); +module_kconfig(VIRTIO_GPU); static void virtio_register_types(void) { diff --git a/hw/display/virtio-vga-gl.c b/hw/display/virtio-vga-gl.c index f22549097c..984faa6b39 100644 --- a/hw/display/virtio-vga-gl.c +++ b/hw/display/virtio-vga-gl.c @@ -37,6 +37,7 @@ static VirtioPCIDeviceTypeInfo virtio_vga_gl_info = { .instance_init = virtio_vga_gl_inst_initfn, }; module_obj(TYPE_VIRTIO_VGA_GL); +module_kconfig(VIRTIO_VGA); static void virtio_vga_register_types(void) { diff --git a/hw/display/virtio-vga.c b/hw/display/virtio-vga.c index 7b55c8d0e7..c206b5da38 100644 --- a/hw/display/virtio-vga.c +++ b/hw/display/virtio-vga.c @@ -231,6 +231,7 @@ static const TypeInfo virtio_vga_base_info = { .abstract = true, }; module_obj(TYPE_VIRTIO_VGA_BASE); +module_kconfig(VIRTIO_VGA); #define TYPE_VIRTIO_VGA "virtio-vga" diff --git a/hw/s390x/virtio-ccw-gpu.c b/hw/s390x/virtio-ccw-gpu.c index 8d995fcb33..0642c5281d 100644 --- a/hw/s390x/virtio-ccw-gpu.c +++ b/hw/s390x/virtio-ccw-gpu.c @@ -69,6 +69,7 @@ static const TypeInfo virtio_ccw_gpu = { .class_init = virtio_ccw_gpu_class_init, }; module_obj(TYPE_VIRTIO_GPU_CCW); +module_kconfig(VIRTIO_CCW); static void virtio_ccw_gpu_register(void) { diff --git a/hw/usb/ccid-card-emulated.c b/hw/usb/ccid-card-emulated.c index 6c8c0355e0..1ddf7297f6 100644 --- a/hw/usb/ccid-card-emulated.c +++ b/hw/usb/ccid-card-emulated.c @@ -613,6 +613,7 @@ static const TypeInfo emulated_card_info = { .class_init = emulated_class_initfn, }; module_obj(TYPE_EMULATED_CCID); +module_kconfig(USB); static void ccid_card_emulated_register_types(void) { diff --git a/hw/usb/ccid-card-passthru.c b/hw/usb/ccid-card-passthru.c index f530ab2565..07ee42f304 100644 --- a/hw/usb/ccid-card-passthru.c +++ b/hw/usb/ccid-card-passthru.c @@ -415,6 +415,7 @@ static const TypeInfo passthru_card_info = { .class_init = passthru_class_initfn, }; module_obj(TYPE_CCID_PASSTHRU); +module_kconfig(USB); static void ccid_card_passthru_register_types(void) { diff --git a/hw/usb/host-libusb.c b/hw/usb/host-libusb.c index 2b35cb6cdd..28f8af8941 100644 --- a/hw/usb/host-libusb.c +++ b/hw/usb/host-libusb.c @@ -1809,6 +1809,7 @@ static const TypeInfo usb_host_dev_info = { .instance_init = usb_host_instance_init, }; module_obj(TYPE_USB_HOST_DEVICE); +module_kconfig(USB); static void usb_host_register_types(void) { diff --git a/hw/usb/redirect.c b/hw/usb/redirect.c index 3bc4dee7fe..fd7df599bc 100644 --- a/hw/usb/redirect.c +++ b/hw/usb/redirect.c @@ -2620,6 +2620,7 @@ static const TypeInfo usbredir_dev_info = { .instance_init = usbredir_instance_init, }; module_obj(TYPE_USB_REDIR); +module_kconfig(USB); static void usbredir_register_types(void) { diff --git a/include/qemu/module.h b/include/qemu/module.h index 5fcc323b2a..bd73607104 100644 --- a/include/qemu/module.h +++ b/include/qemu/module.h @@ -135,6 +135,16 @@ void module_allow_arch(const char *arch); */ #define module_opts(name) modinfo(opts, name) +/** + * module_kconfig + * + * @name: Kconfig requirement necessary to load the module + * + * This module requires a core module that should be implemented and + * enabled in Kconfig. + */ +#define module_kconfig(name) modinfo(kconfig, name) + /* * module info database * diff --git a/scripts/modinfo-generate.py b/scripts/modinfo-generate.py index f559eed007..689f33c0f2 100755 --- a/scripts/modinfo-generate.py +++ b/scripts/modinfo-generate.py @@ -48,6 +48,8 @@ def generate(name, lines): opts.append(data) elif kind == 'arch': arch = data; + elif kind == 'kconfig': + pass # ignore else: print("unknown:", kind) exit(1) From 05d6814c3eb16524e992bb7048d3385f8e99dd6a Mon Sep 17 00:00:00 2001 From: "Jose R. Ziviani" Date: Sat, 28 May 2022 00:20:35 +0200 Subject: [PATCH 05/29] modules: generates per-target modinfo This patch changes the way modinfo is generated and built. Instead of one modinfo.c it generates one modinfo--softmmu.c per target. It aims a fine-tune control of modules by configuring Kconfig. Signed-off-by: Jose R. Ziviani Signed-off-by: Dario Faggioli Message-Id: <165369003038.5857.13084289285185196779.stgit@work> Signed-off-by: Paolo Bonzini --- meson.build | 25 +++++++++++++------ scripts/modinfo-generate.py | 49 ++++++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 22 deletions(-) diff --git a/meson.build b/meson.build index 9b3fd4e52b..d738391810 100644 --- a/meson.build +++ b/meson.build @@ -3175,14 +3175,23 @@ foreach d, list : target_modules endforeach if enable_modules - modinfo_src = custom_target('modinfo.c', - output: 'modinfo.c', - input: modinfo_files, - command: [modinfo_generate, '@INPUT@'], - capture: true) - modinfo_lib = static_library('modinfo', modinfo_src) - modinfo_dep = declare_dependency(link_whole: modinfo_lib) - softmmu_ss.add(modinfo_dep) + foreach target : target_dirs + if target.endswith('-softmmu') + config_target = config_target_mak[target] + config_devices_mak = target + '-config-devices.mak' + modinfo_src = custom_target('modinfo-' + target + '.c', + output: 'modinfo-' + target + '.c', + input: modinfo_files, + command: [modinfo_generate, '--devices', config_devices_mak, '@INPUT@'], + capture: true) + + modinfo_lib = static_library('modinfo-' + target + '.c', modinfo_src) + modinfo_dep = declare_dependency(link_with: modinfo_lib) + + arch = config_target['TARGET_NAME'] == 'sparc64' ? 'sparc64' : config_target['TARGET_BASE_ARCH'] + hw_arch[arch].add(modinfo_dep) + endif + endforeach endif nm = find_program('nm') diff --git a/scripts/modinfo-generate.py b/scripts/modinfo-generate.py index 689f33c0f2..b1538fcced 100755 --- a/scripts/modinfo-generate.py +++ b/scripts/modinfo-generate.py @@ -32,7 +32,7 @@ def parse_line(line): continue return (kind, data) -def generate(name, lines): +def generate(name, lines, enabled): arch = "" objs = [] deps = [] @@ -49,7 +49,13 @@ def generate(name, lines): elif kind == 'arch': arch = data; elif kind == 'kconfig': - pass # ignore + # don't add a module which dependency is not enabled + # in kconfig + if data.strip() not in enabled: + print(" /* module {} isn't enabled in Kconfig. */" + .format(data.strip())) + print("/* },{ */") + return None else: print("unknown:", kind) exit(1) @@ -60,8 +66,8 @@ def generate(name, lines): print_array("objs", objs) print_array("deps", deps) print_array("opts", opts) - print("},{"); - return deps + print("},{") + return {dep.strip('" ') for dep in deps} def print_pre(): print("/* generated by scripts/modinfo-generate.py */") @@ -74,23 +80,38 @@ def print_post(): print("}};") def main(args): - deps = {} + if len(args) < 3 or args[0] != '--devices': + print('Expected: modinfo-generate.py --devices ' + 'config-device.mak [modinfo files]', file=sys.stderr) + exit(1) + + # get all devices enabled in kconfig, from *-config-device.mak + enabled = set() + with open(args[1]) as file: + for line in file.readlines(): + config = line.split('=') + if config[1].rstrip() == 'y': + enabled.add(config[0][7:]) # remove CONFIG_ + + deps = set() + modules = set() print_pre() - for modinfo in args: + for modinfo in args[2:]: with open(modinfo) as f: lines = f.readlines() print(" /* %s */" % modinfo) - (basename, ext) = os.path.splitext(modinfo) - deps[basename] = generate(basename, lines) + (basename, _) = os.path.splitext(modinfo) + moddeps = generate(basename, lines, enabled) + if moddeps is not None: + modules.add(basename) + deps.update(moddeps) print_post() - flattened_deps = {flat.strip('" ') for dep in deps.values() for flat in dep} error = False - for dep in flattened_deps: - if dep not in deps.keys(): - print("Dependency {} cannot be satisfied".format(dep), - file=sys.stderr) - error = True + for dep in deps.difference(modules): + print("Dependency {} cannot be satisfied".format(dep), + file=sys.stderr) + error = True if error: exit(1) From c4b8ffcbb8531206e12cf3ad92fa90f7c80ed464 Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Fri, 27 May 2022 13:46:07 +0300 Subject: [PATCH 06/29] replay: fix event queue flush for qemu shutdown This patch fixes event queue flush in the case of emulator shutdown. replay_finish_events should be called when replay_mode is not cleared. Signed-off-by: Pavel Dovgalyuk Reviewed-by: Richard Henderson Message-Id: <165364836758.688121.7959245442743676491.stgit@pasha-ThinkPad-X280> Signed-off-by: Paolo Bonzini --- replay/replay.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/replay/replay.c b/replay/replay.c index 6df2abc18c..2d3607998a 100644 --- a/replay/replay.c +++ b/replay/replay.c @@ -387,9 +387,8 @@ void replay_finish(void) g_free(replay_snapshot); replay_snapshot = NULL; - replay_mode = REPLAY_MODE_NONE; - replay_finish_events(); + replay_mode = REPLAY_MODE_NONE; } void replay_add_blocker(Error *reason) From 75bbe5e5ec2867f098a31bfd553a1fb084289cc2 Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Fri, 27 May 2022 13:46:13 +0300 Subject: [PATCH 07/29] replay: notify vCPU when BH is scheduled vCPU execution should be suspended when new BH is scheduled. This is needed to avoid guest timeouts caused by the long cycles of the execution. In replay mode execution may hang when vCPU sleeps and block event comes to the queue. This patch adds notification which wakes up vCPU or interrupts execution of guest code. Signed-off-by: Pavel Dovgalyuk Reviewed-by: Richard Henderson -- v2: changed first_cpu to current_cpu (suggested by Richard Henderson) v4: moved vCPU notification to aio_bh_enqueue (suggested by Paolo Bonzini) Message-Id: <165364837317.688121.17680519919871405281.stgit@pasha-ThinkPad-X280> Signed-off-by: Paolo Bonzini --- include/sysemu/cpu-timers.h | 1 + softmmu/icount.c | 8 ++++++++ stubs/icount.c | 4 ++++ util/async.c | 8 ++++++++ 4 files changed, 21 insertions(+) diff --git a/include/sysemu/cpu-timers.h b/include/sysemu/cpu-timers.h index ed6ee5c46c..2e786fe7fb 100644 --- a/include/sysemu/cpu-timers.h +++ b/include/sysemu/cpu-timers.h @@ -59,6 +59,7 @@ int64_t icount_round(int64_t count); /* if the CPUs are idle, start accounting real time to virtual clock. */ void icount_start_warp_timer(void); void icount_account_warp_timer(void); +void icount_notify_exit(void); /* * CPU Ticks and Clock diff --git a/softmmu/icount.c b/softmmu/icount.c index 5ca271620d..1cafec5014 100644 --- a/softmmu/icount.c +++ b/softmmu/icount.c @@ -486,3 +486,11 @@ void icount_configure(QemuOpts *opts, Error **errp) qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) + NANOSECONDS_PER_SECOND / 10); } + +void icount_notify_exit(void) +{ + if (icount_enabled() && current_cpu) { + qemu_cpu_kick(current_cpu); + qemu_clock_notify(QEMU_CLOCK_VIRTUAL); + } +} diff --git a/stubs/icount.c b/stubs/icount.c index f13c43568b..6df8c2bf7d 100644 --- a/stubs/icount.c +++ b/stubs/icount.c @@ -43,3 +43,7 @@ void icount_account_warp_timer(void) { abort(); } + +void icount_notify_exit(void) +{ +} diff --git a/util/async.c b/util/async.c index 554ba70cca..63434ddae4 100644 --- a/util/async.c +++ b/util/async.c @@ -33,6 +33,7 @@ #include "block/raw-aio.h" #include "qemu/coroutine_int.h" #include "qemu/coroutine-tls.h" +#include "sysemu/cpu-timers.h" #include "trace.h" /***********************************************************/ @@ -84,6 +85,13 @@ static void aio_bh_enqueue(QEMUBH *bh, unsigned new_flags) } aio_notify(ctx); + /* + * Workaround for record/replay. + * vCPU execution should be suspended when new BH is set. + * This is needed to avoid guest timeouts caused + * by the long cycles of the execution. + */ + icount_notify_exit(); } /* Only called from aio_bh_poll() and aio_ctx_finalize() */ From 60618e2d77691e44bb78e23b2b0cf07b5c405e56 Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Fri, 27 May 2022 13:46:18 +0300 Subject: [PATCH 08/29] replay: rewrite async event handling This patch decouples checkpoints and async events. It was a tricky part of replay implementation. Now it becomes much simpler and easier to maintain. Signed-off-by: Pavel Dovgalyuk Acked-by: Richard Henderson Message-Id: <165364837856.688121.8785039478408995979.stgit@pasha-ThinkPad-X280> Signed-off-by: Paolo Bonzini --- accel/tcg/tcg-accel-ops-icount.c | 5 +-- docs/replay.txt | 11 ++--- include/sysemu/replay.h | 9 +++- replay/replay-events.c | 20 +++------ replay/replay-internal.h | 6 +-- replay/replay-snapshot.c | 1 - replay/replay.c | 74 +++++++++++++------------------- softmmu/icount.c | 4 +- 8 files changed, 54 insertions(+), 76 deletions(-) diff --git a/accel/tcg/tcg-accel-ops-icount.c b/accel/tcg/tcg-accel-ops-icount.c index 24520ea112..8f1dda4344 100644 --- a/accel/tcg/tcg-accel-ops-icount.c +++ b/accel/tcg/tcg-accel-ops-icount.c @@ -84,8 +84,7 @@ void icount_handle_deadline(void) * Don't interrupt cpu thread, when these events are waiting * (i.e., there is no checkpoint) */ - if (deadline == 0 - && (replay_mode != REPLAY_MODE_PLAY || replay_has_checkpoint())) { + if (deadline == 0) { icount_notify_aio_contexts(); } } @@ -109,7 +108,7 @@ void icount_prepare_for_run(CPUState *cpu) replay_mutex_lock(); - if (cpu->icount_budget == 0 && replay_has_checkpoint()) { + if (cpu->icount_budget == 0) { icount_notify_aio_contexts(); } } diff --git a/docs/replay.txt b/docs/replay.txt index 5b008ca491..6c9fdff09d 100644 --- a/docs/replay.txt +++ b/docs/replay.txt @@ -366,11 +366,9 @@ Here is the list of events that are written into the log: Argument: 4-byte number of executed instructions. - EVENT_INTERRUPT. Used to synchronize interrupt processing. - EVENT_EXCEPTION. Used to synchronize exception handling. - - EVENT_ASYNC. This is a group of events. They are always processed - together with checkpoints. When such an event is generated, it is - stored in the queue and processed only when checkpoint occurs. - Every such event is followed by 1-byte checkpoint id and 1-byte - async event id from the following list: + - EVENT_ASYNC. This is a group of events. When such an event is generated, + it is stored in the queue and processed in icount_account_warp_timer(). + Every such event has it's own id from the following list: - REPLAY_ASYNC_EVENT_BH. Bottom-half callback. This event synchronizes callbacks that affect virtual machine state, but normally called asynchronously. @@ -405,6 +403,5 @@ Here is the list of events that are written into the log: - EVENT_CLOCK + clock_id. Group of events for host clock read operations. Argument: 8-byte clock value. - EVENT_CHECKPOINT + checkpoint_id. Checkpoint for synchronization of - CPU, internal threads, and asynchronous input events. May be followed - by one or more EVENT_ASYNC events. + CPU, internal threads, and asynchronous input events. - EVENT_END. Last event in the log. diff --git a/include/sysemu/replay.h b/include/sysemu/replay.h index 0f3b0f7eac..73dee9ccdf 100644 --- a/include/sysemu/replay.h +++ b/include/sysemu/replay.h @@ -160,9 +160,14 @@ void replay_shutdown_request(ShutdownCause cause); Returns 0 in PLAY mode if checkpoint was not found. Returns 1 in all other cases. */ bool replay_checkpoint(ReplayCheckpoint checkpoint); -/*! Used to determine that checkpoint is pending. +/*! Used to determine that checkpoint or async event is pending. Does not proceed to the next event in the log. */ -bool replay_has_checkpoint(void); +bool replay_has_event(void); +/* + * Processes the async events added to the queue (while recording) + * or reads the events from the file (while replaying). + */ +void replay_async_events(void); /* Asynchronous events queue */ diff --git a/replay/replay-events.c b/replay/replay-events.c index ac47c89834..db1decf9dd 100644 --- a/replay/replay-events.c +++ b/replay/replay-events.c @@ -170,12 +170,11 @@ void replay_block_event(QEMUBH *bh, uint64_t id) } } -static void replay_save_event(Event *event, int checkpoint) +static void replay_save_event(Event *event) { if (replay_mode != REPLAY_MODE_PLAY) { /* put the event into the file */ replay_put_event(EVENT_ASYNC); - replay_put_byte(checkpoint); replay_put_byte(event->event_kind); /* save event-specific data */ @@ -206,34 +205,27 @@ static void replay_save_event(Event *event, int checkpoint) } /* Called with replay mutex locked */ -void replay_save_events(int checkpoint) +void replay_save_events(void) { g_assert(replay_mutex_locked()); - g_assert(checkpoint != CHECKPOINT_CLOCK_WARP_START); - g_assert(checkpoint != CHECKPOINT_CLOCK_VIRTUAL); while (!QTAILQ_EMPTY(&events_list)) { Event *event = QTAILQ_FIRST(&events_list); - replay_save_event(event, checkpoint); + replay_save_event(event); replay_run_event(event); QTAILQ_REMOVE(&events_list, event, events); g_free(event); } } -static Event *replay_read_event(int checkpoint) +static Event *replay_read_event(void) { Event *event; if (replay_state.read_event_kind == -1) { - replay_state.read_event_checkpoint = replay_get_byte(); replay_state.read_event_kind = replay_get_byte(); replay_state.read_event_id = -1; replay_check_error(); } - if (checkpoint != replay_state.read_event_checkpoint) { - return NULL; - } - /* Events that has not to be in the queue */ switch (replay_state.read_event_kind) { case REPLAY_ASYNC_EVENT_BH: @@ -294,11 +286,11 @@ static Event *replay_read_event(int checkpoint) } /* Called with replay mutex locked */ -void replay_read_events(int checkpoint) +void replay_read_events(void) { g_assert(replay_mutex_locked()); while (replay_state.data_kind == EVENT_ASYNC) { - Event *event = replay_read_event(checkpoint); + Event *event = replay_read_event(); if (!event) { break; } diff --git a/replay/replay-internal.h b/replay/replay-internal.h index 97649ed8d7..d6e631a394 100644 --- a/replay/replay-internal.h +++ b/replay/replay-internal.h @@ -87,8 +87,6 @@ typedef struct ReplayState { int32_t read_event_kind; /*! Asynchronous event id read from the log */ uint64_t read_event_id; - /*! Asynchronous event checkpoint id read from the log */ - int32_t read_event_checkpoint; } ReplayState; extern ReplayState replay_state; @@ -152,9 +150,9 @@ void replay_finish_events(void); /*! Returns true if there are any unsaved events in the queue */ bool replay_has_events(void); /*! Saves events from queue into the file */ -void replay_save_events(int checkpoint); +void replay_save_events(void); /*! Read events from the file into the input queue */ -void replay_read_events(int checkpoint); +void replay_read_events(void); /*! Adds specified async event to the queue */ void replay_add_event(ReplayAsyncEventKind event_kind, void *opaque, void *opaque2, uint64_t id); diff --git a/replay/replay-snapshot.c b/replay/replay-snapshot.c index e8767a1937..7e935deb15 100644 --- a/replay/replay-snapshot.c +++ b/replay/replay-snapshot.c @@ -61,7 +61,6 @@ static const VMStateDescription vmstate_replay = { VMSTATE_UINT64(block_request_id, ReplayState), VMSTATE_INT32(read_event_kind, ReplayState), VMSTATE_UINT64(read_event_id, ReplayState), - VMSTATE_INT32(read_event_checkpoint, ReplayState), VMSTATE_END_OF_LIST() }, }; diff --git a/replay/replay.c b/replay/replay.c index 2d3607998a..ccd7edec76 100644 --- a/replay/replay.c +++ b/replay/replay.c @@ -22,7 +22,7 @@ /* Current version of the replay mechanism. Increase it when file format changes. */ -#define REPLAY_VERSION 0xe0200a +#define REPLAY_VERSION 0xe0200b /* Size of replay log header */ #define HEADER_SIZE (sizeof(uint32_t) + sizeof(uint64_t)) @@ -171,64 +171,49 @@ void replay_shutdown_request(ShutdownCause cause) bool replay_checkpoint(ReplayCheckpoint checkpoint) { - bool res = false; - static bool in_checkpoint; assert(EVENT_CHECKPOINT + checkpoint <= EVENT_CHECKPOINT_LAST); - if (!replay_file) { - return true; - } - - if (in_checkpoint) { - /* - Recursion occurs when HW event modifies timers. - Prevent performing icount warp in this case and - wait for another invocation of the checkpoint. - */ - g_assert(replay_mode == REPLAY_MODE_PLAY); - return false; - } - in_checkpoint = true; - replay_save_instructions(); if (replay_mode == REPLAY_MODE_PLAY) { g_assert(replay_mutex_locked()); if (replay_next_event_is(EVENT_CHECKPOINT + checkpoint)) { replay_finish_event(); - } else if (replay_state.data_kind != EVENT_ASYNC) { - res = false; - goto out; + } else { + return false; } - replay_read_events(checkpoint); - /* replay_read_events may leave some unread events. - Return false if not all of the events associated with - checkpoint were processed */ - res = replay_state.data_kind != EVENT_ASYNC; } else if (replay_mode == REPLAY_MODE_RECORD) { g_assert(replay_mutex_locked()); replay_put_event(EVENT_CHECKPOINT + checkpoint); - /* This checkpoint belongs to several threads. - Processing events from different threads is - non-deterministic */ - if (checkpoint != CHECKPOINT_CLOCK_WARP_START - /* FIXME: this is temporary fix, other checkpoints - may also be invoked from the different threads someday. - Asynchronous event processing should be refactored - to create additional replay event kind which is - nailed to the one of the threads and which processes - the event queue. */ - && checkpoint != CHECKPOINT_CLOCK_VIRTUAL) { - replay_save_events(checkpoint); - } - res = true; } -out: - in_checkpoint = false; - return res; + return true; } -bool replay_has_checkpoint(void) +void replay_async_events(void) +{ + static bool processing = false; + /* + * If we are already processing the events, recursion may occur + * in case of incorrect implementation when HW event modifies timers. + * Timer modification may invoke the icount warp, event processing, + * and cause the recursion. + */ + g_assert(!processing); + processing = true; + + replay_save_instructions(); + + if (replay_mode == REPLAY_MODE_PLAY) { + g_assert(replay_mutex_locked()); + replay_read_events(); + } else if (replay_mode == REPLAY_MODE_RECORD) { + g_assert(replay_mutex_locked()); + replay_save_events(); + } + processing = false; +} + +bool replay_has_event(void) { bool res = false; if (replay_mode == REPLAY_MODE_PLAY) { @@ -236,6 +221,7 @@ bool replay_has_checkpoint(void) replay_account_executed_instructions(); res = EVENT_CHECKPOINT <= replay_state.data_kind && replay_state.data_kind <= EVENT_CHECKPOINT_LAST; + res = res || replay_state.data_kind == EVENT_ASYNC; } return res; } diff --git a/softmmu/icount.c b/softmmu/icount.c index 1cafec5014..4504433e16 100644 --- a/softmmu/icount.c +++ b/softmmu/icount.c @@ -322,7 +322,7 @@ void icount_start_warp_timer(void) * to vCPU was processed in advance and vCPU went to sleep. * Therefore we have to wake it up for doing someting. */ - if (replay_has_checkpoint()) { + if (replay_has_event()) { qemu_clock_notify(QEMU_CLOCK_VIRTUAL); } return; @@ -404,6 +404,8 @@ void icount_account_warp_timer(void) return; } + replay_async_events(); + /* warp clock deterministically in record/replay mode */ if (!replay_checkpoint(CHECKPOINT_CLOCK_WARP_ACCOUNT)) { return; From 3e21408b0596aa2babe1e26996a15e7ad5eee71a Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Fri, 27 May 2022 13:46:23 +0300 Subject: [PATCH 09/29] replay: simplify async event processing This patch joins replay event id and async event id into single byte in the log. It makes processing a bit faster and log a bit smaller. Signed-off-by: Pavel Dovgalyuk Reviewed-by: Richard Henderson -- v2: minor enum fixes (suggested by Richard Henderson) Message-Id: <165364838393.688121.8191379555130516329.stgit@pasha-ThinkPad-X280> Signed-off-by: Paolo Bonzini --- replay/replay-events.c | 36 ++++++++++++++---------------------- replay/replay-internal.h | 31 ++++++++++++++----------------- replay/replay-snapshot.c | 1 - replay/replay.c | 5 +++-- 4 files changed, 31 insertions(+), 42 deletions(-) diff --git a/replay/replay-events.c b/replay/replay-events.c index db1decf9dd..af0721cc1a 100644 --- a/replay/replay-events.c +++ b/replay/replay-events.c @@ -174,8 +174,8 @@ static void replay_save_event(Event *event) { if (replay_mode != REPLAY_MODE_PLAY) { /* put the event into the file */ - replay_put_event(EVENT_ASYNC); - replay_put_byte(event->event_kind); + g_assert(event->event_kind < REPLAY_ASYNC_COUNT); + replay_put_event(EVENT_ASYNC + event->event_kind); /* save event-specific data */ switch (event->event_kind) { @@ -220,14 +220,10 @@ void replay_save_events(void) static Event *replay_read_event(void) { Event *event; - if (replay_state.read_event_kind == -1) { - replay_state.read_event_kind = replay_get_byte(); - replay_state.read_event_id = -1; - replay_check_error(); - } + ReplayAsyncEventKind event_kind = replay_state.data_kind - EVENT_ASYNC; /* Events that has not to be in the queue */ - switch (replay_state.read_event_kind) { + switch (event_kind) { case REPLAY_ASYNC_EVENT_BH: case REPLAY_ASYNC_EVENT_BH_ONESHOT: if (replay_state.read_event_id == -1) { @@ -236,17 +232,17 @@ static Event *replay_read_event(void) break; case REPLAY_ASYNC_EVENT_INPUT: event = g_new0(Event, 1); - event->event_kind = replay_state.read_event_kind; + event->event_kind = event_kind; event->opaque = replay_read_input_event(); return event; case REPLAY_ASYNC_EVENT_INPUT_SYNC: event = g_new0(Event, 1); - event->event_kind = replay_state.read_event_kind; + event->event_kind = event_kind; event->opaque = 0; return event; case REPLAY_ASYNC_EVENT_CHAR_READ: event = g_new0(Event, 1); - event->event_kind = replay_state.read_event_kind; + event->event_kind = event_kind; event->opaque = replay_event_char_read_load(); return event; case REPLAY_ASYNC_EVENT_BLOCK: @@ -256,18 +252,17 @@ static Event *replay_read_event(void) break; case REPLAY_ASYNC_EVENT_NET: event = g_new0(Event, 1); - event->event_kind = replay_state.read_event_kind; + event->event_kind = event_kind; event->opaque = replay_event_net_load(); return event; default: - error_report("Unknown ID %d of replay event", - replay_state.read_event_kind); + error_report("Unknown ID %d of replay event", event_kind); exit(1); break; } QTAILQ_FOREACH(event, &events_list, events) { - if (event->event_kind == replay_state.read_event_kind + if (event->event_kind == event_kind && (replay_state.read_event_id == -1 || replay_state.read_event_id == event->id)) { break; @@ -276,12 +271,8 @@ static Event *replay_read_event(void) if (event) { QTAILQ_REMOVE(&events_list, event, events); - } else { - return NULL; } - /* Read event-specific data */ - return event; } @@ -289,13 +280,14 @@ static Event *replay_read_event(void) void replay_read_events(void) { g_assert(replay_mutex_locked()); - while (replay_state.data_kind == EVENT_ASYNC) { + while (replay_state.data_kind >= EVENT_ASYNC + && replay_state.data_kind <= EVENT_ASYNC_LAST) { Event *event = replay_read_event(); if (!event) { break; } replay_finish_event(); - replay_state.read_event_kind = -1; + replay_state.read_event_id = -1; replay_run_event(event); g_free(event); @@ -304,7 +296,7 @@ void replay_read_events(void) void replay_init_events(void) { - replay_state.read_event_kind = -1; + replay_state.read_event_id = -1; } void replay_finish_events(void) diff --git a/replay/replay-internal.h b/replay/replay-internal.h index d6e631a394..89e377be90 100644 --- a/replay/replay-internal.h +++ b/replay/replay-internal.h @@ -12,6 +12,19 @@ * */ +/* Asynchronous events IDs */ + +typedef enum ReplayAsyncEventKind { + REPLAY_ASYNC_EVENT_BH, + REPLAY_ASYNC_EVENT_BH_ONESHOT, + REPLAY_ASYNC_EVENT_INPUT, + REPLAY_ASYNC_EVENT_INPUT_SYNC, + REPLAY_ASYNC_EVENT_CHAR_READ, + REPLAY_ASYNC_EVENT_BLOCK, + REPLAY_ASYNC_EVENT_NET, + REPLAY_ASYNC_COUNT +} ReplayAsyncEventKind; + /* Any changes to order/number of events will need to bump REPLAY_VERSION */ enum ReplayEvents { /* for instruction event */ @@ -22,6 +35,7 @@ enum ReplayEvents { EVENT_EXCEPTION, /* for async events */ EVENT_ASYNC, + EVENT_ASYNC_LAST = EVENT_ASYNC + REPLAY_ASYNC_COUNT - 1, /* for shutdown requests, range allows recovery of ShutdownCause */ EVENT_SHUTDOWN, EVENT_SHUTDOWN_LAST = EVENT_SHUTDOWN + SHUTDOWN_CAUSE__MAX, @@ -49,21 +63,6 @@ enum ReplayEvents { EVENT_COUNT }; -/* Asynchronous events IDs */ - -enum ReplayAsyncEventKind { - REPLAY_ASYNC_EVENT_BH, - REPLAY_ASYNC_EVENT_BH_ONESHOT, - REPLAY_ASYNC_EVENT_INPUT, - REPLAY_ASYNC_EVENT_INPUT_SYNC, - REPLAY_ASYNC_EVENT_CHAR_READ, - REPLAY_ASYNC_EVENT_BLOCK, - REPLAY_ASYNC_EVENT_NET, - REPLAY_ASYNC_COUNT -}; - -typedef enum ReplayAsyncEventKind ReplayAsyncEventKind; - typedef struct ReplayState { /*! Cached clock values. */ int64_t cached_clock[REPLAY_CLOCK_COUNT]; @@ -83,8 +82,6 @@ typedef struct ReplayState { uint64_t block_request_id; /*! Prior value of the host clock */ uint64_t host_clock_last; - /*! Asynchronous event type read from the log */ - int32_t read_event_kind; /*! Asynchronous event id read from the log */ uint64_t read_event_id; } ReplayState; diff --git a/replay/replay-snapshot.c b/replay/replay-snapshot.c index 7e935deb15..10a7cf7992 100644 --- a/replay/replay-snapshot.c +++ b/replay/replay-snapshot.c @@ -59,7 +59,6 @@ static const VMStateDescription vmstate_replay = { VMSTATE_UINT32(has_unread_data, ReplayState), VMSTATE_UINT64(file_offset, ReplayState), VMSTATE_UINT64(block_request_id, ReplayState), - VMSTATE_INT32(read_event_kind, ReplayState), VMSTATE_UINT64(read_event_id, ReplayState), VMSTATE_END_OF_LIST() }, diff --git a/replay/replay.c b/replay/replay.c index ccd7edec76..4c396bb376 100644 --- a/replay/replay.c +++ b/replay/replay.c @@ -22,7 +22,7 @@ /* Current version of the replay mechanism. Increase it when file format changes. */ -#define REPLAY_VERSION 0xe0200b +#define REPLAY_VERSION 0xe0200c /* Size of replay log header */ #define HEADER_SIZE (sizeof(uint32_t) + sizeof(uint64_t)) @@ -221,7 +221,8 @@ bool replay_has_event(void) replay_account_executed_instructions(); res = EVENT_CHECKPOINT <= replay_state.data_kind && replay_state.data_kind <= EVENT_CHECKPOINT_LAST; - res = res || replay_state.data_kind == EVENT_ASYNC; + res = res || (EVENT_ASYNC <= replay_state.data_kind + && replay_state.data_kind <= EVENT_ASYNC_LAST); } return res; } From 04d0583a4f5d00061bf57d17947aa0d5c6a6cecf Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Fri, 27 May 2022 13:46:30 +0300 Subject: [PATCH 10/29] docs: convert docs/devel/replay page to rst This patch converts prior .txt replay devel documentation to .rst. Signed-off-by: Pavel Dovgalyuk Reviewed-by: Richard Henderson Message-Id: <165364839013.688121.11935249420738873044.stgit@pasha-ThinkPad-X280> Signed-off-by: Paolo Bonzini --- docs/devel/index-tcg.rst | 1 + docs/devel/{replay.txt => replay.rst} | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 4 deletions(-) rename docs/devel/{replay.txt => replay.rst} (90%) diff --git a/docs/devel/index-tcg.rst b/docs/devel/index-tcg.rst index 0b0ad12c22..7b9760b26f 100644 --- a/docs/devel/index-tcg.rst +++ b/docs/devel/index-tcg.rst @@ -13,3 +13,4 @@ are only implementing things for HW accelerated hypervisors. multi-thread-tcg tcg-icount tcg-plugins + replay diff --git a/docs/devel/replay.txt b/docs/devel/replay.rst similarity index 90% rename from docs/devel/replay.txt rename to docs/devel/replay.rst index e641c35add..dd8bf3b195 100644 --- a/docs/devel/replay.txt +++ b/docs/devel/replay.rst @@ -1,3 +1,11 @@ +.. + Copyright (c) 2022, ISP RAS + Written by Pavel Dovgalyuk + +======================= +Execution Record/Replay +======================= + Record/replay mechanism, that could be enabled through icount mode, expects the virtual devices to satisfy the following requirements. @@ -5,7 +13,7 @@ The main idea behind this document is that everything that affects the guest state during execution in icount mode should be deterministic. Timers -====== +------ All virtual devices should use virtual clock for timers that change the guest state. Virtual clock is deterministic, therefore such timers are deterministic @@ -19,7 +27,7 @@ the virtual devices (e.g., slirp routing device) that lie outside the replayed guest. Bottom halves -============= +------------- Bottom half callbacks, that affect the guest state, should be invoked through replay_bh_schedule_event or replay_bh_schedule_oneshot_event functions. @@ -27,7 +35,7 @@ Their invocations are saved in record mode and synchronized with the existing log in replay mode. Saving/restoring the VM state -============================= +----------------------------- All fields in the device state structure (including virtual timers) should be restored by loadvm to the same values they had before savevm. @@ -38,7 +46,7 @@ is not defined. It means that you should not call functions like the dependencies that may make restoring the VM state non-deterministic. Stopping the VM -=============== +--------------- Stopping the guest should not interfere with its state (with the exception of the network connections, that could be broken by the remote timeouts). From 43185f7bd4cbe730e7d5a6abcefbf83a5c98f3db Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Fri, 27 May 2022 13:46:36 +0300 Subject: [PATCH 11/29] docs: move replay docs to docs/system/replay.rst This patch adds replay description page, converting prior text from docs/replay.txt. The text was also updated and some sections were moved to devel part of the docs. Signed-off-by: Pavel Dovgalyuk Acked-by: Richard Henderson Message-Id: <165364839601.688121.5131456980322853233.stgit@pasha-ThinkPad-X280> Signed-off-by: Paolo Bonzini --- docs/devel/replay.rst | 264 +++++++++++++++++++++++++- docs/replay.txt | 407 ----------------------------------------- docs/system/index.rst | 1 + docs/system/replay.rst | 237 ++++++++++++++++++++++++ 4 files changed, 496 insertions(+), 413 deletions(-) delete mode 100644 docs/replay.txt create mode 100644 docs/system/replay.rst diff --git a/docs/devel/replay.rst b/docs/devel/replay.rst index dd8bf3b195..0244be8b9c 100644 --- a/docs/devel/replay.rst +++ b/docs/devel/replay.rst @@ -1,20 +1,149 @@ .. Copyright (c) 2022, ISP RAS - Written by Pavel Dovgalyuk + Written by Pavel Dovgalyuk and Alex Bennée ======================= Execution Record/Replay ======================= -Record/replay mechanism, that could be enabled through icount mode, expects -the virtual devices to satisfy the following requirements. +Core concepts +============= -The main idea behind this document is that everything that affects +Record/replay functions are used for the deterministic replay of qemu +execution. Execution recording writes a non-deterministic events log, which +can be later used for replaying the execution anywhere and for unlimited +number of times. Execution replaying reads the log and replays all +non-deterministic events including external input, hardware clocks, +and interrupts. + +Several parts of QEMU include function calls to make event log recording +and replaying. +Devices' models that have non-deterministic input from external devices were +changed to write every external event into the execution log immediately. +E.g. network packets are written into the log when they arrive into the virtual +network adapter. + +All non-deterministic events are coming from these devices. But to +replay them we need to know at which moments they occur. We specify +these moments by counting the number of instructions executed between +every pair of consecutive events. + +Academic papers with description of deterministic replay implementation: + +* `Deterministic Replay of System's Execution with Multi-target QEMU Simulator for Dynamic Analysis and Reverse Debugging `_ +* `Don't panic: reverse debugging of kernel drivers `_ + +Modifications of qemu include: + + * wrappers for clock and time functions to save their return values in the log + * saving different asynchronous events (e.g. system shutdown) into the log + * synchronization of the bottom halves execution + * synchronization of the threads from thread pool + * recording/replaying user input (mouse, keyboard, and microphone) + * adding internal checkpoints for cpu and io synchronization + * network filter for recording and replaying the packets + * block driver for making block layer deterministic + * serial port input record and replay + * recording of random numbers obtained from the external sources + +Instruction counting +-------------------- + +QEMU should work in icount mode to use record/replay feature. icount was +designed to allow deterministic execution in absence of external inputs +of the virtual machine. We also use icount to control the occurrence of the +non-deterministic events. The number of instructions elapsed from the last event +is written to the log while recording the execution. In replay mode we +can predict when to inject that event using the instruction counter. + +Locking and thread synchronisation +---------------------------------- + +Previously the synchronisation of the main thread and the vCPU thread +was ensured by the holding of the BQL. However the trend has been to +reduce the time the BQL was held across the system including under TCG +system emulation. As it is important that batches of events are kept +in sequence (e.g. expiring timers and checkpoints in the main thread +while instruction checkpoints are written by the vCPU thread) we need +another lock to keep things in lock-step. This role is now handled by +the replay_mutex_lock. It used to be held only for each event being +written but now it is held for a whole execution period. This results +in a deterministic ping-pong between the two main threads. + +As the BQL is now a finer grained lock than the replay_lock it is almost +certainly a bug, and a source of deadlocks, to take the +replay_mutex_lock while the BQL is held. This is enforced by an assert. +While the unlocks are usually in the reverse order, this is not +necessary; you can drop the replay_lock while holding the BQL, without +doing a more complicated unlock_iothread/replay_unlock/lock_iothread +sequence. + +Checkpoints +----------- + +Replaying the execution of virtual machine is bound by sources of +non-determinism. These are inputs from clock and peripheral devices, +and QEMU thread scheduling. Thread scheduling affect on processing events +from timers, asynchronous input-output, and bottom halves. + +Invocations of timers are coupled with clock reads and changing the state +of the virtual machine. Reads produce non-deterministic data taken from +host clock. And VM state changes should preserve their order. Their relative +order in replay mode must replicate the order of callbacks in record mode. +To preserve this order we use checkpoints. When a specific clock is processed +in record mode we save to the log special "checkpoint" event. +Checkpoints here do not refer to virtual machine snapshots. They are just +record/replay events used for synchronization. + +QEMU in replay mode will try to invoke timers processing in random moment +of time. That's why we do not process a group of timers until the checkpoint +event will be read from the log. Such an event allows synchronizing CPU +execution and timer events. + +Two other checkpoints govern the "warping" of the virtual clock. +While the virtual machine is idle, the virtual clock increments at +1 ns per *real time* nanosecond. This is done by setting up a timer +(called the warp timer) on the virtual real time clock, so that the +timer fires at the next deadline of the virtual clock; the virtual clock +is then incremented (which is called "warping" the virtual clock) as +soon as the timer fires or the CPUs need to go out of the idle state. +Two functions are used for this purpose; because these actions change +virtual machine state and must be deterministic, each of them creates a +checkpoint. ``icount_start_warp_timer`` checks if the CPUs are idle and if so +starts accounting real time to virtual clock. ``icount_account_warp_timer`` +is called when the CPUs get an interrupt or when the warp timer fires, +and it warps the virtual clock by the amount of real time that has passed +since ``icount_start_warp_timer``. + +Virtual devices +=============== + +Record/replay mechanism, that could be enabled through icount mode, expects +the virtual devices to satisfy the following requirement: +everything that affects the guest state during execution in icount mode should be deterministic. Timers ------ +Timers are used to execute callbacks from different subsystems of QEMU +at the specified moments of time. There are several kinds of timers: + + * Real time clock. Based on host time and used only for callbacks that + do not change the virtual machine state. For this reason real time + clock and timers does not affect deterministic replay at all. + * Virtual clock. These timers run only during the emulation. In icount + mode virtual clock value is calculated using executed instructions counter. + That is why it is completely deterministic and does not have to be recorded. + * Host clock. This clock is used by device models that simulate real time + sources (e.g. real time clock chip). Host clock is the one of the sources + of non-determinism. Host clock read operations should be logged to + make the execution deterministic. + * Virtual real time clock. This clock is similar to real time clock but + it is used only for increasing virtual clock while virtual machine is + sleeping. Due to its nature it is also non-deterministic as the host clock + and has to be logged too. + All virtual devices should use virtual clock for timers that change the guest state. Virtual clock is deterministic, therefore such timers are deterministic too. @@ -26,14 +155,50 @@ but its speed depends on the guest execution. This clock is used by the virtual devices (e.g., slirp routing device) that lie outside the replayed guest. +Block devices +------------- + +Block devices record/replay module (``blkreplay``) intercepts calls of +bdrv coroutine functions at the top of block drivers stack. + +All block completion operations are added to the queue in the coroutines. +When the queue is flushed the information about processed requests +is recorded to the log. In replay phase the queue is matched with +events read from the log. Therefore block devices requests are processed +deterministically. + Bottom halves ------------- Bottom half callbacks, that affect the guest state, should be invoked through -replay_bh_schedule_event or replay_bh_schedule_oneshot_event functions. +``replay_bh_schedule_event`` or ``replay_bh_schedule_oneshot_event`` functions. Their invocations are saved in record mode and synchronized with the existing log in replay mode. +Disk I/O events are completely deterministic in our model, because +in both record and replay modes we start virtual machine from the same +disk state. But callbacks that virtual disk controller uses for reading and +writing the disk may occur at different moments of time in record and replay +modes. + +Reading and writing requests are created by CPU thread of QEMU. Later these +requests proceed to block layer which creates "bottom halves". Bottom +halves consist of callback and its parameters. They are processed when +main loop locks the global mutex. These locks are not synchronized with +replaying process because main loop also processes the events that do not +affect the virtual machine state (like user interaction with monitor). + +That is why we had to implement saving and replaying bottom halves callbacks +synchronously to the CPU execution. When the callback is about to execute +it is added to the queue in the replay module. This queue is written to the +log when its callbacks are executed. In replay mode callbacks are not processed +until the corresponding event is read from the events log file. + +Sometimes the block layer uses asynchronous callbacks for its internal purposes +(like reading or writing VM snapshots or disk image cluster tables). In this +case bottom halves are not marked as "replayable" and do not saved +into the log. + Saving/restoring the VM state ----------------------------- @@ -42,7 +207,7 @@ should be restored by loadvm to the same values they had before savevm. Avoid accessing other devices' state, because the order of saving/restoring is not defined. It means that you should not call functions like -'update_irq' in post_load callback. Save everything explicitly to avoid +``update_irq`` in ``post_load`` callback. Save everything explicitly to avoid the dependencies that may make restoring the VM state non-deterministic. Stopping the VM @@ -52,3 +217,90 @@ Stopping the guest should not interfere with its state (with the exception of the network connections, that could be broken by the remote timeouts). VM can be stopped at any moment of replay by the user. Restarting the VM after that stop should not break the replay by the unneeded guest state change. + +Replay log format +================= + +Record/replay log consists of the header and the sequence of execution +events. The header includes 4-byte replay version id and 8-byte reserved +field. Version is updated every time replay log format changes to prevent +using replay log created by another build of qemu. + +The sequence of the events describes virtual machine state changes. +It includes all non-deterministic inputs of VM, synchronization marks and +instruction counts used to correctly inject inputs at replay. + +Synchronization marks (checkpoints) are used for synchronizing qemu threads +that perform operations with virtual hardware. These operations may change +system's state (e.g., change some register or generate interrupt) and +therefore should execute synchronously with CPU thread. + +Every event in the log includes 1-byte event id and optional arguments. +When argument is an array, it is stored as 4-byte array length +and corresponding number of bytes with data. +Here is the list of events that are written into the log: + + - EVENT_INSTRUCTION. Instructions executed since last event. Followed by: + + - 4-byte number of executed instructions. + + - EVENT_INTERRUPT. Used to synchronize interrupt processing. + - EVENT_EXCEPTION. Used to synchronize exception handling. + - EVENT_ASYNC. This is a group of events. When such an event is generated, + it is stored in the queue and processed in icount_account_warp_timer(). + Every such event has it's own id from the following list: + + - REPLAY_ASYNC_EVENT_BH. Bottom-half callback. This event synchronizes + callbacks that affect virtual machine state, but normally called + asynchronously. Followed by: + + - 8-byte operation id. + + - REPLAY_ASYNC_EVENT_INPUT. Input device event. Contains + parameters of keyboard and mouse input operations + (key press/release, mouse pointer movement). Followed by: + + - 9-16 bytes depending of input event. + + - REPLAY_ASYNC_EVENT_INPUT_SYNC. Internal input synchronization event. + - REPLAY_ASYNC_EVENT_CHAR_READ. Character (e.g., serial port) device input + initiated by the sender. Followed by: + + - 1-byte character device id. + - Array with bytes were read. + + - REPLAY_ASYNC_EVENT_BLOCK. Block device operation. Used to synchronize + operations with disk and flash drives with CPU. Followed by: + + - 8-byte operation id. + + - REPLAY_ASYNC_EVENT_NET. Incoming network packet. Followed by: + + - 1-byte network adapter id. + - 4-byte packet flags. + - Array with packet bytes. + + - EVENT_SHUTDOWN. Occurs when user sends shutdown event to qemu, + e.g., by closing the window. + - EVENT_CHAR_WRITE. Used to synchronize character output operations. Followed by: + + - 4-byte output function return value. + - 4-byte offset in the output array. + + - EVENT_CHAR_READ_ALL. Used to synchronize character input operations, + initiated by qemu. Followed by: + + - Array with bytes that were read. + + - EVENT_CHAR_READ_ALL_ERROR. Unsuccessful character input operation, + initiated by qemu. Followed by: + + - 4-byte error code. + + - EVENT_CLOCK + clock_id. Group of events for host clock read operations. Followed by: + + - 8-byte clock value. + + - EVENT_CHECKPOINT + checkpoint_id. Checkpoint for synchronization of + CPU, internal threads, and asynchronous input events. + - EVENT_END. Last event in the log. diff --git a/docs/replay.txt b/docs/replay.txt deleted file mode 100644 index 6c9fdff09d..0000000000 --- a/docs/replay.txt +++ /dev/null @@ -1,407 +0,0 @@ -Copyright (c) 2010-2015 Institute for System Programming - of the Russian Academy of Sciences. - -This work is licensed under the terms of the GNU GPL, version 2 or later. -See the COPYING file in the top-level directory. - -Record/replay -------------- - -Record/replay functions are used for the deterministic replay of qemu execution. -Execution recording writes a non-deterministic events log, which can be later -used for replaying the execution anywhere and for unlimited number of times. -It also supports checkpointing for faster rewind to the specific replay moment. -Execution replaying reads the log and replays all non-deterministic events -including external input, hardware clocks, and interrupts. - -Deterministic replay has the following features: - * Deterministically replays whole system execution and all contents of - the memory, state of the hardware devices, clocks, and screen of the VM. - * Writes execution log into the file for later replaying for multiple times - on different machines. - * Supports i386, x86_64, and Arm hardware platforms. - * Performs deterministic replay of all operations with keyboard and mouse - input devices. - -Usage of the record/replay: - * First, record the execution with the following command line: - qemu-system-i386 \ - -icount shift=7,rr=record,rrfile=replay.bin \ - -drive file=disk.qcow2,if=none,snapshot,id=img-direct \ - -drive driver=blkreplay,if=none,image=img-direct,id=img-blkreplay \ - -device ide-hd,drive=img-blkreplay \ - -netdev user,id=net1 -device rtl8139,netdev=net1 \ - -object filter-replay,id=replay,netdev=net1 - * After recording, you can replay it by using another command line: - qemu-system-i386 \ - -icount shift=7,rr=replay,rrfile=replay.bin \ - -drive file=disk.qcow2,if=none,snapshot,id=img-direct \ - -drive driver=blkreplay,if=none,image=img-direct,id=img-blkreplay \ - -device ide-hd,drive=img-blkreplay \ - -netdev user,id=net1 -device rtl8139,netdev=net1 \ - -object filter-replay,id=replay,netdev=net1 - The only difference with recording is changing the rr option - from record to replay. - * Block device images are not actually changed in the recording mode, - because all of the changes are written to the temporary overlay file. - This behavior is enabled by using blkreplay driver. It should be used - for every enabled block device, as described in 'Block devices' section. - * '-net none' option should be specified when network is not used, - because QEMU adds network card by default. When network is needed, - it should be configured explicitly with replay filter, as described - in 'Network devices' section. - * Interaction with audio devices and serial ports are recorded and replayed - automatically when such devices are enabled. - -Academic papers with description of deterministic replay implementation: -http://www.computer.org/csdl/proceedings/csmr/2012/4666/00/4666a553-abs.html -http://dl.acm.org/citation.cfm?id=2786805.2803179 - -Modifications of qemu include: - * wrappers for clock and time functions to save their return values in the log - * saving different asynchronous events (e.g. system shutdown) into the log - * synchronization of the bottom halves execution - * synchronization of the threads from thread pool - * recording/replaying user input (mouse, keyboard, and microphone) - * adding internal checkpoints for cpu and io synchronization - * network filter for recording and replaying the packets - * block driver for making block layer deterministic - * serial port input record and replay - * recording of random numbers obtained from the external sources - -Locking and thread synchronisation ----------------------------------- - -Previously the synchronisation of the main thread and the vCPU thread -was ensured by the holding of the BQL. However the trend has been to -reduce the time the BQL was held across the system including under TCG -system emulation. As it is important that batches of events are kept -in sequence (e.g. expiring timers and checkpoints in the main thread -while instruction checkpoints are written by the vCPU thread) we need -another lock to keep things in lock-step. This role is now handled by -the replay_mutex_lock. It used to be held only for each event being -written but now it is held for a whole execution period. This results -in a deterministic ping-pong between the two main threads. - -As the BQL is now a finer grained lock than the replay_lock it is almost -certainly a bug, and a source of deadlocks, to take the -replay_mutex_lock while the BQL is held. This is enforced by an assert. -While the unlocks are usually in the reverse order, this is not -necessary; you can drop the replay_lock while holding the BQL, without -doing a more complicated unlock_iothread/replay_unlock/lock_iothread -sequence. - -Non-deterministic events ------------------------- - -Our record/replay system is based on saving and replaying non-deterministic -events (e.g. keyboard input) and simulating deterministic ones (e.g. reading -from HDD or memory of the VM). Saving only non-deterministic events makes -log file smaller and simulation faster. - -The following non-deterministic data from peripheral devices is saved into -the log: mouse and keyboard input, network packets, audio controller input, -serial port input, and hardware clocks (they are non-deterministic -too, because their values are taken from the host machine). Inputs from -simulated hardware, memory of VM, software interrupts, and execution of -instructions are not saved into the log, because they are deterministic and -can be replayed by simulating the behavior of virtual machine starting from -initial state. - -We had to solve three tasks to implement deterministic replay: recording -non-deterministic events, replaying non-deterministic events, and checking -that there is no divergence between record and replay modes. - -We changed several parts of QEMU to make event log recording and replaying. -Devices' models that have non-deterministic input from external devices were -changed to write every external event into the execution log immediately. -E.g. network packets are written into the log when they arrive into the virtual -network adapter. - -All non-deterministic events are coming from these devices. But to -replay them we need to know at which moments they occur. We specify -these moments by counting the number of instructions executed between -every pair of consecutive events. - -Instruction counting --------------------- - -QEMU should work in icount mode to use record/replay feature. icount was -designed to allow deterministic execution in absence of external inputs -of the virtual machine. We also use icount to control the occurrence of the -non-deterministic events. The number of instructions elapsed from the last event -is written to the log while recording the execution. In replay mode we -can predict when to inject that event using the instruction counter. - -Timers ------- - -Timers are used to execute callbacks from different subsystems of QEMU -at the specified moments of time. There are several kinds of timers: - * Real time clock. Based on host time and used only for callbacks that - do not change the virtual machine state. For this reason real time - clock and timers does not affect deterministic replay at all. - * Virtual clock. These timers run only during the emulation. In icount - mode virtual clock value is calculated using executed instructions counter. - That is why it is completely deterministic and does not have to be recorded. - * Host clock. This clock is used by device models that simulate real time - sources (e.g. real time clock chip). Host clock is the one of the sources - of non-determinism. Host clock read operations should be logged to - make the execution deterministic. - * Virtual real time clock. This clock is similar to real time clock but - it is used only for increasing virtual clock while virtual machine is - sleeping. Due to its nature it is also non-deterministic as the host clock - and has to be logged too. - -Checkpoints ------------ - -Replaying of the execution of virtual machine is bound by sources of -non-determinism. These are inputs from clock and peripheral devices, -and QEMU thread scheduling. Thread scheduling affect on processing events -from timers, asynchronous input-output, and bottom halves. - -Invocations of timers are coupled with clock reads and changing the state -of the virtual machine. Reads produce non-deterministic data taken from -host clock. And VM state changes should preserve their order. Their relative -order in replay mode must replicate the order of callbacks in record mode. -To preserve this order we use checkpoints. When a specific clock is processed -in record mode we save to the log special "checkpoint" event. -Checkpoints here do not refer to virtual machine snapshots. They are just -record/replay events used for synchronization. - -QEMU in replay mode will try to invoke timers processing in random moment -of time. That's why we do not process a group of timers until the checkpoint -event will be read from the log. Such an event allows synchronizing CPU -execution and timer events. - -Two other checkpoints govern the "warping" of the virtual clock. -While the virtual machine is idle, the virtual clock increments at -1 ns per *real time* nanosecond. This is done by setting up a timer -(called the warp timer) on the virtual real time clock, so that the -timer fires at the next deadline of the virtual clock; the virtual clock -is then incremented (which is called "warping" the virtual clock) as -soon as the timer fires or the CPUs need to go out of the idle state. -Two functions are used for this purpose; because these actions change -virtual machine state and must be deterministic, each of them creates a -checkpoint. icount_start_warp_timer checks if the CPUs are idle and if so -starts accounting real time to virtual clock. icount_account_warp_timer -is called when the CPUs get an interrupt or when the warp timer fires, -and it warps the virtual clock by the amount of real time that has passed -since icount_start_warp_timer. - -Bottom halves -------------- - -Disk I/O events are completely deterministic in our model, because -in both record and replay modes we start virtual machine from the same -disk state. But callbacks that virtual disk controller uses for reading and -writing the disk may occur at different moments of time in record and replay -modes. - -Reading and writing requests are created by CPU thread of QEMU. Later these -requests proceed to block layer which creates "bottom halves". Bottom -halves consist of callback and its parameters. They are processed when -main loop locks the global mutex. These locks are not synchronized with -replaying process because main loop also processes the events that do not -affect the virtual machine state (like user interaction with monitor). - -That is why we had to implement saving and replaying bottom halves callbacks -synchronously to the CPU execution. When the callback is about to execute -it is added to the queue in the replay module. This queue is written to the -log when its callbacks are executed. In replay mode callbacks are not processed -until the corresponding event is read from the events log file. - -Sometimes the block layer uses asynchronous callbacks for its internal purposes -(like reading or writing VM snapshots or disk image cluster tables). In this -case bottom halves are not marked as "replayable" and do not saved -into the log. - -Block devices -------------- - -Block devices record/replay module intercepts calls of -bdrv coroutine functions at the top of block drivers stack. -To record and replay block operations the drive must be configured -as following: - -drive file=disk.qcow2,if=none,snapshot,id=img-direct - -drive driver=blkreplay,if=none,image=img-direct,id=img-blkreplay - -device ide-hd,drive=img-blkreplay - -blkreplay driver should be inserted between disk image and virtual driver -controller. Therefore all disk requests may be recorded and replayed. - -All block completion operations are added to the queue in the coroutines. -Queue is flushed at checkpoints and information about processed requests -is recorded to the log. In replay phase the queue is matched with -events read from the log. Therefore block devices requests are processed -deterministically. - -Snapshotting ------------- - -New VM snapshots may be created in replay mode. They can be used later -to recover the desired VM state. All VM states created in replay mode -are associated with the moment of time in the replay scenario. -After recovering the VM state replay will start from that position. - -Default starting snapshot name may be specified with icount field -rrsnapshot as follows: - -icount shift=7,rr=record,rrfile=replay.bin,rrsnapshot=snapshot_name - -This snapshot is created at start of recording and restored at start -of replaying. It also can be loaded while replaying to roll back -the execution. - -'snapshot' flag of the disk image must be removed to save the snapshots -in the overlay (or original image) instead of using the temporary overlay. - -drive file=disk.ovl,if=none,id=img-direct - -drive driver=blkreplay,if=none,image=img-direct,id=img-blkreplay - -device ide-hd,drive=img-blkreplay - -Use QEMU monitor to create additional snapshots. 'savevm ' command -created the snapshot and 'loadvm ' restores it. To prevent corruption -of the original disk image, use overlay files linked to the original images. -Therefore all new snapshots (including the starting one) will be saved in -overlays and the original image remains unchanged. - -When you need to use snapshots with diskless virtual machine, -it must be started with 'orphan' qcow2 image. This image will be used -for storing VM snapshots. Here is the example of the command line for this: - - qemu-system-i386 -icount shift=3,rr=replay,rrfile=record.bin,rrsnapshot=init \ - -net none -drive file=empty.qcow2,if=none,id=rr - -empty.qcow2 drive does not connected to any virtual block device and used -for VM snapshots only. - -Network devices ---------------- - -Record and replay for network interactions is performed with the network filter. -Each backend must have its own instance of the replay filter as follows: - -netdev user,id=net1 -device rtl8139,netdev=net1 - -object filter-replay,id=replay,netdev=net1 - -Replay network filter is used to record and replay network packets. While -recording the virtual machine this filter puts all packets coming from -the outer world into the log. In replay mode packets from the log are -injected into the network device. All interactions with network backend -in replay mode are disabled. - -Audio devices -------------- - -Audio data is recorded and replay automatically. The command line for recording -and replaying must contain identical specifications of audio hardware, e.g.: - -soundhw ac97 - -Serial ports ------------- - -Serial ports input is recorded and replay automatically. The command lines -for recording and replaying must contain identical number of ports in record -and replay modes, but their backends may differ. -E.g., '-serial stdio' in record mode, and '-serial null' in replay mode. - -Reverse debugging ------------------ - -Reverse debugging allows "executing" the program in reverse direction. -GDB remote protocol supports "reverse step" and "reverse continue" -commands. The first one steps single instruction backwards in time, -and the second one finds the last breakpoint in the past. - -Recorded executions may be used to enable reverse debugging. QEMU can't -execute the code in backwards direction, but can load a snapshot and -replay forward to find the desired position or breakpoint. - -The following GDB commands are supported: - - reverse-stepi (or rsi) - step one instruction backwards - - reverse-continue (or rc) - find last breakpoint in the past - -Reverse step loads the nearest snapshot and replays the execution until -the required instruction is met. - -Reverse continue may include several passes of examining the execution -between the snapshots. Each of the passes include the following steps: - 1. loading the snapshot - 2. replaying to examine the breakpoints - 3. if breakpoint or watchpoint was met - - loading the snapshot again - - replaying to the required breakpoint - 4. else - - proceeding to the p.1 with the earlier snapshot - -Therefore usage of the reverse debugging requires at least one snapshot -created in advance. This can be done by omitting 'snapshot' option -for the block drives and adding 'rrsnapshot' for both record and replay -command lines. -See the "Snapshotting" section to learn more about running record/replay -and creating the snapshot in these modes. - -Replay log format ------------------ - -Record/replay log consists of the header and the sequence of execution -events. The header includes 4-byte replay version id and 8-byte reserved -field. Version is updated every time replay log format changes to prevent -using replay log created by another build of qemu. - -The sequence of the events describes virtual machine state changes. -It includes all non-deterministic inputs of VM, synchronization marks and -instruction counts used to correctly inject inputs at replay. - -Synchronization marks (checkpoints) are used for synchronizing qemu threads -that perform operations with virtual hardware. These operations may change -system's state (e.g., change some register or generate interrupt) and -therefore should execute synchronously with CPU thread. - -Every event in the log includes 1-byte event id and optional arguments. -When argument is an array, it is stored as 4-byte array length -and corresponding number of bytes with data. -Here is the list of events that are written into the log: - - - EVENT_INSTRUCTION. Instructions executed since last event. - Argument: 4-byte number of executed instructions. - - EVENT_INTERRUPT. Used to synchronize interrupt processing. - - EVENT_EXCEPTION. Used to synchronize exception handling. - - EVENT_ASYNC. This is a group of events. When such an event is generated, - it is stored in the queue and processed in icount_account_warp_timer(). - Every such event has it's own id from the following list: - - REPLAY_ASYNC_EVENT_BH. Bottom-half callback. This event synchronizes - callbacks that affect virtual machine state, but normally called - asynchronously. - Argument: 8-byte operation id. - - REPLAY_ASYNC_EVENT_INPUT. Input device event. Contains - parameters of keyboard and mouse input operations - (key press/release, mouse pointer movement). - Arguments: 9-16 bytes depending of input event. - - REPLAY_ASYNC_EVENT_INPUT_SYNC. Internal input synchronization event. - - REPLAY_ASYNC_EVENT_CHAR_READ. Character (e.g., serial port) device input - initiated by the sender. - Arguments: 1-byte character device id. - Array with bytes were read. - - REPLAY_ASYNC_EVENT_BLOCK. Block device operation. Used to synchronize - operations with disk and flash drives with CPU. - Argument: 8-byte operation id. - - REPLAY_ASYNC_EVENT_NET. Incoming network packet. - Arguments: 1-byte network adapter id. - 4-byte packet flags. - Array with packet bytes. - - EVENT_SHUTDOWN. Occurs when user sends shutdown event to qemu, - e.g., by closing the window. - - EVENT_CHAR_WRITE. Used to synchronize character output operations. - Arguments: 4-byte output function return value. - 4-byte offset in the output array. - - EVENT_CHAR_READ_ALL. Used to synchronize character input operations, - initiated by qemu. - Argument: Array with bytes that were read. - - EVENT_CHAR_READ_ALL_ERROR. Unsuccessful character input operation, - initiated by qemu. - Argument: 4-byte error code. - - EVENT_CLOCK + clock_id. Group of events for host clock read operations. - Argument: 8-byte clock value. - - EVENT_CHECKPOINT + checkpoint_id. Checkpoint for synchronization of - CPU, internal threads, and asynchronous input events. - - EVENT_END. Last event in the log. diff --git a/docs/system/index.rst b/docs/system/index.rst index 23e30e26e5..e3695649c5 100644 --- a/docs/system/index.rst +++ b/docs/system/index.rst @@ -27,6 +27,7 @@ or Hypervisor.Framework. secrets authz gdb + replay managed-startup bootindex cpu-hotplug diff --git a/docs/system/replay.rst b/docs/system/replay.rst new file mode 100644 index 0000000000..3105327423 --- /dev/null +++ b/docs/system/replay.rst @@ -0,0 +1,237 @@ +.. _replay: + +.. + Copyright (c) 2010-2022 Institute for System Programming + of the Russian Academy of Sciences. + + This work is licensed under the terms of the GNU GPL, version 2 or later. + See the COPYING file in the top-level directory. + +Record/replay +============= + +Record/replay functions are used for the deterministic replay of qemu execution. +Execution recording writes a non-deterministic events log, which can be later +used for replaying the execution anywhere and for unlimited number of times. +It also supports checkpointing for faster rewind to the specific replay moment. +Execution replaying reads the log and replays all non-deterministic events +including external input, hardware clocks, and interrupts. + +Deterministic replay has the following features: + + * Deterministically replays whole system execution and all contents of + the memory, state of the hardware devices, clocks, and screen of the VM. + * Writes execution log into the file for later replaying for multiple times + on different machines. + * Supports i386, x86_64, ARM, AArch64, Risc-V, MIPS, MIPS64, S390X, Alpha, + PowerPC, PowerPC64, M68000, Microblaze, OpenRISC, Nios II, SPARC, + and Xtensa hardware platforms. + * Performs deterministic replay of all operations with keyboard and mouse + input devices, serial ports, and network. + +Usage of the record/replay: + + * First, record the execution with the following command line: + + .. parsed-literal:: + |qemu_system| \\ + -icount shift=auto,rr=record,rrfile=replay.bin \\ + -drive file=disk.qcow2,if=none,snapshot,id=img-direct \\ + -drive driver=blkreplay,if=none,image=img-direct,id=img-blkreplay \\ + -device ide-hd,drive=img-blkreplay \\ + -netdev user,id=net1 -device rtl8139,netdev=net1 \\ + -object filter-replay,id=replay,netdev=net1 + + * After recording, you can replay it by using another command line: + + .. parsed-literal:: + |qemu_system| \\ + -icount shift=auto,rr=replay,rrfile=replay.bin \\ + -drive file=disk.qcow2,if=none,snapshot,id=img-direct \\ + -drive driver=blkreplay,if=none,image=img-direct,id=img-blkreplay \\ + -device ide-hd,drive=img-blkreplay \\ + -netdev user,id=net1 -device rtl8139,netdev=net1 \\ + -object filter-replay,id=replay,netdev=net1 + + The only difference with recording is changing the rr option + from record to replay. + * Block device images are not actually changed in the recording mode, + because all of the changes are written to the temporary overlay file. + This behavior is enabled by using blkreplay driver. It should be used + for every enabled block device, as described in :ref:`block-label` section. + * ``-net none`` option should be specified when network is not used, + because QEMU adds network card by default. When network is needed, + it should be configured explicitly with replay filter, as described + in :ref:`network-label` section. + * Interaction with audio devices and serial ports are recorded and replayed + automatically when such devices are enabled. + +Core idea +--------- + +Record/replay system is based on saving and replaying non-deterministic +events (e.g. keyboard input) and simulating deterministic ones (e.g. reading +from HDD or memory of the VM). Saving only non-deterministic events makes +log file smaller and simulation faster. + +The following non-deterministic data from peripheral devices is saved into +the log: mouse and keyboard input, network packets, audio controller input, +serial port input, and hardware clocks (they are non-deterministic +too, because their values are taken from the host machine). Inputs from +simulated hardware, memory of VM, software interrupts, and execution of +instructions are not saved into the log, because they are deterministic and +can be replayed by simulating the behavior of virtual machine starting from +initial state. + +Instruction counting +-------------------- + +QEMU should work in icount mode to use record/replay feature. icount was +designed to allow deterministic execution in absence of external inputs +of the virtual machine. Record/replay feature is enabled through ``-icount`` +command-line option, making possible deterministic execution of the machine, +interacting with user or network. + +.. _block-label: + +Block devices +------------- + +Block devices record/replay module intercepts calls of +bdrv coroutine functions at the top of block drivers stack. +To record and replay block operations the drive must be configured +as following: + +.. parsed-literal:: + -drive file=disk.qcow2,if=none,snapshot,id=img-direct + -drive driver=blkreplay,if=none,image=img-direct,id=img-blkreplay + -device ide-hd,drive=img-blkreplay + +blkreplay driver should be inserted between disk image and virtual driver +controller. Therefore all disk requests may be recorded and replayed. + +.. _snapshotting-label: + +Snapshotting +------------ + +New VM snapshots may be created in replay mode. They can be used later +to recover the desired VM state. All VM states created in replay mode +are associated with the moment of time in the replay scenario. +After recovering the VM state replay will start from that position. + +Default starting snapshot name may be specified with icount field +rrsnapshot as follows: + +.. parsed-literal:: + -icount shift=auto,rr=record,rrfile=replay.bin,rrsnapshot=snapshot_name + +This snapshot is created at start of recording and restored at start +of replaying. It also can be loaded while replaying to roll back +the execution. + +``snapshot`` flag of the disk image must be removed to save the snapshots +in the overlay (or original image) instead of using the temporary overlay. + +.. parsed-literal:: + -drive file=disk.ovl,if=none,id=img-direct + -drive driver=blkreplay,if=none,image=img-direct,id=img-blkreplay + -device ide-hd,drive=img-blkreplay + +Use QEMU monitor to create additional snapshots. ``savevm `` command +created the snapshot and ``loadvm `` restores it. To prevent corruption +of the original disk image, use overlay files linked to the original images. +Therefore all new snapshots (including the starting one) will be saved in +overlays and the original image remains unchanged. + +When you need to use snapshots with diskless virtual machine, +it must be started with "orphan" qcow2 image. This image will be used +for storing VM snapshots. Here is the example of the command line for this: + +.. parsed-literal:: + |qemu_system| \\ + -icount shift=auto,rr=replay,rrfile=record.bin,rrsnapshot=init \\ + -net none -drive file=empty.qcow2,if=none,id=rr + +``empty.qcow2`` drive does not connected to any virtual block device and used +for VM snapshots only. + +.. _network-label: + +Network devices +--------------- + +Record and replay for network interactions is performed with the network filter. +Each backend must have its own instance of the replay filter as follows: + +.. parsed-literal:: + -netdev user,id=net1 -device rtl8139,netdev=net1 + -object filter-replay,id=replay,netdev=net1 + +Replay network filter is used to record and replay network packets. While +recording the virtual machine this filter puts all packets coming from +the outer world into the log. In replay mode packets from the log are +injected into the network device. All interactions with network backend +in replay mode are disabled. + +Audio devices +------------- + +Audio data is recorded and replay automatically. The command line for recording +and replaying must contain identical specifications of audio hardware, e.g.: + +.. parsed-literal:: + -soundhw ac97 + +Serial ports +------------ + +Serial ports input is recorded and replay automatically. The command lines +for recording and replaying must contain identical number of ports in record +and replay modes, but their backends may differ. +E.g., ``-serial stdio`` in record mode, and ``-serial null`` in replay mode. + +Reverse debugging +----------------- + +Reverse debugging allows "executing" the program in reverse direction. +GDB remote protocol supports "reverse step" and "reverse continue" +commands. The first one steps single instruction backwards in time, +and the second one finds the last breakpoint in the past. + +Recorded executions may be used to enable reverse debugging. QEMU can't +execute the code in backwards direction, but can load a snapshot and +replay forward to find the desired position or breakpoint. + +The following GDB commands are supported: + + - ``reverse-stepi`` (or ``rsi``) - step one instruction backwards + - ``reverse-continue`` (or ``rc``) - find last breakpoint in the past + +Reverse step loads the nearest snapshot and replays the execution until +the required instruction is met. + +Reverse continue may include several passes of examining the execution +between the snapshots. Each of the passes include the following steps: + + #. loading the snapshot + #. replaying to examine the breakpoints + #. if breakpoint or watchpoint was met + + * loading the snapshot again + * replaying to the required breakpoint + + #. else + + * proceeding to the p.1 with the earlier snapshot + +Therefore usage of the reverse debugging requires at least one snapshot +created. This can be done by omitting ``snapshot`` option +for the block drives and adding ``rrsnapshot`` for both record and replay +command lines. +See the :ref:`snapshotting-label` section to learn more about running record/replay +and creating the snapshot in these modes. + +When ``rrsnapshot`` is not used, then snapshot named ``start_debugging`` +created in temporary overlay. This allows using reverse debugging, but with +temporary snapshots (existing within the session). From c9d27a0f9f2ee7e6a57167de7d0dcab406f8bfa7 Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Fri, 27 May 2022 13:46:42 +0300 Subject: [PATCH 12/29] tests/avocado: update replay_linux test This patch updates replay_linux test to make it compatible with new LinuxTest class. Signed-off-by: Pavel Dovgalyuk Message-Id: <165364840253.688121.10404266209986316381.stgit@pasha-ThinkPad-X280> Signed-off-by: Paolo Bonzini --- tests/avocado/replay_linux.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/avocado/replay_linux.py b/tests/avocado/replay_linux.py index 15953f9e49..1099b5647f 100644 --- a/tests/avocado/replay_linux.py +++ b/tests/avocado/replay_linux.py @@ -32,9 +32,16 @@ class ReplayLinux(LinuxTest): bus = 'ide' def setUp(self): - super(ReplayLinux, self).setUp() + # LinuxTest does many replay-incompatible things, but includes + # useful methods. Do not setup LinuxTest here and just + # call some functions. + super(LinuxTest, self).setUp() + self._set_distro() self.boot_path = self.download_boot() - self.cloudinit_path = self.prepare_cloudinit() + self.phone_server = cloudinit.PhoneHomeServer(('0.0.0.0', 0), + self.name) + ssh_pubkey, self.ssh_key = self.set_up_existing_ssh_keys() + self.cloudinit_path = self.prepare_cloudinit(ssh_pubkey) def vm_add_disk(self, vm, path, id, device): bus_string = '' @@ -50,7 +57,9 @@ class ReplayLinux(LinuxTest): vm = self.get_vm() vm.add_args('-smp', '1') vm.add_args('-m', '1024') - vm.add_args('-object', 'filter-replay,id=replay,netdev=hub0port0') + vm.add_args('-netdev', 'user,id=vnet,hostfwd=:127.0.0.1:0-:22', + '-device', 'virtio-net,netdev=vnet') + vm.add_args('-object', 'filter-replay,id=replay,netdev=vnet') if args: vm.add_args(*args) self.vm_add_disk(vm, self.boot_path, 0, self.hdd) @@ -75,8 +84,8 @@ class ReplayLinux(LinuxTest): stop_check=(lambda : not vm.is_running())) console_drainer.start() if record: - cloudinit.wait_for_phone_home(('0.0.0.0', self.phone_home_port), - self.name) + while not self.phone_server.instance_phoned_back: + self.phone_server.handle_request() vm.shutdown() logger.info('finished the recording with log size %s bytes' % os.path.getsize(replay_path)) From 3cd1dce33d2359fe410bb20f6f980f3f91da79bd Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Fri, 27 May 2022 13:46:48 +0300 Subject: [PATCH 13/29] tests/avocado: add replay Linux tests for virtio machine This patch adds two tests for replaying Linux boot process on x86_64 virtio platform. Signed-off-by: Pavel Dovgalyuk Message-Id: <165364840811.688121.11931681195199516354.stgit@pasha-ThinkPad-X280> Signed-off-by: Paolo Bonzini --- tests/avocado/replay_linux.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/tests/avocado/replay_linux.py b/tests/avocado/replay_linux.py index 1099b5647f..3bb1bc8816 100644 --- a/tests/avocado/replay_linux.py +++ b/tests/avocado/replay_linux.py @@ -123,3 +123,29 @@ class ReplayLinuxX8664(ReplayLinux): :avocado: tags=machine:q35 """ self.run_rr(shift=3) + +@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') +class ReplayLinuxX8664Virtio(ReplayLinux): + """ + :avocado: tags=arch:x86_64 + :avocado: tags=virtio + :avocado: tags=accel:tcg + """ + + hdd = 'virtio-blk-pci' + cd = 'virtio-blk-pci' + bus = None + + chksum = 'e3c1b309d9203604922d6e255c2c5d098a309c2d46215d8fc026954f3c5c27a0' + + def test_pc_i440fx(self): + """ + :avocado: tags=machine:pc + """ + self.run_rr(shift=1) + + def test_pc_q35(self): + """ + :avocado: tags=machine:q35 + """ + self.run_rr(shift=3) From 9b59af628c97eada68eebc1a9af0c72b75ecf1b8 Mon Sep 17 00:00:00 2001 From: Pavel Dovgalyuk Date: Fri, 27 May 2022 13:46:53 +0300 Subject: [PATCH 14/29] tests/avocado: add replay Linux test for Aarch64 machines This patch adds two tests for replaying Linux boot process on Aarch64 platform. Signed-off-by: Pavel Dovgalyuk Message-Id: <165364841373.688121.8868079200312201658.stgit@pasha-ThinkPad-X280> Signed-off-by: Paolo Bonzini --- tests/avocado/replay_linux.py | 40 +++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/tests/avocado/replay_linux.py b/tests/avocado/replay_linux.py index 3bb1bc8816..40e4f6908e 100644 --- a/tests/avocado/replay_linux.py +++ b/tests/avocado/replay_linux.py @@ -13,6 +13,7 @@ import logging import time from avocado import skipUnless +from avocado_qemu import BUILD_DIR from avocado.utils import cloudinit from avocado.utils import network from avocado.utils import vmimage @@ -149,3 +150,42 @@ class ReplayLinuxX8664Virtio(ReplayLinux): :avocado: tags=machine:q35 """ self.run_rr(shift=3) + +@skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout') +class ReplayLinuxAarch64(ReplayLinux): + """ + :avocado: tags=accel:tcg + :avocado: tags=arch:aarch64 + :avocado: tags=machine:virt + :avocado: tags=cpu:max + """ + + chksum = '1e18d9c0cf734940c4b5d5ec592facaed2af0ad0329383d5639c997fdf16fe49' + + hdd = 'virtio-blk-device' + cd = 'virtio-blk-device' + bus = None + + def get_common_args(self): + return ('-bios', + os.path.join(BUILD_DIR, 'pc-bios', 'edk2-aarch64-code.fd'), + "-cpu", "max,lpa2=off", + '-device', 'virtio-rng-pci,rng=rng0', + '-object', 'rng-builtin,id=rng0') + + def test_virt_gicv2(self): + """ + :avocado: tags=machine:gic-version=2 + """ + + self.run_rr(shift=3, + args=(*self.get_common_args(), + "-machine", "virt,gic-version=2")) + + def test_virt_gicv3(self): + """ + :avocado: tags=machine:gic-version=3 + """ + + self.run_rr(shift=3, + args=(*self.get_common_args(), From d426ff938b3f95e0b04bb410d47ae50359340a16 Mon Sep 17 00:00:00 2001 From: Dario Faggioli Date: Fri, 27 May 2022 17:30:54 +0200 Subject: [PATCH 15/29] tests/Makefile.include: Fix 'make check-help' output MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since commit 3d2f73ef75e ("build: use "meson test" as the test harness"), check-report.tap is no more, and we have check-report.junit.xml. Update the output of 'make check-help', which was still listing 'check-report.tap', accordingly. Fixes: 3d2f73ef75e Signed-off-by: Dario Faggioli Message-Id: <165366545439.6869.11633009118019728798.stgit@work> Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Paolo Bonzini --- tests/Makefile.include | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/Makefile.include b/tests/Makefile.include index 6a1688e33e..2a1c77440a 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -3,28 +3,28 @@ .PHONY: check-help check-help: @echo "Regression testing targets:" - @echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest and decodetree tests" - @echo " $(MAKE) bench Run speed tests" + @echo " $(MAKE) check Run block, qapi-schema, unit, softfloat, qtest and decodetree tests" + @echo " $(MAKE) bench Run speed tests" @echo @echo "Individual test suites:" - @echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target" - @echo " $(MAKE) check-qtest Run qtest tests" - @echo " $(MAKE) check-unit Run qobject tests" - @echo " $(MAKE) check-qapi-schema Run QAPI schema tests" - @echo " $(MAKE) check-block Run block tests" + @echo " $(MAKE) check-qtest-TARGET Run qtest tests for given target" + @echo " $(MAKE) check-qtest Run qtest tests" + @echo " $(MAKE) check-unit Run qobject tests" + @echo " $(MAKE) check-qapi-schema Run QAPI schema tests" + @echo " $(MAKE) check-block Run block tests" ifneq ($(filter $(all-check-targets), check-softfloat),) - @echo " $(MAKE) check-tcg Run TCG tests" - @echo " $(MAKE) check-softfloat Run FPU emulation tests" + @echo " $(MAKE) check-tcg Run TCG tests" + @echo " $(MAKE) check-softfloat Run FPU emulation tests" endif - @echo " $(MAKE) check-avocado Run avocado (integration) tests for currently configured targets" + @echo " $(MAKE) check-avocado Run avocado (integration) tests for currently configured targets" @echo - @echo " $(MAKE) check-report.tap Generates an aggregated TAP test report" - @echo " $(MAKE) check-venv Creates a Python venv for tests" - @echo " $(MAKE) check-clean Clean the tests and related data" + @echo " $(MAKE) check-report.junit.xml Generates an aggregated XML test report" + @echo " $(MAKE) check-venv Creates a Python venv for tests" + @echo " $(MAKE) check-clean Clean the tests and related data" @echo @echo "The following are useful for CI builds" - @echo " $(MAKE) check-build Build most test binaries" - @echo " $(MAKE) get-vm-images Downloads all images used by avocado tests, according to configured targets (~350 MB each, 1.5 GB max)" + @echo " $(MAKE) check-build Build most test binaries" + @echo " $(MAKE) get-vm-images Downloads all images used by avocado tests, according to configured targets (~350 MB each, 1.5 GB max)" @echo @echo @echo "The variable SPEED can be set to control the gtester speed setting." From efb3934adf9ee7794db7e0ade9f576c634592891 Mon Sep 17 00:00:00 2001 From: Igor Mammedov Date: Tue, 24 May 2022 11:10:19 -0400 Subject: [PATCH 16/29] x86: cpu: make sure number of addressable IDs for processor cores meets the spec Accourding Intel's CPUID[EAX=04H] resulting bits 31 - 26 in EAX should be: " **** The nearest power-of-2 integer that is not smaller than (1 + EAX[31:26]) is the number of unique Core_IDs reserved for addressing different processor cores in a physical package. Core ID is a subset of bits of the initial APIC ID. " ensure that values stored in EAX[31-26] always meets this condition. Signed-off-by: Igor Mammedov Message-Id: <20220524151020.2541698-2-imammedo@redhat.com> Signed-off-by: Paolo Bonzini --- target/i386/cpu.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 9fdfec9d8b..94cc4a8700 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -5287,7 +5287,7 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count, /* QEMU gives out its own APIC IDs, never pass down bits 31..26. */ *eax &= ~0xFC000000; if ((*eax & 31) && cs->nr_cores > 1) { - *eax |= (cs->nr_cores - 1) << 26; + *eax |= (pow2ceil(cs->nr_cores) - 1) << 26; } } else if (cpu->vendor_cpuid_only && IS_AMD_CPU(env)) { *eax = *ebx = *ecx = *edx = 0; From d7caf13b5fcf742e5680c1d3448ba070fc811644 Mon Sep 17 00:00:00 2001 From: Igor Mammedov Date: Tue, 24 May 2022 11:10:20 -0400 Subject: [PATCH 17/29] x86: cpu: fixup number of addressable IDs for logical processors sharing cache When QEMU is started with '-cpu host,host-cache-info=on', it will passthrough host's number of logical processors sharing cache and number of processor cores in the physical package. QEMU already fixes up the later to correctly reflect number of configured cores for VM, however number of logical processors sharing cache is still comes from host CPU, which confuses guest started with: -machine q35,accel=kvm \ -cpu host,host-cache-info=on,l3-cache=off \ -smp 20,sockets=2,dies=1,cores=10,threads=1 \ -numa node,nodeid=0,memdev=ram-node0 \ -numa node,nodeid=1,memdev=ram-node1 \ -numa cpu,socket-id=0,node-id=0 \ -numa cpu,socket-id=1,node-id=1 on 2 socket Xeon 4210R host with 10 cores per socket with CPUID[04H]: ... --- cache 3 --- cache type = unified cache (3) cache level = 0x3 (3) self-initializing cache level = true fully associative cache = false maximum IDs for CPUs sharing cache = 0x1f (31) maximum IDs for cores in pkg = 0xf (15) ... that doesn't match number of logical processors VM was configured with and as result RHEL 9.0 guest complains: sched: CPU #10's llc-sibling CPU #0 is not on the same node! [node: 1 != 0]. Ignoring dependency. WARNING: CPU: 10 PID: 0 at arch/x86/kernel/smpboot.c:421 topology_sane.isra.0+0x67/0x80 ... Call Trace: set_cpu_sibling_map+0x176/0x590 start_secondary+0x5b/0x150 secondary_startup_64_no_verify+0xc2/0xcb Fix it by capping max number of logical processors to vcpus/socket as it was configured, which fixes the issue. Signed-off-by: Igor Mammedov Fixes: https://bugzilla.redhat.com/show_bug.cgi?id=2088311 Message-Id: <20220524151020.2541698-3-imammedo@redhat.com> Signed-off-by: Paolo Bonzini --- target/i386/cpu.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/target/i386/cpu.c b/target/i386/cpu.c index 94cc4a8700..6a57ef13af 100644 --- a/target/i386/cpu.c +++ b/target/i386/cpu.c @@ -5284,10 +5284,22 @@ void cpu_x86_cpuid(CPUX86State *env, uint32_t index, uint32_t count, /* cache info: needed for Core compatibility */ if (cpu->cache_info_passthrough) { x86_cpu_get_cache_cpuid(index, count, eax, ebx, ecx, edx); - /* QEMU gives out its own APIC IDs, never pass down bits 31..26. */ - *eax &= ~0xFC000000; - if ((*eax & 31) && cs->nr_cores > 1) { - *eax |= (pow2ceil(cs->nr_cores) - 1) << 26; + /* + * QEMU has its own number of cores/logical cpus, + * set 24..14, 31..26 bit to configured values + */ + if (*eax & 31) { + int host_vcpus_per_cache = 1 + ((*eax & 0x3FFC000) >> 14); + int vcpus_per_socket = env->nr_dies * cs->nr_cores * + cs->nr_threads; + if (cs->nr_cores > 1) { + *eax &= ~0xFC000000; + *eax |= (pow2ceil(cs->nr_cores) - 1) << 26; + } + if (host_vcpus_per_cache > vcpus_per_socket) { + *eax &= ~0x3FFC000; + *eax |= (pow2ceil(vcpus_per_socket) - 1) << 14; + } } } else if (cpu->vendor_cpuid_only && IS_AMD_CPU(env)) { *eax = *ebx = *ecx = *edx = 0; From e7874a50ff3f5b20fb46f36958ad88334e4d55cd Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 25 May 2022 20:09:13 -0400 Subject: [PATCH 18/29] python: update for mypy 0.950 typeshed (included in mypy) recently updated to improve the typing for WriteTransport objects. I was working around this, but now there's a version where I shouldn't work around it. Unfortunately this creates some minor ugliness if I want to support both pre- and post-0.950 versions. For now, for my sanity, just disable the unused-ignores warning. Signed-off-by: John Snow Reviewed-by: Paolo Bonzini Message-Id: <20220526000921.1581503-2-jsnow@redhat.com> Signed-off-by: Paolo Bonzini --- python/qemu/qmp/util.py | 4 +++- python/setup.cfg | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/python/qemu/qmp/util.py b/python/qemu/qmp/util.py index eaa5fc7d5f..ca6225e9cd 100644 --- a/python/qemu/qmp/util.py +++ b/python/qemu/qmp/util.py @@ -40,7 +40,9 @@ async def flush(writer: asyncio.StreamWriter) -> None: drain. The flow control limits are restored after the call is completed. """ - transport = cast(asyncio.WriteTransport, writer.transport) + transport = cast( # type: ignore[redundant-cast] + asyncio.WriteTransport, writer.transport + ) # https://github.com/python/typeshed/issues/5779 low, high = transport.get_write_buffer_limits() # type: ignore diff --git a/python/setup.cfg b/python/setup.cfg index e877ea5647..c2c61c7519 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -79,6 +79,7 @@ strict = True python_version = 3.6 warn_unused_configs = True namespace_packages = True +warn_unused_ignores = False [mypy-qemu.utils.qom_fuse] # fusepy has no type stubs: From caa0f7cbaf4159186d614849ddfbdd89f26d597c Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 25 May 2022 20:09:14 -0400 Subject: [PATCH 19/29] tests: add "TESTS_PYTHON" variable to Makefile This is a convenience feature: $(PYTHON) points to the Python executable we were instructed to use by the configure script. We use that Python to create a virtual environment with the "check-venv" target in tests/Makefile.include. $(TESTS_PYTHON) points to the Python executable belonging to the virtual environment tied to the build. This Python executable is a symlink to the binary used to create the venv, which will be the version provided at configure time. Using $(TESTS_PYTHON) therefore uses the $(PYTHON) executable, but with paths modified to use packages installed to the venv. Signed-off-by: John Snow Reviewed-by: Paolo Bonzini Message-Id: <20220526000921.1581503-3-jsnow@redhat.com> Signed-off-by: Paolo Bonzini --- tests/Makefile.include | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/Makefile.include b/tests/Makefile.include index 2a1c77440a..50237f073a 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -89,6 +89,7 @@ TARGETS=$(patsubst libqemu-%.fa, %, $(filter libqemu-%.fa, $(ninja-targets))) TESTS_VENV_DIR=$(BUILD_DIR)/tests/venv TESTS_VENV_REQ=$(SRC_PATH)/tests/requirements.txt TESTS_RESULTS_DIR=$(BUILD_DIR)/tests/results +TESTS_PYTHON=$(TESTS_VENV_DIR)/bin/python ifndef AVOCADO_TESTS AVOCADO_TESTS=tests/avocado endif @@ -108,7 +109,7 @@ $(TESTS_VENV_DIR): $(TESTS_VENV_REQ) $(PYTHON) -m venv $@, \ VENV, $@) $(call quiet-command, \ - $(TESTS_VENV_DIR)/bin/python -m pip -q install -r $(TESTS_VENV_REQ), \ + $(TESTS_PYTHON) -m pip -q install -r $(TESTS_VENV_REQ), \ PIP, $(TESTS_VENV_REQ)) $(call quiet-command, touch $@) @@ -126,7 +127,7 @@ FEDORA_31_DOWNLOAD=$(filter $(FEDORA_31_ARCHES),$(FEDORA_31_ARCHES_CANDIDATES)) # download one specific Fedora 31 image get-vm-image-fedora-31-%: check-venv $(call quiet-command, \ - $(TESTS_VENV_DIR)/bin/python -m avocado vmimage get \ + $(TESTS_PYTHON) -m avocado vmimage get \ --distro=fedora --distro-version=31 --arch=$*, \ "AVOCADO", "Downloading avocado tests VM image for $*") @@ -135,7 +136,7 @@ get-vm-images: check-venv $(patsubst %,get-vm-image-fedora-31-%, $(FEDORA_31_DOW check-avocado: check-venv $(TESTS_RESULTS_DIR) get-vm-images $(call quiet-command, \ - $(TESTS_VENV_DIR)/bin/python -m avocado \ + $(TESTS_PYTHON) -m avocado \ --show=$(AVOCADO_SHOW) run --job-results-dir=$(TESTS_RESULTS_DIR) \ $(if $(AVOCADO_TAGS),, --filter-by-tags-include-empty \ --filter-by-tags-include-empty-key) \ From aa19dd33ce616c1a34bb569da8e642f0b6f51fa0 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 25 May 2022 20:09:15 -0400 Subject: [PATCH 20/29] tests: use python3 as the python executable name Use "python3" instead of "python" as per PEP0394: https://peps.python.org/pep-0394/ This should always be defined (in a venv, at least!), matching the preferred python shebang of "#!/usr/bin/env python3". Signed-off-by: John Snow Reviewed-by: Paolo Bonzini Message-Id: <20220526000921.1581503-4-jsnow@redhat.com> Signed-off-by: Paolo Bonzini --- tests/Makefile.include | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/Makefile.include b/tests/Makefile.include index 50237f073a..fae2dd0f3c 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -89,7 +89,7 @@ TARGETS=$(patsubst libqemu-%.fa, %, $(filter libqemu-%.fa, $(ninja-targets))) TESTS_VENV_DIR=$(BUILD_DIR)/tests/venv TESTS_VENV_REQ=$(SRC_PATH)/tests/requirements.txt TESTS_RESULTS_DIR=$(BUILD_DIR)/tests/results -TESTS_PYTHON=$(TESTS_VENV_DIR)/bin/python +TESTS_PYTHON=$(TESTS_VENV_DIR)/bin/python3 ifndef AVOCADO_TESTS AVOCADO_TESTS=tests/avocado endif From a547c82701925400949856488a2c8c8750a3276e Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 25 May 2022 20:09:16 -0400 Subject: [PATCH 21/29] tests: silence pip upgrade warnings during venv creation Turn off the nag warning coaxing us to upgrade pip. It's not really that interesting to see in CI logs, and as long as nothing is broken -- nothing is broken. Signed-off-by: John Snow Reviewed-by: Paolo Bonzini Message-Id: <20220526000921.1581503-5-jsnow@redhat.com> Signed-off-by: Paolo Bonzini --- tests/Makefile.include | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/Makefile.include b/tests/Makefile.include index fae2dd0f3c..ba9f5bd65c 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -109,8 +109,8 @@ $(TESTS_VENV_DIR): $(TESTS_VENV_REQ) $(PYTHON) -m venv $@, \ VENV, $@) $(call quiet-command, \ - $(TESTS_PYTHON) -m pip -q install -r $(TESTS_VENV_REQ), \ - PIP, $(TESTS_VENV_REQ)) + $(TESTS_PYTHON) -m pip -q --disable-pip-version-check install \ + -r $(TESTS_VENV_REQ), PIP, $(TESTS_VENV_REQ)) $(call quiet-command, touch $@) $(TESTS_RESULTS_DIR): From 5a6f1199cb5fca105ca99356ae05f8e48f5c8d97 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 25 May 2022 20:09:17 -0400 Subject: [PATCH 22/29] tests: add quiet-venv-pip macro Factor out the "test venv pip" macro; rewrite the "check-venv" rule to be a little more compact. Replace the "PIP" pseudo-command output with "VENVPIP" to make it 1% more clear that we are talking about using pip to install something into a venv. Signed-off-by: John Snow Reviewed-by: Paolo Bonzini Message-Id: <20220526000921.1581503-6-jsnow@redhat.com> Signed-off-by: Paolo Bonzini --- tests/Makefile.include | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/Makefile.include b/tests/Makefile.include index ba9f5bd65c..fa46c0c61b 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -104,13 +104,13 @@ else AVOCADO_CMDLINE_TAGS=$(addprefix -t , $(AVOCADO_TAGS)) endif +quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \ + $(TESTS_PYTHON) -m pip -q --disable-pip-version-check $1, \ + "VENVPIP","$1") + $(TESTS_VENV_DIR): $(TESTS_VENV_REQ) - $(call quiet-command, \ - $(PYTHON) -m venv $@, \ - VENV, $@) - $(call quiet-command, \ - $(TESTS_PYTHON) -m pip -q --disable-pip-version-check install \ - -r $(TESTS_VENV_REQ), PIP, $(TESTS_VENV_REQ)) + $(call quiet-command, $(PYTHON) -m venv $@, VENV, $@) + $(call quiet-venv-pip,install -r $(TESTS_VENV_REQ)) $(call quiet-command, touch $@) $(TESTS_RESULTS_DIR): From 0e7647aa8a5d5b99a0ffb401ba75f0ae8c171c96 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 25 May 2022 20:09:18 -0400 Subject: [PATCH 23/29] tests: install "qemu" namespace package into venv This patch adds the "qemu" namespace package to the $build/tests/venv directory. It does so in "editable" mode, which means that changes to the source python directory will actively be reflected by the venv. This patch also then removes any sys.path hacking from the avocado test scripts directly. By doing this, the environment of where to find these packages is managed entirely by the virtual environment and not by the scripts themselves. Signed-off-by: John Snow Reviewed-by: Paolo Bonzini Message-Id: <20220526000921.1581503-7-jsnow@redhat.com> Signed-off-by: Paolo Bonzini --- tests/Makefile.include | 1 + tests/avocado/avocado_qemu/__init__.py | 11 +++++------ tests/avocado/virtio_check_params.py | 1 - tests/avocado/virtio_version.py | 1 - tests/requirements.txt | 1 + 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/Makefile.include b/tests/Makefile.include index fa46c0c61b..3accb83b13 100644 --- a/tests/Makefile.include +++ b/tests/Makefile.include @@ -110,6 +110,7 @@ quiet-venv-pip = $(quiet-@)$(call quiet-command-run, \ $(TESTS_VENV_DIR): $(TESTS_VENV_REQ) $(call quiet-command, $(PYTHON) -m venv $@, VENV, $@) + $(call quiet-venv-pip,install -e "$(SRC_PATH)/python/") $(call quiet-venv-pip,install -r $(TESTS_VENV_REQ)) $(call quiet-command, touch $@) diff --git a/tests/avocado/avocado_qemu/__init__.py b/tests/avocado/avocado_qemu/__init__.py index 39f15c1d51..b656a70c55 100644 --- a/tests/avocado/avocado_qemu/__init__.py +++ b/tests/avocado/avocado_qemu/__init__.py @@ -21,6 +21,11 @@ import avocado from avocado.utils import cloudinit, datadrainer, process, ssh, vmimage from avocado.utils.path import find_command +from qemu.machine import QEMUMachine +from qemu.utils import (get_info_usernet_hostfwd_port, kvm_available, + tcg_available) + + #: The QEMU build root directory. It may also be the source directory #: if building from the source dir, but it's safer to use BUILD_DIR for #: that purpose. Be aware that if this code is moved outside of a source @@ -35,12 +40,6 @@ if os.path.islink(os.path.dirname(os.path.dirname(__file__))): else: SOURCE_DIR = BUILD_DIR -sys.path.append(os.path.join(SOURCE_DIR, 'python')) - -from qemu.machine import QEMUMachine -from qemu.utils import (get_info_usernet_hostfwd_port, kvm_available, - tcg_available) - def has_cmd(name, args=None): """ diff --git a/tests/avocado/virtio_check_params.py b/tests/avocado/virtio_check_params.py index e869690473..4093da8a67 100644 --- a/tests/avocado/virtio_check_params.py +++ b/tests/avocado/virtio_check_params.py @@ -22,7 +22,6 @@ import os import re import logging -sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) from qemu.machine import QEMUMachine from avocado_qemu import QemuSystemTest from avocado import skip diff --git a/tests/avocado/virtio_version.py b/tests/avocado/virtio_version.py index 208910bb84..c84e48813a 100644 --- a/tests/avocado/virtio_version.py +++ b/tests/avocado/virtio_version.py @@ -11,7 +11,6 @@ Check compatibility of virtio device types import sys import os -sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) from qemu.machine import QEMUMachine from avocado_qemu import QemuSystemTest diff --git a/tests/requirements.txt b/tests/requirements.txt index a21b59b443..0ba561b6bd 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,6 @@ # Add Python module requirements, one per line, to be installed # in the tests/venv Python virtual environment. For more info, # refer to: https://pip.pypa.io/en/stable/user_guide/#id1 +# Note that qemu.git/python/ is always implicitly installed. avocado-framework==88.1 pycdlib==1.11.0 From f4c66f17051f0dee0e6070c6b78a78e7425b5c96 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 25 May 2022 20:09:19 -0400 Subject: [PATCH 24/29] tests: use tests/venv to run basevm.py-based scripts This patch co-opts the virtual environment being used by avocado tests to also run the basevm.py tests. This is being done in preparation for for the qemu.qmp package being removed from qemu.git. As part of the change, remove any sys.path() hacks and treat "qemu" as a normal third-party import. Signed-off-by: John Snow Reviewed-by: Paolo Bonzini Message-Id: <20220526000921.1581503-8-jsnow@redhat.com> Signed-off-by: Paolo Bonzini --- tests/vm/Makefile.include | 13 +++++++------ tests/vm/basevm.py | 6 +++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/vm/Makefile.include b/tests/vm/Makefile.include index ae91f5043e..588bc999cc 100644 --- a/tests/vm/Makefile.include +++ b/tests/vm/Makefile.include @@ -84,10 +84,11 @@ vm-clean-all: $(IMAGES_DIR)/%.img: $(SRC_PATH)/tests/vm/% \ $(SRC_PATH)/tests/vm/basevm.py \ - $(SRC_PATH)/tests/vm/Makefile.include + $(SRC_PATH)/tests/vm/Makefile.include \ + check-venv @mkdir -p $(IMAGES_DIR) $(call quiet-command, \ - $(PYTHON) $< \ + $(TESTS_PYTHON) $< \ $(if $(V)$(DEBUG), --debug) \ $(if $(GENISOIMAGE),--genisoimage $(GENISOIMAGE)) \ $(if $(QEMU_LOCAL),--build-path $(BUILD_DIR)) \ @@ -101,9 +102,9 @@ $(IMAGES_DIR)/%.img: $(SRC_PATH)/tests/vm/% \ # Build in VM $(IMAGE) -vm-build-%: $(IMAGES_DIR)/%.img +vm-build-%: $(IMAGES_DIR)/%.img check-venv $(call quiet-command, \ - $(PYTHON) $(SRC_PATH)/tests/vm/$* \ + $(TESTS_PYTHON) $(SRC_PATH)/tests/vm/$* \ $(if $(V)$(DEBUG), --debug) \ $(if $(DEBUG), --interactive) \ $(if $(J),--jobs $(J)) \ @@ -127,9 +128,9 @@ vm-boot-serial-%: $(IMAGES_DIR)/%.img -device virtio-net-pci,netdev=vnet \ || true -vm-boot-ssh-%: $(IMAGES_DIR)/%.img +vm-boot-ssh-%: $(IMAGES_DIR)/%.img check-venv $(call quiet-command, \ - $(PYTHON) $(SRC_PATH)/tests/vm/$* \ + $(TESTS_PYTHON) $(SRC_PATH)/tests/vm/$* \ $(if $(J),--jobs $(J)) \ $(if $(V)$(DEBUG), --debug) \ $(if $(QEMU_LOCAL),--build-path $(BUILD_DIR)) \ diff --git a/tests/vm/basevm.py b/tests/vm/basevm.py index 254e11c932..d7d0413df3 100644 --- a/tests/vm/basevm.py +++ b/tests/vm/basevm.py @@ -18,9 +18,6 @@ import socket import logging import time import datetime -sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu.machine import QEMUMachine -from qemu.utils import get_info_usernet_hostfwd_port, kvm_available import subprocess import hashlib import argparse @@ -31,6 +28,9 @@ import multiprocessing import traceback import shlex +from qemu.machine import QEMUMachine +from qemu.utils import get_info_usernet_hostfwd_port, kvm_available + SSH_KEY_FILE = os.path.join(os.path.dirname(__file__), "..", "keys", "id_rsa") SSH_PUB_KEY_FILE = os.path.join(os.path.dirname(__file__), From e184036a0e12cbfb8980362cd4de18268b4d0c38 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 25 May 2022 20:09:20 -0400 Subject: [PATCH 25/29] tests: add python3-venv to debian10.docker This is needed to be able to add a venv-building step to 'make check'; the clang-user job in particular needs this to be able to run check-unit. Signed-off-by: John Snow Reviewed-by: Paolo Bonzini Message-Id: <20220526000921.1581503-9-jsnow@redhat.com> Signed-off-by: Paolo Bonzini --- tests/docker/dockerfiles/debian10.docker | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/docker/dockerfiles/debian10.docker b/tests/docker/dockerfiles/debian10.docker index b414af1b9f..03be923066 100644 --- a/tests/docker/dockerfiles/debian10.docker +++ b/tests/docker/dockerfiles/debian10.docker @@ -34,4 +34,5 @@ RUN apt update && \ python3 \ python3-sphinx \ python3-sphinx-rtd-theme \ + python3-venv \ $(apt-get -s build-dep --arch-only qemu | egrep ^Inst | fgrep '[all]' | cut -d\ -f2) From 1d8cf47e5b8b4faa95954824cb6610d81c50d7d2 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 25 May 2022 20:09:21 -0400 Subject: [PATCH 26/29] tests: run 'device-crash-test' from tests/venv Remove the sys.path hacking from device-crash-test, and add in a little user-friendly message for anyone who was used to running this script directly from the source tree. Modify the GitLab job recipes to create the tests/venv first, then run device-crash-test from that venv. Signed-off-by: John Snow Reviewed-by: Paolo Bonzini Message-Id: <20220526000921.1581503-10-jsnow@redhat.com> Signed-off-by: Paolo Bonzini --- .gitlab-ci.d/buildtest.yml | 8 +++++--- scripts/device-crash-test | 14 +++++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.d/buildtest.yml b/.gitlab-ci.d/buildtest.yml index ecac3ec50c..544385f5be 100644 --- a/.gitlab-ci.d/buildtest.yml +++ b/.gitlab-ci.d/buildtest.yml @@ -110,7 +110,8 @@ crash-test-debian: IMAGE: debian-amd64 script: - cd build - - scripts/device-crash-test -q ./qemu-system-i386 + - make check-venv + - tests/venv/bin/python3 scripts/device-crash-test -q ./qemu-system-i386 build-system-fedora: extends: .native_build_job_template @@ -155,8 +156,9 @@ crash-test-fedora: IMAGE: fedora script: - cd build - - scripts/device-crash-test -q ./qemu-system-ppc - - scripts/device-crash-test -q ./qemu-system-riscv32 + - make check-venv + - tests/venv/bin/python3 scripts/device-crash-test -q ./qemu-system-ppc + - tests/venv/bin/python3 scripts/device-crash-test -q ./qemu-system-riscv32 build-system-centos: extends: .native_build_job_template diff --git a/scripts/device-crash-test b/scripts/device-crash-test index a203b3fdea..73bcb98693 100755 --- a/scripts/device-crash-test +++ b/scripts/device-crash-test @@ -33,10 +33,18 @@ import re import random import argparse from itertools import chain +from pathlib import Path -sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'python')) -from qemu.machine import QEMUMachine -from qemu.qmp import ConnectError +try: + from qemu.machine import QEMUMachine + from qemu.qmp import ConnectError +except ModuleNotFoundError as exc: + path = Path(__file__).resolve() + print(f"Module '{exc.name}' not found.") + print(" Try 'make check-venv' from your build directory,") + print(" and then one way to run this script is like so:") + print(f' > $builddir/tests/venv/bin/python3 "{path}"') + sys.exit(1) logger = logging.getLogger('device-crash-test') dbg = logger.debug From d4bb4432ab43febc0eb41b031c0c3aaf5113a0c6 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 6 Jun 2022 12:44:45 +0200 Subject: [PATCH 27/29] regenerate meson-buildoptions.sh Signed-off-by: Paolo Bonzini --- scripts/meson-buildoptions.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/meson-buildoptions.sh b/scripts/meson-buildoptions.sh index 731e5ea1cf..00ea4d8cd1 100644 --- a/scripts/meson-buildoptions.sh +++ b/scripts/meson-buildoptions.sh @@ -156,13 +156,13 @@ meson_options_help() { printf "%s\n" ' vhost-kernel vhost kernel backend support' printf "%s\n" ' vhost-net vhost-net kernel acceleration support' printf "%s\n" ' vhost-user vhost-user backend support' - printf "%s\n" ' vmnet vmnet.framework network backend support' printf "%s\n" ' vhost-user-blk-server' printf "%s\n" ' build vhost-user-blk server' printf "%s\n" ' vhost-vdpa vhost-vdpa kernel backend support' printf "%s\n" ' virglrenderer virgl rendering support' printf "%s\n" ' virtfs virtio-9p support' printf "%s\n" ' virtiofsd build virtiofs daemon (virtiofsd)' + printf "%s\n" ' vmnet vmnet.framework network backend support' printf "%s\n" ' vnc VNC server' printf "%s\n" ' vnc-jpeg JPEG lossy compression for VNC server' printf "%s\n" ' vnc-sasl SASL authentication for VNC server' @@ -430,6 +430,8 @@ _meson_option_parse() { --disable-virtfs) printf "%s" -Dvirtfs=disabled ;; --enable-virtiofsd) printf "%s" -Dvirtiofsd=enabled ;; --disable-virtiofsd) printf "%s" -Dvirtiofsd=disabled ;; + --enable-vmnet) printf "%s" -Dvmnet=enabled ;; + --disable-vmnet) printf "%s" -Dvmnet=disabled ;; --enable-vnc) printf "%s" -Dvnc=enabled ;; --disable-vnc) printf "%s" -Dvnc=disabled ;; --enable-vnc-jpeg) printf "%s" -Dvnc_jpeg=enabled ;; From 7632a38e7c21c993da74d17afa8dd66d9552da06 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 6 Jun 2022 12:44:57 +0200 Subject: [PATCH 28/29] configure: remove reference to removed option Signed-off-by: Paolo Bonzini --- configure | 1 - 1 file changed, 1 deletion(-) diff --git a/configure b/configure index b9ccff9067..ac18ed4f3a 100755 --- a/configure +++ b/configure @@ -1035,7 +1035,6 @@ Advanced options (experts only): --with-git-submodules=ignore do not update or check git submodules (default if no .git dir) --static enable static build [$static] --bindir=PATH install binaries in PATH - --efi-aarch64=PATH PATH of efi file to use for aarch64 VMs. --with-suffix=SUFFIX suffix for QEMU data inside datadir/libdir/sysconfdir/docdir [$qemu_suffix] --without-default-features default all --enable-* options to "disabled" --without-default-devices do not include any device that is not needed to From ca5765c8521b8284b8277ed4b811ac0ab9010fc8 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Mon, 6 Jun 2022 16:03:38 +0200 Subject: [PATCH 29/29] meson: qga: do not use deprecated meson.build_root() The function will return the build root of the parent project if called from a subproject; that is irrelevant for QEMU's usage but rarely desirable, and therefore the function was deprecated and replaced by two functions project_build_root() and global_build_root(). Replace it with the former. Signed-off-by: Paolo Bonzini --- qga/meson.build | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qga/meson.build b/qga/meson.build index 619ff095bc..65c1e93846 100644 --- a/qga/meson.build +++ b/qga/meson.build @@ -125,7 +125,7 @@ if targetos == 'windows' wixl, '-o', '@OUTPUT0@', '@INPUT0@', qemu_ga_msi_arch[cpu], qemu_ga_msi_vss, - '-D', 'BUILD_DIR=' + meson.build_root(), + '-D', 'BUILD_DIR=' + meson.project_build_root(), '-D', 'BIN_DIR=' + glib.get_variable('bindir'), '-D', 'QEMU_GA_VERSION=' + config_host['QEMU_GA_VERSION'], '-D', 'QEMU_GA_MANUFACTURER=' + config_host['QEMU_GA_MANUFACTURER'],