From d60c75d28fe9d49bef28f7fa863481e1d9f6ff96 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Wed, 2 Jun 2021 16:41:05 +0800 Subject: [PATCH 01/12] vhost-vdpa: skip ram device from the IOTLB mapping vDPA is not tie to any specific hardware, for safety and simplicity, vhost-vDPA doesn't allow MMIO area to be mapped via IOTLB. Only the doorbell could be mapped via mmap(). So this patch exclude skip the ram device from the IOTLB mapping. Reviewed-by: Si-Wei Liu Signed-off-by: Jason Wang --- hw/virtio/vhost-vdpa.c | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c index ee51863d28..aef5055abf 100644 --- a/hw/virtio/vhost-vdpa.c +++ b/hw/virtio/vhost-vdpa.c @@ -28,6 +28,8 @@ static bool vhost_vdpa_listener_skipped_section(MemoryRegionSection *section) { return (!memory_region_is_ram(section->mr) && !memory_region_is_iommu(section->mr)) || + /* vhost-vDPA doesn't allow MMIO to be mapped */ + memory_region_is_ram_device(section->mr) || /* * Sizing an enabled 64-bit BAR can cause spurious mappings to * addresses in the upper part of the 64-bit address space. These @@ -172,22 +174,12 @@ static void vhost_vdpa_listener_region_add(MemoryListener *listener, vaddr, section->readonly); if (ret) { error_report("vhost vdpa map fail!"); - if (memory_region_is_ram_device(section->mr)) { - /* Allow unexpected mappings not to be fatal for RAM devices */ - error_report("map ram fail!"); - return ; - } goto fail; } return; fail: - if (memory_region_is_ram_device(section->mr)) { - error_report("failed to vdpa_dma_map. pci p2p may not work"); - return; - - } /* * On the initfn path, store the first error in the container so we * can gracefully fail. Runtime, there's not much we can do other From d0416d487bd561526486c2e43910ea9494855e16 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Thu, 15 Apr 2021 15:33:56 +0800 Subject: [PATCH 02/12] vhost-vdpa: map virtqueue notification area if possible This patch implements the vq notification mapping support for vhost-vDPA. This is simply done by using mmap()/munmap() for the vhost-vDPA fd during device start/stop. For the device without notification mapping support, we fall back to eventfd based notification gracefully. Reviewed-by: Si-Wei Liu Signed-off-by: Jason Wang --- hw/virtio/vhost-vdpa.c | 85 ++++++++++++++++++++++++++++++++++ include/hw/virtio/vhost-vdpa.h | 6 +++ 2 files changed, 91 insertions(+) diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c index aef5055abf..7bcccf10fc 100644 --- a/hw/virtio/vhost-vdpa.c +++ b/hw/virtio/vhost-vdpa.c @@ -286,12 +286,95 @@ static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque) return 0; } +static void vhost_vdpa_host_notifier_uninit(struct vhost_dev *dev, + int queue_index) +{ + size_t page_size = qemu_real_host_page_size; + struct vhost_vdpa *v = dev->opaque; + VirtIODevice *vdev = dev->vdev; + VhostVDPAHostNotifier *n; + + n = &v->notifier[queue_index]; + + if (n->addr) { + virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, false); + object_unparent(OBJECT(&n->mr)); + munmap(n->addr, page_size); + n->addr = NULL; + } +} + +static void vhost_vdpa_host_notifiers_uninit(struct vhost_dev *dev, int n) +{ + int i; + + for (i = 0; i < n; i++) { + vhost_vdpa_host_notifier_uninit(dev, i); + } +} + +static int vhost_vdpa_host_notifier_init(struct vhost_dev *dev, int queue_index) +{ + size_t page_size = qemu_real_host_page_size; + struct vhost_vdpa *v = dev->opaque; + VirtIODevice *vdev = dev->vdev; + VhostVDPAHostNotifier *n; + int fd = v->device_fd; + void *addr; + char *name; + + vhost_vdpa_host_notifier_uninit(dev, queue_index); + + n = &v->notifier[queue_index]; + + addr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, fd, + queue_index * page_size); + if (addr == MAP_FAILED) { + goto err; + } + + name = g_strdup_printf("vhost-vdpa/host-notifier@%p mmaps[%d]", + v, queue_index); + memory_region_init_ram_device_ptr(&n->mr, OBJECT(vdev), name, + page_size, addr); + g_free(name); + + if (virtio_queue_set_host_notifier_mr(vdev, queue_index, &n->mr, true)) { + munmap(addr, page_size); + goto err; + } + n->addr = addr; + + return 0; + +err: + return -1; +} + +static void vhost_vdpa_host_notifiers_init(struct vhost_dev *dev) +{ + int i; + + for (i = dev->vq_index; i < dev->vq_index + dev->nvqs; i++) { + if (vhost_vdpa_host_notifier_init(dev, i)) { + goto err; + } + } + + return; + +err: + vhost_vdpa_host_notifiers_uninit(dev, i); + return; +} + static int vhost_vdpa_cleanup(struct vhost_dev *dev) { struct vhost_vdpa *v; assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); v = dev->opaque; trace_vhost_vdpa_cleanup(dev, v); + vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); memory_listener_unregister(&v->listener); dev->opaque = NULL; @@ -468,6 +551,7 @@ static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started) if (started) { uint8_t status = 0; memory_listener_register(&v->listener, &address_space_memory); + vhost_vdpa_host_notifiers_init(dev); vhost_vdpa_set_vring_ready(dev); vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_DRIVER_OK); vhost_vdpa_call(dev, VHOST_VDPA_GET_STATUS, &status); @@ -477,6 +561,7 @@ static int vhost_vdpa_dev_start(struct vhost_dev *dev, bool started) vhost_vdpa_reset_device(dev); vhost_vdpa_add_status(dev, VIRTIO_CONFIG_S_ACKNOWLEDGE | VIRTIO_CONFIG_S_DRIVER); + vhost_vdpa_host_notifiers_uninit(dev, dev->nvqs); memory_listener_unregister(&v->listener); return 0; diff --git a/include/hw/virtio/vhost-vdpa.h b/include/hw/virtio/vhost-vdpa.h index ae9ee7adb2..9188226d8b 100644 --- a/include/hw/virtio/vhost-vdpa.h +++ b/include/hw/virtio/vhost-vdpa.h @@ -14,11 +14,17 @@ #include "hw/virtio/virtio.h" +typedef struct VhostVDPAHostNotifier { + MemoryRegion mr; + void *addr; +} VhostVDPAHostNotifier; + typedef struct vhost_vdpa { int device_fd; uint32_t msg_type; MemoryListener listener; struct vhost_dev *dev; + VhostVDPAHostNotifier notifier[VIRTIO_QUEUE_MAX]; } VhostVDPA; #endif From c33f23a419f95da16ab4faaf08be635c89b96ff0 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Wed, 2 Jun 2021 11:31:26 +0800 Subject: [PATCH 03/12] vhost-vdpa: don't initialize backend_features We used to initialize backend_features during vhost_vdpa_init() regardless whether or not it was supported by vhost. This will lead the unsupported features like VIRTIO_F_IN_ORDER to be included and set to the vhost-vdpa during vhost_dev_start. Because the VIRTIO_F_IN_ORDER is not supported by vhost-vdpa so it won't be advertised to guest which will break the datapath. Fix this by not initializing the backend_features, so the acked_features could be built only from guest features via vhost_net_ack_features(). Fixes: 108a64818e69b ("vhost-vdpa: introduce vhost-vdpa backend") Cc: qemu-stable@nongnu.org Cc: Gautam Dawar Signed-off-by: Jason Wang --- hw/virtio/vhost-vdpa.c | 3 --- 1 file changed, 3 deletions(-) diff --git a/hw/virtio/vhost-vdpa.c b/hw/virtio/vhost-vdpa.c index 7bcccf10fc..61ba313331 100644 --- a/hw/virtio/vhost-vdpa.c +++ b/hw/virtio/vhost-vdpa.c @@ -268,15 +268,12 @@ static void vhost_vdpa_add_status(struct vhost_dev *dev, uint8_t status) static int vhost_vdpa_init(struct vhost_dev *dev, void *opaque) { struct vhost_vdpa *v; - uint64_t features; assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_VDPA); trace_vhost_vdpa_init(dev, opaque); v = opaque; v->dev = dev; dev->opaque = opaque ; - vhost_vdpa_call(dev, VHOST_GET_FEATURES, &features); - dev->backend_features = features; v->listener = vhost_vdpa_memory_listener; v->msg_type = VHOST_IOTLB_MSG_V2; From 1b031172119c8996af5fa84b6e10627ea4122d64 Mon Sep 17 00:00:00 2001 From: Jason Wang Date: Wed, 2 Jun 2021 11:31:27 +0800 Subject: [PATCH 04/12] vhost-vdpa: remove the unused vhost_vdpa_get_acked_features() No user for this helper, let's remove it. Signed-off-by: Jason Wang --- include/net/vhost-vdpa.h | 1 - net/vhost-vdpa.c | 9 --------- 2 files changed, 10 deletions(-) diff --git a/include/net/vhost-vdpa.h b/include/net/vhost-vdpa.h index 45e34b7cfc..b81f9a6f2a 100644 --- a/include/net/vhost-vdpa.h +++ b/include/net/vhost-vdpa.h @@ -15,7 +15,6 @@ #define TYPE_VHOST_VDPA "vhost-vdpa" struct vhost_net *vhost_vdpa_get_vhost_net(NetClientState *nc); -uint64_t vhost_vdpa_get_acked_features(NetClientState *nc); extern const int vdpa_feature_bits[]; diff --git a/net/vhost-vdpa.c b/net/vhost-vdpa.c index 8b14215549..19187dce8c 100644 --- a/net/vhost-vdpa.c +++ b/net/vhost-vdpa.c @@ -68,15 +68,6 @@ VHostNetState *vhost_vdpa_get_vhost_net(NetClientState *nc) return s->vhost_net; } -uint64_t vhost_vdpa_get_acked_features(NetClientState *nc) -{ - VhostVDPAState *s = DO_UPCAST(VhostVDPAState, nc, nc); - assert(nc->info->type == NET_CLIENT_DRIVER_VHOST_VDPA); - s->acked_features = vhost_net_get_acked_features(s->vhost_net); - - return s->acked_features; -} - static int vhost_vdpa_net_check_device_id(struct vhost_net *net) { uint32_t device_id; From 48f596c5a711d62f8b0c78b95569b2d8f9ad2704 Mon Sep 17 00:00:00 2001 From: Paolo Bonzini Date: Tue, 11 May 2021 11:39:55 -0400 Subject: [PATCH 05/12] netdev: add more commands to preconfig mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Creating and destroying network backend does not require a fully constructed machine. Allow the related monitor commands to run before machine initialization has concluded. Signed-off-by: Paolo Bonzini Reviewed-by: Daniel P. Berrangé Signed-off-by: Jason Wang --- hmp-commands.hx | 2 ++ qapi/net.json | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/hmp-commands.hx b/hmp-commands.hx index 84dcc3aae6..8e45bce2cd 100644 --- a/hmp-commands.hx +++ b/hmp-commands.hx @@ -1269,6 +1269,7 @@ ERST .help = "add host network device", .cmd = hmp_netdev_add, .command_completion = netdev_add_completion, + .flags = "p", }, SRST @@ -1283,6 +1284,7 @@ ERST .help = "remove host network device", .cmd = hmp_netdev_del, .command_completion = netdev_del_completion, + .flags = "p", }, SRST diff --git a/qapi/net.json b/qapi/net.json index af3f5b0fda..7fab2e7cd8 100644 --- a/qapi/net.json +++ b/qapi/net.json @@ -55,7 +55,8 @@ # <- { "return": {} } # ## -{ 'command': 'netdev_add', 'data': 'Netdev', 'boxed': true } +{ 'command': 'netdev_add', 'data': 'Netdev', 'boxed': true, + 'allow-preconfig': true } ## # @netdev_del: @@ -75,7 +76,8 @@ # <- { "return": {} } # ## -{ 'command': 'netdev_del', 'data': {'id': 'str'} } +{ 'command': 'netdev_del', 'data': {'id': 'str'}, + 'allow-preconfig': true } ## # @NetLegacyNicOptions: From f1ff9c437711803f4862e580a211b3af8bb369c7 Mon Sep 17 00:00:00 2001 From: "Rao, Lei" Date: Tue, 8 Jun 2021 16:23:25 +0800 Subject: [PATCH 06/12] Remove some duplicate trace code. There is the same trace code in the colo_compare_packet_payload. Signed-off-by: Lei Rao Reviewed-by: Li Zhijian Reviewed-by: Zhang Chen Reviewed-by: Lukas Straub Tested-by: Lukas Straub Signed-off-by: Zhang Chen Signed-off-by: Jason Wang --- net/colo-compare.c | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/net/colo-compare.c b/net/colo-compare.c index 9d1ad99941..c142c08dc6 100644 --- a/net/colo-compare.c +++ b/net/colo-compare.c @@ -590,19 +590,6 @@ static int colo_packet_compare_other(Packet *spkt, Packet *ppkt) uint16_t offset = ppkt->vnet_hdr_len; trace_colo_compare_main("compare other"); - if (trace_event_get_state_backends(TRACE_COLO_COMPARE_IP_INFO)) { - char pri_ip_src[20], pri_ip_dst[20], sec_ip_src[20], sec_ip_dst[20]; - - strcpy(pri_ip_src, inet_ntoa(ppkt->ip->ip_src)); - strcpy(pri_ip_dst, inet_ntoa(ppkt->ip->ip_dst)); - strcpy(sec_ip_src, inet_ntoa(spkt->ip->ip_src)); - strcpy(sec_ip_dst, inet_ntoa(spkt->ip->ip_dst)); - - trace_colo_compare_ip_info(ppkt->size, pri_ip_src, - pri_ip_dst, spkt->size, - sec_ip_src, sec_ip_dst); - } - if (ppkt->size != spkt->size) { trace_colo_compare_main("Other: payload size of packets are different"); return -1; From 229620d52e389bc7690f11e4f5fa9b249b5ca65d Mon Sep 17 00:00:00 2001 From: "Rao, Lei" Date: Tue, 8 Jun 2021 16:23:26 +0800 Subject: [PATCH 07/12] Fix the qemu crash when guest shutdown during checkpoint This patch fixes the following: qemu-system-x86_64: invalid runstate transition: 'colo' ->'shutdown' Aborted (core dumped) Signed-off-by: Lei Rao Reviewed-by: Li Zhijian Reviewed-by: Zhang Chen Reviewed-by: Lukas Straub Tested-by: Lukas Straub Signed-off-by: Zhang Chen Signed-off-by: Jason Wang --- softmmu/runstate.c | 1 + 1 file changed, 1 insertion(+) diff --git a/softmmu/runstate.c b/softmmu/runstate.c index ce8977c6a2..15640572c0 100644 --- a/softmmu/runstate.c +++ b/softmmu/runstate.c @@ -126,6 +126,7 @@ static const RunStateTransition runstate_transitions_def[] = { { RUN_STATE_RESTORE_VM, RUN_STATE_PRELAUNCH }, { RUN_STATE_COLO, RUN_STATE_RUNNING }, + { RUN_STATE_COLO, RUN_STATE_SHUTDOWN}, { RUN_STATE_RUNNING, RUN_STATE_DEBUG }, { RUN_STATE_RUNNING, RUN_STATE_INTERNAL_ERROR }, From 7c2eae9a77234f92be96d33cc20b990db714d86d Mon Sep 17 00:00:00 2001 From: "Rao, Lei" Date: Tue, 8 Jun 2021 16:23:27 +0800 Subject: [PATCH 08/12] Optimize the function of filter_send The iov_size has been calculated in filter_send(). we can directly return the size.In this way, this is no need to repeat calculations in filter_redirector_receive_iov(); Signed-off-by: Lei Rao Reviewed-by: Li Zhijian Reviewed-by: Zhang Chen Reviewed-by: Lukas Straub Tested-by: Lukas Straub Signed-off-by: Zhang Chen Signed-off-by: Jason Wang --- net/filter-mirror.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/net/filter-mirror.c b/net/filter-mirror.c index f8e65007c0..f20240cc9f 100644 --- a/net/filter-mirror.c +++ b/net/filter-mirror.c @@ -88,7 +88,7 @@ static int filter_send(MirrorState *s, goto err; } - return 0; + return size; err: return ret < 0 ? ret : -EIO; @@ -159,7 +159,7 @@ static ssize_t filter_mirror_receive_iov(NetFilterState *nf, int ret; ret = filter_send(s, iov, iovcnt); - if (ret) { + if (ret < 0) { error_report("filter mirror send failed(%s)", strerror(-ret)); } @@ -182,10 +182,10 @@ static ssize_t filter_redirector_receive_iov(NetFilterState *nf, if (qemu_chr_fe_backend_connected(&s->chr_out)) { ret = filter_send(s, iov, iovcnt); - if (ret) { + if (ret < 0) { error_report("filter redirector send failed(%s)", strerror(-ret)); } - return iov_size(iov, iovcnt); + return ret; } else { return 0; } From 3ba024457facdb6b0ef9c5c742261d4080a80a11 Mon Sep 17 00:00:00 2001 From: "Rao, Lei" Date: Tue, 8 Jun 2021 16:23:28 +0800 Subject: [PATCH 09/12] Remove migrate_set_block_enabled in checkpoint We can detect disk migration in migrate_prepare, if disk migration is enabled in COLO mode, we can directly report an error.and there is no need to disable block migration at every checkpoint. Signed-off-by: Lei Rao Signed-off-by: Zhang Chen Reviewed-by: Li Zhijian Reviewed-by: Zhang Chen Reviewed-by: Lukas Straub Tested-by: Lukas Straub Signed-off-by: Jason Wang --- migration/colo.c | 6 ------ migration/migration.c | 4 ++++ 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/migration/colo.c b/migration/colo.c index e498fdb125..79fa1f6619 100644 --- a/migration/colo.c +++ b/migration/colo.c @@ -435,12 +435,6 @@ static int colo_do_checkpoint_transaction(MigrationState *s, if (failover_get_state() != FAILOVER_STATUS_NONE) { goto out; } - - /* Disable block migration */ - migrate_set_block_enabled(false, &local_err); - if (local_err) { - goto out; - } qemu_mutex_lock_iothread(); #ifdef CONFIG_REPLICATION diff --git a/migration/migration.c b/migration/migration.c index 393299e150..4828997f63 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -2217,6 +2217,10 @@ static bool migrate_prepare(MigrationState *s, bool blk, bool blk_inc, } if (blk || blk_inc) { + if (migrate_colo_enabled()) { + error_setg(errp, "No disk migration is required in COLO mode"); + return false; + } if (migrate_use_block() || migrate_use_block_incremental()) { error_setg(errp, "Command options are incompatible with " "current migration capabilities"); From 9b492719dd0445c676da6805c84f9a5893583d1c Mon Sep 17 00:00:00 2001 From: "Rao, Lei" Date: Tue, 8 Jun 2021 16:23:29 +0800 Subject: [PATCH 10/12] Add a function named packet_new_nocopy for COLO. Use the packet_new_nocopy instead of packet_new in the filter-rewriter module. There will be one less memory copy in the processing of each network packet. Signed-off-by: Lei Rao Signed-off-by: Zhang Chen Reviewed-by: Zhang Chen Signed-off-by: Jason Wang --- net/colo.c | 25 +++++++++++++++++-------- net/colo.h | 1 + net/filter-rewriter.c | 3 +-- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/net/colo.c b/net/colo.c index ef00609848..3a3e6e89a0 100644 --- a/net/colo.c +++ b/net/colo.c @@ -157,19 +157,28 @@ void connection_destroy(void *opaque) Packet *packet_new(const void *data, int size, int vnet_hdr_len) { - Packet *pkt = g_slice_new(Packet); + Packet *pkt = g_slice_new0(Packet); pkt->data = g_memdup(data, size); pkt->size = size; pkt->creation_ms = qemu_clock_get_ms(QEMU_CLOCK_HOST); pkt->vnet_hdr_len = vnet_hdr_len; - pkt->tcp_seq = 0; - pkt->tcp_ack = 0; - pkt->seq_end = 0; - pkt->header_size = 0; - pkt->payload_size = 0; - pkt->offset = 0; - pkt->flags = 0; + + return pkt; +} + +/* + * packet_new_nocopy will not copy data, so the caller can't release + * the data. And it will be released in packet_destroy. + */ +Packet *packet_new_nocopy(void *data, int size, int vnet_hdr_len) +{ + Packet *pkt = g_slice_new0(Packet); + + pkt->data = data; + pkt->size = size; + pkt->creation_ms = qemu_clock_get_ms(QEMU_CLOCK_HOST); + pkt->vnet_hdr_len = vnet_hdr_len; return pkt; } diff --git a/net/colo.h b/net/colo.h index 573ab91785..d91cd245c4 100644 --- a/net/colo.h +++ b/net/colo.h @@ -101,6 +101,7 @@ bool connection_has_tracked(GHashTable *connection_track_table, ConnectionKey *key); void connection_hashtable_reset(GHashTable *connection_track_table); Packet *packet_new(const void *data, int size, int vnet_hdr_len); +Packet *packet_new_nocopy(void *data, int size, int vnet_hdr_len); void packet_destroy(void *opaque, void *user_data); void packet_destroy_partial(void *opaque, void *user_data); diff --git a/net/filter-rewriter.c b/net/filter-rewriter.c index 10fe3939b1..cb3a96cde1 100644 --- a/net/filter-rewriter.c +++ b/net/filter-rewriter.c @@ -270,8 +270,7 @@ static ssize_t colo_rewriter_receive_iov(NetFilterState *nf, vnet_hdr_len = nf->netdev->vnet_hdr_len; } - pkt = packet_new(buf, size, vnet_hdr_len); - g_free(buf); + pkt = packet_new_nocopy(buf, size, vnet_hdr_len); /* * if we get tcp packet From 0c7af1a778d036402ec0829783afee1ce6ea942c Mon Sep 17 00:00:00 2001 From: "Rao, Lei" Date: Tue, 8 Jun 2021 16:23:30 +0800 Subject: [PATCH 11/12] Add the function of colo_compare_cleanup This patch fixes the following: #0 __GI_raise (sig=sig@entry=6) at ../sysdeps/unix/sysv/linux/raise.c:50 #1 0x00007f6ae4559859 in __GI_abort () at abort.c:79 #2 0x0000559aaa386720 in error_exit (err=16, msg=0x559aaa5973d0 <__func__.16227> "qemu_mutex_destroy") at util/qemu-thread-posix.c:36 #3 0x0000559aaa3868c5 in qemu_mutex_destroy (mutex=0x559aabffe828) at util/qemu-thread-posix.c:69 #4 0x0000559aaa2f93a8 in char_finalize (obj=0x559aabffe800) at chardev/char.c:285 #5 0x0000559aaa23318a in object_deinit (obj=0x559aabffe800, type=0x559aabfd7d20) at qom/object.c:606 #6 0x0000559aaa2331b8 in object_deinit (obj=0x559aabffe800, type=0x559aabfd9060) at qom/object.c:610 #7 0x0000559aaa233200 in object_finalize (data=0x559aabffe800) at qom/object.c:620 #8 0x0000559aaa234202 in object_unref (obj=0x559aabffe800) at qom/object.c:1074 #9 0x0000559aaa2356b6 in object_finalize_child_property (obj=0x559aac0dac10, name=0x559aac778760 "compare0-0", opaque=0x559aabffe800) at qom/object.c:1584 #10 0x0000559aaa232f70 in object_property_del_all (obj=0x559aac0dac10) at qom/object.c:557 #11 0x0000559aaa2331ed in object_finalize (data=0x559aac0dac10) at qom/object.c:619 #12 0x0000559aaa234202 in object_unref (obj=0x559aac0dac10) at qom/object.c:1074 #13 0x0000559aaa2356b6 in object_finalize_child_property (obj=0x559aac0c75c0, name=0x559aac0dadc0 "chardevs", opaque=0x559aac0dac10) at qom/object.c:1584 #14 0x0000559aaa233071 in object_property_del_child (obj=0x559aac0c75c0, child=0x559aac0dac10, errp=0x0) at qom/object.c:580 #15 0x0000559aaa233155 in object_unparent (obj=0x559aac0dac10) at qom/object.c:599 #16 0x0000559aaa2fb721 in qemu_chr_cleanup () at chardev/char.c:1159 #17 0x0000559aa9f9b110 in main (argc=54, argv=0x7ffeb62fa998, envp=0x7ffeb62fab50) at vl.c:4539 When chardev is cleaned up, chr_write_lock needs to be destroyed. But the colo-compare module is not cleaned up normally before it when the guest poweroff. It is holding chr_write_lock at this time. This will cause qemu crash.So we add the function of colo_compare_cleanup() before qemu_chr_cleanup() to fix the bug. Signed-off-by: Lei Rao Reviewed-by: Zhang Chen Reviewed-by: Lukas Straub Tested-by: Lukas Straub Signed-off-by: Zhang Chen Signed-off-by: Jason Wang --- net/colo-compare.c | 10 ++++++++++ net/colo-compare.h | 1 + net/net.c | 4 ++++ 3 files changed, 15 insertions(+) diff --git a/net/colo-compare.c b/net/colo-compare.c index c142c08dc6..5b538f4e0b 100644 --- a/net/colo-compare.c +++ b/net/colo-compare.c @@ -1402,6 +1402,16 @@ static void colo_compare_init(Object *obj) compare_set_vnet_hdr); } +void colo_compare_cleanup(void) +{ + CompareState *tmp = NULL; + CompareState *n = NULL; + + QTAILQ_FOREACH_SAFE(tmp, &net_compares, next, n) { + object_unparent(OBJECT(tmp)); + } +} + static void colo_compare_finalize(Object *obj) { CompareState *s = COLO_COMPARE(obj); diff --git a/net/colo-compare.h b/net/colo-compare.h index 22ddd512e2..b055270da2 100644 --- a/net/colo-compare.h +++ b/net/colo-compare.h @@ -20,5 +20,6 @@ void colo_notify_compares_event(void *opaque, int event, Error **errp); void colo_compare_register_notifier(Notifier *notify); void colo_compare_unregister_notifier(Notifier *notify); +void colo_compare_cleanup(void); #endif /* QEMU_COLO_COMPARE_H */ diff --git a/net/net.c b/net/net.c index 2a472604ec..76bbb7c31b 100644 --- a/net/net.c +++ b/net/net.c @@ -52,6 +52,7 @@ #include "qapi/error.h" #include "qapi/opts-visitor.h" #include "sysemu/runstate.h" +#include "net/colo-compare.h" #include "net/filter.h" #include "qapi/string-output-visitor.h" @@ -1402,6 +1403,9 @@ void net_cleanup(void) { NetClientState *nc; + /*cleanup colo compare module for COLO*/ + colo_compare_cleanup(); + /* We may del multiple entries during qemu_del_net_client(), * so QTAILQ_FOREACH_SAFE() is also not safe here. */ From 5a2d9929ac1f01a1e8ef2a3f56f69e6069863dad Mon Sep 17 00:00:00 2001 From: "Rao, Lei" Date: Tue, 8 Jun 2021 16:23:31 +0800 Subject: [PATCH 12/12] Fixed calculation error of pkt->header_size in fill_pkt_tcp_info() The data pointer has skipped vnet_hdr_len in the function of parse_packet_early().So, we can not subtract vnet_hdr_len again when calculating pkt->header_size in fill_pkt_tcp_info(). Otherwise, it will cause network packet comparsion errors and greatly increase the frequency of checkpoints. Signed-off-by: Lei Rao Signed-off-by: Zhang Chen Reviewed-by: Li Zhijian Reviewed-by: Zhang Chen Reviewed-by: Lukas Straub Tested-by: Lukas Straub Signed-off-by: Jason Wang --- net/colo-compare.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/net/colo-compare.c b/net/colo-compare.c index 5b538f4e0b..b100e7b51f 100644 --- a/net/colo-compare.c +++ b/net/colo-compare.c @@ -211,7 +211,7 @@ static void fill_pkt_tcp_info(void *data, uint32_t *max_ack) pkt->tcp_ack = ntohl(tcphd->th_ack); *max_ack = *max_ack > pkt->tcp_ack ? *max_ack : pkt->tcp_ack; pkt->header_size = pkt->transport_header - (uint8_t *)pkt->data - + (tcphd->th_off << 2) - pkt->vnet_hdr_len; + + (tcphd->th_off << 2); pkt->payload_size = pkt->size - pkt->header_size; pkt->seq_end = pkt->tcp_seq + pkt->payload_size; pkt->flags = tcphd->th_flags;