diff --git a/include/exec/target_page.h b/include/exec/target_page.h index 96726c36a4..bbf37aea17 100644 --- a/include/exec/target_page.h +++ b/include/exec/target_page.h @@ -18,4 +18,5 @@ size_t qemu_target_page_size(void); int qemu_target_page_bits(void); int qemu_target_page_bits_min(void); +size_t qemu_target_pages_to_MiB(size_t pages); #endif diff --git a/migration/block.c b/migration/block.c index a37678ce95..12617b4152 100644 --- a/migration/block.c +++ b/migration/block.c @@ -747,7 +747,7 @@ static int block_save_setup(QEMUFile *f, void *opaque) static int block_save_iterate(QEMUFile *f, void *opaque) { int ret; - uint64_t last_bytes = qemu_file_total_transferred(f); + uint64_t last_bytes = qemu_file_transferred(f); trace_migration_block_save("iterate", block_mig_state.submitted, block_mig_state.transferred); @@ -799,7 +799,7 @@ static int block_save_iterate(QEMUFile *f, void *opaque) } qemu_put_be64(f, BLK_MIG_FLAG_EOS); - uint64_t delta_bytes = qemu_file_total_transferred(f) - last_bytes; + uint64_t delta_bytes = qemu_file_transferred(f) - last_bytes; return (delta_bytes > 0); } diff --git a/migration/dirtyrate.c b/migration/dirtyrate.c index 388337a332..c06f12c39d 100644 --- a/migration/dirtyrate.c +++ b/migration/dirtyrate.c @@ -14,9 +14,8 @@ #include "qemu/error-report.h" #include #include "qapi/error.h" -#include "cpu.h" #include "exec/ramblock.h" -#include "exec/ram_addr.h" +#include "exec/target_page.h" #include "qemu/rcu_queue.h" #include "qemu/main-loop.h" #include "qapi/qapi-commands-migration.h" @@ -29,6 +28,7 @@ #include "sysemu/kvm.h" #include "sysemu/runstate.h" #include "exec/memory.h" +#include "qemu/xxhash.h" /* * total_dirty_pages is procted by BQL and is used @@ -74,13 +74,11 @@ static inline void record_dirtypages(DirtyPageRecord *dirty_pages, static int64_t do_calculate_dirtyrate(DirtyPageRecord dirty_pages, int64_t calc_time_ms) { - uint64_t memory_size_MB; uint64_t increased_dirty_pages = dirty_pages.end_pages - dirty_pages.start_pages; + uint64_t memory_size_MiB = qemu_target_pages_to_MiB(increased_dirty_pages); - memory_size_MB = (increased_dirty_pages * TARGET_PAGE_SIZE) >> 20; - - return memory_size_MB * 1000 / calc_time_ms; + return memory_size_MiB * 1000 / calc_time_ms; } void global_dirty_log_change(unsigned int flag, bool start) @@ -291,8 +289,8 @@ static void update_dirtyrate_stat(struct RamblockDirtyInfo *info) DirtyStat.page_sampling.total_dirty_samples += info->sample_dirty_count; DirtyStat.page_sampling.total_sample_count += info->sample_pages_count; /* size of total pages in MB */ - DirtyStat.page_sampling.total_block_mem_MB += (info->ramblock_pages * - TARGET_PAGE_SIZE) >> 20; + DirtyStat.page_sampling.total_block_mem_MB += + qemu_target_pages_to_MiB(info->ramblock_pages); } static void update_dirtyrate(uint64_t msec) @@ -308,6 +306,34 @@ static void update_dirtyrate(uint64_t msec) DirtyStat.dirty_rate = dirtyrate; } +/* + * Compute hash of a single page of size TARGET_PAGE_SIZE. + */ +static uint32_t compute_page_hash(void *ptr) +{ + size_t page_size = qemu_target_page_size(); + uint32_t i; + uint64_t v1, v2, v3, v4; + uint64_t res; + const uint64_t *p = ptr; + + v1 = QEMU_XXHASH_SEED + XXH_PRIME64_1 + XXH_PRIME64_2; + v2 = QEMU_XXHASH_SEED + XXH_PRIME64_2; + v3 = QEMU_XXHASH_SEED + 0; + v4 = QEMU_XXHASH_SEED - XXH_PRIME64_1; + for (i = 0; i < page_size / 8; i += 4) { + v1 = XXH64_round(v1, p[i + 0]); + v2 = XXH64_round(v2, p[i + 1]); + v3 = XXH64_round(v3, p[i + 2]); + v4 = XXH64_round(v4, p[i + 3]); + } + res = XXH64_mergerounds(v1, v2, v3, v4); + res += page_size; + res = XXH64_avalanche(res); + return (uint32_t)(res & UINT32_MAX); +} + + /* * get hash result for the sampled memory with length of TARGET_PAGE_SIZE * in ramblock, which starts from ramblock base address. @@ -315,13 +341,13 @@ static void update_dirtyrate(uint64_t msec) static uint32_t get_ramblock_vfn_hash(struct RamblockDirtyInfo *info, uint64_t vfn) { - uint32_t crc; + uint32_t hash; - crc = crc32(0, (info->ramblock_addr + - vfn * TARGET_PAGE_SIZE), TARGET_PAGE_SIZE); + hash = compute_page_hash(info->ramblock_addr + + vfn * qemu_target_page_size()); - trace_get_ramblock_vfn_hash(info->idstr, vfn, crc); - return crc; + trace_get_ramblock_vfn_hash(info->idstr, vfn, hash); + return hash; } static bool save_ramblock_hash(struct RamblockDirtyInfo *info) @@ -373,7 +399,7 @@ static void get_ramblock_dirty_info(RAMBlock *block, sample_pages_per_gigabytes) >> 30; /* Right shift TARGET_PAGE_BITS to calc page count */ info->ramblock_pages = qemu_ram_get_used_length(block) >> - TARGET_PAGE_BITS; + qemu_target_page_bits(); info->ramblock_addr = qemu_ram_get_host_addr(block); strcpy(info->idstr, qemu_ram_get_idstr(block)); } @@ -454,13 +480,13 @@ out: static void calc_page_dirty_rate(struct RamblockDirtyInfo *info) { - uint32_t crc; + uint32_t hash; int i; for (i = 0; i < info->sample_pages_count; i++) { - crc = get_ramblock_vfn_hash(info, info->sample_page_vfn[i]); - if (crc != info->hash_result[i]) { - trace_calc_page_dirty_rate(info->idstr, crc, info->hash_result[i]); + hash = get_ramblock_vfn_hash(info, info->sample_page_vfn[i]); + if (hash != info->hash_result[i]) { + trace_calc_page_dirty_rate(info->idstr, hash, info->hash_result[i]); info->sample_dirty_count++; } } @@ -484,7 +510,7 @@ find_block_matched(RAMBlock *block, int count, if (infos[i].ramblock_addr != qemu_ram_get_host_addr(block) || infos[i].ramblock_pages != - (qemu_ram_get_used_length(block) >> TARGET_PAGE_BITS)) { + (qemu_ram_get_used_length(block) >> qemu_target_page_bits())) { trace_find_page_matched(block->idstr); return NULL; } diff --git a/migration/meson.build b/migration/meson.build index eb41b77db9..dc8b1daef5 100644 --- a/migration/meson.build +++ b/migration/meson.build @@ -13,6 +13,7 @@ softmmu_ss.add(files( 'block-dirty-bitmap.c', 'channel.c', 'channel-block.c', + 'dirtyrate.c', 'exec.c', 'fd.c', 'global_state.c', @@ -42,6 +43,5 @@ endif softmmu_ss.add(when: zstd, if_true: files('multifd-zstd.c')) specific_ss.add(when: 'CONFIG_SOFTMMU', - if_true: files('dirtyrate.c', - 'ram.c', + if_true: files('ram.c', 'target.c')) diff --git a/migration/migration.c b/migration/migration.c index 439e8651df..00d8ba8da0 100644 --- a/migration/migration.c +++ b/migration/migration.c @@ -2140,12 +2140,7 @@ static int postcopy_start(MigrationState *ms) * will notice we're in POSTCOPY_ACTIVE and not actually * wrap their state up here */ - /* 0 max-postcopy-bandwidth means unlimited */ - if (!bandwidth) { - qemu_file_set_rate_limit(ms->to_dst_file, INT64_MAX); - } else { - qemu_file_set_rate_limit(ms->to_dst_file, bandwidth / XFER_LIMIT_RATIO); - } + qemu_file_set_rate_limit(ms->to_dst_file, bandwidth); if (migrate_postcopy_ram()) { /* Ping just for debugging, helps line traces up */ qemu_savevm_send_ping(ms->to_dst_file, 2); @@ -2653,7 +2648,7 @@ static MigThrError migration_detect_error(MigrationState *s) /* How many bytes have we transferred since the beginning of the migration */ static uint64_t migration_total_bytes(MigrationState *s) { - return qemu_file_total_transferred(s->to_dst_file) + + return qemu_file_transferred(s->to_dst_file) + stat64_get(&mig_stats.multifd_bytes); } @@ -3236,11 +3231,10 @@ void migrate_fd_connect(MigrationState *s, Error *error_in) if (resume) { /* This is a resumed migration */ - rate_limit = migrate_max_postcopy_bandwidth() / - XFER_LIMIT_RATIO; + rate_limit = migrate_max_postcopy_bandwidth(); } else { /* This is a fresh new migration */ - rate_limit = migrate_max_bandwidth() / XFER_LIMIT_RATIO; + rate_limit = migrate_max_bandwidth(); /* Notify before starting migration thread */ notifier_list_notify(&migration_state_notifiers, s); diff --git a/migration/options.c b/migration/options.c index 7ed88b7b32..c2a278ee2d 100644 --- a/migration/options.c +++ b/migration/options.c @@ -1243,7 +1243,7 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp) s->parameters.max_bandwidth = params->max_bandwidth; if (s->to_dst_file && !migration_in_postcopy()) { qemu_file_set_rate_limit(s->to_dst_file, - s->parameters.max_bandwidth / XFER_LIMIT_RATIO); + s->parameters.max_bandwidth); } } @@ -1273,7 +1273,7 @@ static void migrate_params_apply(MigrateSetParameters *params, Error **errp) s->parameters.max_postcopy_bandwidth = params->max_postcopy_bandwidth; if (s->to_dst_file && migration_in_postcopy()) { qemu_file_set_rate_limit(s->to_dst_file, - s->parameters.max_postcopy_bandwidth / XFER_LIMIT_RATIO); + s->parameters.max_postcopy_bandwidth); } } if (params->has_max_cpu_throttle) { diff --git a/migration/qemu-file.c b/migration/qemu-file.c index 61fb580342..597054759d 100644 --- a/migration/qemu-file.c +++ b/migration/qemu-file.c @@ -29,6 +29,7 @@ #include "migration.h" #include "qemu-file.h" #include "trace.h" +#include "options.h" #include "qapi/error.h" #define IO_BUF_SIZE 32768 @@ -43,12 +44,12 @@ struct QEMUFile { * Maximum amount of data in bytes to transfer during one * rate limiting time window */ - int64_t rate_limit_max; + uint64_t rate_limit_max; /* * Total amount of data in bytes queued for transfer * during this rate limiting time window */ - int64_t rate_limit_used; + uint64_t rate_limit_used; /* The sum of bytes transferred on the wire */ uint64_t total_transferred; @@ -708,7 +709,7 @@ int coroutine_mixed_fn qemu_get_byte(QEMUFile *f) return result; } -uint64_t qemu_file_total_transferred_fast(QEMUFile *f) +uint64_t qemu_file_transferred_fast(QEMUFile *f) { uint64_t ret = f->total_transferred; int i; @@ -720,7 +721,7 @@ uint64_t qemu_file_total_transferred_fast(QEMUFile *f) return ret; } -uint64_t qemu_file_total_transferred(QEMUFile *f) +uint64_t qemu_file_transferred(QEMUFile *f) { qemu_fflush(f); return f->total_transferred; @@ -737,14 +738,17 @@ int qemu_file_rate_limit(QEMUFile *f) return 0; } -int64_t qemu_file_get_rate_limit(QEMUFile *f) +uint64_t qemu_file_get_rate_limit(QEMUFile *f) { return f->rate_limit_max; } -void qemu_file_set_rate_limit(QEMUFile *f, int64_t limit) +void qemu_file_set_rate_limit(QEMUFile *f, uint64_t limit) { - f->rate_limit_max = limit; + /* + * 'limit' is per second. But we check it each 100 miliseconds. + */ + f->rate_limit_max = limit / XFER_LIMIT_RATIO; } void qemu_file_reset_rate_limit(QEMUFile *f) @@ -752,7 +756,7 @@ void qemu_file_reset_rate_limit(QEMUFile *f) f->rate_limit_used = 0; } -void qemu_file_acct_rate_limit(QEMUFile *f, int64_t len) +void qemu_file_acct_rate_limit(QEMUFile *f, uint64_t len) { f->rate_limit_used += len; } diff --git a/migration/qemu-file.h b/migration/qemu-file.h index 4ee58a87dd..bcc39081f2 100644 --- a/migration/qemu-file.h +++ b/migration/qemu-file.h @@ -68,7 +68,7 @@ void qemu_file_set_hooks(QEMUFile *f, const QEMUFileHooks *hooks); int qemu_fclose(QEMUFile *f); /* - * qemu_file_total_transferred: + * qemu_file_transferred: * * Report the total number of bytes transferred with * this file. @@ -83,19 +83,19 @@ int qemu_fclose(QEMUFile *f); * * Returns: the total bytes transferred */ -uint64_t qemu_file_total_transferred(QEMUFile *f); +uint64_t qemu_file_transferred(QEMUFile *f); /* - * qemu_file_total_transferred_fast: + * qemu_file_transferred_fast: * - * As qemu_file_total_transferred except for writable + * As qemu_file_transferred except for writable * files, where no flush is performed and the reported * amount will include the size of any queued buffers, * on top of the amount actually transferred. * * Returns: the total bytes transferred and queued */ -uint64_t qemu_file_total_transferred_fast(QEMUFile *f); +uint64_t qemu_file_transferred_fast(QEMUFile *f); /* * put_buffer without copying the buffer. @@ -138,9 +138,9 @@ void qemu_file_reset_rate_limit(QEMUFile *f); * out of band from the main file object I/O methods, and * need to be applied to the rate limiting calcuations */ -void qemu_file_acct_rate_limit(QEMUFile *f, int64_t len); -void qemu_file_set_rate_limit(QEMUFile *f, int64_t new_rate); -int64_t qemu_file_get_rate_limit(QEMUFile *f); +void qemu_file_acct_rate_limit(QEMUFile *f, uint64_t len); +void qemu_file_set_rate_limit(QEMUFile *f, uint64_t new_rate); +uint64_t qemu_file_get_rate_limit(QEMUFile *f); int qemu_file_get_error_obj(QEMUFile *f, Error **errp); int qemu_file_get_error_obj_any(QEMUFile *f1, QEMUFile *f2, Error **errp); void qemu_file_set_error_obj(QEMUFile *f, int ret, Error *err); diff --git a/migration/savevm.c b/migration/savevm.c index 032044b1d5..e33788343a 100644 --- a/migration/savevm.c +++ b/migration/savevm.c @@ -927,9 +927,9 @@ static int vmstate_load(QEMUFile *f, SaveStateEntry *se) static void vmstate_save_old_style(QEMUFile *f, SaveStateEntry *se, JSONWriter *vmdesc) { - uint64_t old_offset = qemu_file_total_transferred_fast(f); + uint64_t old_offset = qemu_file_transferred_fast(f); se->ops->save_state(f, se->opaque); - uint64_t size = qemu_file_total_transferred_fast(f) - old_offset; + uint64_t size = qemu_file_transferred_fast(f) - old_offset; if (vmdesc) { json_writer_int64(vmdesc, "size", size); @@ -2956,7 +2956,7 @@ bool save_snapshot(const char *name, bool overwrite, const char *vmstate, goto the_end; } ret = qemu_savevm_state(f, errp); - vm_state_size = qemu_file_total_transferred(f); + vm_state_size = qemu_file_transferred(f); ret2 = qemu_fclose(f); if (ret < 0) { goto the_end; diff --git a/migration/trace-events b/migration/trace-events index 92161eeac5..f39818c329 100644 --- a/migration/trace-events +++ b/migration/trace-events @@ -342,8 +342,8 @@ dirty_bitmap_load_success(void) "" # dirtyrate.c dirtyrate_set_state(const char *new_state) "new state %s" query_dirty_rate_info(const char *new_state) "current state %s" -get_ramblock_vfn_hash(const char *idstr, uint64_t vfn, uint32_t crc) "ramblock name: %s, vfn: %"PRIu64 ", crc: %" PRIu32 -calc_page_dirty_rate(const char *idstr, uint32_t new_crc, uint32_t old_crc) "ramblock name: %s, new crc: %" PRIu32 ", old crc: %" PRIu32 +get_ramblock_vfn_hash(const char *idstr, uint64_t vfn, uint32_t hash) "ramblock name: %s, vfn: %"PRIu64 ", hash: %" PRIu32 +calc_page_dirty_rate(const char *idstr, uint32_t new_hash, uint32_t old_hash) "ramblock name: %s, new hash: %" PRIu32 ", old hash: %" PRIu32 skip_sample_ramblock(const char *idstr, uint64_t ramblock_size) "ramblock name: %s, ramblock size: %" PRIu64 find_page_matched(const char *idstr) "ramblock %s addr or size changed" dirtyrate_calculate(int64_t dirtyrate) "dirty rate: %" PRIi64 " MB/s" diff --git a/migration/vmstate.c b/migration/vmstate.c index 351f56104e..af01d54b6f 100644 --- a/migration/vmstate.c +++ b/migration/vmstate.c @@ -361,7 +361,7 @@ int vmstate_save_state_v(QEMUFile *f, const VMStateDescription *vmsd, void *curr_elem = first_elem + size * i; vmsd_desc_field_start(vmsd, vmdesc_loop, field, i, n_elems); - old_offset = qemu_file_total_transferred_fast(f); + old_offset = qemu_file_transferred_fast(f); if (field->flags & VMS_ARRAY_OF_POINTER) { assert(curr_elem); curr_elem = *(void **)curr_elem; @@ -391,8 +391,7 @@ int vmstate_save_state_v(QEMUFile *f, const VMStateDescription *vmsd, return ret; } - written_bytes = qemu_file_total_transferred_fast(f) - - old_offset; + written_bytes = qemu_file_transferred_fast(f) - old_offset; vmsd_desc_field_end(vmsd, vmdesc_loop, field, written_bytes, i); /* Compressed arrays only care about the first element */ diff --git a/softmmu/dirtylimit.c b/softmmu/dirtylimit.c index 71bf6dc7a4..015a9038d1 100644 --- a/softmmu/dirtylimit.c +++ b/softmmu/dirtylimit.c @@ -235,20 +235,15 @@ bool dirtylimit_vcpu_index_valid(int cpu_index) static uint64_t dirtylimit_dirty_ring_full_time(uint64_t dirtyrate) { static uint64_t max_dirtyrate; - unsigned target_page_bits = qemu_target_page_bits(); - uint64_t dirty_ring_size_MB; + uint64_t dirty_ring_size_MiB; - /* So far, the largest (non-huge) page size is 64k, i.e. 16 bits. */ - assert(target_page_bits < 20); - - /* Convert ring size (pages) to MiB (2**20). */ - dirty_ring_size_MB = kvm_dirty_ring_size() >> (20 - target_page_bits); + dirty_ring_size_MiB = qemu_target_pages_to_MiB(kvm_dirty_ring_size()); if (max_dirtyrate < dirtyrate) { max_dirtyrate = dirtyrate; } - return dirty_ring_size_MB * 1000000 / max_dirtyrate; + return dirty_ring_size_MiB * 1000000 / max_dirtyrate; } static inline bool dirtylimit_done(uint64_t quota, diff --git a/softmmu/physmem.c b/softmmu/physmem.c index 0e0182d9f2..efaed36773 100644 --- a/softmmu/physmem.c +++ b/softmmu/physmem.c @@ -3357,6 +3357,17 @@ int qemu_target_page_bits_min(void) return TARGET_PAGE_BITS_MIN; } +/* Convert target pages to MiB (2**20). */ +size_t qemu_target_pages_to_MiB(size_t pages) +{ + int page_bits = TARGET_PAGE_BITS; + + /* So far, the largest (non-huge) page size is 64k, i.e. 16 bits. */ + g_assert(page_bits < 20); + + return pages >> (20 - page_bits); +} + bool cpu_physical_memory_is_io(hwaddr phys_addr) { MemoryRegion*mr;