Various fixes and updates:

- editor config tweak for shell scripts
   - iotest updates (still not default for make check)
   - various docker updates
   - gcc/ubsan updates for travis
   - some clean-ups for tests/vm (no serial autoinstall)
   - semihosting fix for Coverity
   - fixes for cputlb in 64-on-32 cases
   - gdbstub re-factor + maintainership update
 -----BEGIN PGP SIGNATURE-----
 
 iQEzBAABCgAdFiEEZoWumedRZ7yvyN81+9DbCVqeKkQFAl0BLmgACgkQ+9DbCVqe
 KkRtRQf/RCD20OYfA++TxGuj68/SIJXc+mir6KViRzmPbGoJKTCbgt9GInLc2nwm
 RvwLHWEoLQ/u8O9XWgj8KIwLeiDZS2or1BjAiV5sbfWFEzUTvfhZGPX55dGYw2ON
 Yj7xL/fS+UFBR+YvGtJmqQb38FmY9n8JB/jpT6rbi+bigXbLLVxvmk01tbVw/IKH
 ona1U+lYJFYGPp7xt6wbwwao3NgOo2PGM0L07lNy3k2sq1EFbtnWVJH9CjdiJ9bn
 wEbk2S78Du+NVnqF7peOFPl7NRgzsgUv1+m6NPGmO/kbgMBHwczcG+QDO5t7EJ4n
 7s5K8x6C3yQxav811L1+Lz3/4angkQ==
 =cBKA
 -----END PGP SIGNATURE-----

Merge remote-tracking branch 'remotes/stsquad/tags/pull-testing-gdbstub-cputlb-120619-3' into staging

Various fixes and updates:

  - editor config tweak for shell scripts
  - iotest updates (still not default for make check)
  - various docker updates
  - gcc/ubsan updates for travis
  - some clean-ups for tests/vm (no serial autoinstall)
  - semihosting fix for Coverity
  - fixes for cputlb in 64-on-32 cases
  - gdbstub re-factor + maintainership update

# gpg: Signature made Wed 12 Jun 2019 17:55:04 BST
# gpg:                using RSA key 6685AE99E75167BCAFC8DF35FBD0DB095A9E2A44
# gpg: Good signature from "Alex Bennée (Master Work Key) <alex.bennee@linaro.org>" [full]
# Primary key fingerprint: 6685 AE99 E751 67BC AFC8  DF35 FBD0 DB09 5A9E 2A44

* remotes/stsquad/tags/pull-testing-gdbstub-cputlb-120619-3: (40 commits)
  gdbstub: Implement qemu physical memory mode
  gdbstub: Clear unused variables in gdb_handle_packet
  gdbstub: Implement target halted (? pkt) with new infra
  gdbstub: Implement generic set/query (Q/q pkt) with new infra
  gdbstub: Implement v commands with new infra
  gdbstub: Implement step (s pkt) with new infra
  gdbstub: Implement file io (F pkt) with new infra
  gdbstub: Implement read all registers (g pkt) with new infra
  gdbstub: Implement write all registers (G pkt) with new infra
  gdbstub: Implement read memory (m pkt) with new infra
  gdbstub: Implement write memory (M pkt) with new infra
  gdbstub: Implement get register (p pkt) with new infra
  gdbstub: Implement set register (P pkt) with new infra
  gdbstub: Implement breakpoint commands (Z/z pkt) with new infra
  gdbstub: Implement set_thread (H pkt) with new infra
  gdbstub: Implement continue with signal (C pkt) with new infra
  gdbstub: Implement continue (c pkt) with new infra
  gdbstub: Implement thread_alive (T pkt) with new infra
  gdbstub: Implement deatch (D pkt) with new infra
  gdbstub: Add infrastructure to parse cmd packets
  ...

Signed-off-by: Peter Maydell <peter.maydell@linaro.org>

.editorconfig

@@ -26,6 +26,10 @@ file_type_emacs = makefile
 indent_style = space
 indent_size = 4
 
+[*.sh]
+indent_style = space
+indent_size = 4
+
 [*.{s,S}]
 indent_style = tab
 indent_size = 8

.travis.yml

@@ -152,6 +152,13 @@ matrix:
       compiler: clang
 
 
+    - env:
+        - CONFIG="--target-list=${MAIN_SOFTMMU_TARGETS} "
+      compiler: clang
+      before_script:
+        - ./configure ${CONFIG} --extra-cflags="-fsanitize=undefined -Werror" || { cat config.log && exit 1; }
+
+
     - env:
         - CONFIG="--disable-user --target-list-exclude=${MAIN_SOFTMMU_TARGETS}"
       compiler: clang
@@ -240,8 +247,8 @@ matrix:
             - ubuntu-toolchain-r-test
           packages:
             # Extra toolchains
-            - gcc-7
+            - gcc-9
-            - g++-7
+            - g++-9
             # Build dependencies
             - libaio-dev
             - libattr1-dev
@@ -270,11 +277,11 @@ matrix:
       language: generic
       compiler: none
       env:
-        - COMPILER_NAME=gcc CXX=g++-7 CC=gcc-7
+        - COMPILER_NAME=gcc CXX=g++-9 CC=gcc-9
-        - CONFIG="--cc=gcc-7 --cxx=g++-7 --disable-pie --disable-linux-user"
+        - CONFIG="--cc=gcc-9 --cxx=g++-9 --disable-pie --disable-linux-user"
         - TEST_CMD=""
       before_script:
-        - ./configure ${CONFIG} --extra-cflags="-g3 -O0 -fsanitize=thread -fuse-ld=gold" || { cat config.log && exit 1; }
+        - ./configure ${CONFIG} --extra-cflags="-g3 -O0 -Wno-error=stringop-truncation -fsanitize=thread -fuse-ld=gold" || { cat config.log && exit 1; }
 
 
     # Run check-tcg against linux-user

MAINTAINERS

@@ -1865,7 +1865,9 @@ F: util/error.c
 F: util/qemu-error.c
 
 GDB stub
-S: Orphan
+M: Alex Bennée <alex.bennee@linaro.org>
+R: Philippe Mathieu-Daudé <philmd@redhat.com>
+S: Maintained
 F: gdbstub*
 F: gdb-xml/
 

accel/tcg/cputlb.c

@@ -1315,10 +1315,10 @@ load_helper(CPUArchState *env, target_ulong addr, TCGMemOpIdx oi,
         && unlikely((addr & ~TARGET_PAGE_MASK) + size - 1
                     >= TARGET_PAGE_SIZE)) {
         target_ulong addr1, addr2;
-        tcg_target_ulong r1, r2;
+        uint64_t r1, r2;
         unsigned shift;
     do_unaligned_access:
-        addr1 = addr & ~(size - 1);
+        addr1 = addr & ~((target_ulong)size - 1);
         addr2 = addr1 + size;
         r1 = full_load(env, addr1, oi, retaddr);
         r2 = full_load(env, addr2, oi, retaddr);

hw/semihosting/console.c

@@ -36,26 +36,24 @@ int qemu_semihosting_log_out(const char *s, int len)
 /*
  * A re-implementation of lock_user_string that we can use locally
  * instead of relying on softmmu-semi. Hopefully we can deprecate that
- * in time. We either copy len bytes if specified or until we find a NULL.
+ * in time. Copy string until we find a 0 or address error.
  */
-static GString *copy_user_string(CPUArchState *env, target_ulong addr, int len)
+static GString *copy_user_string(CPUArchState *env, target_ulong addr)
 {
     CPUState *cpu = env_cpu(env);
-    GString *s = g_string_sized_new(len ? len : 128);
+    GString *s = g_string_sized_new(128);
     uint8_t c;
-    bool done;
 
     do {
         if (cpu_memory_rw_debug(cpu, addr++, &c, 1, 0) == 0) {
             s = g_string_append_c(s, c);
-            done = len ? s->len == len : c == 0;
         } else {
             qemu_log_mask(LOG_GUEST_ERROR,
                           "%s: passed inaccessible address " TARGET_FMT_lx,
                           __func__, addr);
-            done = true;
+            break;
         }
-    } while (!done);
+    } while (c!=0);
 
     return s;
 }
@@ -68,9 +66,9 @@ static void semihosting_cb(CPUState *cs, target_ulong ret, target_ulong err)
     }
 }
 
-int qemu_semihosting_console_out(CPUArchState *env, target_ulong addr, int len)
+int qemu_semihosting_console_outs(CPUArchState *env, target_ulong addr)
 {
-    GString *s = copy_user_string(env, addr, len);
+    GString *s = copy_user_string(env, addr);
     int out = s->len;
 
     if (use_gdb_syscalls()) {
@@ -82,3 +80,21 @@ int qemu_semihosting_console_out(CPUArchState *env, target_ulong addr, int len)
     g_string_free(s, true);
     return out;
 }
+
+void qemu_semihosting_console_outc(CPUArchState *env, target_ulong addr)
+{
+    CPUState *cpu = env_cpu(env);
+    uint8_t c;
+
+    if (cpu_memory_rw_debug(cpu, addr, &c, 1, 0) == 0) {
+        if (use_gdb_syscalls()) {
+            gdb_do_syscall(semihosting_cb, "write,2,%x,%x", addr, 1);
+        } else {
+            qemu_semihosting_log_out((const char *) &c, 1);
+        }
+    } else {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: passed inaccessible address " TARGET_FMT_lx,
+                      __func__, addr);
+    }
+}

include/hw/semihosting/console.h

@@ -10,17 +10,30 @@
 #define SEMIHOST_CONSOLE_H
 
 /**
- * qemu_semihosting_console_out:
+ * qemu_semihosting_console_outs:
  * @env: CPUArchState
- * @s: host address of guest string
+ * @s: host address of null terminated guest string
- * @len: length of string or 0 (string is null terminated)
  *
- * Send a guest string to the debug console. This may be the remote
+ * Send a null terminated guest string to the debug console. This may
- * gdb session if a softmmu guest is currently being debugged.
+ * be the remote gdb session if a softmmu guest is currently being
+ * debugged.
  *
  * Returns: number of bytes written.
  */
-int qemu_semihosting_console_out(CPUArchState *env, target_ulong s, int len);
+int qemu_semihosting_console_outs(CPUArchState *env, target_ulong s);
+
+/**
+ * qemu_semihosting_console_outc:
+ * @env: CPUArchState
+ * @s: host address of null terminated guest string
+ *
+ * Send single character from guest memory to the debug console. This
+ * may be the remote gdb session if a softmmu guest is currently being
+ * debugged.
+ *
+ * Returns: nothing
+ */
+void qemu_semihosting_console_outc(CPUArchState *env, target_ulong c);
 
 /**
  * qemu_semihosting_log_out:

linux-user/arm/semihost.c

@@ -15,10 +15,35 @@
 #include "hw/semihosting/console.h"
 #include "qemu.h"
 
-int qemu_semihosting_console_out(CPUArchState *env, target_ulong addr, int len)
+int qemu_semihosting_console_outs(CPUArchState *env, target_ulong addr)
 {
-    void *s = lock_user_string(addr);
+    int len = target_strlen(addr);
-    len = write(STDERR_FILENO, s, len ? len : strlen(s));
+    void *s;
+    if (len < 0){
+       qemu_log_mask(LOG_GUEST_ERROR,
+                     "%s: passed inaccessible address " TARGET_FMT_lx,
+                     __func__, addr);
+       return 0;
+    }
+    s = lock_user(VERIFY_READ, addr, (long)(len + 1), 1);
+    g_assert(s);  /* target_strlen has already verified this will work */
+    len = write(STDERR_FILENO, s, len);
     unlock_user(s, addr, 0);
     return len;
 }
+
+void qemu_semihosting_console_outc(CPUArchState *env, target_ulong addr)
+{
+    char c;
+
+    if (get_user_u8(c, addr)) {
+        qemu_log_mask(LOG_GUEST_ERROR,
+                      "%s: passed inaccessible address " TARGET_FMT_lx,
+                      __func__, addr);
+    } else {
+        if (write(STDERR_FILENO, &c, 1) != 1) {
+            qemu_log_mask(LOG_UNIMP, "%s: unexpected write to stdout failure",
+                          __func__);
+        }
+    }
+}

qemu-io-cmds.c

@@ -248,20 +248,21 @@ static void cvtstr(double value, char *str, size_t size)
 
 
 
-static struct timeval tsub(struct timeval t1, struct timeval t2)
+static struct timespec tsub(struct timespec t1, struct timespec t2)
 {
-    t1.tv_usec -= t2.tv_usec;
+    t1.tv_nsec -= t2.tv_nsec;
-    if (t1.tv_usec < 0) {
+    if (t1.tv_nsec < 0) {
-        t1.tv_usec += 1000000;
+        t1.tv_nsec += NANOSECONDS_PER_SECOND;
         t1.tv_sec--;
     }
     t1.tv_sec -= t2.tv_sec;
     return t1;
 }
 
-static double tdiv(double value, struct timeval tv)
+static double tdiv(double value, struct timespec tv)
 {
-    return value / ((double)tv.tv_sec + ((double)tv.tv_usec / 1000000.0));
+    double seconds = tv.tv_sec + (tv.tv_nsec / 1e9);
+    return value / seconds;
 }
 
 #define HOURS(sec)      ((sec) / (60 * 60))
@@ -274,29 +275,27 @@ enum {
     VERBOSE_FIXED_TIME  = 0x2,
 };
 
-static void timestr(struct timeval *tv, char *ts, size_t size, int format)
+static void timestr(struct timespec *tv, char *ts, size_t size, int format)
 {
-    double usec = (double)tv->tv_usec / 1000000.0;
+    double frac_sec = tv->tv_nsec / 1e9;
 
     if (format & TERSE_FIXED_TIME) {
         if (!HOURS(tv->tv_sec)) {
-            snprintf(ts, size, "%u:%02u.%02u",
+            snprintf(ts, size, "%u:%05.2f",
                      (unsigned int) MINUTES(tv->tv_sec),
-                    (unsigned int) SECONDS(tv->tv_sec),
+                     SECONDS(tv->tv_sec) + frac_sec);
-                    (unsigned int) (usec * 100));
             return;
         }
         format |= VERBOSE_FIXED_TIME; /* fallback if hours needed */
     }
 
     if ((format & VERBOSE_FIXED_TIME) || tv->tv_sec) {
-        snprintf(ts, size, "%u:%02u:%02u.%02u",
+        snprintf(ts, size, "%u:%02u:%05.2f",
                 (unsigned int) HOURS(tv->tv_sec),
                 (unsigned int) MINUTES(tv->tv_sec),
-                (unsigned int) SECONDS(tv->tv_sec),
+                 SECONDS(tv->tv_sec) + frac_sec);
-                (unsigned int) (usec * 100));
     } else {
-        snprintf(ts, size, "0.%04u sec", (unsigned int) (usec * 10000));
+        snprintf(ts, size, "%05.2f sec", frac_sec);
     }
 }
 
@@ -376,7 +375,7 @@ static void dump_buffer(const void *buffer, int64_t offset, int64_t len)
     }
 }
 
-static void print_report(const char *op, struct timeval *t, int64_t offset,
+static void print_report(const char *op, struct timespec *t, int64_t offset,
                          int64_t count, int64_t total, int cnt, bool Cflag)
 {
     char s1[64], s2[64], ts[64];
@@ -649,7 +648,7 @@ static const cmdinfo_t read_cmd = {
 
 static int read_f(BlockBackend *blk, int argc, char **argv)
 {
-    struct timeval t1, t2;
+    struct timespec t1, t2;
     bool Cflag = false, qflag = false, vflag = false;
     bool Pflag = false, sflag = false, lflag = false, bflag = false;
     int c, cnt, ret;
@@ -758,13 +757,13 @@ static int read_f(BlockBackend *blk, int argc, char **argv)
 
     buf = qemu_io_alloc(blk, count, 0xab);
 
-    gettimeofday(&t1, NULL);
+    clock_gettime(CLOCK_MONOTONIC, &t1);
     if (bflag) {
         ret = do_load_vmstate(blk, buf, offset, count, &total);
     } else {
         ret = do_pread(blk, buf, offset, count, &total);
     }
-    gettimeofday(&t2, NULL);
+    clock_gettime(CLOCK_MONOTONIC, &t2);
 
     if (ret < 0) {
         printf("read failed: %s\n", strerror(-ret));
@@ -836,7 +835,7 @@ static const cmdinfo_t readv_cmd = {
 
 static int readv_f(BlockBackend *blk, int argc, char **argv)
 {
-    struct timeval t1, t2;
+    struct timespec t1, t2;
     bool Cflag = false, qflag = false, vflag = false;
     int c, cnt, ret;
     char *buf;
@@ -891,9 +890,9 @@ static int readv_f(BlockBackend *blk, int argc, char **argv)
         return -EINVAL;
     }
 
-    gettimeofday(&t1, NULL);
+    clock_gettime(CLOCK_MONOTONIC, &t1);
     ret = do_aio_readv(blk, &qiov, offset, &total);
-    gettimeofday(&t2, NULL);
+    clock_gettime(CLOCK_MONOTONIC, &t2);
 
     if (ret < 0) {
         printf("readv failed: %s\n", strerror(-ret));
@@ -972,7 +971,7 @@ static const cmdinfo_t write_cmd = {
 
 static int write_f(BlockBackend *blk, int argc, char **argv)
 {
-    struct timeval t1, t2;
+    struct timespec t1, t2;
     bool Cflag = false, qflag = false, bflag = false;
     bool Pflag = false, zflag = false, cflag = false;
     int flags = 0;
@@ -1091,7 +1090,7 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
         buf = qemu_io_alloc(blk, count, pattern);
     }
 
-    gettimeofday(&t1, NULL);
+    clock_gettime(CLOCK_MONOTONIC, &t1);
     if (bflag) {
         ret = do_save_vmstate(blk, buf, offset, count, &total);
     } else if (zflag) {
@@ -1101,7 +1100,7 @@ static int write_f(BlockBackend *blk, int argc, char **argv)
     } else {
         ret = do_pwrite(blk, buf, offset, count, flags, &total);
     }
-    gettimeofday(&t2, NULL);
+    clock_gettime(CLOCK_MONOTONIC, &t2);
 
     if (ret < 0) {
         printf("write failed: %s\n", strerror(-ret));
@@ -1160,7 +1159,7 @@ static const cmdinfo_t writev_cmd = {
 
 static int writev_f(BlockBackend *blk, int argc, char **argv)
 {
-    struct timeval t1, t2;
+    struct timespec t1, t2;
     bool Cflag = false, qflag = false;
     int flags = 0;
     int c, cnt, ret;
@@ -1213,9 +1212,9 @@ static int writev_f(BlockBackend *blk, int argc, char **argv)
         return -EINVAL;
     }
 
-    gettimeofday(&t1, NULL);
+    clock_gettime(CLOCK_MONOTONIC, &t1);
     ret = do_aio_writev(blk, &qiov, offset, flags, &total);
-    gettimeofday(&t2, NULL);
+    clock_gettime(CLOCK_MONOTONIC, &t2);
 
     if (ret < 0) {
         printf("writev failed: %s\n", strerror(-ret));
@@ -1250,15 +1249,15 @@ struct aio_ctx {
     bool zflag;
     BlockAcctCookie acct;
     int pattern;
-    struct timeval t1;
+    struct timespec t1;
 };
 
 static void aio_write_done(void *opaque, int ret)
 {
     struct aio_ctx *ctx = opaque;
-    struct timeval t2;
+    struct timespec t2;
 
-    gettimeofday(&t2, NULL);
+    clock_gettime(CLOCK_MONOTONIC, &t2);
 
 
     if (ret < 0) {
@@ -1288,9 +1287,9 @@ out:
 static void aio_read_done(void *opaque, int ret)
 {
     struct aio_ctx *ctx = opaque;
-    struct timeval t2;
+    struct timespec t2;
 
-    gettimeofday(&t2, NULL);
+    clock_gettime(CLOCK_MONOTONIC, &t2);
 
     if (ret < 0) {
         printf("readv failed: %s\n", strerror(-ret));
@@ -1425,7 +1424,7 @@ static int aio_read_f(BlockBackend *blk, int argc, char **argv)
         return -EINVAL;
     }
 
-    gettimeofday(&ctx->t1, NULL);
+    clock_gettime(CLOCK_MONOTONIC, &ctx->t1);
     block_acct_start(blk_get_stats(blk), &ctx->acct, ctx->qiov.size,
                      BLOCK_ACCT_READ);
     blk_aio_preadv(blk, ctx->offset, &ctx->qiov, 0, aio_read_done, ctx);
@@ -1570,7 +1569,7 @@ static int aio_write_f(BlockBackend *blk, int argc, char **argv)
             return -EINVAL;
         }
 
-        gettimeofday(&ctx->t1, NULL);
+        clock_gettime(CLOCK_MONOTONIC, &ctx->t1);
         block_acct_start(blk_get_stats(blk), &ctx->acct, ctx->qiov.size,
                          BLOCK_ACCT_WRITE);
 
@@ -1746,7 +1745,7 @@ static const cmdinfo_t discard_cmd = {
 
 static int discard_f(BlockBackend *blk, int argc, char **argv)
 {
-    struct timeval t1, t2;
+    struct timespec t1, t2;
     bool Cflag = false, qflag = false;
     int c, ret;
     int64_t offset, bytes;
@@ -1787,9 +1786,9 @@ static int discard_f(BlockBackend *blk, int argc, char **argv)
         return -EINVAL;
     }
 
-    gettimeofday(&t1, NULL);
+    clock_gettime(CLOCK_MONOTONIC, &t1);
     ret = blk_pdiscard(blk, offset, bytes);
-    gettimeofday(&t2, NULL);
+    clock_gettime(CLOCK_MONOTONIC, &t2);
 
     if (ret < 0) {
         printf("discard failed: %s\n", strerror(-ret));

scripts/archive-source.sh

@@ -19,16 +19,25 @@ if test $# -lt 1; then
 fi
 
 tar_file=$(realpath "$1")
-list_file="${tar_file}.list"
+sub_tdir=$(mktemp -d "${tar_file%.tar}.sub.XXXXXXXX")
-vroot_dir="${tar_file}.vroot"
+sub_file="${sub_tdir}/submodule.tar"
 
 # We want a predictable list of submodules for builds, that is
 # independent of what the developer currently has initialized
 # in their checkout, because the build environment is completely
 # different to the host OS.
 submodules="dtc slirp ui/keycodemapdb tests/fp/berkeley-softfloat-3 tests/fp/berkeley-testfloat-3"
+sub_deinit=""
 
-trap "status=$?; rm -rf \"$list_file\" \"$vroot_dir\"; exit \$status" 0 1 2 3 15
+function cleanup() {
+    local status=$?
+    rm -rf "$sub_tdir"
+    if test "$sub_deinit" != ""; then
+        git submodule deinit $sub_deinit
+    fi
+    exit $status
+}
+trap "cleanup" 0 1 2 3 15
 
 if git diff-index --quiet HEAD -- &>/dev/null
 then
@@ -36,45 +45,26 @@ then
 else
     HEAD=$(git stash create)
 fi
-git clone --shared . "$vroot_dir"
+
-test $? -ne 0 && error "failed to clone into '$vroot_dir'"
+git archive --format tar $HEAD > "$tar_file"
+test $? -ne 0 && error "failed to archive qemu"
 for sm in $submodules; do
-    if test -d "$sm/.git"
+    status="$(git submodule status "$sm")"
-    then
+    smhash="${status#[ +-]}"
-       git clone --shared "$sm" "$vroot_dir/$sm"
+    smhash="${smhash%% *}"
-       test $? -ne 0 && error "failed to clone submodule $sm"
+    case "$status" in
-    fi
+        -*)
+            sub_deinit="$sub_deinit $sm"
+            git submodule update --init "$sm"
+            test $? -ne 0 && error "failed to update submodule $sm"
+            ;;
+        +*)
+            echo "WARNING: submodule $sm is out of sync"
+            ;;
+    esac
+    (cd $sm; git archive --format tar --prefix "$sm/" $smhash) > "$sub_file"
+    test $? -ne 0 && error "failed to archive submodule $sm ($smhash)"
+    tar --concatenate --file "$tar_file" "$sub_file"
+    test $? -ne 0 && error "failed append submodule $sm to $tar_file"
 done
-
-cd "$vroot_dir"
-test $? -ne 0 && error "failed to change into '$vroot_dir'"
-
-git checkout $HEAD
-test $? -ne 0 && error "failed to checkout $HEAD revision"
-
-for sm in $submodules; do
-    git submodule update --init $sm
-    test $? -ne 0 && error "failed to init submodule $sm"
-done
-
-if test -n "$submodules"; then
-    {
-        git ls-files || error "git ls-files failed"
-        for sm in $submodules; do
-            (cd $sm; git ls-files) | sed "s:^:$sm/:"
-            if test "${PIPESTATUS[*]}" != "0 0"; then
-                error "git ls-files in submodule $sm failed"
-            fi
-        done
-    } | grep -x -v $(for sm in $submodules; do echo "-e $sm"; done) > "$list_file"
-else
-    git ls-files > "$list_file"
-fi
-
-if test $? -ne 0; then
-    error "failed to generate list file"
-fi
-
-tar -cf "$tar_file" -T "$list_file" || error "failed to create tar file"
-
 exit 0

target/arm/arm-semi.c

@@ -314,10 +314,10 @@ target_ulong do_arm_semihosting(CPUARMState *env)
             return set_swi_errno(ts, close(arg0));
         }
     case TARGET_SYS_WRITEC:
-        qemu_semihosting_console_out(env, args, 1);
+        qemu_semihosting_console_outc(env, args);
         return 0xdeadbeef;
     case TARGET_SYS_WRITE0:
-        return qemu_semihosting_console_out(env, args, 0);
+        return qemu_semihosting_console_outs(env, args);
     case TARGET_SYS_WRITE:
         GET_ARG(0);
         GET_ARG(1);

tests/docker/dockerfiles/fedora-cris-cross.docker

@@ -2,7 +2,7 @@
 # Cross compiler for cris system tests
 #
 
-FROM fedora:latest
+FROM fedora:30
 ENV PACKAGES gcc-cris-linux-gnu
 RUN dnf install -y $PACKAGES
 RUN rpm -q $PACKAGES | sort > /packages.txt

tests/docker/dockerfiles/fedora-i386-cross.docker

@@ -1,4 +1,4 @@
-FROM fedora:29
+FROM fedora:30
 ENV PACKAGES \
     gcc \
     glib2-devel.i686 \

tests/docker/dockerfiles/fedora.docker

@@ -1,4 +1,4 @@
-FROM fedora:29
+FROM fedora:30
 ENV PACKAGES \
     bc \
     bison \

tests/docker/dockerfiles/ubuntu.docker

@@ -1,6 +1,15 @@
-FROM ubuntu:16.04
+#
-RUN echo "deb http://archive.ubuntu.com/ubuntu/ trusty universe multiverse" >> \
+# Latest Ubuntu Release
-    /etc/apt/sources.list
+#
+# Useful for testing against relatively bleeding edge libraries and
+# compilers. We also have seperate recipe for the most recent LTS
+# release.
+#
+# When updating use the full tag not :latest otherwise the build
+# system won't pick up that it has changed.
+#
+
+FROM ubuntu:19.04
 ENV PACKAGES flex bison \
     ccache \
     clang \
@@ -21,7 +30,7 @@ ENV PACKAGES flex bison \
     libepoxy-dev \
     libfdt-dev \
     libgbm-dev \
-    libgnutls-dev \
+    libgnutls28-dev \
     libgtk-3-dev \
     libibverbs-dev \
     libiscsi-dev \
@@ -34,7 +43,7 @@ ENV PACKAGES flex bison \
     libnss3-dev \
     libnuma-dev \
     libpixman-1-dev \
-    libpng12-dev \
+    libpng-dev \
     librados-dev \
     librbd-dev \
     librdmacm-dev \

tests/tcg/i386/Makefile.softmmu-target

@@ -8,17 +8,13 @@
 
 I386_SYSTEM_SRC=$(SRC_PATH)/tests/tcg/i386/system
 X64_SYSTEM_SRC=$(SRC_PATH)/tests/tcg/x86_64/system
-# Set search path for all sources
-VPATH+=$(I386_SYSTEM_SRC)
 
 # These objects provide the basic boot code and helper functions for all tests
 CRT_OBJS=boot.o
 
-X86_TEST_SRCS=$(wildcard $(I386_SYSTEM_SRC)/*.c)
-X86_TESTS = $(patsubst $(I386_SYSTEM_SRC)/%.c, %, $(X86_TEST_SRCS))
-
 ifeq ($(TARGET_X86_64), y)
 CRT_PATH=$(X64_SYSTEM_SRC)
+CFLAGS=-march=x86-64
 LINK_SCRIPT=$(X64_SYSTEM_SRC)/kernel.ld
 LDFLAGS=-Wl,-T$(LINK_SCRIPT) -Wl,-melf_x86_64
 else
@@ -26,12 +22,12 @@ CRT_PATH=$(I386_SYSTEM_SRC)
 CFLAGS+=-m32
 LINK_SCRIPT=$(I386_SYSTEM_SRC)/kernel.ld
 LDFLAGS=-Wl,-T$(LINK_SCRIPT) -Wl,-melf_i386
-# FIXME: move to common once x86_64 is bootstrapped
-TESTS+=$(X86_TESTS) $(MULTIARCH_TESTS)
 endif
 CFLAGS+=-nostdlib -ggdb -O0 $(MINILIB_INC)
 LDFLAGS+=-static -nostdlib $(CRT_OBJS) $(MINILIB_OBJS) -lgcc
 
+TESTS+=$(MULTIARCH_TESTS)
+
 # building head blobs
 .PRECIOUS: $(CRT_OBJS)
 

tests/tcg/multiarch/system/memory.c

@@ -208,6 +208,7 @@ static bool read_test_data_u32(int offset)
 
     for (i = 0; i < max; i++) {
         uint8_t b1, b2, b3, b4;
+        int zeros = 0;
         word = *ptr++;
 
         b1 = word >> 24 & 0xff;
@@ -215,6 +216,16 @@ static bool read_test_data_u32(int offset)
         b3 = word >> 8 & 0xff;
         b4 = word & 0xff;
 
+        zeros += (b1 == 0 ? 1 : 0);
+        zeros += (b2 == 0 ? 1 : 0);
+        zeros += (b3 == 0 ? 1 : 0);
+        zeros += (b4 == 0 ? 1 : 0);
+        if (zeros > 1) {
+            ml_printf("Error @ %p, more zeros than expected: %d, %d, %d, %d",
+                      ptr - 1, b1, b2, b3, b4);
+            return false;
+        }
+
         if ((b1 < b2 && b1 != 0) ||
             (b2 < b3 && b2 != 0) ||
             (b3 < b4 && b3 != 0)) {
@@ -238,6 +249,7 @@ static bool read_test_data_u64(int offset)
 
     for (i = 0; i < max; i++) {
         uint8_t b1, b2, b3, b4, b5, b6, b7, b8;
+        int zeros = 0;
         word = *ptr++;
 
         b1 = ((uint64_t) (word >> 56)) & 0xff;
@@ -249,6 +261,20 @@ static bool read_test_data_u64(int offset)
         b7 = (word >> 8)  & 0xff;
         b8 = (word >> 0)  & 0xff;
 
+        zeros += (b1 == 0 ? 1 : 0);
+        zeros += (b2 == 0 ? 1 : 0);
+        zeros += (b3 == 0 ? 1 : 0);
+        zeros += (b4 == 0 ? 1 : 0);
+        zeros += (b5 == 0 ? 1 : 0);
+        zeros += (b6 == 0 ? 1 : 0);
+        zeros += (b7 == 0 ? 1 : 0);
+        zeros += (b8 == 0 ? 1 : 0);
+        if (zeros > 1) {
+            ml_printf("Error @ %p, more zeros than expected: %d, %d, %d, %d, %d, %d, %d, %d",
+                      ptr - 1, b1, b2, b3, b4, b5, b6, b7, b8);
+            return false;
+        }
+
         if ((b1 < b2 && b1 != 0) ||
             (b2 < b3 && b2 != 0) ||
             (b3 < b4 && b3 != 0) ||
@@ -272,7 +298,7 @@ read_ufn read_ufns[] = { read_test_data_u16,
                          read_test_data_u32,
                          read_test_data_u64 };
 
-bool do_unsigned_reads(void)
+bool do_unsigned_reads(int start_off)
 {
     int i;
     bool ok = true;
@@ -280,11 +306,11 @@ bool do_unsigned_reads(void)
     for (i = 0; i < ARRAY_SIZE(read_ufns) && ok; i++) {
 #if CHECK_UNALIGNED
         int off;
-        for (off = 0; off < 8 && ok; off++) {
+        for (off = start_off; off < 8 && ok; off++) {
             ok = read_ufns[i](off);
         }
 #else
-        ok = read_ufns[i](0);
+        ok = read_ufns[i](start_off);
 #endif
     }
 
@@ -298,11 +324,11 @@ static bool do_unsigned_test(init_ufn fn)
     int i;
     for (i = 0; i < 8 && ok; i++) {
         fn(i);
-        ok = do_unsigned_reads();
+        ok = do_unsigned_reads(i);
     }
 #else
     fn(0);
-    return do_unsigned_reads();
+    return do_unsigned_reads(0);
 #endif
 }
 

tests/tcg/x86_64/system/boot.S

@@ -0,0 +1,277 @@
+/*
+ * x86_64 boot and support code
+ *
+ * Copyright 2019 Linaro
+ *
+ * This work is licensed under the terms of the GNU GPL, version 3 or later.
+ * See the COPYING file in the top-level directory.
+ *
+ * Unlike the i386 version we instead use Xen's PVHVM booting header
+ * which should drop us automatically into 32 bit mode ready to go. I've
+ * nabbed bits of the Linux kernel setup to achieve this.
+ *
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+        .section .head
+
+#define ELFNOTE_START(name, type, flags)	\
+.pushsection .note.name, flags,@note	;	\
+  .balign 4				;	\
+  .long 2f - 1f		/* namesz */	;	\
+  .long 4484f - 3f	/* descsz */	;	\
+  .long type				;	\
+1:.asciz #name				;	\
+2:.balign 4				;	\
+3:
+
+#define ELFNOTE_END				\
+4484:.balign 4				;	\
+.popsection				;
+
+#define ELFNOTE(name, type, desc)		\
+	ELFNOTE_START(name, type, "")		\
+		desc			;	\
+	ELFNOTE_END
+
+#define XEN_ELFNOTE_ENTRY          1
+#define XEN_ELFNOTE_HYPERCALL_PAGE 2
+#define XEN_ELFNOTE_VIRT_BASE      3
+#define XEN_ELFNOTE_PADDR_OFFSET   4
+#define XEN_ELFNOTE_PHYS32_ENTRY  18
+
+#define __ASM_FORM(x)	x
+#define __ASM_FORM_RAW(x)     x
+#define __ASM_FORM_COMMA(x) x,
+#define __ASM_SEL(a,b)           __ASM_FORM(b)
+#define __ASM_SEL_RAW(a,b)      __ASM_FORM_RAW(b)
+#define _ASM_PTR	__ASM_SEL(.long, .quad)
+
+	ELFNOTE(Xen, XEN_ELFNOTE_VIRT_BASE,      _ASM_PTR 0x100000)
+	ELFNOTE(Xen, XEN_ELFNOTE_ENTRY,          _ASM_PTR _start)
+	ELFNOTE(Xen, XEN_ELFNOTE_PHYS32_ENTRY,   _ASM_PTR _start)    /* entry == virtbase */
+	ELFNOTE(Xen, XEN_ELFNOTE_PADDR_OFFSET,   _ASM_PTR 0)
+
+       /*
+	* Entry point for PVH guests.
+	*
+	* Xen ABI specifies the following register state when we come here:
+	*
+	* - `ebx`: contains the physical memory address where the loader has placed
+	*          the boot start info structure.
+	* - `cr0`: bit 0 (PE) must be set. All the other writeable bits are cleared.
+	* - `cr4`: all bits are cleared.
+	* - `cs `: must be a 32-bit read/execute code segment with a base of ‘0’
+	*          and a limit of ‘0xFFFFFFFF’. The selector value is unspecified.
+	* - `ds`, `es`: must be a 32-bit read/write data segment with a base of
+	*               ‘0’ and a limit of ‘0xFFFFFFFF’. The selector values are all
+	*               unspecified.
+	* - `tr`: must be a 32-bit TSS (active) with a base of '0' and a limit
+	*         of '0x67'.
+	* - `eflags`: bit 17 (VM) must be cleared. Bit 9 (IF) must be cleared.
+	*             Bit 8 (TF) must be cleared. Other bits are all unspecified.
+	*
+	* All other processor registers and flag bits are unspecified. The OS is in
+	* charge of setting up it's own stack, GDT and IDT.
+	*/
+        .code32
+        .section .text
+
+.global _start
+_start:
+	cld
+        lgdt gdtr
+
+        ljmp $0x8,$.Lloadcs
+.Lloadcs:
+        mov $0x10,%eax
+        mov %eax,%ds
+        mov %eax,%es
+        mov %eax,%fs
+        mov %eax,%gs
+        mov %eax,%ss
+
+	/* Enable PAE mode (bit 5). */
+	mov %cr4, %eax
+	btsl $5, %eax
+	mov %eax, %cr4
+
+#define MSR_EFER		0xc0000080 /* extended feature register */
+
+	/* Enable Long mode. */
+	mov $MSR_EFER, %ecx
+	rdmsr
+	btsl $8, %eax
+	wrmsr
+
+	/* Enable paging */
+	mov $.Lpml4, %ecx
+	mov %ecx, %cr3
+
+	mov %cr0, %eax
+	btsl $31, %eax
+	mov %eax, %cr0
+
+	/* Jump to 64-bit mode. */
+        lgdt gdtr64
+        ljmp $0x8,$.Lenter64
+
+        .code64
+        .section .text
+.Lenter64:
+
+
+	// Setup stack ASAP
+	movq $stack_end,%rsp
+
+        /* don't worry about stack frame, assume everthing is garbage when we return */
+	call main
+
+        /* output any non-zero result in eax to isa-debug-exit device */
+        test %al, %al
+        jz 1f
+        out %ax, $0xf4
+
+1:      /* QEMU ACPI poweroff */
+	mov $0x604,%edx
+	mov $0x2000,%eax
+	out %ax,%dx
+	hlt
+	jmp 1b
+
+        /*
+         * Helper Functions
+         *
+         * x86_64 calling convention is rdi, rsi, rdx, rcx, r8, r9
+         */
+
+        /* Output a single character to serial port */
+        .global __sys_outc
+__sys_outc:
+        pushq %rax
+        mov %rax, %rdx
+	out %al,$0xE9
+        popq %rax
+        ret
+
+	/* Interrupt Descriptor Table */
+
+        .section .data
+        .align 16
+
+idt_00: .int 0, 0
+idt_01: .int 0, 0
+idt_02: .int 0, 0
+idt_03: .int 0, 0
+idt_04: .int 0, 0
+idt_05: .int 0, 0
+idt_06: .int 0, 0 /* intr_6_opcode, Invalid Opcode */
+idt_07: .int 0, 0
+idt_08: .int 0, 0
+idt_09: .int 0, 0
+idt_0A: .int 0, 0
+idt_0B: .int 0, 0
+idt_0C: .int 0, 0
+idt_0D: .int 0, 0
+idt_0E: .int 0, 0
+idt_0F: .int 0, 0
+idt_10: .int 0, 0
+idt_11: .int 0, 0
+idt_12: .int 0, 0
+idt_13: .int 0, 0
+idt_14: .int 0, 0
+idt_15: .int 0, 0
+idt_16: .int 0, 0
+idt_17: .int 0, 0
+idt_18: .int 0, 0
+idt_19: .int 0, 0
+idt_1A: .int 0, 0
+idt_1B: .int 0, 0
+idt_1C: .int 0, 0
+idt_1D: .int 0, 0
+idt_1E: .int 0, 0
+idt_1F: .int 0, 0
+
+
+	/*
+	 * Global Descriptor Table (GDT)
+	 *
+	 * This describes various memory areas (segments) through
+	 * segment descriptors. In 32 bit mode each segment each
+	 * segement is associated with segment registers which are
+	 * implicitly (or explicitly) referenced depending on the
+	 * instruction. However in 64 bit mode selectors are flat and
+	 * segmented addressing isn't used.
+	 */
+gdt:
+        .short 0
+gdtr:
+        .short gdt_en - gdt - 1
+        .int gdt
+
+        // Code cs:
+        .short 0xFFFF
+        .short 0
+        .byte 0
+        .byte 0x9b
+        .byte 0xCF
+        .byte 0
+
+        // Data  ds:, ss:, es:, fs:, and gs:
+        .short 0xFFFF
+        .short 0
+        .byte 0
+        .byte 0x93
+        .byte 0xCF
+        .byte 0
+gdt_en:
+
+gdt64:
+        .short 0
+gdtr64:
+        .short gdt64_en - gdt64 - 1
+        .int gdt64
+
+        // Code
+        .short 0xFFFF
+        .short 0
+        .byte 0
+        .byte 0x9b
+        .byte 0xAF
+        .byte 0
+
+        // Data
+        .short 0xFFFF
+        .short 0
+        .byte 0
+        .byte 0x93
+        .byte 0xCF
+        .byte 0
+gdt64_en:
+
+	.section .bss
+        .align 16
+
+stack: .space 65536
+stack_end:
+
+	.section .data
+
+.align 4096
+.Lpd:
+i = 0
+        .rept 512 * 4
+        .quad 0x1e7 | (i << 21)
+        i = i + 1
+        .endr
+
+.align 4096
+.Lpdp:
+        .quad .Lpd + 7 + 0 * 4096 /* 0-1 GB */
+        .quad .Lpd + 7 + 1 * 4096 /* 1-2 GB */
+        .quad .Lpd + 7 + 2 * 4096 /* 2-3 GB */
+        .quad .Lpd + 7 + 3 * 4096 /* 3-4 GB */
+
+.align 4096
+.Lpml4:
+        .quad .Lpdp + 7 /* 0-512 GB */

tests/tcg/x86_64/system/kernel.ld

@@ -0,0 +1,33 @@
+PHDRS {
+	text PT_LOAD FLAGS(5);          /* R_E */
+	note PT_NOTE FLAGS(0);          /* ___ */
+}
+
+SECTIONS {
+	. = 0x100000;
+
+	.text : {
+		__load_st = .;
+		*(.head)
+		*(.text)
+	} :text
+
+	.rodata : {
+		*(.rodata)
+	} :text
+
+        /* Keep build ID and PVH notes in same section */
+        .notes :  {
+               *(.note.*)
+        } :note
+
+        .data : {
+		*(.data)
+		__load_en = .;
+	} :text
+
+	.bss : {
+		*(.bss)
+		__bss_en = .;
+	}
+}

tests/vm/Makefile.include

@@ -21,9 +21,13 @@ vm-test:
 	@echo "  vm-clean-all                    - Clean up VM images"
 	@echo
 	@echo "Special variables:"
-	@echo "    BUILD_TARGET=foo		 - override the build target"
+	@echo "    BUILD_TARGET=foo		 - Override the build target"
-	@echo "    TARGET_LIST=a,b,c    	 - Override target list in builds."
+	@echo "    TARGET_LIST=a,b,c    	 - Override target list in builds"
 	@echo '    EXTRA_CONFIGURE_OPTS="..."'
+	@echo "    J=[0..9]*            	 - Override the -jN parameter for make commands"
+	@echo "    DEBUG=1              	 - Enable verbose output on host and interactive debugging"
+	@echo "    V=1				 - Enable verbose ouput on host and guest commands"
+	@echo "    QEMU=/path/to/qemu		 - Change path to QEMU binary"
 
 vm-build-all: $(addprefix vm-build-, $(IMAGES))
 
@@ -35,7 +39,7 @@ $(IMAGES_DIR)/%.img:	$(SRC_PATH)/tests/vm/% \
 			$(SRC_PATH)/tests/vm/Makefile.include
 	@mkdir -p $(IMAGES_DIR)
 	$(call quiet-command, \
-		$< \
+		$(PYTHON) $< \
 		$(if $(V)$(DEBUG), --debug) \
 		--image "$@" \
 		--force \
@@ -46,7 +50,7 @@ $(IMAGES_DIR)/%.img:	$(SRC_PATH)/tests/vm/% \
 # Build in VM $(IMAGE)
 vm-build-%: $(IMAGES_DIR)/%.img
 	$(call quiet-command, \
-		$(SRC_PATH)/tests/vm/$* \
+		$(PYTHON) $(SRC_PATH)/tests/vm/$* \
 		$(if $(V)$(DEBUG), --debug) \
 		$(if $(DEBUG), --interactive) \
 		$(if $(J),--jobs $(J)) \

tests/vm/basevm.py

@@ -73,7 +73,7 @@ class BaseVM(object):
             "-vnc", "127.0.0.1:0,to=20",
             "-serial", "file:%s" % os.path.join(self._tmpdir, "serial.out")]
         if vcpus and vcpus > 1:
-            self._args += ["-smp", str(vcpus)]
+            self._args += ["-smp", "%d" % vcpus]
         if kvm_available(self.arch):
             self._args += ["-enable-kvm"]
         else:
@@ -85,12 +85,13 @@ class BaseVM(object):
             if not sha256sum:
                 return True
             checksum = subprocess.check_output(["sha256sum", fname]).split()[0]
-            return sha256sum == checksum
+            return sha256sum == checksum.decode("utf-8")
 
         cache_dir = os.path.expanduser("~/.cache/qemu-vm/download")
         if not os.path.exists(cache_dir):
             os.makedirs(cache_dir)
-        fname = os.path.join(cache_dir, hashlib.sha1(url).hexdigest())
+        fname = os.path.join(cache_dir,
+                             hashlib.sha1(url.encode("utf-8")).hexdigest())
         if os.path.exists(fname) and check_sha256sum(fname):
             return fname
         logging.debug("Downloading %s to %s...", url, fname)
@@ -134,7 +135,7 @@ class BaseVM(object):
         raise NotImplementedError
 
     def add_source_dir(self, src_dir):
-        name = "data-" + hashlib.sha1(src_dir).hexdigest()[:5]
+        name = "data-" + hashlib.sha1(src_dir.encode("utf-8")).hexdigest()[:5]
         tarfile = os.path.join(self._tmpdir, name + ".tar")
         logging.debug("Creating archive %s for src_dir dir: %s", tarfile, src_dir)
         subprocess.check_call(["./scripts/archive-source.sh", tarfile],
@@ -204,7 +205,7 @@ def parse_args(vmcls):
 
     def get_default_jobs():
         if kvm_available(vmcls.arch):
-            return multiprocessing.cpu_count() / 2
+            return multiprocessing.cpu_count() // 2
         else:
             return 1
 
@@ -256,7 +257,7 @@ def main(vmcls):
             vm.add_source_dir(args.build_qemu)
             cmd = [vm.BUILD_SCRIPT.format(
                    configure_opts = " ".join(argv),
-                   jobs=args.jobs,
+                   jobs=int(args.jobs),
                    target=args.build_target,
                    verbose = "V=1" if args.verbose else "")]
         else:

tests/vm/centos

@@ -26,9 +26,9 @@ class CentosVM(basevm.BaseVM):
         export SRC_ARCHIVE=/dev/vdb;
         sudo chmod a+r $SRC_ARCHIVE;
         tar -xf $SRC_ARCHIVE;
-        make docker-test-block@centos7 V={verbose} J={jobs};
+        make docker-test-block@centos7 {verbose} J={jobs} NETWORK=1;
-        make docker-test-quick@centos7 V={verbose} J={jobs};
+        make docker-test-quick@centos7 {verbose} J={jobs} NETWORK=1;
-        make docker-test-mingw@fedora V={verbose} J={jobs};
+        make docker-test-mingw@fedora  {verbose} J={jobs} NETWORK=1;
     """
 
     def _gen_cloud_init_iso(self):