* Lots of functional test improvements (clean-ups, don't fail on

temporary download errors, etc.)
 * Convert some more avocado tests to the functional framework
 * Disallow building with libnfs v6 due to an API breakage
 -----BEGIN PGP SIGNATURE-----
 
 iQJFBAABCAAvFiEEJ7iIR+7gJQEY8+q5LtnXdP5wLbUFAmdirOIRHHRodXRoQHJl
 ZGhhdC5jb20ACgkQLtnXdP5wLbU0NRAAke8X0B6OOD+99lY5nc7Hrh7N1m+sw5Lw
 TVwIpxdhxU11vgdlCodfdoVJCV1NGVHwkR57lLNr+bdspWDBBwlmUWn0+t2QCXGe
 oyQsV+boznsjG9pan6v6DcU/gOu7/7ZydhJi+M8Msf8ah0lcn/otAdC4ZFB93JLh
 6xPnj69y8HomCW+wMyXl7WTjcWX0wQFzweEYY8p7X7p1rtjYyseiZlRjNAvPgTMI
 jznZ6v9/qU54xR9RnKdW+0m1Qu06nx26Wz+ZBlvrJS1Llloe23X9+LY1tDD0Xh1D
 9P0v9PuaBWRRF+UjVjl37LMyn9h1aaKFKBoWQiKMbyvOVr4ncobjRgN8r5kdNxDP
 FZ/fA1GiX8O3foN9uB9JLKd6Hl49LAqQSPzAneEc3pfQLH3NdAjPxJDbJH5fyMa7
 qVOQC0Bdy8+2kCxFfKbemrwDOFcyq1fVYcADPDZySjMiPnwFJ1Qpni1tXY1PZ+Tl
 Q18AsFJanyAAn7L+8R3Yl54983SuR5eXIFxO+Tq9mw1V1V2h+Cm09HGcS8y5bxFG
 Xh+jhMsMB98NFLR87W6olwl57gKllSbTYuGtiz9TrbnuT/THhUJ0k/B76L7C9HWE
 ZefkFxC5Zy8jrcz3pgarO+19V+eXg5rwGtEngRQrji/3cY5CbK7Jeh5nvZQeASpb
 nZ/gJ/gC8Gs=
 =SWw6
 -----END PGP SIGNATURE-----

Merge tag 'pull-request-2024-12-18' of https://gitlab.com/thuth/qemu into staging

* Lots of functional test improvements (clean-ups, don't fail on
  temporary download errors, etc.)
* Convert some more avocado tests to the functional framework
* Disallow building with libnfs v6 due to an API breakage

# -----BEGIN PGP SIGNATURE-----
#
# iQJFBAABCAAvFiEEJ7iIR+7gJQEY8+q5LtnXdP5wLbUFAmdirOIRHHRodXRoQHJl
# ZGhhdC5jb20ACgkQLtnXdP5wLbU0NRAAke8X0B6OOD+99lY5nc7Hrh7N1m+sw5Lw
# TVwIpxdhxU11vgdlCodfdoVJCV1NGVHwkR57lLNr+bdspWDBBwlmUWn0+t2QCXGe
# oyQsV+boznsjG9pan6v6DcU/gOu7/7ZydhJi+M8Msf8ah0lcn/otAdC4ZFB93JLh
# 6xPnj69y8HomCW+wMyXl7WTjcWX0wQFzweEYY8p7X7p1rtjYyseiZlRjNAvPgTMI
# jznZ6v9/qU54xR9RnKdW+0m1Qu06nx26Wz+ZBlvrJS1Llloe23X9+LY1tDD0Xh1D
# 9P0v9PuaBWRRF+UjVjl37LMyn9h1aaKFKBoWQiKMbyvOVr4ncobjRgN8r5kdNxDP
# FZ/fA1GiX8O3foN9uB9JLKd6Hl49LAqQSPzAneEc3pfQLH3NdAjPxJDbJH5fyMa7
# qVOQC0Bdy8+2kCxFfKbemrwDOFcyq1fVYcADPDZySjMiPnwFJ1Qpni1tXY1PZ+Tl
# Q18AsFJanyAAn7L+8R3Yl54983SuR5eXIFxO+Tq9mw1V1V2h+Cm09HGcS8y5bxFG
# Xh+jhMsMB98NFLR87W6olwl57gKllSbTYuGtiz9TrbnuT/THhUJ0k/B76L7C9HWE
# ZefkFxC5Zy8jrcz3pgarO+19V+eXg5rwGtEngRQrji/3cY5CbK7Jeh5nvZQeASpb
# nZ/gJ/gC8Gs=
# =SWw6
# -----END PGP SIGNATURE-----
# gpg: Signature made Wed 18 Dec 2024 06:07:14 EST
# gpg:                using RSA key 27B88847EEE0250118F3EAB92ED9D774FE702DB5
# gpg:                issuer "thuth@redhat.com"
# gpg: Good signature from "Thomas Huth <th.huth@gmx.de>" [full]
# gpg:                 aka "Thomas Huth <thuth@redhat.com>" [full]
# gpg:                 aka "Thomas Huth <huth@tuxfamily.org>" [full]
# gpg:                 aka "Thomas Huth <th.huth@posteo.de>" [unknown]
# Primary key fingerprint: 27B8 8847 EEE0 2501 18F3  EAB9 2ED9 D774 FE70 2DB5

* tag 'pull-request-2024-12-18' of https://gitlab.com/thuth/qemu: (38 commits)
  meson.build: Disallow libnfs v6 to fix the broken macOS build
  tests/functional: Convert the hotplug_cpu avocado test
  tests/functional: Convert the intel_iommu avocado test
  tests/functional: Add a helper function for retrieving the hostfwd port
  tests/functional: Convert the arm virt avocado test
  tests/functional: Convert the quanta-gsj avocado test
  MAINTAINERS: add myself as reviewer for functional test suite
  tests/functional: ignore errors when caching assets, except for 404
  tests/functional: skip tests if assets are not available
  tests/functional: remove now unused 'run_cmd' helper
  tests/functional: replace 'run_cmd' with subprocess helpers
  tests/functional: drop back compat imports from utils.py
  tests/functional: convert tests to new uncompress helper
  tests/functional: add 'uncompress' to QemuBaseTest
  tests/functional: add a generalized uncompress helper
  tests/functional: convert tests to new archive_extract helper
  tests/functional: add 'archive_extract' to QemuBaseTest
  tests/functional: add a generalized archive_extract
  tests/functional: let cpio_extract accept filenames
  tests/functional: add common deb_extract helper
  ...

Signed-off-by: Stefan Hajnoczi <stefanha@redhat.com>

MAINTAINERS

@@ -872,6 +872,7 @@ F: tests/qtest/adm1266-test.c
 F: pc-bios/npcm7xx_bootrom.bin
 F: roms/vbootrom
 F: docs/system/arm/nuvoton.rst
+F: tests/functional/test_arm_quanta_gsj.py
 
 Raspberry Pi
 M: Peter Maydell <peter.maydell@linaro.org>
@@ -3681,6 +3682,7 @@ S: Supported
 F: hw/i386/intel_iommu.c
 F: hw/i386/intel_iommu_internal.h
 F: include/hw/i386/intel_iommu.h
+F: tests/functional/test_intel_iommu.py
 
 AMD-Vi Emulation
 S: Orphan
@@ -4157,6 +4159,7 @@ W: https://cirrus-ci.com/github/qemu/qemu
 Functional testing framework
 M: Thomas Huth <thuth@redhat.com>
 R: Philippe Mathieu-Daudé <philmd@linaro.org>
+R: Daniel P. Berrange <berrange@redhat.com>
 F: tests/functional/qemu_test/
 
 Windows Hosted Continuous Integration

meson.build

@@ -1145,7 +1145,7 @@ endif
 
 libnfs = not_found
 if not get_option('libnfs').auto() or have_block
-  libnfs = dependency('libnfs', version: '>=1.9.3',
+  libnfs = dependency('libnfs', version: ['>=1.9.3', '<6.0.0'],
                       required: get_option('libnfs'),
                       method: 'pkg-config')
 endif

tests/avocado/boot_linux_console.py

@@ -94,110 +94,3 @@ class BootLinuxConsole(LinuxKernelTest):
         self.vm.launch()
         console_pattern = 'Kernel command line: %s' % kernel_command_line
         self.wait_for_console_pattern(console_pattern)
-
-    def test_arm_virt(self):
-        """
-        :avocado: tags=arch:arm
-        :avocado: tags=machine:virt
-        :avocado: tags=accel:tcg
-        """
-        kernel_url = ('https://archives.fedoraproject.org/pub/archive/fedora'
-                      '/linux/releases/29/Everything/armhfp/os/images/pxeboot'
-                      '/vmlinuz')
-        kernel_hash = 'e9826d741b4fb04cadba8d4824d1ed3b7fb8b4d4'
-        kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
-
-        self.vm.set_console()
-        kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
-                               'console=ttyAMA0')
-        self.vm.add_args('-kernel', kernel_path,
-                         '-append', kernel_command_line)
-        self.vm.launch()
-        console_pattern = 'Kernel command line: %s' % kernel_command_line
-        self.wait_for_console_pattern(console_pattern)
-
-    @skipUnless(os.getenv('AVOCADO_TIMEOUT_EXPECTED'), 'Test might timeout')
-    def test_arm_quanta_gsj(self):
-        """
-        :avocado: tags=arch:arm
-        :avocado: tags=machine:quanta-gsj
-        :avocado: tags=accel:tcg
-        """
-        # 25 MiB compressed, 32 MiB uncompressed.
-        image_url = (
-                'https://github.com/hskinnemoen/openbmc/releases/download/'
-                '20200711-gsj-qemu-0/obmc-phosphor-image-gsj.static.mtd.gz')
-        image_hash = '14895e634923345cb5c8776037ff7876df96f6b1'
-        image_path_gz = self.fetch_asset(image_url, asset_hash=image_hash)
-        image_name = 'obmc.mtd'
-        image_path = os.path.join(self.workdir, image_name)
-        archive.gzip_uncompress(image_path_gz, image_path)
-
-        self.vm.set_console()
-        drive_args = 'file=' + image_path + ',if=mtd,bus=0,unit=0'
-        self.vm.add_args('-drive', drive_args)
-        self.vm.launch()
-
-        # Disable drivers and services that stall for a long time during boot,
-        # to avoid running past the 90-second timeout. These may be removed
-        # as the corresponding device support is added.
-        kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + (
-                'console=${console} '
-                'mem=${mem} '
-                'initcall_blacklist=npcm_i2c_bus_driver_init '
-                'systemd.mask=systemd-random-seed.service '
-                'systemd.mask=dropbearkey.service '
-        )
-
-        self.wait_for_console_pattern('> BootBlock by Nuvoton')
-        self.wait_for_console_pattern('>Device: Poleg BMC NPCM730')
-        self.wait_for_console_pattern('>Skip DDR init.')
-        self.wait_for_console_pattern('U-Boot ')
-        interrupt_interactive_console_until_pattern(
-                self, 'Hit any key to stop autoboot:', 'U-Boot>')
-        exec_command_and_wait_for_pattern(
-                self, "setenv bootargs ${bootargs} " + kernel_command_line,
-                'U-Boot>')
-        exec_command_and_wait_for_pattern(
-                self, 'run romboot', 'Booting Kernel from flash')
-        self.wait_for_console_pattern('Booting Linux on physical CPU 0x0')
-        self.wait_for_console_pattern('CPU1: thread -1, cpu 1, socket 0')
-        self.wait_for_console_pattern('OpenBMC Project Reference Distro')
-        self.wait_for_console_pattern('gsj login:')
-
-    def test_arm_quanta_gsj_initrd(self):
-        """
-        :avocado: tags=arch:arm
-        :avocado: tags=machine:quanta-gsj
-        :avocado: tags=accel:tcg
-        """
-        initrd_url = (
-                'https://github.com/hskinnemoen/openbmc/releases/download/'
-                '20200711-gsj-qemu-0/obmc-phosphor-initramfs-gsj.cpio.xz')
-        initrd_hash = '98fefe5d7e56727b1eb17d5c00311b1b5c945300'
-        initrd_path = self.fetch_asset(initrd_url, asset_hash=initrd_hash)
-        kernel_url = (
-                'https://github.com/hskinnemoen/openbmc/releases/download/'
-                '20200711-gsj-qemu-0/uImage-gsj.bin')
-        kernel_hash = 'fa67b2f141d56d39b3c54305c0e8a899c99eb2c7'
-        kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
-        dtb_url = (
-                'https://github.com/hskinnemoen/openbmc/releases/download/'
-                '20200711-gsj-qemu-0/nuvoton-npcm730-gsj.dtb')
-        dtb_hash = '18315f7006d7b688d8312d5c727eecd819aa36a4'
-        dtb_path = self.fetch_asset(dtb_url, asset_hash=dtb_hash)
-
-        self.vm.set_console()
-        kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
-                               'console=ttyS0,115200n8 '
-                               'earlycon=uart8250,mmio32,0xf0001000')
-        self.vm.add_args('-kernel', kernel_path,
-                         '-initrd', initrd_path,
-                         '-dtb', dtb_path,
-                         '-append', kernel_command_line)
-        self.vm.launch()
-
-        self.wait_for_console_pattern('Booting Linux on physical CPU 0x0')
-        self.wait_for_console_pattern('CPU1: thread -1, cpu 1, socket 0')
-        self.wait_for_console_pattern(
-                'Give root password for system maintenance')

tests/avocado/hotplug_cpu.py

@@ -1,37 +0,0 @@
-# Functional test that hotplugs a CPU and checks it on a Linux guest
-#
-# Copyright (c) 2021 Red Hat, Inc.
-#
-# Author:
-#  Cleber Rosa <crosa@redhat.com>
-#
-# This work is licensed under the terms of the GNU GPL, version 2 or
-# later.  See the COPYING file in the top-level directory.
-
-from avocado_qemu.linuxtest import LinuxTest
-
-
-class HotPlugCPU(LinuxTest):
-
-    def test(self):
-        """
-        :avocado: tags=arch:x86_64
-        :avocado: tags=machine:q35
-        :avocado: tags=accel:kvm
-        """
-        self.require_accelerator('kvm')
-        self.vm.add_args('-accel', 'kvm')
-        self.vm.add_args('-cpu', 'Haswell')
-        self.vm.add_args('-smp', '1,sockets=1,cores=2,threads=1,maxcpus=2')
-        self.launch_and_wait()
-
-        self.ssh_command('test -e /sys/devices/system/cpu/cpu0')
-        with self.assertRaises(AssertionError):
-            self.ssh_command('test -e /sys/devices/system/cpu/cpu1')
-
-        self.vm.cmd('device_add',
-                    driver='Haswell-x86_64-cpu',
-                    socket_id=0,
-                    core_id=1,
-                    thread_id=0)
-        self.ssh_command('test -e /sys/devices/system/cpu/cpu1')

tests/avocado/intel_iommu.py

@@ -1,122 +0,0 @@
-# INTEL_IOMMU Functional tests
-#
-# Copyright (c) 2021 Red Hat, Inc.
-#
-# Author:
-#  Eric Auger <eric.auger@redhat.com>
-#
-# This work is licensed under the terms of the GNU GPL, version 2 or
-# later.  See the COPYING file in the top-level directory.
-import os
-
-from avocado import skipUnless
-from avocado_qemu.linuxtest import LinuxTest
-
-@skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
-class IntelIOMMU(LinuxTest):
-    """
-    :avocado: tags=arch:x86_64
-    :avocado: tags=distro:fedora
-    :avocado: tags=distro_version:31
-    :avocado: tags=machine:q35
-    :avocado: tags=accel:kvm
-    :avocado: tags=intel_iommu
-    :avocado: tags=flaky
-    """
-
-    IOMMU_ADDON = ',iommu_platform=on,disable-modern=off,disable-legacy=on'
-    kernel_path = None
-    initrd_path = None
-    kernel_params = None
-
-    def set_up_boot(self):
-        path = self.download_boot()
-        self.vm.add_args('-device', 'virtio-blk-pci,bus=pcie.0,' +
-                         'drive=drv0,id=virtio-disk0,bootindex=1,'
-                         'werror=stop,rerror=stop' + self.IOMMU_ADDON)
-        self.vm.add_args('-device', 'virtio-gpu-pci' + self.IOMMU_ADDON)
-        self.vm.add_args('-drive',
-                         'file=%s,if=none,cache=writethrough,id=drv0' % path)
-
-    def setUp(self):
-        super(IntelIOMMU, self).setUp(None, 'virtio-net-pci' + self.IOMMU_ADDON)
-
-    def add_common_args(self):
-        self.vm.add_args('-device', 'virtio-rng-pci,rng=rng0')
-        self.vm.add_args('-object',
-                         'rng-random,id=rng0,filename=/dev/urandom')
-
-    def common_vm_setup(self, custom_kernel=None):
-        self.require_accelerator("kvm")
-        self.add_common_args()
-        self.vm.add_args("-accel", "kvm")
-
-        if custom_kernel is None:
-            return
-
-        kernel_url = self.distro.pxeboot_url + 'vmlinuz'
-        kernel_hash = '5b6f6876e1b5bda314f93893271da0d5777b1f3c'
-        initrd_url = self.distro.pxeboot_url + 'initrd.img'
-        initrd_hash = 'dd0340a1b39bd28f88532babd4581c67649ec5b1'
-        self.kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash)
-        self.initrd_path = self.fetch_asset(initrd_url, asset_hash=initrd_hash)
-
-    def run_and_check(self):
-        if self.kernel_path:
-            self.vm.add_args('-kernel', self.kernel_path,
-                             '-append', self.kernel_params,
-                             '-initrd', self.initrd_path)
-        self.launch_and_wait()
-        self.ssh_command('cat /proc/cmdline')
-        self.ssh_command('dmesg | grep -e DMAR -e IOMMU')
-        self.ssh_command('find /sys/kernel/iommu_groups/ -type l')
-        self.ssh_command('dnf -y install numactl-devel')
-
-    def test_intel_iommu(self):
-        """
-        :avocado: tags=intel_iommu_intremap
-        """
-
-        self.common_vm_setup(True)
-        self.vm.add_args('-device', 'intel-iommu,intremap=on')
-        self.vm.add_args('-machine', 'kernel_irqchip=split')
-
-        self.kernel_params = (self.distro.default_kernel_params +
-                              ' quiet intel_iommu=on')
-        self.run_and_check()
-
-    def test_intel_iommu_strict(self):
-        """
-        :avocado: tags=intel_iommu_strict
-        """
-
-        self.common_vm_setup(True)
-        self.vm.add_args('-device', 'intel-iommu,intremap=on')
-        self.vm.add_args('-machine', 'kernel_irqchip=split')
-        self.kernel_params = (self.distro.default_kernel_params +
-                              ' quiet intel_iommu=on,strict')
-        self.run_and_check()
-
-    def test_intel_iommu_strict_cm(self):
-        """
-        :avocado: tags=intel_iommu_strict_cm
-        """
-
-        self.common_vm_setup(True)
-        self.vm.add_args('-device', 'intel-iommu,intremap=on,caching-mode=on')
-        self.vm.add_args('-machine', 'kernel_irqchip=split')
-        self.kernel_params = (self.distro.default_kernel_params +
-                              ' quiet intel_iommu=on,strict')
-        self.run_and_check()
-
-    def test_intel_iommu_pt(self):
-        """
-        :avocado: tags=intel_iommu_pt
-        """
-
-        self.common_vm_setup(True)
-        self.vm.add_args('-device', 'intel-iommu,intremap=on')
-        self.vm.add_args('-machine', 'kernel_irqchip=split')
-        self.kernel_params = (self.distro.default_kernel_params +
-                              ' quiet intel_iommu=on iommu=pt')
-        self.run_and_check()

tests/functional/meson.build

@@ -27,9 +27,11 @@ test_timeouts = {
   'arm_collie' : 180,
   'arm_cubieboard' : 360,
   'arm_orangepi' : 540,
+  'arm_quanta_gsj' : 240,
   'arm_raspi2' : 120,
   'arm_tuxrun' : 240,
   'arm_sx1' : 360,
+  'intel_iommu': 300,
   'mips_malta' : 120,
   'netdev_ethtool' : 180,
   'ppc_40p' : 240,
@@ -85,10 +87,12 @@ tests_arm_system_thorough = [
   'arm_emcraft_sf2',
   'arm_integratorcp',
   'arm_orangepi',
+  'arm_quanta_gsj',
   'arm_raspi2',
   'arm_smdkc210',
   'arm_sx1',
   'arm_vexpress',
+  'arm_virt',
   'arm_tuxrun',
 ]
 
@@ -224,11 +228,13 @@ tests_x86_64_system_quick = [
 
 tests_x86_64_system_thorough = [
   'acpi_bits',
-  'x86_64_tuxrun',
+  'intel_iommu',
   'linux_initrd',
   'multiprocess',
   'netdev_ethtool',
   'virtio_gpu',
+  'x86_64_hotplug_cpu',
+  'x86_64_tuxrun',
 ]
 
 tests_xtensa_system_thorough = [

tests/functional/qemu_test/__init__.py

@@ -8,8 +8,13 @@
 
 from .asset import Asset
 from .config import BUILD_DIR
-from .cmd import has_cmd, has_cmds, run_cmd, is_readable_executable_file, \
+from .cmd import is_readable_executable_file, \
     interrupt_interactive_console_until_pattern, wait_for_console_pattern, \
-    exec_command, exec_command_and_wait_for_pattern, get_qemu_img
+    exec_command, exec_command_and_wait_for_pattern, get_qemu_img, which
 from .testcase import QemuBaseTest, QemuUserTest, QemuSystemTest
 from .linuxkernel import LinuxKernelTest
+from .decorators import skipIfMissingCommands, skipIfNotMachine, \
+    skipFlakyTest, skipUntrustedTest, skipBigDataTest, \
+    skipIfMissingImports
+from .archive import archive_extract
+from .uncompress import uncompress

tests/functional/qemu_test/archive.py

@@ -0,0 +1,117 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# Utilities for python-based QEMU tests
+#
+# Copyright 2024 Red Hat, Inc.
+#
+# Authors:
+#  Thomas Huth <thuth@redhat.com>
+
+import os
+from subprocess import check_call, run, DEVNULL
+import tarfile
+from urllib.parse import urlparse
+import zipfile
+
+from .asset import Asset
+
+
+def tar_extract(archive, dest_dir, member=None):
+    with tarfile.open(archive) as tf:
+        if hasattr(tarfile, 'data_filter'):
+            tf.extraction_filter = getattr(tarfile, 'data_filter',
+                                           (lambda member, path: member))
+        if member:
+            tf.extract(member=member, path=dest_dir)
+        else:
+            tf.extractall(path=dest_dir)
+
+def cpio_extract(archive, output_path):
+    cwd = os.getcwd()
+    os.chdir(output_path)
+    # Not passing 'check=True' as cpio exits with non-zero
+    # status if the archive contains any device nodes :-(
+    if type(archive) == str:
+        run(['cpio', '-i', '-F', archive],
+            stdout=DEVNULL, stderr=DEVNULL)
+    else:
+        run(['cpio', '-i'],
+            input=archive.read(),
+            stdout=DEVNULL, stderr=DEVNULL)
+    os.chdir(cwd)
+
+def zip_extract(archive, dest_dir, member=None):
+    with zipfile.ZipFile(archive, 'r') as zf:
+        if member:
+            zf.extract(member=member, path=dest_dir)
+        else:
+            zf.extractall(path=dest_dir)
+
+def deb_extract(archive, dest_dir, member=None):
+    cwd = os.getcwd()
+    os.chdir(dest_dir)
+    try:
+        proc = run(['ar', 't', archive],
+                   check=True, capture_output=True, encoding='utf8')
+        file_path = proc.stdout.split()[2]
+        check_call(['ar', 'x', archive, file_path],
+                   stdout=DEVNULL, stderr=DEVNULL)
+        tar_extract(file_path, dest_dir, member)
+    finally:
+        os.chdir(cwd)
+
+'''
+@params archive: filename, Asset, or file-like object to extract
+@params dest_dir: target directory to extract into
+@params member: optional member file to limit extraction to
+
+Extracts @archive into @dest_dir. All files are extracted
+unless @member specifies a limit.
+
+If @format is None, heuristics will be applied to guess the format
+from the filename or Asset URL. @format must be non-None if @archive
+is a file-like object.
+'''
+def archive_extract(archive, dest_dir, format=None, member=None):
+    if format is None:
+        format = guess_archive_format(archive)
+    if type(archive) == Asset:
+        archive = str(archive)
+
+    if format == "tar":
+        tar_extract(archive, dest_dir, member)
+    elif format == "zip":
+        zip_extract(archive, dest_dir, member)
+    elif format == "cpio":
+        if member is not None:
+            raise Exception("Unable to filter cpio extraction")
+        cpio_extract(archive, dest_dir)
+    elif format == "deb":
+        if type(archive) != str:
+            raise Exception("Unable to use file-like object with deb archives")
+        deb_extract(archive, dest_dir, "./" + member)
+    else:
+        raise Exception(f"Unknown archive format {format}")
+
+'''
+@params archive: filename, or Asset to guess
+
+Guess the format of @compressed, raising an exception if
+no format can be determined
+'''
+def guess_archive_format(archive):
+    if type(archive) == Asset:
+        archive = urlparse(archive.url).path
+    elif type(archive) != str:
+        raise Exception(f"Unable to guess archive format for {archive}")
+
+    if ".tar." in archive or archive.endswith("tgz"):
+        return "tar"
+    elif archive.endswith(".zip"):
+        return "zip"
+    elif archive.endswith(".cpio"):
+        return "cpio"
+    elif archive.endswith(".deb") or archive.endswith(".udeb"):
+        return "deb"
+    else:
+        raise Exception(f"Unknown archive format for {archive}")

tests/functional/qemu_test/asset.py

@@ -9,13 +9,13 @@ import hashlib
 import logging
 import os
 import stat
-import subprocess
 import sys
 import unittest
 import urllib.request
 from time import sleep
 from pathlib import Path
 from shutil import copyfileobj
+from urllib.error import HTTPError
 
 
 # Instances of this class must be declared as class level variables
@@ -40,6 +40,9 @@ class Asset:
         return "Asset: url=%s hash=%s cache=%s" % (
             self.url, self.hash, self.cache_file)
 
+    def __str__(self):
+        return str(self.cache_file)
+
     def _check(self, cache_file):
         if self.hash is None:
             return True
@@ -63,6 +66,12 @@ class Asset:
     def valid(self):
         return self.cache_file.exists() and self._check(self.cache_file)
 
+    def fetchable(self):
+        return not os.environ.get("QEMU_TEST_NO_DOWNLOAD", False)
+
+    def available(self):
+        return self.valid() or self.fetchable()
+
     def _wait_for_other_download(self, tmp_cache_file):
         # Another thread already seems to download the asset, so wait until
         # it is done, while also checking the size to see whether it is stuck
@@ -101,7 +110,7 @@ class Asset:
                            self.cache_file, self.url)
             return str(self.cache_file)
 
-        if os.environ.get("QEMU_TEST_NO_DOWNLOAD", False):
+        if not self.fetchable():
             raise Exception("Asset cache is invalid and downloads disabled")
 
         self.log.info("Downloading %s to %s...", self.url, self.cache_file)
@@ -162,7 +171,18 @@ class Asset:
         for name, asset in vars(test.__class__).items():
             if name.startswith("ASSET_") and type(asset) == Asset:
                 log.info("Attempting to cache '%s'" % asset)
-                asset.fetch()
+                try:
+                    asset.fetch()
+                except HTTPError as e:
+                    # Treat 404 as fatal, since it is highly likely to
+                    # indicate a broken test rather than a transient
+                    # server or networking problem
+                    if e.code == 404:
+                        raise
+
+                    log.debug(f"HTTP error {e.code} from {asset.url} " +
+                              "skipping asset precache")
+
         log.removeHandler(handler)
 
     def precache_suite(suite):

tests/functional/qemu_test/cmd.py

@@ -14,66 +14,18 @@
 import logging
 import os
 import os.path
-import subprocess
-
-from .config import BUILD_DIR
 
 
-def has_cmd(name, args=None):
+def which(tool):
+    """ looks up the full path for @tool, returns None if not found
+        or if @tool does not have executable permissions.
     """
-    This function is for use in a @skipUnless decorator, e.g.:
+    paths=os.getenv('PATH')
-
+    for p in paths.split(os.path.pathsep):
-        @skipUnless(*has_cmd('sudo -n', ('sudo', '-n', 'true')))
+        p = os.path.join(p, tool)
-        def test_something_that_needs_sudo(self):
+        if os.access(p, os.X_OK):
-            ...
+            return p
-    """
+    return None
-
-    if args is None:
-        args = ('which', name)
-
-    try:
-        _, stderr, exitcode = run_cmd(args)
-    except Exception as e:
-        exitcode = -1
-        stderr = str(e)
-
-    if exitcode != 0:
-        cmd_line = ' '.join(args)
-        err = f'{name} required, but "{cmd_line}" failed: {stderr.strip()}'
-        return (False, err)
-    else:
-        return (True, '')
-
-def has_cmds(*cmds):
-    """
-    This function is for use in a @skipUnless decorator and
-    allows checking for the availability of multiple commands, e.g.:
-
-        @skipUnless(*has_cmds(('cmd1', ('cmd1', '--some-parameter')),
-                              'cmd2', 'cmd3'))
-        def test_something_that_needs_cmd1_and_cmd2(self):
-            ...
-    """
-
-    for cmd in cmds:
-        if isinstance(cmd, str):
-            cmd = (cmd,)
-
-        ok, errstr = has_cmd(*cmd)
-        if not ok:
-            return (False, errstr)
-
-    return (True, '')
-
-def run_cmd(args):
-    subp = subprocess.Popen(args,
-                            stdout=subprocess.PIPE,
-                            stderr=subprocess.PIPE,
-                            universal_newlines=True)
-    stdout, stderr = subp.communicate()
-    ret = subp.returncode
-
-    return (stdout, stderr, ret)
 
 def is_readable_executable_file(path):
     return os.path.isfile(path) and os.access(path, os.R_OK | os.X_OK)
@@ -241,10 +193,10 @@ def get_qemu_img(test):
 
     # If qemu-img has been built, use it, otherwise the system wide one
     # will be used.
-    qemu_img = os.path.join(BUILD_DIR, 'qemu-img')
+    qemu_img = test.build_file('qemu-img')
     if os.path.exists(qemu_img):
         return qemu_img
-    (has_system_qemu_img, errmsg) = has_cmd('qemu-img')
+    qemu_img = which('qemu-img')
-    if has_system_qemu_img:
+    if qemu_img is not None:
-        return 'qemu-img'
+        return qemu_img
-    test.skipTest(errmsg)
+    test.skipTest(f"qemu-img not found in build dir or '$PATH'")

tests/functional/qemu_test/decorators.py

@@ -0,0 +1,107 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# Decorators useful in functional tests
+
+import os
+import platform
+from unittest import skipUnless
+
+from .cmd import which
+
+'''
+Decorator to skip execution of a test if the list
+of command binaries is not available in $PATH.
+Example:
+
+  @skipIfMissingCommands("mkisofs", "losetup")
+'''
+def skipIfMissingCommands(*args):
+    def has_cmds(cmdlist):
+        for cmd in cmdlist:
+            if not which(cmd):
+                return False
+        return True
+
+    return skipUnless(lambda: has_cmds(args),
+                      'required command(s) "%s" not installed' %
+                      ", ".join(args))
+
+'''
+Decorator to skip execution of a test if the current
+host machine does not match one of the permitted
+machines.
+Example
+
+  @skipIfNotMachine("x86_64", "aarch64")
+'''
+def skipIfNotMachine(*args):
+    return skipUnless(lambda: platform.machine() in args,
+                        'not running on one of the required machine(s) "%s"' %
+                        ", ".join(args))
+
+'''
+Decorator to skip execution of flaky tests, unless
+the $QEMU_TEST_FLAKY_TESTS environment variable is set.
+A bug URL must be provided that documents the observed
+failure behaviour, so it can be tracked & re-evaluated
+in future.
+
+Historical tests may be providing "None" as the bug_url
+but this should not be done for new test.
+
+Example:
+
+  @skipFlakyTest("https://gitlab.com/qemu-project/qemu/-/issues/NNN")
+'''
+def skipFlakyTest(bug_url):
+    if bug_url is None:
+        bug_url = "FIXME: reproduce flaky test and file bug report or remove"
+    return skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'),
+                      f'Test is unstable: {bug_url}')
+
+'''
+Decorator to skip execution of tests which are likely
+to execute untrusted commands on the host, or commands
+which process untrusted code, unless the
+$QEMU_TEST_ALLOW_UNTRUSTED_CODE env var is set.
+Example:
+
+  @skipUntrustedTest()
+'''
+def skipUntrustedTest():
+    return skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'),
+                      'Test runs untrusted code / processes untrusted data')
+
+'''
+Decorator to skip execution of tests which need large
+data storage (over around 500MB-1GB mark) on the host,
+unless the $QEMU_TEST_ALLOW_LARGE_STORAGE environment
+variable is set
+
+Example:
+
+  @skipBigDataTest()
+'''
+def skipBigDataTest():
+    return skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'),
+                      'Test requires large host storage space')
+
+'''
+Decorator to skip execution of a test if the list
+of python imports is not available.
+Example:
+
+  @skipIfMissingImports("numpy", "cv2")
+'''
+def skipIfMissingImports(*args):
+    def has_imports(importlist):
+        for impname in importlist:
+            try:
+                import impname
+            except ImportError:
+                return False
+        return True
+
+    return skipUnless(lambda: has_imports(args),
+                      'required import(s) "%s" not installed' %
+                      ", ".join(args))

tests/functional/qemu_test/linuxkernel.py

@@ -3,11 +3,9 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later.  See the COPYING file in the top-level directory.
 
-import os
-
 from .testcase import QemuSystemTest
-from .cmd import run_cmd, wait_for_console_pattern
+from .cmd import wait_for_console_pattern
-from .utils import archive_extract
+
 
 class LinuxKernelTest(QemuSystemTest):
     KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 '
@@ -28,26 +26,3 @@ class LinuxKernelTest(QemuSystemTest):
         self.vm.launch()
         if wait_for:
                 self.wait_for_console_pattern(wait_for)
-
-    def extract_from_deb(self, deb_path, path):
-        """
-        Extracts a file from a deb package into the test workdir
-
-        :param deb_path: path to the deb archive
-        :param path: path within the deb archive of the file to be extracted
-        :returns: path of the extracted file
-        """
-        cwd = os.getcwd()
-        os.chdir(self.workdir)
-        (stdout, stderr, ret) = run_cmd(['ar', 't', deb_path])
-        file_path = stdout.split()[2]
-        run_cmd(['ar', 'x', deb_path, file_path])
-        archive_extract(file_path, self.workdir)
-        os.chdir(cwd)
-        # Return complete path to extracted file.  Because callers to
-        # extract_from_deb() specify 'path' with a leading slash, it is
-        # necessary to use os.path.relpath() as otherwise os.path.join()
-        # interprets it as an absolute path and drops the self.workdir part.
-        return os.path.normpath(os.path.join(self.workdir,
-                                             os.path.relpath(path, '/')))
-

tests/functional/qemu_test/tesseract.py

@@ -5,30 +5,19 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later. See the COPYING file in the top-level directory.
 
-import re
 import logging
+from subprocess import run
 
-from . import has_cmd, run_cmd
-
-def tesseract_available(expected_version):
-    (has_tesseract, _) = has_cmd('tesseract')
-    if not has_tesseract:
-        return False
-    (stdout, stderr, ret) = run_cmd([ 'tesseract', '--version'])
-    if ret:
-        return False
-    version = stdout.split()[1]
-    return int(version.split('.')[0]) >= expected_version
 
 def tesseract_ocr(image_path, tesseract_args=''):
     console_logger = logging.getLogger('console')
     console_logger.debug(image_path)
-    (stdout, stderr, ret) = run_cmd(['tesseract', image_path,
+    proc = run(['tesseract', image_path, 'stdout'],
-                                     'stdout'])
+               capture_output=True, encoding='utf8')
-    if ret:
+    if proc.returncode:
         return None
     lines = []
-    for line in stdout.split('\n'):
+    for line in proc.stdout.split('\n'):
         sline = line.strip()
         if len(sline):
             console_logger.debug(sline)

tests/functional/qemu_test/testcase.py

@@ -13,19 +13,22 @@
 
 import logging
 import os
+from pathlib import Path
 import pycotap
 import shutil
-import subprocess
+from subprocess import run
 import sys
+import tempfile
 import unittest
 import uuid
 
 from qemu.machine import QEMUMachine
 from qemu.utils import kvm_available, tcg_available
 
+from .archive import archive_extract
 from .asset import Asset
-from .cmd import run_cmd
 from .config import BUILD_DIR
+from .uncompress import uncompress
 
 
 class QemuBaseTest(unittest.TestCase):
@@ -37,17 +40,169 @@ class QemuBaseTest(unittest.TestCase):
     log = None
     logdir = None
 
+    '''
+    @params compressed: filename, Asset, or file-like object to uncompress
+    @params format: optional compression format (gzip, lzma)
+
+    Uncompresses @compressed into the scratch directory.
+
+    If @format is None, heuristics will be applied to guess the format
+    from the filename or Asset URL. @format must be non-None if @uncompressed
+    is a file-like object.
+
+    Returns the fully qualified path to the uncompressed file
+    '''
+    def uncompress(self, compressed, format=None):
+        self.log.debug(f"Uncompress {compressed} format={format}")
+        if type(compressed) == Asset:
+            compressed.fetch()
+
+        (name, ext) = os.path.splitext(str(compressed))
+        uncompressed = self.scratch_file(os.path.basename(name))
+
+        uncompress(compressed, uncompressed, format)
+
+        return uncompressed
+
+    '''
+    @params archive: filename, Asset, or file-like object to extract
+    @params format: optional archive format (tar, zip, deb, cpio)
+    @params sub_dir: optional sub-directory to extract into
+    @params member: optional member file to limit extraction to
+
+    Extracts @archive into the scratch directory, or a directory beneath
+    named by @sub_dir. All files are extracted unless @member specifies
+    a limit.
+
+    If @format is None, heuristics will be applied to guess the format
+    from the filename or Asset URL. @format must be non-None if @archive
+    is a file-like object.
+
+    If @member is non-None, returns the fully qualified path to @member
+    '''
+    def archive_extract(self, archive, format=None, sub_dir=None, member=None):
+        self.log.debug(f"Extract {archive} format={format}" +
+                       f"sub_dir={sub_dir} member={member}")
+        if type(archive) == Asset:
+            archive.fetch()
+        if sub_dir is None:
+            archive_extract(archive, self.scratch_file(), format, member)
+        else:
+            archive_extract(archive, self.scratch_file(sub_dir),
+                            format, member)
+
+        if member is not None:
+            return self.scratch_file(member)
+        return None
+
+    '''
+    Create a temporary directory suitable for storing UNIX
+    socket paths.
+
+    Returns: a tempfile.TemporaryDirectory instance
+    '''
+    def socket_dir(self):
+        if self.socketdir is None:
+            self.socketdir = tempfile.TemporaryDirectory(
+                prefix="qemu_func_test_sock_")
+        return self.socketdir
+
+    '''
+    @params args list of zero or more subdirectories or file
+
+    Construct a path for accessing a data file located
+    relative to the source directory that is the root for
+    functional tests.
+
+    @args may be an empty list to reference the root dir
+    itself, may be a single element to reference a file in
+    the root directory, or may be multiple elements to
+    reference a file nested below. The path components
+    will be joined using the platform appropriate path
+    separator.
+
+    Returns: string representing a file path
+    '''
+    def data_file(self, *args):
+        return str(Path(Path(__file__).parent.parent, *args))
+
+    '''
+    @params args list of zero or more subdirectories or file
+
+    Construct a path for accessing a data file located
+    relative to the build directory root.
+
+    @args may be an empty list to reference the build dir
+    itself, may be a single element to reference a file in
+    the build directory, or may be multiple elements to
+    reference a file nested below. The path components
+    will be joined using the platform appropriate path
+    separator.
+
+    Returns: string representing a file path
+    '''
+    def build_file(self, *args):
+        return str(Path(BUILD_DIR, *args))
+
+    '''
+    @params args list of zero or more subdirectories or file
+
+    Construct a path for accessing/creating a scratch file
+    located relative to a temporary directory dedicated to
+    this test case. The directory and its contents will be
+    purged upon completion of the test.
+
+    @args may be an empty list to reference the scratch dir
+    itself, may be a single element to reference a file in
+    the scratch directory, or may be multiple elements to
+    reference a file nested below. The path components
+    will be joined using the platform appropriate path
+    separator.
+
+    Returns: string representing a file path
+    '''
+    def scratch_file(self, *args):
+        return str(Path(self.workdir, *args))
+
+    '''
+    @params args list of zero or more subdirectories or file
+
+    Construct a path for accessing/creating a log file
+    located relative to a temporary directory dedicated to
+    this test case. The directory and its log files will be
+    preserved upon completion of the test.
+
+    @args may be an empty list to reference the log dir
+    itself, may be a single element to reference a file in
+    the log directory, or may be multiple elements to
+    reference a file nested below. The path components
+    will be joined using the platform appropriate path
+    separator.
+
+    Returns: string representing a file path
+    '''
+    def log_file(self, *args):
+        return str(Path(self.outputdir, *args))
+
+    def assets_available(self):
+        for name, asset in vars(self.__class__).items():
+            if name.startswith("ASSET_") and type(asset) == Asset:
+                if not asset.available():
+                    self.log.debug(f"Asset {asset.url} not available")
+                    return False
+        return True
+
     def setUp(self, bin_prefix):
         self.assertIsNotNone(self.qemu_bin, 'QEMU_TEST_QEMU_BINARY must be set')
         self.arch = self.qemu_bin.split('-')[-1]
+        self.socketdir = None
 
-        self.outputdir = os.path.join(BUILD_DIR, 'tests', 'functional',
+        self.outputdir = self.build_file('tests', 'functional',
-                                      self.arch, self.id())
+                                         self.arch, self.id())
         self.workdir = os.path.join(self.outputdir, 'scratch')
         os.makedirs(self.workdir, exist_ok=True)
 
-        self.logdir = self.outputdir
+        self.log_filename = self.log_file('base.log')
-        self.log_filename = os.path.join(self.logdir, 'base.log')
         self.log = logging.getLogger('qemu-test')
         self.log.setLevel(logging.DEBUG)
         self._log_fh = logging.FileHandler(self.log_filename, mode='w')
@@ -62,9 +217,15 @@ class QemuBaseTest(unittest.TestCase):
         self.machinelog.setLevel(logging.DEBUG)
         self.machinelog.addHandler(self._log_fh)
 
+        if not self.assets_available():
+            self.skipTest('One or more assets is not available')
+
     def tearDown(self):
         if "QEMU_TEST_KEEP_SCRATCH" not in os.environ:
             shutil.rmtree(self.workdir)
+        if self.socketdir is not None:
+            shutil.rmtree(self.socketdir.name)
+            self.socketdir = None
         self.machinelog.removeHandler(self._log_fh)
         self.log.removeHandler(self._log_fh)
 
@@ -100,11 +261,11 @@ class QemuUserTest(QemuBaseTest):
         self._ldpath.append(os.path.abspath(ldpath))
 
     def run_cmd(self, bin_path, args=[]):
-        return subprocess.run([self.qemu_bin]
+        return run([self.qemu_bin]
-                              + ["-L %s" % ldpath for ldpath in self._ldpath]
+                   + ["-L %s" % ldpath for ldpath in self._ldpath]
-                              + [bin_path]
+                   + [bin_path]
-                              + args,
+                   + args,
-                              text=True, capture_output=True)
+                   text=True, capture_output=True)
 
 class QemuSystemTest(QemuBaseTest):
     """Facilitates system emulation tests."""
@@ -120,7 +281,7 @@ class QemuSystemTest(QemuBaseTest):
 
         console_log = logging.getLogger('console')
         console_log.setLevel(logging.DEBUG)
-        self.console_log_name = os.path.join(self.logdir, 'console.log')
+        self.console_log_name = self.log_file('console.log')
         self._console_log_fh = logging.FileHandler(self.console_log_name,
                                                    mode='w')
         self._console_log_fh.setLevel(logging.DEBUG)
@@ -131,7 +292,9 @@ class QemuSystemTest(QemuBaseTest):
     def set_machine(self, machinename):
         # TODO: We should use QMP to get the list of available machines
         if not self._machinehelp:
-            self._machinehelp = run_cmd([self.qemu_bin, '-M', 'help'])[0];
+            self._machinehelp = run(
+                [self.qemu_bin, '-M', 'help'],
+                capture_output=True, check=True, encoding='utf8').stdout
         if self._machinehelp.find(machinename) < 0:
             self.skipTest('no support for machine ' + machinename)
         self.machine = machinename
@@ -159,22 +322,24 @@ class QemuSystemTest(QemuBaseTest):
                           "available" % accelerator)
 
     def require_netdev(self, netdevname):
-        netdevhelp = run_cmd([self.qemu_bin,
+        help = run([self.qemu_bin,
-                             '-M', 'none', '-netdev', 'help'])[0];
+                    '-M', 'none', '-netdev', 'help'],
-        if netdevhelp.find('\n' + netdevname + '\n') < 0:
+                   capture_output=True, check=True, encoding='utf8').stdout;
+        if help.find('\n' + netdevname + '\n') < 0:
             self.skipTest('no support for " + netdevname + " networking')
 
     def require_device(self, devicename):
-        devhelp = run_cmd([self.qemu_bin,
+        help = run([self.qemu_bin,
-                           '-M', 'none', '-device', 'help'])[0];
+                    '-M', 'none', '-device', 'help'],
-        if devhelp.find(devicename) < 0:
+                   capture_output=True, check=True, encoding='utf8').stdout;
+        if help.find(devicename) < 0:
             self.skipTest('no support for device ' + devicename)
 
     def _new_vm(self, name, *args):
         vm = QEMUMachine(self.qemu_bin,
                          name=name,
                          base_temp_dir=self.workdir,
-                         log_dir=self.logdir)
+                         log_dir=self.log_file())
         self.log.debug('QEMUMachine "%s" created', name)
         self.log.debug('QEMUMachine "%s" temp_dir: %s', name, vm.temp_dir)
 

tests/functional/qemu_test/tuxruntest.py

@@ -11,12 +11,12 @@
 
 import os
 import stat
-import time
+from subprocess import check_call, DEVNULL
 
 from qemu_test import QemuSystemTest
-from qemu_test import exec_command, exec_command_and_wait_for_pattern
+from qemu_test import exec_command_and_wait_for_pattern
 from qemu_test import wait_for_console_pattern
-from qemu_test import has_cmd, run_cmd, get_qemu_img
+from qemu_test import which, get_qemu_img
 
 class TuxRunBaselineTest(QemuSystemTest):
 
@@ -39,10 +39,8 @@ class TuxRunBaselineTest(QemuSystemTest):
         super().setUp()
 
         # We need zstd for all the tuxrun tests
-        (has_zstd, msg) = has_cmd('zstd')
+        if which('zstd') is None:
-        if has_zstd is False:
+            self.skipTest("zstd not found in $PATH")
-            self.skipTest(msg)
-        self.zstd = 'zstd'
 
         # Pre-init TuxRun specific settings: Most machines work with
         # reasonable defaults but we sometimes need to tweak the
@@ -77,10 +75,11 @@ class TuxRunBaselineTest(QemuSystemTest):
         kernel_image =  kernel_asset.fetch()
         disk_image_zst = rootfs_asset.fetch()
 
-        disk_image = self.workdir + "/rootfs.ext4"
+        disk_image = self.scratch_file("rootfs.ext4")
 
-        run_cmd([self.zstd, "-f", "-d", disk_image_zst,
+        check_call(['zstd', "-f", "-d", disk_image_zst,
-                 "-o", disk_image])
+                    "-o", disk_image],
+                   stdout=DEVNULL, stderr=DEVNULL)
         # zstd copies source archive permissions for the output
         # file, so must make this writable for QEMU
         os.chmod(disk_image, stat.S_IRUSR | stat.S_IWUSR)

tests/functional/qemu_test/uncompress.py

@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+#
+# Utilities for python-based QEMU tests
+#
+# Copyright 2024 Red Hat, Inc.
+#
+# Authors:
+#  Thomas Huth <thuth@redhat.com>
+
+import gzip
+import lzma
+import os
+import shutil
+from urllib.parse import urlparse
+
+from .asset import Asset
+
+
+def gzip_uncompress(gz_path, output_path):
+    if os.path.exists(output_path):
+        return
+    with gzip.open(gz_path, 'rb') as gz_in:
+        try:
+            with open(output_path, 'wb') as raw_out:
+                shutil.copyfileobj(gz_in, raw_out)
+        except:
+            os.remove(output_path)
+            raise
+
+def lzma_uncompress(xz_path, output_path):
+    if os.path.exists(output_path):
+        return
+    with lzma.open(xz_path, 'rb') as lzma_in:
+        try:
+            with open(output_path, 'wb') as raw_out:
+                shutil.copyfileobj(lzma_in, raw_out)
+        except:
+            os.remove(output_path)
+            raise
+
+'''
+@params compressed: filename, Asset, or file-like object to uncompress
+@params uncompressed: filename to uncompress into
+@params format: optional compression format (gzip, lzma)
+
+Uncompresses @compressed into @uncompressed
+
+If @format is None, heuristics will be applied to guess the format
+from the filename or Asset URL. @format must be non-None if @uncompressed
+is a file-like object.
+
+Returns the fully qualified path to the uncompessed file
+'''
+def uncompress(compressed, uncompressed, format=None):
+    if format is None:
+        format = guess_uncompress_format(compressed)
+
+    if format == "xz":
+        lzma_uncompress(str(compressed), uncompressed)
+    elif format == "gz":
+        gzip_uncompress(str(compressed), uncompressed)
+    else:
+        raise Exception(f"Unknown compression format {format}")
+
+'''
+@params compressed: filename, Asset, or file-like object to guess
+
+Guess the format of @compressed, raising an exception if
+no format can be determined
+'''
+def guess_uncompress_format(compressed):
+    if type(compressed) == Asset:
+        compressed = urlparse(compressed.url).path
+    elif type(compressed) != str:
+        raise Exception(f"Unable to guess compression cformat for {compressed}")
+
+    (name, ext) = os.path.splitext(compressed)
+    if ext == ".xz":
+        return "xz"
+    elif ext == ".gz":
+        return "gz"
+    else:
+        raise Exception(f"Unknown compression format for {compressed}")

tests/functional/qemu_test/utils.py

@@ -8,12 +8,14 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later.  See the COPYING file in the top-level directory.
 
-import gzip
-import lzma
 import os
-import shutil
+
-import subprocess
+from qemu.utils import get_info_usernet_hostfwd_port
-import tarfile
+
+
+def get_usernet_hostfwd_port(vm):
+    res = vm.cmd('human-monitor-command', command_line='info usernet')
+    return get_info_usernet_hostfwd_port(res)
 
 """
 Round up to next power of 2
@@ -35,43 +37,3 @@ def image_pow2ceil_expand(path):
         if size != size_aligned:
             with open(path, 'ab+') as fd:
                 fd.truncate(size_aligned)
-
-def archive_extract(archive, dest_dir, member=None):
-    with tarfile.open(archive) as tf:
-        if hasattr(tarfile, 'data_filter'):
-            tf.extraction_filter = getattr(tarfile, 'data_filter',
-                                           (lambda member, path: member))
-        if member:
-            tf.extract(member=member, path=dest_dir)
-        else:
-            tf.extractall(path=dest_dir)
-
-def gzip_uncompress(gz_path, output_path):
-    if os.path.exists(output_path):
-        return
-    with gzip.open(gz_path, 'rb') as gz_in:
-        try:
-            with open(output_path, 'wb') as raw_out:
-                shutil.copyfileobj(gz_in, raw_out)
-        except:
-            os.remove(output_path)
-            raise
-
-def lzma_uncompress(xz_path, output_path):
-    if os.path.exists(output_path):
-        return
-    with lzma.open(xz_path, 'rb') as lzma_in:
-        try:
-            with open(output_path, 'wb') as raw_out:
-                shutil.copyfileobj(lzma_in, raw_out)
-        except:
-            os.remove(output_path)
-            raise
-
-def cpio_extract(cpio_handle, output_path):
-    cwd = os.getcwd()
-    os.chdir(output_path)
-    subprocess.run(['cpio', '-i'],
-                   input=cpio_handle.read(),
-                   stderr=subprocess.DEVNULL)
-    os.chdir(cwd)

tests/functional/test_aarch64_aspeed.py

@@ -6,13 +6,12 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import sys
 import os
 
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import wait_for_console_pattern
 from qemu_test import exec_command_and_wait_for_pattern
-from qemu_test.utils import archive_extract
+
 
 class AST2x00MachineSDK(QemuSystemTest):
 
@@ -35,30 +34,31 @@ class AST2x00MachineSDK(QemuSystemTest):
     def test_aarch64_ast2700_evb_sdk_v09_02(self):
         self.set_machine('ast2700-evb')
 
-        image_path = self.ASSET_SDK_V902_AST2700.fetch()
+        self.archive_extract(self.ASSET_SDK_V902_AST2700)
-        archive_extract(image_path, self.workdir)
 
         num_cpu = 4
-        image_dir = self.workdir + '/ast2700-default/'
+        uboot_size = os.path.getsize(self.scratch_file('ast2700-default',
-        uboot_size = os.path.getsize(image_dir + 'u-boot-nodtb.bin')
+                                                       'u-boot-nodtb.bin'))
         uboot_dtb_load_addr = hex(0x400000000 + uboot_size)
 
         load_images_list = [
             {
                 'addr': '0x400000000',
-                'file': image_dir + 'u-boot-nodtb.bin'
+                'file': self.scratch_file('ast2700-default',
+                                          'u-boot-nodtb.bin')
             },
             {
                 'addr': str(uboot_dtb_load_addr),
-                'file': image_dir + 'u-boot.dtb'
+                'file': self.scratch_file('ast2700-default', 'u-boot.dtb')
             },
             {
                 'addr': '0x430000000',
-                'file': image_dir + 'bl31.bin'
+                'file': self.scratch_file('ast2700-default', 'bl31.bin')
             },
             {
                 'addr': '0x430080000',
-                'file': image_dir + 'optee/tee-raw.bin'
+                'file': self.scratch_file('ast2700-default', 'optee',
+                                          'tee-raw.bin')
             }
         ]
 
@@ -75,7 +75,8 @@ class AST2x00MachineSDK(QemuSystemTest):
         self.vm.add_args('-smp', str(num_cpu))
         self.vm.add_args('-device',
                          'tmp105,bus=aspeed.i2c.bus.1,address=0x4d,id=tmp-test')
-        self.do_test_aarch64_aspeed_sdk_start(image_dir + 'image-bmc')
+        self.do_test_aarch64_aspeed_sdk_start(
+            self.scratch_file('ast2700-default', 'image-bmc'))
 
         wait_for_console_pattern(self, 'ast2700-default login:')
 

tests/functional/test_aarch64_raspi3.py

@@ -7,9 +7,6 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import os
-from zipfile import ZipFile
-
 from qemu_test import LinuxKernelTest, Asset
 
 
@@ -22,11 +19,7 @@ class Aarch64Raspi3Machine(LinuxKernelTest):
 
     def test_aarch64_raspi3_atf(self):
         efi_name = 'RPI_EFI.fd'
-        zip_path = self.ASSET_RPI3_UEFI.fetch()
+        efi_fd = self.archive_extract(self.ASSET_RPI3_UEFI, member=efi_name)
-
-        with ZipFile(zip_path, 'r') as zf:
-                     zf.extract(efi_name, path=self.workdir)
-        efi_fd = os.path.join(self.workdir, efi_name)
 
         self.set_machine('raspi3b')
         self.vm.set_console(console_index=1)

tests/functional/test_aarch64_raspi4.py

@@ -5,11 +5,8 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import os
-
 from qemu_test import LinuxKernelTest, Asset
 from qemu_test import exec_command_and_wait_for_pattern
-from qemu_test.utils import gzip_uncompress
 
 
 class Aarch64Raspi4Machine(LinuxKernelTest):
@@ -32,9 +29,10 @@ class Aarch64Raspi4Machine(LinuxKernelTest):
         '7c0b16d1853772f6f4c3ca63e789b3b9ff4936efac9c8a01fb0c98c05c7a7648')
 
     def test_arm_raspi4(self):
-        deb_path = self.ASSET_KERNEL_20190215.fetch()
+        kernel_path = self.archive_extract(self.ASSET_KERNEL_20190215,
-        kernel_path = self.extract_from_deb(deb_path, '/boot/kernel8.img')
+                                           member='boot/kernel8.img')
-        dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2711-rpi-4-b.dtb')
+        dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215,
+                                        member='boot/bcm2711-rpi-4-b.dtb')
 
         self.set_machine('raspi4b')
         self.vm.set_console()
@@ -60,12 +58,11 @@ class Aarch64Raspi4Machine(LinuxKernelTest):
 
 
     def test_arm_raspi4_initrd(self):
-        deb_path = self.ASSET_KERNEL_20190215.fetch()
+        kernel_path = self.archive_extract(self.ASSET_KERNEL_20190215,
-        kernel_path = self.extract_from_deb(deb_path, '/boot/kernel8.img')
+                                           member='boot/kernel8.img')
-        dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2711-rpi-4-b.dtb')
+        dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215,
-        initrd_path_gz = self.ASSET_INITRD.fetch()
+                                        member='boot/bcm2711-rpi-4-b.dtb')
-        initrd_path = os.path.join(self.workdir, 'rootfs.cpio')
+        initrd_path = self.uncompress(self.ASSET_INITRD)
-        gzip_uncompress(initrd_path_gz, initrd_path)
 
         self.set_machine('raspi4b')
         self.vm.set_console()

tests/functional/test_aarch64_sbsaref.py

@@ -8,12 +8,10 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import os
-
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import wait_for_console_pattern
 from qemu_test import interrupt_interactive_console_until_pattern
-from qemu_test.utils import lzma_uncompress
+
 
 def fetch_firmware(test):
     """
@@ -31,14 +29,10 @@ def fetch_firmware(test):
     """
 
     # Secure BootRom (TF-A code)
-    fs0_xz_path = Aarch64SbsarefMachine.ASSET_FLASH0.fetch()
+    fs0_path = test.uncompress(Aarch64SbsarefMachine.ASSET_FLASH0)
-    fs0_path = os.path.join(test.workdir, "SBSA_FLASH0.fd")
-    lzma_uncompress(fs0_xz_path, fs0_path)
 
     # Non-secure rom (UEFI and EFI variables)
-    fs1_xz_path = Aarch64SbsarefMachine.ASSET_FLASH1.fetch()
+    fs1_path = test.uncompress(Aarch64SbsarefMachine.ASSET_FLASH1)
-    fs1_path = os.path.join(test.workdir, "SBSA_FLASH1.fd")
-    lzma_uncompress(fs1_xz_path, fs1_path)
 
     for path in [fs0_path, fs1_path]:
         with open(path, "ab+") as fd:

tests/functional/test_aarch64_sbsaref_alpine.py

@@ -12,7 +12,6 @@ import os
 
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import wait_for_console_pattern
-from qemu_test import interrupt_interactive_console_until_pattern
 from unittest import skipUnless
 from test_aarch64_sbsaref import fetch_firmware
 

tests/functional/test_aarch64_sbsaref_freebsd.py

@@ -12,7 +12,6 @@ import os
 
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import wait_for_console_pattern
-from qemu_test import interrupt_interactive_console_until_pattern
 from unittest import skipUnless
 from test_aarch64_sbsaref import fetch_firmware
 

tests/functional/test_aarch64_virt.py

@@ -11,13 +11,12 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 import time
-import os
 import logging
+from subprocess import check_call, DEVNULL
 
-from qemu_test import BUILD_DIR
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import exec_command, wait_for_console_pattern
-from qemu_test import get_qemu_img, run_cmd
+from qemu_test import get_qemu_img
 
 
 class Aarch64VirtMachine(QemuSystemTest):
@@ -54,8 +53,8 @@ class Aarch64VirtMachine(QemuSystemTest):
                          "mte=on,"
                          "gic-version=max,iommu=smmuv3")
         self.vm.add_args("-smp", "2", "-m", "1024")
-        self.vm.add_args('-bios', os.path.join(BUILD_DIR, 'pc-bios',
+        self.vm.add_args('-bios', self.build_file('pc-bios',
-                                               'edk2-aarch64-code.fd'))
+                                                  'edk2-aarch64-code.fd'))
         self.vm.add_args("-drive", f"file={iso_path},media=cdrom,format=raw")
         self.vm.add_args('-device', 'virtio-rng-pci,rng=rng0')
         self.vm.add_args('-object', 'rng-random,id=rng0,filename=/dev/urandom')
@@ -96,9 +95,10 @@ class Aarch64VirtMachine(QemuSystemTest):
 
         # Also add a scratch block device
         logger.info('creating scratch qcow2 image')
-        image_path = os.path.join(self.workdir, 'scratch.qcow2')
+        image_path = self.scratch_file('scratch.qcow2')
         qemu_img = get_qemu_img(self)
-        run_cmd([qemu_img, 'create', '-f', 'qcow2', image_path, '8M'])
+        check_call([qemu_img, 'create', '-f', 'qcow2', image_path, '8M'],
+                   stdout=DEVNULL, stderr=DEVNULL)
 
         # Add the device
         self.vm.add_args('-blockdev',

tests/functional/test_acpi_bits.py

@@ -31,56 +31,24 @@ including an upgraded acpica. The fork is located here:
 https://gitlab.com/qemu-project/biosbits-bits .
 """
 
-import logging
 import os
-import platform
 import re
 import shutil
 import subprocess
-import tarfile
-import tempfile
-import zipfile
 
-from pathlib import Path
 from typing import (
     List,
     Optional,
     Sequence,
 )
 from qemu.machine import QEMUMachine
-from unittest import skipIf
+from qemu_test import (QemuSystemTest, Asset, skipIfMissingCommands,
-from qemu_test import QemuSystemTest, Asset
+                       skipIfNotMachine)
 
-deps = ["xorriso", "mformat"] # dependent tools needed in the test setup/box.
-supported_platforms = ['x86_64'] # supported test platforms.
 
 # default timeout of 120 secs is sometimes not enough for bits test.
 BITS_TIMEOUT = 200
 
-def which(tool):
-    """ looks up the full path for @tool, returns None if not found
-        or if @tool does not have executable permissions.
-    """
-    paths=os.getenv('PATH')
-    for p in paths.split(os.path.pathsep):
-        p = os.path.join(p, tool)
-        if os.path.exists(p) and os.access(p, os.X_OK):
-            return p
-    return None
-
-def missing_deps():
-    """ returns True if any of the test dependent tools are absent.
-    """
-    for dep in deps:
-        if which(dep) is None:
-            return True
-    return False
-
-def supported_platform():
-    """ checks if the test is running on a supported platform.
-    """
-    return platform.machine() in supported_platforms
-
 class QEMUBitsMachine(QEMUMachine): # pylint: disable=too-few-public-methods
     """
     A QEMU VM, with isa-debugcon enabled and bits iso passed
@@ -123,9 +91,8 @@ class QEMUBitsMachine(QEMUMachine): # pylint: disable=too-few-public-methods
         """return the base argument to QEMU binary"""
         return self._base_args
 
-@skipIf(not supported_platform() or missing_deps(),
+@skipIfMissingCommands("xorriso", "mformat")
-        'unsupported platform or dependencies (%s) not installed' \
+@skipIfNotMachine("x86_64")
-        % ','.join(deps))
 class AcpiBitsTest(QemuSystemTest): #pylint: disable=too-many-instance-attributes
     """
     ACPI and SMBIOS tests using biosbits.
@@ -149,7 +116,6 @@ class AcpiBitsTest(QemuSystemTest): #pylint: disable=too-many-instance-attribute
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         self._vm = None
-        self._baseDir = None
 
         self._debugcon_addr = '0x403'
         self._debugcon_log = 'debugcon-log.txt'
@@ -164,29 +130,24 @@ class AcpiBitsTest(QemuSystemTest): #pylint: disable=too-many-instance-attribute
     def copy_bits_config(self):
         """ copies the bios bits config file into bits.
         """
-        config_file = 'bits-cfg.txt'
+        bits_config_file = self.data_file('acpi-bits',
-        bits_config_dir = os.path.join(self._baseDir, 'acpi-bits',
+                                          'bits-config',
-                                       'bits-config')
+                                          'bits-cfg.txt')
-        target_config_dir = os.path.join(self.workdir,
+        target_config_dir = self.scratch_file('bits-%d' %
-                                         'bits-%d' %self.BITS_INTERNAL_VER,
+                                              self.BITS_INTERNAL_VER,
-                                         'boot')
+                                              'boot')
-        self.assertTrue(os.path.exists(bits_config_dir))
+        self.assertTrue(os.path.exists(bits_config_file))
         self.assertTrue(os.path.exists(target_config_dir))
-        self.assertTrue(os.access(os.path.join(bits_config_dir,
+        shutil.copy2(bits_config_file, target_config_dir)
-                                               config_file), os.R_OK))
-        shutil.copy2(os.path.join(bits_config_dir, config_file),
-                     target_config_dir)
         self.logger.info('copied config file %s to %s',
-                         config_file, target_config_dir)
+                         bits_config_file, target_config_dir)
 
     def copy_test_scripts(self):
         """copies the python test scripts into bits. """
 
-        bits_test_dir = os.path.join(self._baseDir, 'acpi-bits',
+        bits_test_dir = self.data_file('acpi-bits', 'bits-tests')
-                                     'bits-tests')
+        target_test_dir = self.scratch_file('bits-%d' % self.BITS_INTERNAL_VER,
-        target_test_dir = os.path.join(self.workdir,
+                                            'boot', 'python')
-                                       'bits-%d' %self.BITS_INTERNAL_VER,
-                                       'boot', 'python')
 
         self.assertTrue(os.path.exists(bits_test_dir))
         self.assertTrue(os.path.exists(target_test_dir))
@@ -223,8 +184,8 @@ class AcpiBitsTest(QemuSystemTest): #pylint: disable=too-many-instance-attribute
             the directory where we have extracted our pre-built bits grub
             tarball.
         """
-        grub_x86_64_mods = os.path.join(self.workdir, 'grub-inst-x86_64-efi')
+        grub_x86_64_mods = self.scratch_file('grub-inst-x86_64-efi')
-        grub_i386_mods = os.path.join(self.workdir, 'grub-inst')
+        grub_i386_mods = self.scratch_file('grub-inst')
 
         self.assertTrue(os.path.exists(grub_x86_64_mods))
         self.assertTrue(os.path.exists(grub_i386_mods))
@@ -245,13 +206,11 @@ class AcpiBitsTest(QemuSystemTest): #pylint: disable=too-many-instance-attribute
         """ Uses grub-mkrescue to generate a fresh bits iso with the python
             test scripts
         """
-        bits_dir = os.path.join(self.workdir,
+        bits_dir = self.scratch_file('bits-%d' % self.BITS_INTERNAL_VER)
-                                'bits-%d' %self.BITS_INTERNAL_VER)
+        iso_file = self.scratch_file('bits-%d.iso' % self.BITS_INTERNAL_VER)
-        iso_file = os.path.join(self.workdir,
+        mkrescue_script = self.scratch_file('grub-inst-x86_64-efi',
-                                'bits-%d.iso' %self.BITS_INTERNAL_VER)
+                                            'bin',
-        mkrescue_script = os.path.join(self.workdir,
+                                            'grub-mkrescue')
-                                       'grub-inst-x86_64-efi', 'bin',
-                                       'grub-mkrescue')
 
         self.assertTrue(os.access(mkrescue_script,
                                   os.R_OK | os.W_OK | os.X_OK))
@@ -286,33 +245,25 @@ class AcpiBitsTest(QemuSystemTest): #pylint: disable=too-many-instance-attribute
         super().setUp()
         self.logger = self.log
 
-        self._baseDir = Path(__file__).parent
+        prebuiltDir = self.scratch_file('prebuilt')
-
-        prebuiltDir = os.path.join(self.workdir, 'prebuilt')
         if not os.path.isdir(prebuiltDir):
             os.mkdir(prebuiltDir, mode=0o775)
 
-        bits_zip_file = os.path.join(prebuiltDir, 'bits-%d-%s.zip'
+        bits_zip_file = self.scratch_file('prebuilt',
-                                     %(self.BITS_INTERNAL_VER,
+                                          'bits-%d-%s.zip'
-                                       self.BITS_COMMIT_HASH))
+                                          %(self.BITS_INTERNAL_VER,
-        grub_tar_file = os.path.join(prebuiltDir,
+                                            self.BITS_COMMIT_HASH))
-                                     'bits-%d-%s-grub.tar.gz'
+        grub_tar_file = self.scratch_file('prebuilt',
-                                     %(self.BITS_INTERNAL_VER,
+                                          'bits-%d-%s-grub.tar.gz'
-                                       self.BITS_COMMIT_HASH))
+                                          %(self.BITS_INTERNAL_VER,
-
+                                            self.BITS_COMMIT_HASH))
-        bitsLocalArtLoc = self.ASSET_BITS.fetch()
-        self.logger.info("downloaded bits artifacts to %s", bitsLocalArtLoc)
 
         # extract the bits artifact in the temp working directory
-        with zipfile.ZipFile(bitsLocalArtLoc, 'r') as zref:
+        self.archive_extract(self.ASSET_BITS, sub_dir='prebuilt', format='zip')
-            zref.extractall(prebuiltDir)
 
         # extract the bits software in the temp working directory
-        with zipfile.ZipFile(bits_zip_file, 'r') as zref:
+        self.archive_extract(bits_zip_file)
-            zref.extractall(self.workdir)
+        self.archive_extract(grub_tar_file)
-
-        with tarfile.open(grub_tar_file, 'r', encoding='utf-8') as tarball:
-            tarball.extractall(self.workdir)
 
         self.copy_test_scripts()
         self.copy_bits_config()
@@ -322,7 +273,7 @@ class AcpiBitsTest(QemuSystemTest): #pylint: disable=too-many-instance-attribute
         """parse the log generated by running bits tests and
            check for failures.
         """
-        debugconf = os.path.join(self.workdir, self._debugcon_log)
+        debugconf = self.scratch_file(self._debugcon_log)
         log = ""
         with open(debugconf, 'r', encoding='utf-8') as filehandle:
             log = filehandle.read()
@@ -354,8 +305,7 @@ class AcpiBitsTest(QemuSystemTest): #pylint: disable=too-many-instance-attribute
         """The main test case implementation."""
 
         self.set_machine('pc')
-        iso_file = os.path.join(self.workdir,
+        iso_file = self.scratch_file('bits-%d.iso' % self.BITS_INTERNAL_VER)
-                                'bits-%d.iso' %self.BITS_INTERNAL_VER)
 
         self.assertTrue(os.access(iso_file, os.R_OK))
 

tests/functional/test_alpha_clipper.py

@@ -5,10 +5,7 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import os
-
 from qemu_test import LinuxKernelTest, Asset
-from qemu_test.utils import gzip_uncompress
 
 
 class AlphaClipperTest(LinuxKernelTest):
@@ -22,8 +19,7 @@ class AlphaClipperTest(LinuxKernelTest):
         self.set_machine('clipper')
         kernel_path = self.ASSET_KERNEL.fetch()
 
-        uncompressed_kernel = os.path.join(self.workdir, 'vmlinux')
+        uncompressed_kernel = self.uncompress(self.ASSET_KERNEL, format="gz")
-        gzip_uncompress(kernel_path, uncompressed_kernel)
 
         self.vm.set_console()
         kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + 'console=ttyS0'

tests/functional/test_arm_aspeed_ast1030.py

@@ -6,11 +6,9 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import os
-
 from qemu_test import LinuxKernelTest, Asset
 from qemu_test import exec_command_and_wait_for_pattern
-from zipfile import ZipFile
+
 
 class AST1030Machine(LinuxKernelTest):
 
@@ -22,12 +20,9 @@ class AST1030Machine(LinuxKernelTest):
     def test_ast1030_zephyros_1_04(self):
         self.set_machine('ast1030-evb')
 
-        zip_file = self.ASSET_ZEPHYR_1_04.fetch()
-
         kernel_name = "ast1030-evb-demo/zephyr.elf"
-        with ZipFile(zip_file, 'r') as zf:
+        kernel_file = self.archive_extract(
-                     zf.extract(kernel_name, path=self.workdir)
+            self.ASSET_ZEPHYR_1_04, member=kernel_name)
-        kernel_file = os.path.join(self.workdir, kernel_name)
 
         self.vm.set_console()
         self.vm.add_args('-kernel', kernel_file, '-nographic')
@@ -44,12 +39,9 @@ class AST1030Machine(LinuxKernelTest):
     def test_ast1030_zephyros_1_07(self):
         self.set_machine('ast1030-evb')
 
-        zip_file = self.ASSET_ZEPHYR_1_07.fetch()
-
         kernel_name = "ast1030-evb-demo/zephyr.bin"
-        with ZipFile(zip_file, 'r') as zf:
+        kernel_file = self.archive_extract(
-                     zf.extract(kernel_name, path=self.workdir)
+            self.ASSET_ZEPHYR_1_07, member=kernel_name)
-        kernel_file = os.path.join(self.workdir, kernel_name)
 
         self.vm.set_console()
         self.vm.add_args('-kernel', kernel_file, '-nographic')

tests/functional/test_arm_aspeed_ast2500.py

@@ -7,7 +7,7 @@
 from qemu_test import Asset
 from aspeed import AspeedTest
 from qemu_test import exec_command_and_wait_for_pattern
-from qemu_test.utils import archive_extract
+
 
 class AST2500Machine(AspeedTest):
 
@@ -45,12 +45,10 @@ class AST2500Machine(AspeedTest):
     def test_arm_ast2500_evb_sdk(self):
         self.set_machine('ast2500-evb')
 
-        image_path = self.ASSET_SDK_V806_AST2500.fetch()
+        self.archive_extract(self.ASSET_SDK_V806_AST2500)
-
-        archive_extract(image_path, self.workdir)
 
         self.do_test_arm_aspeed_sdk_start(
-            self.workdir + '/ast2500-default/image-bmc')
+            self.scratch_file("ast2500-default", "image-bmc"))
 
         self.wait_for_console_pattern('ast2500-default login:')
 

tests/functional/test_arm_aspeed_ast2600.py

@@ -11,10 +11,8 @@ import subprocess
 
 from qemu_test import Asset
 from aspeed import AspeedTest
-from qemu_test import exec_command_and_wait_for_pattern
+from qemu_test import exec_command_and_wait_for_pattern, skipIfMissingCommands
-from qemu_test import has_cmd
+
-from qemu_test.utils import archive_extract
-from unittest import skipUnless
 
 class AST2600Machine(AspeedTest):
 
@@ -68,7 +66,7 @@ class AST2600Machine(AspeedTest):
          'images/ast2600-evb/buildroot-2023.02-tpm/flash.img'),
         'a46009ae8a5403a0826d607215e731a8c68d27c14c41e55331706b8f9c7bd997')
 
-    @skipUnless(*has_cmd('swtpm'))
+    @skipIfMissingCommands('swtpm')
     def test_arm_ast2600_evb_buildroot_tpm(self):
         self.set_machine('ast2600-evb')
 
@@ -106,16 +104,14 @@ class AST2600Machine(AspeedTest):
     def test_arm_ast2600_evb_sdk(self):
         self.set_machine('ast2600-evb')
 
-        image_path = self.ASSET_SDK_V806_AST2600_A2.fetch()
+        self.archive_extract(self.ASSET_SDK_V806_AST2600_A2)
-
-        archive_extract(image_path, self.workdir)
 
         self.vm.add_args('-device',
             'tmp105,bus=aspeed.i2c.bus.5,address=0x4d,id=tmp-test');
         self.vm.add_args('-device',
             'ds1338,bus=aspeed.i2c.bus.5,address=0x32');
         self.do_test_arm_aspeed_sdk_start(
-            self.workdir + '/ast2600-a2/image-bmc')
+            self.scratch_file("ast2600-a2", "image-bmc"))
 
         self.wait_for_console_pattern('ast2600-a2 login:')
 

tests/functional/test_arm_aspeed_rainier.py

@@ -43,11 +43,12 @@ class RainierMachine(AspeedTest):
     def test_arm_debian_kernel_boot(self):
         self.set_machine('rainier-bmc')
 
-        deb_path = self.ASSET_DEBIAN_LINUX_ARMHF_DEB.fetch()
+        kernel_path = self.archive_extract(
-
+            self.ASSET_DEBIAN_LINUX_ARMHF_DEB,
-        kernel_path = self.extract_from_deb(deb_path, '/boot/vmlinuz-5.17.0-2-armmp')
+            member='boot/vmlinuz-5.17.0-2-armmp')
-        dtb_path = self.extract_from_deb(deb_path,
+        dtb_path = self.archive_extract(
-                '/usr/lib/linux-image-5.17.0-2-armmp/aspeed-bmc-ibm-rainier.dtb')
+            self.ASSET_DEBIAN_LINUX_ARMHF_DEB,
+            member='usr/lib/linux-image-5.17.0-2-armmp/aspeed-bmc-ibm-rainier.dtb')
 
         self.vm.set_console()
         self.vm.add_args('-kernel', kernel_path,

tests/functional/test_arm_bflt.py

@@ -6,13 +6,10 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import os
 import bz2
 
 from qemu_test import QemuUserTest, Asset
-from qemu_test import has_cmd
+from qemu_test import skipIfMissingCommands, skipUntrustedTest
-from qemu_test.utils import cpio_extract
-from unittest import skipUnless
 
 
 class LoadBFLT(QemuUserTest):
@@ -21,15 +18,15 @@ class LoadBFLT(QemuUserTest):
         ('https://elinux.org/images/5/51/Stm32_mini_rootfs.cpio.bz2'),
          'eefb788e4980c9e8d6c9d60ce7d15d4da6bf4fbc6a80f487673824600d5ba9cc')
 
-    @skipUnless(*has_cmd('cpio'))
+    @skipIfMissingCommands('cpio')
-    @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code')
+    @skipUntrustedTest()
     def test_stm32(self):
         # See https://elinux.org/STM32#User_Space
         rootfs_path_bz2 = self.ASSET_ROOTFS.fetch()
-        busybox_path = os.path.join(self.workdir, "bin/busybox")
+        busybox_path = self.scratch_file("bin", "busybox")
 
         with bz2.open(rootfs_path_bz2, 'rb') as cpio_handle:
-            cpio_extract(cpio_handle, self.workdir)
+            self.archive_extract(cpio_handle, format="cpio")
 
         res = self.run_cmd(busybox_path)
         ver = 'BusyBox v1.24.0.git (2015-02-03 22:17:13 CET) multi-call binary.'

tests/functional/test_arm_bpim2u.py

@@ -9,9 +9,9 @@ import os
 
 from qemu_test import LinuxKernelTest, exec_command_and_wait_for_pattern
 from qemu_test import Asset, interrupt_interactive_console_until_pattern
-from qemu_test.utils import archive_extract, gzip_uncompress, lzma_uncompress
+from qemu_test import skipBigDataTest
 from qemu_test.utils import image_pow2ceil_expand
-from unittest import skipUnless
+
 
 class BananaPiMachine(LinuxKernelTest):
 
@@ -38,12 +38,11 @@ class BananaPiMachine(LinuxKernelTest):
 
     def test_arm_bpim2u(self):
         self.set_machine('bpim2u')
-        deb_path = self.ASSET_DEB.fetch()
+        kernel_path = self.archive_extract(
-        kernel_path = self.extract_from_deb(deb_path,
+            self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi')
-                                            '/boot/vmlinuz-6.6.16-current-sunxi')
+        dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/'
-        dtb_path = ('/usr/lib/linux-image-6.6.16-current-sunxi/'
                     'sun8i-r40-bananapi-m2-ultra.dtb')
-        dtb_path = self.extract_from_deb(deb_path, dtb_path)
+        dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path)
 
         self.vm.set_console()
         kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
@@ -60,15 +59,12 @@ class BananaPiMachine(LinuxKernelTest):
 
     def test_arm_bpim2u_initrd(self):
         self.set_machine('bpim2u')
-        deb_path = self.ASSET_DEB.fetch()
+        kernel_path = self.archive_extract(
-        kernel_path = self.extract_from_deb(deb_path,
+            self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi')
-                                            '/boot/vmlinuz-6.6.16-current-sunxi')
+        dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/'
-        dtb_path = ('/usr/lib/linux-image-6.6.16-current-sunxi/'
                     'sun8i-r40-bananapi-m2-ultra.dtb')
-        dtb_path = self.extract_from_deb(deb_path, dtb_path)
+        dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path)
-        initrd_path_gz = self.ASSET_INITRD.fetch()
+        initrd_path = self.uncompress(self.ASSET_INITRD)
-        initrd_path = os.path.join(self.workdir, 'rootfs.cpio')
-        gzip_uncompress(initrd_path_gz, initrd_path)
 
         self.vm.set_console()
         kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
@@ -99,14 +95,12 @@ class BananaPiMachine(LinuxKernelTest):
         self.require_netdev('user')
 
         deb_path = self.ASSET_DEB.fetch()
-        kernel_path = self.extract_from_deb(deb_path,
+        kernel_path = self.archive_extract(
-                                            '/boot/vmlinuz-6.6.16-current-sunxi')
+            self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi')
-        dtb_path = ('/usr/lib/linux-image-6.6.16-current-sunxi/'
+        dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/'
                     'sun8i-r40-bananapi-m2-ultra.dtb')
-        dtb_path = self.extract_from_deb(deb_path, dtb_path)
+        dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path)
-        rootfs_path_xz = self.ASSET_ROOTFS.fetch()
+        rootfs_path = self.uncompress(self.ASSET_ROOTFS)
-        rootfs_path = os.path.join(self.workdir, 'rootfs.cpio')
-        lzma_uncompress(rootfs_path_xz, rootfs_path)
         image_pow2ceil_expand(rootfs_path)
 
         self.vm.set_console()
@@ -143,14 +137,12 @@ class BananaPiMachine(LinuxKernelTest):
         os.remove(dtb_path)
         os.remove(rootfs_path)
 
-    @skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), 'storage limited')
+    @skipBigDataTest()
     def test_arm_bpim2u_openwrt_22_03_3(self):
         self.set_machine('bpim2u')
         # This test download a 8.9 MiB compressed image and expand it
         # to 127 MiB.
-        image_path_gz = self.ASSET_SD_IMAGE.fetch()
+        image_path = self.uncompress(self.ASSET_SD_IMAGE)
-        image_path = os.path.join(self.workdir, 'sdcard.img')
-        gzip_uncompress(image_path_gz, image_path)
         image_pow2ceil_expand(image_path)
 
         self.vm.set_console()

tests/functional/test_arm_canona1100.py

@@ -12,7 +12,7 @@
 
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import wait_for_console_pattern
-from qemu_test.utils import archive_extract
+
 
 class CanonA1100Machine(QemuSystemTest):
     """Boots the barebox firmware and checks that the console is operational"""
@@ -26,12 +26,10 @@ class CanonA1100Machine(QemuSystemTest):
     def test_arm_canona1100(self):
         self.set_machine('canon-a1100')
 
-        file_path = self.ASSET_BIOS.fetch()
+        bios = self.archive_extract(self.ASSET_BIOS,
-        archive_extract(file_path, dest_dir=self.workdir,
+                                    member="day18/barebox.canon-a1100.bin")
-                        member="day18/barebox.canon-a1100.bin")
         self.vm.set_console()
-        self.vm.add_args('-bios',
+        self.vm.add_args('-bios', bios)
-                         self.workdir + '/day18/barebox.canon-a1100.bin')
         self.vm.launch()
         wait_for_console_pattern(self, 'running /env/bin/init')
 

tests/functional/test_arm_collie.py

@@ -6,7 +6,7 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 from qemu_test import LinuxKernelTest, Asset
-from qemu_test.utils import archive_extract
+
 
 class CollieTest(LinuxKernelTest):
 

tests/functional/test_arm_cubieboard.py

@@ -5,12 +5,12 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 import os
-import shutil
 
 from qemu_test import LinuxKernelTest, Asset, exec_command_and_wait_for_pattern
 from qemu_test import interrupt_interactive_console_until_pattern
-from qemu_test.utils import gzip_uncompress, image_pow2ceil_expand
+from qemu_test import skipBigDataTest
-from unittest import skipUnless
+from qemu_test.utils import image_pow2ceil_expand
+
 
 class CubieboardMachine(LinuxKernelTest):
 
@@ -38,14 +38,12 @@ class CubieboardMachine(LinuxKernelTest):
 
     def test_arm_cubieboard_initrd(self):
         self.set_machine('cubieboard')
-        deb_path = self.ASSET_DEB.fetch()
+        kernel_path = self.archive_extract(
-        kernel_path = self.extract_from_deb(deb_path,
+            self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi')
-                                            '/boot/vmlinuz-6.6.16-current-sunxi')
+        dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' +
-        dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun4i-a10-cubieboard.dtb'
+                    'sun4i-a10-cubieboard.dtb')
-        dtb_path = self.extract_from_deb(deb_path, dtb_path)
+        dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path)
-        initrd_path_gz = self.ASSET_INITRD.fetch()
+        initrd_path = self.uncompress(self.ASSET_INITRD)
-        initrd_path = os.path.join(self.workdir, 'rootfs.cpio')
-        gzip_uncompress(initrd_path_gz, initrd_path)
 
         self.vm.set_console()
         kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
@@ -71,15 +69,13 @@ class CubieboardMachine(LinuxKernelTest):
 
     def test_arm_cubieboard_sata(self):
         self.set_machine('cubieboard')
-        deb_path = self.ASSET_DEB.fetch()
+        kernel_path = self.archive_extract(
-        kernel_path = self.extract_from_deb(deb_path,
+            self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi')
-                                            '/boot/vmlinuz-6.6.16-current-sunxi')
+        dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' +
-        dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun4i-a10-cubieboard.dtb'
+                    'sun4i-a10-cubieboard.dtb')
-        dtb_path = self.extract_from_deb(deb_path, dtb_path)
+        dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path)
 
-        rootfs_path_gz = self.ASSET_SATA_ROOTFS.fetch()
+        rootfs_path = self.uncompress(self.ASSET_SATA_ROOTFS)
-        rootfs_path = os.path.join(self.workdir, 'rootfs.cpio')
-        gzip_uncompress(rootfs_path_gz, rootfs_path)
 
         self.vm.set_console()
         kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
@@ -106,14 +102,12 @@ class CubieboardMachine(LinuxKernelTest):
         # Wait for VM to shut down gracefully
         self.vm.wait()
 
-    @skipUnless(os.getenv('AVOCADO_ALLOW_LARGE_STORAGE'), 'storage limited')
+    @skipBigDataTest()
     def test_arm_cubieboard_openwrt_22_03_2(self):
         # This test download a 7.5 MiB compressed image and expand it
         # to 126 MiB.
         self.set_machine('cubieboard')
-        image_path_gz = self.ASSET_OPENWRT.fetch()
+        image_path = self.uncompress(self.ASSET_OPENWRT)
-        image_path = os.path.join(self.workdir, 'sdcard.img')
-        gzip_uncompress(image_path_gz, image_path)
         image_pow2ceil_expand(image_path)
 
         self.vm.set_console()

tests/functional/test_arm_emcraft_sf2.py

@@ -28,7 +28,7 @@ class EmcraftSf2Machine(LinuxKernelTest):
 
         uboot_path = self.ASSET_UBOOT.fetch()
         spi_path = self.ASSET_SPI.fetch()
-        spi_path_rw = os.path.join(self.workdir, 'spi.bin')
+        spi_path_rw = self.scratch_file('spi.bin')
         shutil.copy(spi_path, spi_path_rw)
         os.chmod(spi_path_rw, 0o600)
 

tests/functional/test_arm_integratorcp.py

@@ -12,25 +12,11 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import os
 import logging
 
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import wait_for_console_pattern
-from unittest import skipUnless
+from qemu_test import skipIfMissingImports, skipUntrustedTest
-
-
-NUMPY_AVAILABLE = True
-try:
-    import numpy as np
-except ImportError:
-    NUMPY_AVAILABLE = False
-
-CV2_AVAILABLE = True
-try:
-    import cv2
-except ImportError:
-    CV2_AVAILABLE = False
 
 
 class IntegratorMachine(QemuSystemTest):
@@ -63,7 +49,7 @@ class IntegratorMachine(QemuSystemTest):
                          '-append', 'printk.time=0 console=ttyAMA0')
         self.vm.launch()
 
-    @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code')
+    @skipUntrustedTest()
     def test_integratorcp_console(self):
         """
         Boots the Linux kernel and checks that the console is operational
@@ -71,14 +57,16 @@ class IntegratorMachine(QemuSystemTest):
         self.boot_integratorcp()
         wait_for_console_pattern(self, 'Log in as root')
 
-    @skipUnless(NUMPY_AVAILABLE, 'Python NumPy not installed')
+    @skipIfMissingImports("numpy", "cv2")
-    @skipUnless(CV2_AVAILABLE, 'Python OpenCV not installed')
+    @skipUntrustedTest()
-    @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code')
     def test_framebuffer_tux_logo(self):
         """
         Boot Linux and verify the Tux logo is displayed on the framebuffer.
         """
-        screendump_path = os.path.join(self.workdir, "screendump.pbm")
+        import numpy as np
+        import cv2
+
+        screendump_path = self.scratch_file("screendump.pbm")
         tuxlogo_path = self.ASSET_TUXLOGO.fetch()
 
         self.boot_integratorcp()

tests/functional/test_arm_orangepi.py

@@ -10,10 +10,9 @@ import shutil
 
 from qemu_test import LinuxKernelTest, exec_command_and_wait_for_pattern
 from qemu_test import Asset, interrupt_interactive_console_until_pattern
-from qemu_test import wait_for_console_pattern
+from qemu_test import wait_for_console_pattern, skipBigDataTest
-from qemu_test.utils import archive_extract, gzip_uncompress, lzma_uncompress
 from qemu_test.utils import image_pow2ceil_expand
-from unittest import skipUnless
+
 
 class BananaPiMachine(LinuxKernelTest):
 
@@ -50,11 +49,11 @@ class BananaPiMachine(LinuxKernelTest):
 
     def test_arm_orangepi(self):
         self.set_machine('orangepi-pc')
-        deb_path = self.ASSET_DEB.fetch()
+        kernel_path = self.archive_extract(
-        kernel_path = self.extract_from_deb(deb_path,
+            self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi')
-                                            '/boot/vmlinuz-6.6.16-current-sunxi')
+        dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' +
-        dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun8i-h3-orangepi-pc.dtb'
+                    'sun8i-h3-orangepi-pc.dtb')
-        dtb_path = self.extract_from_deb(deb_path, dtb_path)
+        dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path)
 
         self.vm.set_console()
         kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
@@ -71,14 +70,12 @@ class BananaPiMachine(LinuxKernelTest):
 
     def test_arm_orangepi_initrd(self):
         self.set_machine('orangepi-pc')
-        deb_path = self.ASSET_DEB.fetch()
+        kernel_path = self.archive_extract(
-        kernel_path = self.extract_from_deb(deb_path,
+            self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi')
-                                            '/boot/vmlinuz-6.6.16-current-sunxi')
+        dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' +
-        dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun8i-h3-orangepi-pc.dtb'
+                    'sun8i-h3-orangepi-pc.dtb')
-        dtb_path = self.extract_from_deb(deb_path, dtb_path)
+        dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path)
-        initrd_path_gz = self.ASSET_INITRD.fetch()
+        initrd_path = self.uncompress(self.ASSET_INITRD)
-        initrd_path = os.path.join(self.workdir, 'rootfs.cpio')
-        gzip_uncompress(initrd_path_gz, initrd_path)
 
         self.vm.set_console()
         kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
@@ -107,14 +104,12 @@ class BananaPiMachine(LinuxKernelTest):
     def test_arm_orangepi_sd(self):
         self.set_machine('orangepi-pc')
         self.require_netdev('user')
-        deb_path = self.ASSET_DEB.fetch()
+        kernel_path = self.archive_extract(
-        kernel_path = self.extract_from_deb(deb_path,
+            self.ASSET_DEB, member='boot/vmlinuz-6.6.16-current-sunxi')
-                                            '/boot/vmlinuz-6.6.16-current-sunxi')
+        dtb_path = ('usr/lib/linux-image-6.6.16-current-sunxi/' +
-        dtb_path = '/usr/lib/linux-image-6.6.16-current-sunxi/sun8i-h3-orangepi-pc.dtb'
+                    'sun8i-h3-orangepi-pc.dtb')
-        dtb_path = self.extract_from_deb(deb_path, dtb_path)
+        dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path)
-        rootfs_path_xz = self.ASSET_ROOTFS.fetch()
+        rootfs_path = self.uncompress(self.ASSET_ROOTFS)
-        rootfs_path = os.path.join(self.workdir, 'rootfs.cpio')
-        lzma_uncompress(rootfs_path_xz, rootfs_path)
         image_pow2ceil_expand(rootfs_path)
 
         self.vm.set_console()
@@ -149,15 +144,13 @@ class BananaPiMachine(LinuxKernelTest):
         os.remove(dtb_path)
         os.remove(rootfs_path)
 
-    @skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), 'storage limited')
+    @skipBigDataTest()
     def test_arm_orangepi_armbian(self):
         self.set_machine('orangepi-pc')
         # This test download a 275 MiB compressed image and expand it
         # to 1036 MiB, but the underlying filesystem is 1552 MiB...
         # As we expand it to 2 GiB we are safe.
-        image_path_xz = self.ASSET_ARMBIAN.fetch()
+        image_path = self.uncompress(self.ASSET_ARMBIAN)
-        image_path = os.path.join(self.workdir, 'armbian.img')
-        lzma_uncompress(image_path_xz, image_path)
         image_pow2ceil_expand(image_path)
 
         self.vm.set_console()
@@ -185,20 +178,17 @@ class BananaPiMachine(LinuxKernelTest):
                                       'to <orangepipc>')
         self.wait_for_console_pattern('Starting Load Kernel Modules...')
 
-    @skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), 'storage limited')
+    @skipBigDataTest()
     def test_arm_orangepi_uboot_netbsd9(self):
         self.set_machine('orangepi-pc')
         # This test download a 304MB compressed image and expand it to 2GB
-        deb_path = self.ASSET_UBOOT.fetch()
         # We use the common OrangePi PC 'plus' build of U-Boot for our secondary
         # program loader (SPL). We will then set the path to the more specific
         # OrangePi "PC" device tree blob with 'setenv fdtfile' in U-Boot prompt,
         # before to boot NetBSD.
-        uboot_path = '/usr/lib/u-boot/orangepi_plus/u-boot-sunxi-with-spl.bin'
+        uboot_path = 'usr/lib/u-boot/orangepi_plus/u-boot-sunxi-with-spl.bin'
-        uboot_path = self.extract_from_deb(deb_path, uboot_path)
+        uboot_path = self.archive_extract(self.ASSET_UBOOT, member=uboot_path)
-        image_path_gz = self.ASSET_NETBSD.fetch()
+        image_path = self.uncompress(self.ASSET_NETBSD)
-        image_path = os.path.join(self.workdir, 'armv7.img')
-        gzip_uncompress(image_path_gz, image_path)
         image_pow2ceil_expand(image_path)
         image_drive_args = 'if=sd,format=raw,snapshot=on,file=' + image_path
 

tests/functional/test_arm_quanta_gsj.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+#
+# Functional test that boots a Linux kernel and checks the console
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+import os
+
+from qemu_test import LinuxKernelTest, Asset, exec_command_and_wait_for_pattern
+from qemu_test import interrupt_interactive_console_until_pattern
+from unittest import skipUnless
+
+class EmcraftSf2Machine(LinuxKernelTest):
+
+    ASSET_IMAGE = Asset(
+        ('https://github.com/hskinnemoen/openbmc/releases/download/'
+         '20200711-gsj-qemu-0/obmc-phosphor-image-gsj.static.mtd.gz'),
+        'eccd4e375cde53034c84aece5c511932cacf838d9fd3f63da368a511757da72b')
+
+    ASSET_INITRD = Asset(
+        ('https://github.com/hskinnemoen/openbmc/releases/download/'
+         '20200711-gsj-qemu-0/obmc-phosphor-initramfs-gsj.cpio.xz'),
+        '37b05009fc54db1434beac12bd7ff99a2e751a2f032ee18d9042f991dd0cdeaa')
+
+    ASSET_KERNEL = Asset(
+        ('https://github.com/hskinnemoen/openbmc/releases/download/'
+         '20200711-gsj-qemu-0/uImage-gsj.bin'),
+        'ce6d6b37bff46c74fc7b1e90da10a431cc37a62cdb35ec199fa73473d0790110')
+
+    ASSET_DTB = Asset(
+        ('https://github.com/hskinnemoen/openbmc/releases/download/'
+         '20200711-gsj-qemu-0/nuvoton-npcm730-gsj.dtb'),
+        '3249b2da787d4b9ad4e61f315b160abfceb87b5e1895a7ce898ce7f40c8d4045')
+
+    @skipUnless(os.getenv('QEMU_TEST_TIMEOUT_EXPECTED'), 'Test might timeout')
+    def test_arm_quanta_gsj(self):
+        self.set_machine('quanta-gsj')
+        image_path = self.uncompress(ASSET_IMAGE, 'obmc.mtd', format='gz')
+
+        self.vm.set_console()
+        drive_args = 'file=' + image_path + ',if=mtd,bus=0,unit=0'
+        self.vm.add_args('-drive', drive_args)
+        self.vm.launch()
+
+        # Disable drivers and services that stall for a long time during boot,
+        # to avoid running past the 90-second timeout. These may be removed
+        # as the corresponding device support is added.
+        kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE + (
+                'console=${console} '
+                'mem=${mem} '
+                'initcall_blacklist=npcm_i2c_bus_driver_init '
+                'systemd.mask=systemd-random-seed.service '
+                'systemd.mask=dropbearkey.service '
+        )
+
+        self.wait_for_console_pattern('> BootBlock by Nuvoton')
+        self.wait_for_console_pattern('>Device: Poleg BMC NPCM730')
+        self.wait_for_console_pattern('>Skip DDR init.')
+        self.wait_for_console_pattern('U-Boot ')
+        interrupt_interactive_console_until_pattern(
+                self, 'Hit any key to stop autoboot:', 'U-Boot>')
+        exec_command_and_wait_for_pattern(
+                self, "setenv bootargs ${bootargs} " + kernel_command_line,
+                'U-Boot>')
+        exec_command_and_wait_for_pattern(
+                self, 'run romboot', 'Booting Kernel from flash')
+        self.wait_for_console_pattern('Booting Linux on physical CPU 0x0')
+        self.wait_for_console_pattern('CPU1: thread -1, cpu 1, socket 0')
+        self.wait_for_console_pattern('OpenBMC Project Reference Distro')
+        self.wait_for_console_pattern('gsj login:')
+
+    def test_arm_quanta_gsj_initrd(self):
+        self.set_machine('quanta-gsj')
+        initrd_path = self.ASSET_INITRD.fetch()
+        kernel_path = self.ASSET_KERNEL.fetch()
+        dtb_path = self.ASSET_DTB.fetch()
+
+        self.vm.set_console()
+        kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
+                               'console=ttyS0,115200n8 '
+                               'earlycon=uart8250,mmio32,0xf0001000')
+        self.vm.add_args('-kernel', kernel_path,
+                         '-initrd', initrd_path,
+                         '-dtb', dtb_path,
+                         '-append', kernel_command_line)
+        self.vm.launch()
+
+        self.wait_for_console_pattern('Booting Linux on physical CPU 0x0')
+        self.wait_for_console_pattern('CPU1: thread -1, cpu 1, socket 0')
+        self.wait_for_console_pattern(
+                'Give root password for system maintenance')
+
+if __name__ == '__main__':
+    LinuxKernelTest.main()

tests/functional/test_arm_raspi2.py

@@ -7,11 +7,8 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import os
-
 from qemu_test import LinuxKernelTest, Asset
 from qemu_test import exec_command_and_wait_for_pattern
-from qemu_test.utils import gzip_uncompress
 
 
 class ArmRaspi2Machine(LinuxKernelTest):
@@ -37,9 +34,10 @@ class ArmRaspi2Machine(LinuxKernelTest):
         serial_kernel_cmdline = {
             0: 'earlycon=pl011,0x3f201000 console=ttyAMA0',
         }
-        deb_path = self.ASSET_KERNEL_20190215.fetch()
+        kernel_path = self.archive_extract(self.ASSET_KERNEL_20190215,
-        kernel_path = self.extract_from_deb(deb_path, '/boot/kernel7.img')
+                                           member='boot/kernel7.img')
-        dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2709-rpi-2-b.dtb')
+        dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215,
+                                        member='boot/bcm2709-rpi-2-b.dtb')
 
         self.set_machine('raspi2b')
         self.vm.set_console()
@@ -61,12 +59,11 @@ class ArmRaspi2Machine(LinuxKernelTest):
         self.do_test_arm_raspi2(0)
 
     def test_arm_raspi2_initrd(self):
-        deb_path = self.ASSET_KERNEL_20190215.fetch()
+        kernel_path = self.archive_extract(self.ASSET_KERNEL_20190215,
-        kernel_path = self.extract_from_deb(deb_path, '/boot/kernel7.img')
+                                           member='boot/kernel7.img')
-        dtb_path = self.extract_from_deb(deb_path, '/boot/bcm2709-rpi-2-b.dtb')
+        dtb_path = self.archive_extract(self.ASSET_KERNEL_20190215,
-        initrd_path_gz = self.ASSET_INITRD.fetch()
+                                        member='boot/bcm2709-rpi-2-b.dtb')
-        initrd_path = os.path.join(self.workdir, 'rootfs.cpio')
+        initrd_path = self.uncompress(self.ASSET_INITRD)
-        gzip_uncompress(initrd_path_gz, initrd_path)
 
         self.set_machine('raspi2b')
         self.vm.set_console()

tests/functional/test_arm_smdkc210.py

@@ -5,10 +5,9 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 import os
-import shutil
 
-from qemu_test import LinuxKernelTest, Asset, exec_command_and_wait_for_pattern
+from qemu_test import LinuxKernelTest, Asset
-from qemu_test.utils import gzip_uncompress
+
 
 class Smdkc210Machine(LinuxKernelTest):
 
@@ -26,15 +25,12 @@ class Smdkc210Machine(LinuxKernelTest):
     def test_arm_exynos4210_initrd(self):
         self.set_machine('smdkc210')
 
-        deb_path = self.ASSET_DEB.fetch()
+        kernel_path = self.archive_extract(self.ASSET_DEB,
-        kernel_path = self.extract_from_deb(deb_path,
+                                           member='boot/vmlinuz-4.19.0-6-armmp')
-                                            '/boot/vmlinuz-4.19.0-6-armmp')
+        dtb_path = 'usr/lib/linux-image-4.19.0-6-armmp/exynos4210-smdkv310.dtb'
-        dtb_path = '/usr/lib/linux-image-4.19.0-6-armmp/exynos4210-smdkv310.dtb'
+        dtb_path = self.archive_extract(self.ASSET_DEB, member=dtb_path)
-        dtb_path = self.extract_from_deb(deb_path, dtb_path)
 
-        initrd_path_gz = self.ASSET_ROOTFS.fetch()
+        initrd_path = self.uncompress(self.ASSET_ROOTFS)
-        initrd_path = os.path.join(self.workdir, 'rootfs.cpio')
-        gzip_uncompress(initrd_path_gz, initrd_path)
 
         self.vm.set_console()
         kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +

tests/functional/test_arm_sx1.py

@@ -14,7 +14,7 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 from qemu_test import LinuxKernelTest, Asset
-from qemu_test.utils import archive_extract
+
 
 class SX1Test(LinuxKernelTest):
 

tests/functional/test_arm_vexpress.py

@@ -6,7 +6,7 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 from qemu_test import LinuxKernelTest, Asset
-from qemu_test.utils import archive_extract
+
 
 class VExpressTest(LinuxKernelTest):
 
@@ -16,10 +16,10 @@ class VExpressTest(LinuxKernelTest):
 
     def test_arm_vexpressa9(self):
         self.set_machine('vexpress-a9')
-        file_path = self.ASSET_DAY16.fetch()
+        self.archive_extract(self.ASSET_DAY16)
-        archive_extract(file_path, self.workdir)
+        self.launch_kernel(self.scratch_file('day16', 'winter.zImage'),
-        self.launch_kernel(self.workdir + '/day16/winter.zImage',
+                           dtb=self.scratch_file('day16',
-                           dtb=self.workdir + '/day16/vexpress-v2p-ca9.dtb',
+                                                 'vexpress-v2p-ca9.dtb'),
                            wait_for='QEMU advent calendar')
 
 if __name__ == '__main__':

tests/functional/test_arm_virt.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+#
+# Functional test that boots a Linux kernel and checks the console
+#
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+from qemu_test import LinuxKernelTest, Asset
+
+class ArmVirtMachine(LinuxKernelTest):
+
+    ASSET_KERNEL = Asset(
+        ('https://archives.fedoraproject.org/pub/archive/fedora/linux/'
+         'releases/29/Everything/armhfp/os/images/pxeboot/vmlinuz'),
+        '18dd5f1a9a28bd539f9d047f7c0677211bae528e8712b40ca5a229a4ad8e2591')
+
+    def test_arm_virt(self):
+        self.set_machine('virt')
+        kernel_path = self.ASSET_KERNEL.fetch()
+
+        self.vm.set_console()
+        kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +
+                               'console=ttyAMA0')
+        self.vm.add_args('-kernel', kernel_path,
+                         '-append', kernel_command_line)
+        self.vm.launch()
+        console_pattern = 'Kernel command line: %s' % kernel_command_line
+        self.wait_for_console_pattern(console_pattern)
+
+if __name__ == '__main__':
+    LinuxKernelTest.main()

tests/functional/test_info_usernet.py

@@ -11,8 +11,7 @@
 # later.  See the COPYING file in the top-level directory.
 
 from qemu_test import QemuSystemTest
-
+from qemu_test.utils import get_usernet_hostfwd_port
-from qemu.utils import get_info_usernet_hostfwd_port
 
 
 class InfoUsernet(QemuSystemTest):
@@ -22,9 +21,8 @@ class InfoUsernet(QemuSystemTest):
         self.set_machine('none')
         self.vm.add_args('-netdev', 'user,id=vnet,hostfwd=:127.0.0.1:0-:22')
         self.vm.launch()
-        res = self.vm.cmd('human-monitor-command',
+
-                          command_line='info usernet')
+        port = get_usernet_hostfwd_port(self.vm)
-        port = get_info_usernet_hostfwd_port(res)
         self.assertIsNotNone(port,
                              ('"info usernet" output content does not seem to '
                               'contain the redirected port'))

tests/functional/test_intel_iommu.py

@@ -0,0 +1,175 @@
+#!/usr/bin/env python3
+#
+# INTEL_IOMMU Functional tests
+#
+# Copyright (c) 2021 Red Hat, Inc.
+#
+# Author:
+#  Eric Auger <eric.auger@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or
+# later.  See the COPYING file in the top-level directory.
+
+import hashlib
+import urllib.request
+
+from qemu_test import LinuxKernelTest, Asset, exec_command_and_wait_for_pattern
+from qemu_test.utils import get_usernet_hostfwd_port
+
+
+class IntelIOMMU(LinuxKernelTest):
+
+    ASSET_KERNEL = Asset(
+        ('https://archives.fedoraproject.org/pub/archive/fedora/linux/releases'
+         '/31/Server/x86_64/os/images/pxeboot/vmlinuz'),
+        'd4738d03dbbe083ca610d0821d0a8f1488bebbdccef54ce33e3adb35fda00129')
+
+    ASSET_INITRD = Asset(
+        ('https://archives.fedoraproject.org/pub/archive/fedora/linux/releases'
+         '/31/Server/x86_64/os/images/pxeboot/initrd.img'),
+        '277cd6c7adf77c7e63d73bbb2cded8ef9e2d3a2f100000e92ff1f8396513cd8b')
+
+    ASSET_DISKIMAGE = Asset(
+        ('https://archives.fedoraproject.org/pub/archive/fedora/linux/releases'
+         '/31/Cloud/x86_64/images/Fedora-Cloud-Base-31-1.9.x86_64.qcow2'),
+        'e3c1b309d9203604922d6e255c2c5d098a309c2d46215d8fc026954f3c5c27a0')
+
+    DEFAULT_KERNEL_PARAMS = ('root=/dev/vda1 console=ttyS0 net.ifnames=0 '
+                             'quiet rd.rescue ')
+    GUEST_PORT = 8080
+    IOMMU_ADDON = ',iommu_platform=on,disable-modern=off,disable-legacy=on'
+    kernel_path = None
+    initrd_path = None
+    kernel_params = None
+
+    def add_common_args(self, path):
+        self.vm.add_args('-drive', f'file={path},if=none,id=drv0,snapshot=on')
+        self.vm.add_args('-device', 'virtio-blk-pci,bus=pcie.0,' +
+                         'drive=drv0,id=virtio-disk0,bootindex=1,'
+                         'werror=stop,rerror=stop' + self.IOMMU_ADDON)
+        self.vm.add_args('-device', 'virtio-gpu-pci' + self.IOMMU_ADDON)
+
+        self.vm.add_args('-netdev',
+                         'user,id=n1,hostfwd=tcp:127.0.0.1:0-:%d' %
+                         self.GUEST_PORT)
+        self.vm.add_args('-device',
+                         'virtio-net-pci,netdev=n1' + self.IOMMU_ADDON)
+
+        self.vm.add_args('-device', 'virtio-rng-pci,rng=rng0')
+        self.vm.add_args('-object',
+                         'rng-random,id=rng0,filename=/dev/urandom')
+        self.vm.add_args("-m", "1G")
+        self.vm.add_args("-accel", "kvm")
+
+    def common_vm_setup(self):
+        self.set_machine('q35')
+        self.require_accelerator("kvm")
+        self.require_netdev('user')
+
+        self.kernel_path = self.ASSET_KERNEL.fetch()
+        self.initrd_path = self.ASSET_INITRD.fetch()
+        image_path = self.ASSET_DISKIMAGE.fetch()
+        self.add_common_args(image_path)
+        self.kernel_params = self.DEFAULT_KERNEL_PARAMS
+
+    def run_and_check(self):
+        if self.kernel_path:
+            self.vm.add_args('-kernel', self.kernel_path,
+                             '-append', self.kernel_params,
+                             '-initrd', self.initrd_path)
+        self.vm.set_console()
+        self.vm.launch()
+        self.wait_for_console_pattern('Entering emergency mode.')
+        prompt = '# '
+        self.wait_for_console_pattern(prompt)
+
+        # Copy a file (checked later), umount afterwards to drop disk cache:
+        exec_command_and_wait_for_pattern(self, 'mount /dev/vda1 /sysroot',
+                                          prompt)
+        filename = '/boot/initramfs-5.3.7-301.fc31.x86_64.img'
+        exec_command_and_wait_for_pattern(self, (f'cp /sysroot{filename}'
+                                                 ' /sysroot/root/data'),
+                                          prompt)
+        exec_command_and_wait_for_pattern(self, 'umount /sysroot', prompt)
+
+        # Switch from initrd to the cloud image filesystem:
+        exec_command_and_wait_for_pattern(self, 'mount /dev/vda1 /sysroot',
+                                          prompt)
+        exec_command_and_wait_for_pattern(self,
+                ('for d in dev proc sys run ; do '
+                 'mount -o bind /$d /sysroot/$d ; done'), prompt)
+        exec_command_and_wait_for_pattern(self, 'chroot /sysroot', prompt)
+
+        # Checking for IOMMU enablement:
+        self.log.info("Checking whether IOMMU has been enabled...")
+        exec_command_and_wait_for_pattern(self, 'cat /proc/cmdline',
+                                          'intel_iommu=on')
+        self.wait_for_console_pattern(prompt)
+        exec_command_and_wait_for_pattern(self, 'dmesg | grep DMAR:',
+                                          'IOMMU enabled')
+        self.wait_for_console_pattern(prompt)
+        exec_command_and_wait_for_pattern(self,
+                                    'find /sys/kernel/iommu_groups/ -type l',
+                                    'devices/0000:00:')
+        self.wait_for_console_pattern(prompt)
+
+        # Check hard disk device via sha256sum:
+        self.log.info("Checking hard disk...")
+        hashsum = '0dc7472f879be70b2f3daae279e3ae47175ffe249691e7d97f47222b65b8a720'
+        exec_command_and_wait_for_pattern(self, 'sha256sum ' + filename,
+                                          hashsum)
+        self.wait_for_console_pattern(prompt)
+        exec_command_and_wait_for_pattern(self, 'sha256sum /root/data',
+                                          hashsum)
+        self.wait_for_console_pattern(prompt)
+
+        # Check virtio-net via HTTP:
+        exec_command_and_wait_for_pattern(self, 'dhclient eth0', prompt)
+        exec_command_and_wait_for_pattern(self,
+                        f'python3 -m http.server {self.GUEST_PORT} & sleep 1',
+                        f'Serving HTTP on 0.0.0.0 port {self.GUEST_PORT}')
+        hl = hashlib.sha256()
+        hostport = get_usernet_hostfwd_port(self.vm)
+        url = f'http://localhost:{hostport}{filename}'
+        self.log.info(f'Downloading {url} ...')
+        with urllib.request.urlopen(url) as response:
+            while True:
+                chunk = response.read(1 << 20)
+                if not chunk:
+                    break
+                hl.update(chunk)
+
+        digest = hl.hexdigest()
+        self.log.info(f'sha256sum of download is {digest}.')
+        self.assertEqual(digest, hashsum)
+
+    def test_intel_iommu(self):
+        self.common_vm_setup()
+        self.vm.add_args('-device', 'intel-iommu,intremap=on')
+        self.vm.add_args('-machine', 'kernel_irqchip=split')
+        self.kernel_params += 'intel_iommu=on'
+        self.run_and_check()
+
+    def test_intel_iommu_strict(self):
+        self.common_vm_setup()
+        self.vm.add_args('-device', 'intel-iommu,intremap=on')
+        self.vm.add_args('-machine', 'kernel_irqchip=split')
+        self.kernel_params += 'intel_iommu=on,strict'
+        self.run_and_check()
+
+    def test_intel_iommu_strict_cm(self):
+        self.common_vm_setup()
+        self.vm.add_args('-device', 'intel-iommu,intremap=on,caching-mode=on')
+        self.vm.add_args('-machine', 'kernel_irqchip=split')
+        self.kernel_params += 'intel_iommu=on,strict'
+        self.run_and_check()
+
+    def test_intel_iommu_pt(self):
+        self.common_vm_setup()
+        self.vm.add_args('-device', 'intel-iommu,intremap=on')
+        self.vm.add_args('-machine', 'kernel_irqchip=split')
+        self.kernel_params += 'intel_iommu=on iommu=pt'
+        self.run_and_check()
+
+if __name__ == '__main__':
+    LinuxKernelTest.main()

tests/functional/test_linux_initrd.py

@@ -10,12 +10,10 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later.  See the COPYING file in the top-level directory.
 
-import os
 import logging
 import tempfile
 
-from qemu_test import QemuSystemTest, Asset
+from qemu_test import QemuSystemTest, Asset, skipFlakyTest
-from unittest import skipUnless
 
 
 class LinuxInitrd(QemuSystemTest):
@@ -60,7 +58,8 @@ class LinuxInitrd(QemuSystemTest):
                 max_size + 1)
             self.assertRegex(self.vm.get_log(), expected_msg)
 
-    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
+    # XXX file tracking bug
+    @skipFlakyTest(bug_url=None)
     def test_with_2gib_file_should_work_with_linux_v4_16(self):
         """
         QEMU has supported up to 4 GiB initrd for recent kernel

tests/functional/test_m68k_mcf5208evb.py

@@ -6,7 +6,7 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 from qemu_test import LinuxKernelTest, Asset
-from qemu_test.utils import archive_extract
+
 
 class Mcf5208EvbTest(LinuxKernelTest):
 
@@ -16,10 +16,10 @@ class Mcf5208EvbTest(LinuxKernelTest):
 
     def test_m68k_mcf5208evb(self):
         self.set_machine('mcf5208evb')
-        file_path = self.ASSET_DAY07.fetch()
+        self.archive_extract(self.ASSET_DAY07)
-        archive_extract(file_path, self.workdir)
         self.vm.set_console()
-        self.vm.add_args('-kernel', self.workdir + '/day07/sanity-clause.elf')
+        self.vm.add_args('-kernel',
+                         self.scratch_file('day07', 'sanity-clause.elf'))
         self.vm.launch()
         self.wait_for_console_pattern('QEMU advent calendar')
 

tests/functional/test_m68k_nextcube.py

@@ -7,19 +7,11 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later.  See the COPYING file in the top-level directory.
 
-import os
 import time
 
 from qemu_test import QemuSystemTest, Asset
-from unittest import skipUnless
+from qemu_test import skipIfMissingImports, skipIfMissingCommands
-
+from qemu_test.tesseract import tesseract_ocr
-from qemu_test.tesseract import tesseract_available, tesseract_ocr
-
-PIL_AVAILABLE = True
-try:
-    from PIL import Image
-except ImportError:
-    PIL_AVAILABLE = False
 
 
 class NextCubeMachine(QemuSystemTest):
@@ -43,23 +35,21 @@ class NextCubeMachine(QemuSystemTest):
         self.vm.cmd('human-monitor-command',
                     command_line='screendump %s' % screenshot_path)
 
-    @skipUnless(PIL_AVAILABLE, 'Python PIL not installed')
+    @skipIfMissingImports("PIL")
     def test_bootrom_framebuffer_size(self):
         self.set_machine('next-cube')
-        screenshot_path = os.path.join(self.workdir, "dump.ppm")
+        screenshot_path = self.scratch_file("dump.ppm")
         self.check_bootrom_framebuffer(screenshot_path)
 
+        from PIL import Image
         width, height = Image.open(screenshot_path).size
         self.assertEqual(width, 1120)
         self.assertEqual(height, 832)
 
-    # Tesseract 4 adds a new OCR engine based on LSTM neural networks. The
+    @skipIfMissingCommands('tesseract')
-    # new version is faster and more accurate than version 3. The drawback is
-    # that it is still alpha-level software.
-    @skipUnless(tesseract_available(4), 'tesseract OCR tool not available')
     def test_bootrom_framebuffer_ocr_with_tesseract(self):
         self.set_machine('next-cube')
-        screenshot_path = os.path.join(self.workdir, "dump.ppm")
+        screenshot_path = self.scratch_file("dump.ppm")
         self.check_bootrom_framebuffer(screenshot_path)
         lines = tesseract_ocr(screenshot_path)
         text = '\n'.join(lines)

tests/functional/test_m68k_q800.py

@@ -18,9 +18,8 @@ class Q800MachineTest(LinuxKernelTest):
     def test_m68k_q800(self):
         self.set_machine('q800')
 
-        deb_path = self.ASSET_KERNEL.fetch()
+        kernel_path = self.archive_extract(self.ASSET_KERNEL,
-        kernel_path = self.extract_from_deb(deb_path,
+                                           member='boot/vmlinux-5.3.0-1-m68k')
-                                            '/boot/vmlinux-5.3.0-1-m68k')
 
         self.vm.set_console()
         kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE +

tests/functional/test_microblaze_s3adsp1800.py

@@ -7,10 +7,9 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later. See the COPYING file in the top-level directory.
 
-from qemu_test import exec_command, exec_command_and_wait_for_pattern
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import wait_for_console_pattern
-from qemu_test.utils import archive_extract
+
 
 class MicroblazeMachine(QemuSystemTest):
 
@@ -23,10 +22,10 @@ class MicroblazeMachine(QemuSystemTest):
 
     def test_microblaze_s3adsp1800(self):
         self.set_machine('petalogix-s3adsp1800')
-        file_path = self.ASSET_IMAGE.fetch()
+        self.archive_extract(self.ASSET_IMAGE)
-        archive_extract(file_path, self.workdir)
         self.vm.set_console()
-        self.vm.add_args('-kernel', self.workdir + '/day17/ballerina.bin')
+        self.vm.add_args('-kernel',
+                         self.scratch_file('day17', 'ballerina.bin'))
         self.vm.launch()
         wait_for_console_pattern(self, 'This architecture does not have '
                                        'kernel memory protection')

tests/functional/test_microblazeel_s3adsp1800.py

@@ -11,7 +11,7 @@ import time
 from qemu_test import exec_command, exec_command_and_wait_for_pattern
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import wait_for_console_pattern
-from qemu_test.utils import archive_extract
+
 
 class MicroblazeelMachine(QemuSystemTest):
 
@@ -24,11 +24,11 @@ class MicroblazeelMachine(QemuSystemTest):
     def test_microblazeel_s3adsp1800(self):
         self.require_netdev('user')
         self.set_machine('petalogix-s3adsp1800')
-        file_path = self.ASSET_IMAGE.fetch()
+        self.archive_extract(self.ASSET_IMAGE)
-        archive_extract(file_path, self.workdir)
         self.vm.set_console()
-        self.vm.add_args('-kernel', self.workdir + '/day13/xmaton.bin')
+        self.vm.add_args('-kernel', self.scratch_file('day13', 'xmaton.bin'))
-        self.vm.add_args('-nic', 'user,tftp=' + self.workdir + '/day13/')
+        tftproot = self.scratch_file('day13')
+        self.vm.add_args('-nic', f'user,tftp={tftproot}')
         self.vm.launch()
         wait_for_console_pattern(self, 'QEMU Advent Calendar 2023')
         time.sleep(0.1)

tests/functional/test_mips64el_fuloong2e.py

@@ -13,7 +13,7 @@ import os
 import subprocess
 
 from qemu_test import LinuxKernelTest, Asset
-from qemu_test import wait_for_console_pattern
+from qemu_test import wait_for_console_pattern, skipUntrustedTest
 from unittest import skipUnless
 
 class MipsFuloong2e(LinuxKernelTest):
@@ -26,9 +26,9 @@ class MipsFuloong2e(LinuxKernelTest):
         '2a70f15b397f4ced632b0c15cb22660394190644146d804d60a4796eefbe1f50')
 
     def test_linux_kernel_3_16(self):
-        deb_path = self.ASSET_KERNEL.fetch()
+        kernel_path = self.archive_extract(
-        kernel_path = self.extract_from_deb(deb_path,
+            self.ASSET_KERNEL,
-                                            '/boot/vmlinux-3.16.0-6-loongson-2e')
+            member='boot/vmlinux-3.16.0-6-loongson-2e')
 
         self.set_machine('fuloong2e')
         self.vm.set_console()
@@ -39,7 +39,7 @@ class MipsFuloong2e(LinuxKernelTest):
         console_pattern = 'Kernel command line: %s' % kernel_command_line
         self.wait_for_console_pattern(console_pattern)
 
-    @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code')
+    @skipUntrustedTest()
     @skipUnless(os.getenv('RESCUE_YL_PATH'), 'RESCUE_YL_PATH not available')
     def test_linux_kernel_2_6_27_isa_serial(self):
         # Recovery system for the Yeeloong laptop

tests/functional/test_mips64el_loongson3v.py

@@ -9,11 +9,9 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import os
-
-from unittest import skipUnless
 from qemu_test import QemuSystemTest, Asset
-from qemu_test import wait_for_console_pattern
+from qemu_test import wait_for_console_pattern, skipUntrustedTest
+
 
 class MipsLoongson3v(QemuSystemTest):
     timeout = 60
@@ -23,7 +21,7 @@ class MipsLoongson3v(QemuSystemTest):
          'releases/download/20210112/pmon-3avirt.bin'),
         'fcdf6bb2cb7885a4a62f31fcb0d5e368bac7b6cea28f40c6dfa678af22fea20a')
 
-    @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code')
+    @skipUntrustedTest()
     def test_pmon_serial_console(self):
         self.set_machine('loongson3-virt')
 

tests/functional/test_mips64el_malta.py

@@ -14,20 +14,7 @@ import logging
 
 from qemu_test import LinuxKernelTest, Asset
 from qemu_test import exec_command_and_wait_for_pattern
-from qemu_test.utils import gzip_uncompress
+from qemu_test import skipIfMissingImports, skipFlakyTest, skipUntrustedTest
-from unittest import skipUnless
-
-NUMPY_AVAILABLE = True
-try:
-    import numpy as np
-except ImportError:
-    NUMPY_AVAILABLE = False
-
-CV2_AVAILABLE = True
-try:
-    import cv2
-except ImportError:
-    CV2_AVAILABLE = False
 
 
 class MaltaMachineConsole(LinuxKernelTest):
@@ -51,9 +38,9 @@ class MaltaMachineConsole(LinuxKernelTest):
         [2] https://kernel-team.pages.debian.net/kernel-handbook/
             ch-common-tasks.html#s-common-official
         """
-        deb_path = self.ASSET_KERNEL_2_63_2.fetch()
+        kernel_path = self.archive_extract(
-        kernel_path = self.extract_from_deb(deb_path,
+            self.ASSET_KERNEL_2_63_2,
-                                            '/boot/vmlinux-2.6.32-5-5kc-malta')
+            member='boot/vmlinux-2.6.32-5-5kc-malta')
 
         self.set_machine('malta')
         self.vm.set_console()
@@ -76,12 +63,10 @@ class MaltaMachineConsole(LinuxKernelTest):
          'rootfs.mipsel64r1.cpio.gz'),
         '75ba10cd35fb44e32948eeb26974f061b703c81c4ba2fab1ebcacf1d1bec3b61')
 
-    @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code')
+    @skipUntrustedTest()
     def test_mips64el_malta_5KEc_cpio(self):
         kernel_path = self.ASSET_KERNEL_3_19_3.fetch()
-        initrd_path_gz = self.ASSET_CPIO_R1.fetch()
+        initrd_path = self.uncompress(self.ASSET_CPIO_R1)
-        initrd_path = os.path.join(self.workdir, 'rootfs.cpio')
-        gzip_uncompress(initrd_path_gz, initrd_path)
 
         self.set_machine('malta')
         self.vm.set_console()
@@ -106,8 +91,7 @@ class MaltaMachineConsole(LinuxKernelTest):
         self.vm.wait()
 
 
-@skipUnless(NUMPY_AVAILABLE, 'Python NumPy not installed')
+@skipIfMissingImports('numpy', 'cv2')
-@skipUnless(CV2_AVAILABLE, 'Python OpenCV not installed')
 class MaltaMachineFramebuffer(LinuxKernelTest):
 
     timeout = 30
@@ -126,11 +110,13 @@ class MaltaMachineFramebuffer(LinuxKernelTest):
         """
         Boot Linux kernel and check Tux logo is displayed on the framebuffer.
         """
-        screendump_path = os.path.join(self.workdir, 'screendump.pbm')
 
-        kernel_path_gz = self.ASSET_KERNEL_4_7_0.fetch()
+        import numpy as np
-        kernel_path = self.workdir + "/vmlinux"
+        import cv2
-        gzip_uncompress(kernel_path_gz, kernel_path)
+
+        screendump_path = self.scratch_file('screendump.pbm')
+
+        kernel_path = self.uncompress(self.ASSET_KERNEL_4_7_0)
 
         tuxlogo_path = self.ASSET_TUXLOGO.fetch()
 
@@ -171,11 +157,12 @@ class MaltaMachineFramebuffer(LinuxKernelTest):
     def test_mips_malta_i6400_framebuffer_logo_1core(self):
         self.do_test_i6400_framebuffer_logo(1)
 
-    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
+    # XXX file tracking bug
+    @skipFlakyTest(bug_url=None)
     def test_mips_malta_i6400_framebuffer_logo_7cores(self):
         self.do_test_i6400_framebuffer_logo(7)
 
-    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
+    @skipFlakyTest(bug_url=None)
     def test_mips_malta_i6400_framebuffer_logo_8cores(self):
         self.do_test_i6400_framebuffer_logo(8)
 

tests/functional/test_mips_malta.py

@@ -6,11 +6,8 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import os
-
 from qemu_test import LinuxKernelTest, Asset
 from qemu_test import exec_command_and_wait_for_pattern
-from qemu_test.utils import gzip_uncompress
 
 
 class MaltaMachineConsole(LinuxKernelTest):
@@ -22,9 +19,9 @@ class MaltaMachineConsole(LinuxKernelTest):
         '16ca524148afb0626f483163e5edf352bc1ab0e4fc7b9f9d473252762f2c7a43')
 
     def test_mips_malta(self):
-        deb_path = self.ASSET_KERNEL_2_63_2.fetch()
+        kernel_path = self.archive_extract(
-        kernel_path = self.extract_from_deb(deb_path,
+            self.ASSET_KERNEL_2_63_2,
-                                            '/boot/vmlinux-2.6.32-5-4kc-malta')
+            member='boot/vmlinux-2.6.32-5-4kc-malta')
 
         self.set_machine('malta')
         self.vm.set_console()
@@ -48,12 +45,10 @@ class MaltaMachineConsole(LinuxKernelTest):
         'dcfe3a7fe3200da3a00d176b95caaa086495eb158f2bff64afc67d7e1eb2cddc')
 
     def test_mips_malta_cpio(self):
-        deb_path = self.ASSET_KERNEL_4_5_0.fetch()
+        kernel_path = self.archive_extract(
-        kernel_path = self.extract_from_deb(deb_path,
+            self.ASSET_KERNEL_4_5_0,
-                                            '/boot/vmlinux-4.5.0-2-4kc-malta')
+            member='boot/vmlinux-4.5.0-2-4kc-malta')
-        initrd_path_gz = self.ASSET_INITRD.fetch()
+        initrd_path = self.uncompress(self.ASSET_INITRD)
-        initrd_path = os.path.join(self.workdir, 'rootfs.cpio')
-        gzip_uncompress(initrd_path_gz, initrd_path)
 
         self.set_machine('malta')
         self.vm.set_console()

tests/functional/test_mipsel_malta.py

@@ -9,13 +9,9 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import os
-
 from qemu_test import QemuSystemTest, LinuxKernelTest, Asset
 from qemu_test import interrupt_interactive_console_until_pattern
 from qemu_test import wait_for_console_pattern
-from qemu_test.utils import lzma_uncompress
-from zipfile import ZipFile
 
 
 class MaltaMachineConsole(LinuxKernelTest):
@@ -36,9 +32,8 @@ class MaltaMachineConsole(LinuxKernelTest):
          'generic_nano32r6el_page64k_dbg.xz'),
         'ce21ff4b07a981ecb8a39db2876616f5a2473eb2ab459c6f67465b9914b0c6b6')
 
-    def do_test_mips_malta32el_nanomips(self, kernel_path_xz):
+    def do_test_mips_malta32el_nanomips(self, kernel):
-        kernel_path = os.path.join(self.workdir, 'kernel')
+        kernel_path = self.uncompress(kernel)
-        lzma_uncompress(kernel_path_xz, kernel_path)
 
         self.set_machine('malta')
         self.vm.set_console()
@@ -54,16 +49,13 @@ class MaltaMachineConsole(LinuxKernelTest):
         self.wait_for_console_pattern(console_pattern)
 
     def test_mips_malta32el_nanomips_4k(self):
-        kernel_path_xz = self.ASSET_KERNEL_4K.fetch()
+        self.do_test_mips_malta32el_nanomips(self.ASSET_KERNEL_4K)
-        self.do_test_mips_malta32el_nanomips(kernel_path_xz)
 
     def test_mips_malta32el_nanomips_16k_up(self):
-        kernel_path_xz = self.ASSET_KERNEL_16K.fetch()
+        self.do_test_mips_malta32el_nanomips(self.ASSET_KERNEL_16K)
-        self.do_test_mips_malta32el_nanomips(kernel_path_xz)
 
     def test_mips_malta32el_nanomips_64k_dbg(self):
-        kernel_path_xz = self.ASSET_KERNEL_16K.fetch()
+        self.do_test_mips_malta32el_nanomips(self.ASSET_KERNEL_64K)
-        self.do_test_mips_malta32el_nanomips(kernel_path_xz)
 
 
 class MaltaMachineYAMON(QemuSystemTest):
@@ -75,10 +67,8 @@ class MaltaMachineYAMON(QemuSystemTest):
 
     def test_mipsel_malta_yamon(self):
         yamon_bin = 'yamon-02.22.bin'
-        zip_path = self.ASSET_YAMON_ROM.fetch()
+        self.archive_extract(self.ASSET_YAMON_ROM)
-        with ZipFile(zip_path, 'r') as zf:
+        yamon_path = self.scratch_file(yamon_bin)
-            zf.extract(yamon_bin, path=self.workdir)
-        yamon_path = os.path.join(self.workdir, yamon_bin)
 
         self.set_machine('malta')
         self.vm.set_console()

tests/functional/test_or1k_sim.py

@@ -6,7 +6,7 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 from qemu_test import LinuxKernelTest, Asset
-from qemu_test.utils import archive_extract
+
 
 class OpenRISC1kSimTest(LinuxKernelTest):
 
@@ -16,10 +16,9 @@ class OpenRISC1kSimTest(LinuxKernelTest):
 
     def test_or1k_sim(self):
         self.set_machine('or1k-sim')
-        file_path = self.ASSET_DAY20.fetch()
+        self.archive_extract(self.ASSET_DAY20)
-        archive_extract(file_path, self.workdir)
         self.vm.set_console()
-        self.vm.add_args('-kernel', self.workdir + '/day20/vmlinux')
+        self.vm.add_args('-kernel', self.scratch_file('day20', 'vmlinux'))
         self.vm.launch()
         self.wait_for_console_pattern('QEMU advent calendar')
 

tests/functional/test_ppc64_e500.py

@@ -5,7 +5,7 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 from qemu_test import LinuxKernelTest, Asset
-from qemu_test.utils import archive_extract
+
 
 class E500Test(LinuxKernelTest):
 
@@ -16,9 +16,8 @@ class E500Test(LinuxKernelTest):
     def test_ppc64_e500(self):
         self.set_machine('ppce500')
         self.cpu = 'e5500'
-        file_path = self.ASSET_DAY19.fetch()
+        self.archive_extract(self.ASSET_DAY19)
-        archive_extract(file_path, self.workdir)
+        self.launch_kernel(self.scratch_file('day19', 'uImage'),
-        self.launch_kernel(self.workdir + '/day19/uImage',
                            wait_for='QEMU advent calendar')
 
 if __name__ == '__main__':

tests/functional/test_ppc64_hv.py

@@ -9,35 +9,14 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later.  See the COPYING file in the top-level directory.
 
-from unittest import skipIf, skipUnless
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import wait_for_console_pattern, exec_command
+from qemu_test import skipIfMissingCommands, skipBigDataTest
 import os
 import time
 import subprocess
 from datetime import datetime
 
-deps = ["xorriso"] # dependent tools needed in the test setup/box.
-
-def which(tool):
-    """ looks up the full path for @tool, returns None if not found
-        or if @tool does not have executable permissions.
-    """
-    paths=os.getenv('PATH')
-    for p in paths.split(os.path.pathsep):
-        p = os.path.join(p, tool)
-        if os.path.exists(p) and os.access(p, os.X_OK):
-            return p
-    return None
-
-def missing_deps():
-    """ returns True if any of the test dependent tools are absent.
-    """
-    for dep in deps:
-        if which(dep) is None:
-            return True
-    return False
-
 # Alpine is a light weight distro that supports QEMU. These tests boot
 # that on the machine then run a QEMU guest inside it in KVM mode,
 # that runs the same Alpine distro image.
@@ -45,8 +24,8 @@ def missing_deps():
 # large download, but it may be more polite to create qcow2 image with
 # QEMU already installed and use that.
 # XXX: The order of these tests seems to matter, see git blame.
-@skipIf(missing_deps(), 'dependencies (%s) not installed' % ','.join(deps))
+@skipIfMissingCommands("xorriso")
-@skipUnless(os.getenv('QEMU_TEST_ALLOW_LARGE_STORAGE'), 'storage limited')
+@skipBigDataTest()
 class HypervisorTest(QemuSystemTest):
 
     timeout = 1000
@@ -67,23 +46,15 @@ class HypervisorTest(QemuSystemTest):
         :param path: path within the iso file of the file to be extracted
         :returns: path of the extracted file
         """
-        filename = os.path.basename(path)
+        filename = self.scratch_file(os.path.basename(path))
-
-        cwd = os.getcwd()
-        os.chdir(self.workdir)
 
         cmd = "xorriso -osirrox on -indev %s -cpx %s %s" % (iso, path, filename)
         subprocess.run(cmd.split(),
                        stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
 
         os.chmod(filename, 0o600)
-        os.chdir(cwd)
 
-        # Return complete path to extracted file.  Because callers to
+        return filename
-        # extract_from_iso() specify 'path' with a leading slash, it is
-        # necessary to use os.path.relpath() as otherwise os.path.join()
-        # interprets it as an absolute path and drops the self.workdir part.
-        return os.path.normpath(os.path.join(self.workdir, filename))
 
     def setUp(self):
         super().setUp()

tests/functional/test_ppc64_tuxrun.py

@@ -11,9 +11,10 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
+from subprocess import check_call, DEVNULL
 import tempfile
 
-from qemu_test import run_cmd, Asset
+from qemu_test import Asset
 from qemu_test.tuxruntest import TuxRunBaselineTest
 
 class TuxRunPPC64Test(TuxRunBaselineTest):
@@ -70,7 +71,9 @@ class TuxRunPPC64Test(TuxRunBaselineTest):
         # Create a temporary qcow2 and launch the test-case
         with tempfile.NamedTemporaryFile(prefix=prefix,
                                          suffix='.qcow2') as qcow2:
-            run_cmd([self.qemu_img, 'create', '-f', 'qcow2', qcow2.name, ' 1G'])
+            check_call([self.qemu_img, 'create', '-f', 'qcow2',
+                        qcow2.name, ' 1G'],
+                       stdout=DEVNULL, stderr=DEVNULL)
 
             self.vm.add_args('-drive', 'file=' + qcow2.name +
                          ',format=qcow2,if=none,id='

tests/functional/test_ppc_40p.py

@@ -7,11 +7,8 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later. See the COPYING file in the top-level directory.
 
-import os
-
-from unittest import skipUnless
 from qemu_test import QemuSystemTest, Asset
-from qemu_test import wait_for_console_pattern
+from qemu_test import wait_for_console_pattern, skipUntrustedTest
 
 
 class IbmPrep40pMachine(QemuSystemTest):
@@ -37,7 +34,7 @@ class IbmPrep40pMachine(QemuSystemTest):
     # All rights reserved.
     # U.S. Government Users Restricted Rights - Use, duplication or disclosure
     # restricted by GSA ADP Schedule Contract with IBM Corp.
-    @skipUnless(os.getenv('QEMU_TEST_ALLOW_UNTRUSTED_CODE'), 'untrusted code')
+    @skipUntrustedTest()
     def test_factory_firmware_and_netbsd(self):
         self.set_machine('40p')
         self.require_accelerator("tcg")

tests/functional/test_ppc_amiga.py

@@ -10,8 +10,8 @@
 import subprocess
 
 from qemu_test import QemuSystemTest, Asset
-from qemu_test import wait_for_console_pattern, run_cmd
+from qemu_test import wait_for_console_pattern
-from zipfile import ZipFile
+
 
 class AmigaOneMachine(QemuSystemTest):
 
@@ -26,16 +26,16 @@ class AmigaOneMachine(QemuSystemTest):
         self.require_accelerator("tcg")
         self.set_machine('amigaone')
         tar_name = 'A1Firmware_Floppy_05-Mar-2005.zip'
-        zip_file = self.ASSET_IMAGE.fetch()
+        self.archive_extract(self.ASSET_IMAGE, format="zip")
-        with ZipFile(zip_file, 'r') as zf:
+        bios = self.scratch_file("u-boot-amigaone.bin")
-            zf.extractall(path=self.workdir)
+        with open(bios, "wb") as bios_fh:
-        bios_fh = open(self.workdir + "/u-boot-amigaone.bin", "wb")
+            subprocess.run(['tail', '-c', '524288',
-        subprocess.run(['tail', '-c', '524288',
+                            self.scratch_file("floppy_edition",
-                        self.workdir + "/floppy_edition/updater.image"],
+                                              "updater.image")],
-                        stdout=bios_fh)
+                           stdout=bios_fh)
 
         self.vm.set_console()
-        self.vm.add_args('-bios', self.workdir + '/u-boot-amigaone.bin')
+        self.vm.add_args('-bios', bios)
         self.vm.launch()
         wait_for_console_pattern(self, 'FLASH:')
 

tests/functional/test_ppc_bamboo.py

@@ -7,11 +7,11 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later.  See the COPYING file in the top-level directory.
 
-from qemu_test.utils import archive_extract
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import wait_for_console_pattern
 from qemu_test import exec_command_and_wait_for_pattern
 
+
 class BambooMachine(QemuSystemTest):
 
     timeout = 90
@@ -25,13 +25,14 @@ class BambooMachine(QemuSystemTest):
         self.set_machine('bamboo')
         self.require_accelerator("tcg")
         self.require_netdev('user')
-        file_path = self.ASSET_IMAGE.fetch()
+        self.archive_extract(self.ASSET_IMAGE)
-        archive_extract(file_path, self.workdir)
         self.vm.set_console()
-        self.vm.add_args('-kernel', self.workdir +
+        self.vm.add_args('-kernel',
-                                   '/system-image-powerpc-440fp/linux',
+                         self.scratch_file('system-image-powerpc-440fp',
-                         '-initrd', self.workdir +
+                                           'linux'),
-                                   '/system-image-powerpc-440fp/rootfs.cpio.gz',
+                         '-initrd',
+                         self.scratch_file('system-image-powerpc-440fp',
+                                           'rootfs.cpio.gz'),
                          '-nic', 'user,model=rtl8139,restrict=on')
         self.vm.launch()
         wait_for_console_pattern(self, 'Type exit when done')

tests/functional/test_ppc_mac.py

@@ -5,7 +5,7 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 from qemu_test import LinuxKernelTest, Asset
-from qemu_test.utils import archive_extract
+
 
 class MacTest(LinuxKernelTest):
 
@@ -19,11 +19,9 @@ class MacTest(LinuxKernelTest):
         # we're running kvm_hv or kvm_pr. For now let's disable this test
         # if we don't have TCG support.
         self.require_accelerator("tcg")
-
+        self.archive_extract(self.ASSET_DAY15)
-        file_path = self.ASSET_DAY15.fetch()
-        archive_extract(file_path, self.workdir)
         self.vm.add_args('-M', 'graphics=off')
-        self.launch_kernel(self.workdir + '/day15/invaders.elf',
+        self.launch_kernel(self.scratch_file('day15', 'invaders.elf'),
                            wait_for='QEMU advent calendar')
 
     def test_ppc_g3beige(self):

tests/functional/test_ppc_mpc8544ds.py

@@ -7,10 +7,10 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later.  See the COPYING file in the top-level directory.
 
-from qemu_test.utils import archive_extract
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import wait_for_console_pattern
 
+
 class Mpc8544dsMachine(QemuSystemTest):
 
     timeout = 90
@@ -25,10 +25,10 @@ class Mpc8544dsMachine(QemuSystemTest):
     def test_ppc_mpc8544ds(self):
         self.require_accelerator("tcg")
         self.set_machine('mpc8544ds')
-        file_path = self.ASSET_IMAGE.fetch()
+        kernel_file = self.archive_extract(self.ASSET_IMAGE,
-        archive_extract(file_path, self.workdir, member='creek/creek.bin')
+                                           member='creek/creek.bin')
         self.vm.set_console()
-        self.vm.add_args('-kernel', self.workdir + '/creek/creek.bin')
+        self.vm.add_args('-kernel', kernel_file)
         self.vm.launch()
         wait_for_console_pattern(self, 'QEMU advent calendar 2020',
                                  self.panic_message)

tests/functional/test_ppc_virtex_ml507.py

@@ -7,10 +7,10 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later.  See the COPYING file in the top-level directory.
 
-from qemu_test.utils import archive_extract
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import wait_for_console_pattern
 
+
 class VirtexMl507Machine(QemuSystemTest):
 
     timeout = 90
@@ -25,11 +25,11 @@ class VirtexMl507Machine(QemuSystemTest):
     def test_ppc_virtex_ml507(self):
         self.require_accelerator("tcg")
         self.set_machine('virtex-ml507')
-        file_path = self.ASSET_IMAGE.fetch()
+        self.archive_extract(self.ASSET_IMAGE)
-        archive_extract(file_path, self.workdir)
         self.vm.set_console()
-        self.vm.add_args('-kernel', self.workdir + '/hippo/hippo.linux',
+        self.vm.add_args('-kernel', self.scratch_file('hippo', 'hippo.linux'),
-                         '-dtb', self.workdir + '/hippo/virtex440-ml507.dtb',
+                         '-dtb', self.scratch_file('hippo',
+                                                   'virtex440-ml507.dtb'),
                          '-m', '512')
         self.vm.launch()
         wait_for_console_pattern(self, 'QEMU advent calendar 2020',

tests/functional/test_rx_gdbsim.py

@@ -10,13 +10,9 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later.  See the COPYING file in the top-level directory.
 
-import os
-
-from unittest import skipUnless
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import exec_command_and_wait_for_pattern
-from qemu_test import wait_for_console_pattern
+from qemu_test import wait_for_console_pattern, skipFlakyTest
-from qemu_test.utils import gzip_uncompress
 
 
 class RxGdbSimMachine(QemuSystemTest):
@@ -40,9 +36,7 @@ class RxGdbSimMachine(QemuSystemTest):
         """
         self.set_machine('gdbsim-r5f562n8')
 
-        uboot_path_gz = self.ASSET_UBOOT.fetch()
+        uboot_path = self.uncompress(self.ASSET_UBOOT)
-        uboot_path = os.path.join(self.workdir, 'u-boot.bin')
-        gzip_uncompress(uboot_path_gz, uboot_path)
 
         self.vm.set_console()
         self.vm.add_args('-bios', uboot_path,
@@ -52,9 +46,10 @@ class RxGdbSimMachine(QemuSystemTest):
         wait_for_console_pattern(self, uboot_version)
         gcc_version = 'rx-unknown-linux-gcc (GCC) 9.0.0 20181105 (experimental)'
         # FIXME limit baudrate on chardev, else we type too fast
+        #  https://gitlab.com/qemu-project/qemu/-/issues/2691
         #exec_command_and_wait_for_pattern(self, 'version', gcc_version)
 
-    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable on GitLab')
+    @skipFlakyTest(bug_url="https://gitlab.com/qemu-project/qemu/-/issues/2691")
     def test_linux_sash(self):
         """
         Boots a Linux kernel and checks that the console is operational.

tests/functional/test_s390x_ccw_virtio.py

@@ -17,7 +17,7 @@ import tempfile
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import exec_command_and_wait_for_pattern
 from qemu_test import wait_for_console_pattern
-from qemu_test.utils import lzma_uncompress
+
 
 class S390CCWVirtioMachine(QemuSystemTest):
     KERNEL_COMMON_COMMAND_LINE = 'printk.time=0 '
@@ -174,9 +174,7 @@ class S390CCWVirtioMachine(QemuSystemTest):
 
         kernel_path = self.ASSET_F31_KERNEL.fetch()
 
-        initrd_path_xz = self.ASSET_F31_INITRD.fetch()
+        initrd_path = self.uncompress(self.ASSET_F31_INITRD, format="xz")
-        initrd_path = os.path.join(self.workdir, 'initrd-raw.img')
-        lzma_uncompress(initrd_path_xz, initrd_path)
 
         self.vm.set_console()
         kernel_command_line = (self.KERNEL_COMMON_COMMAND_LINE + ' audit=0 '

tests/functional/test_s390x_topology.py

@@ -10,13 +10,10 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later.  See the COPYING file in the top-level directory.
 
-import os
-
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import exec_command
 from qemu_test import exec_command_and_wait_for_pattern
 from qemu_test import wait_for_console_pattern
-from qemu_test.utils import lzma_uncompress
 
 
 class S390CPUTopology(QemuSystemTest):
@@ -88,9 +85,7 @@ class S390CPUTopology(QemuSystemTest):
         """
         self.require_accelerator("kvm")
         kernel_path = self.ASSET_F35_KERNEL.fetch()
-        initrd_path_xz = self.ASSET_F35_INITRD.fetch()
+        initrd_path = self.uncompress(self.ASSET_F35_INITRD, format="xz")
-        initrd_path = os.path.join(self.workdir, 'initrd-raw.img')
-        lzma_uncompress(initrd_path_xz, initrd_path)
 
         self.vm.set_console()
         kernel_command_line = self.KERNEL_COMMON_COMMAND_LINE

tests/functional/test_sh4_r2d.py

@@ -4,11 +4,8 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import os
+from qemu_test import LinuxKernelTest, Asset, skipFlakyTest
 
-from qemu_test import LinuxKernelTest, Asset
-from qemu_test.utils import archive_extract
-from unittest import skipUnless
 
 class R2dTest(LinuxKernelTest):
 
@@ -18,13 +15,14 @@ class R2dTest(LinuxKernelTest):
 
     # This test has a 6-10% failure rate on various hosts that look
     # like issues with a buggy kernel.
-    @skipUnless(os.getenv('QEMU_TEST_FLAKY_TESTS'), 'Test is unstable')
+    # XXX file tracking bug
+    @skipFlakyTest(bug_url=None)
     def test_r2d(self):
         self.set_machine('r2d')
-        file_path = self.ASSET_DAY09.fetch()
+        self.archive_extract(self.ASSET_DAY09)
-        archive_extract(file_path, self.workdir)
         self.vm.add_args('-append', 'console=ttySC1')
-        self.launch_kernel(self.workdir + '/day09/zImage', console_index=1,
+        self.launch_kernel(self.scratch_file('day09', 'zImage'),
+                           console_index=1,
                            wait_for='QEMU advent calendar')
 
 if __name__ == '__main__':

tests/functional/test_sh4eb_r2d.py

@@ -4,12 +4,9 @@
 #
 # SPDX-License-Identifier: GPL-2.0-or-later
 
-import os
-import shutil
-
 from qemu_test import LinuxKernelTest, Asset
 from qemu_test import exec_command_and_wait_for_pattern
-from qemu_test.utils import archive_extract
+
 
 class R2dEBTest(LinuxKernelTest):
 
@@ -19,14 +16,13 @@ class R2dEBTest(LinuxKernelTest):
 
     def test_sh4eb_r2d(self):
         self.set_machine('r2d')
-        file_path = self.ASSET_TGZ.fetch()
+        self.archive_extract(self.ASSET_TGZ)
-        archive_extract(file_path, self.workdir)
         self.vm.add_args('-append', 'console=ttySC1 noiotrap')
-        self.launch_kernel(os.path.join(self.workdir, 'sh4eb/linux-kernel'),
+        self.launch_kernel(self.scratch_file('sh4eb', 'linux-kernel'),
-                           initrd=os.path.join(self.workdir, 'sh4eb/initramfs.cpio.gz'),
+                           initrd=self.scratch_file('sh4eb',
+                                                    'initramfs.cpio.gz'),
                            console_index=1, wait_for='Type exit when done')
         exec_command_and_wait_for_pattern(self, 'exit', 'Restarting system')
-        shutil.rmtree(os.path.join(self.workdir, 'sh4eb'))
 
 if __name__ == '__main__':
     LinuxKernelTest.main()

tests/functional/test_sparc64_sun4u.py

@@ -10,11 +10,9 @@
 # This work is licensed under the terms of the GNU GPL, version 2 or
 # later. See the COPYING file in the top-level directory.
 
-import os
-
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import wait_for_console_pattern
-from qemu_test.utils import archive_extract
+
 
 class Sun4uMachine(QemuSystemTest):
     """Boots the Linux kernel and checks that the console is operational"""
@@ -28,11 +26,10 @@ class Sun4uMachine(QemuSystemTest):
 
     def test_sparc64_sun4u(self):
         self.set_machine('sun4u')
-        file_path = self.ASSET_IMAGE.fetch()
+        kernel_file = self.archive_extract(self.ASSET_IMAGE,
-        kernel_name = 'day23/vmlinux'
+                                           member='day23/vmlinux')
-        archive_extract(file_path, self.workdir, kernel_name)
         self.vm.set_console()
-        self.vm.add_args('-kernel', os.path.join(self.workdir, kernel_name),
+        self.vm.add_args('-kernel', kernel_file,
                          '-append', 'printk.time=0')
         self.vm.launch()
         wait_for_console_pattern(self, 'Starting logging: OK')

tests/functional/test_sparc_sun4m.py

@@ -6,7 +6,7 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 from qemu_test import LinuxKernelTest, Asset
-from qemu_test.utils import archive_extract
+
 
 class Sun4mTest(LinuxKernelTest):
 
@@ -16,9 +16,8 @@ class Sun4mTest(LinuxKernelTest):
 
     def test_sparc_ss20(self):
         self.set_machine('SS-20')
-        file_path = self.ASSET_DAY11.fetch()
+        self.archive_extract(self.ASSET_DAY11)
-        archive_extract(file_path, self.workdir)
+        self.launch_kernel(self.scratch_file('day11', 'zImage.elf'),
-        self.launch_kernel(self.workdir + '/day11/zImage.elf',
                            wait_for='QEMU advent calendar')
 
 if __name__ == '__main__':

tests/functional/test_virtio_gpu.py

@@ -6,25 +6,19 @@
 # later.  See the COPYING file in the top-level directory.
 
 
-from qemu_test import BUILD_DIR
 from qemu_test import QemuSystemTest, Asset
 from qemu_test import wait_for_console_pattern
 from qemu_test import exec_command_and_wait_for_pattern
 from qemu_test import is_readable_executable_file
 
-from qemu.utils import kvm_available
 
 import os
 import socket
 import subprocess
 
 
-def pick_default_vug_bin():
+def pick_default_vug_bin(test):
-    relative_path = "./contrib/vhost-user-gpu/vhost-user-gpu"
+    bld_dir_path = test.build_file("contrib", "vhost-user-gpu", "vhost-user-gpu")
-    if is_readable_executable_file(relative_path):
-        return relative_path
-
-    bld_dir_path = os.path.join(BUILD_DIR, relative_path)
     if is_readable_executable_file(bld_dir_path):
         return bld_dir_path
 
@@ -87,7 +81,7 @@ class VirtioGPUx86(QemuSystemTest):
         # FIXME: should check presence of vhost-user-gpu, virgl, memfd etc
         self.require_accelerator('kvm')
 
-        vug = pick_default_vug_bin()
+        vug = pick_default_vug_bin(self)
         if not vug:
             self.skipTest("Could not find vhost-user-gpu")
 
@@ -101,9 +95,7 @@ class VirtioGPUx86(QemuSystemTest):
         os.set_inheritable(qemu_sock.fileno(), True)
         os.set_inheritable(vug_sock.fileno(), True)
 
-        self._vug_log_path = os.path.join(
+        self._vug_log_path = self.log_file("vhost-user-gpu.log")
-            self.logdir, "vhost-user-gpu.log"
-        )
         self._vug_log_file = open(self._vug_log_path, "wb")
         self.log.info('Complete vhost-user-gpu.log file can be '
                       'found at %s', self._vug_log_path)

tests/functional/test_x86_64_hotplug_cpu.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+#
+# Functional test that hotplugs a CPU and checks it on a Linux guest
+#
+# Copyright (c) 2021 Red Hat, Inc.
+#
+# Author:
+#  Cleber Rosa <crosa@redhat.com>
+#
+# This work is licensed under the terms of the GNU GPL, version 2 or
+# later.  See the COPYING file in the top-level directory.
+
+from qemu_test import LinuxKernelTest, Asset, exec_command_and_wait_for_pattern
+
+
+class HotPlugCPU(LinuxKernelTest):
+
+    ASSET_KERNEL = Asset(
+        ('https://archives.fedoraproject.org/pub/archive/fedora/linux/releases'
+         '/31/Server/x86_64/os/images/pxeboot/vmlinuz'),
+        'd4738d03dbbe083ca610d0821d0a8f1488bebbdccef54ce33e3adb35fda00129')
+
+    ASSET_INITRD = Asset(
+        ('https://archives.fedoraproject.org/pub/archive/fedora/linux/releases'
+         '/31/Server/x86_64/os/images/pxeboot/initrd.img'),
+        '277cd6c7adf77c7e63d73bbb2cded8ef9e2d3a2f100000e92ff1f8396513cd8b')
+
+    def test_hotplug(self):
+
+        self.require_accelerator('kvm')
+        self.vm.add_args('-accel', 'kvm')
+        self.vm.add_args('-cpu', 'Haswell')
+        self.vm.add_args('-smp', '1,sockets=1,cores=2,threads=1,maxcpus=2')
+        self.vm.add_args('-m', '1G')
+        self.vm.add_args('-append', 'console=ttyS0 rd.rescue')
+
+        self.launch_kernel(self.ASSET_KERNEL.fetch(),
+                           self.ASSET_INITRD.fetch(),
+                           wait_for='Entering emergency mode.')
+        prompt = '# '
+        self.wait_for_console_pattern(prompt)
+
+        exec_command_and_wait_for_pattern(self,
+                                          'cd /sys/devices/system/cpu/cpu0',
+                                          'cpu0#')
+        exec_command_and_wait_for_pattern(self,
+                                          'cd /sys/devices/system/cpu/cpu1',
+                                          'No such file or directory')
+
+        self.vm.cmd('device_add',
+                    driver='Haswell-x86_64-cpu',
+                    id='c1',
+                    socket_id=0,
+                    core_id=1,
+                    thread_id=0)
+        self.wait_for_console_pattern('CPU1 has been hot-added')
+
+        exec_command_and_wait_for_pattern(self,
+                                          'cd /sys/devices/system/cpu/cpu1',
+                                          'cpu1#')
+
+        self.vm.cmd('device_del', id='c1')
+
+        exec_command_and_wait_for_pattern(self,
+                                          'cd /sys/devices/system/cpu/cpu1',
+                                          'No such file or directory')
+
+if __name__ == '__main__':
+    LinuxKernelTest.main()

tests/functional/test_xtensa_lx60.py

@@ -6,7 +6,7 @@
 # SPDX-License-Identifier: GPL-2.0-or-later
 
 from qemu_test import LinuxKernelTest, Asset
-from qemu_test.utils import archive_extract
+
 
 class XTensaLX60Test(LinuxKernelTest):
 
@@ -17,9 +17,9 @@ class XTensaLX60Test(LinuxKernelTest):
     def test_xtensa_lx60(self):
         self.set_machine('lx60')
         self.cpu = 'dc233c'
-        file_path = self.ASSET_DAY02.fetch()
+        self.archive_extract(self.ASSET_DAY02)
-        archive_extract(file_path, self.workdir)
+        self.launch_kernel(self.scratch_file('day02',
-        self.launch_kernel(self.workdir + '/day02/santas-sleigh-ride.elf',
+                                             'santas-sleigh-ride.elf'),
                            wait_for='QEMU advent calendar')
 
 if __name__ == '__main__':