From 9864b7f8ac5f06d7ce610997e01c187c3d01967c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:21 +0100 Subject: [PATCH 01/30] gitlab: reduce targets in cross_user_build_job MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We already limit the scope of the cross system build to reduce the cross build times. With the recent addition of more targets we are also running into timeout issues for some of the cross user builds. I've selected a few of those linux-user targets which are less likely to be in common use as distros don't have pre-built rootfs for them. I've also added the same CROSS_SKIP_TARGETS variable as is occasionally used to further limit cross system builds. Signed-off-by: Alex Bennée Reviewed-by: Richard Henderson Message-Id: <20220914155950.804707-2-alex.bennee@linaro.org> --- .gitlab-ci.d/crossbuild-template.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.d/crossbuild-template.yml b/.gitlab-ci.d/crossbuild-template.yml index 28b2142ec2..5e8892fd49 100644 --- a/.gitlab-ci.d/crossbuild-template.yml +++ b/.gitlab-ci.d/crossbuild-template.yml @@ -46,5 +46,8 @@ - cd build - PKG_CONFIG_PATH=$PKG_CONFIG_PATH ../configure --enable-werror --disable-docs $QEMU_CONFIGURE_OPTS - --disable-system + --disable-system --target-list-exclude="aarch64_be-linux-user + alpha-linux-user cris-linux-user m68k-linux-user microblazeel-linux-user + nios2-linux-user or1k-linux-user ppc-linux-user sparc-linux-user + xtensa-linux-user $CROSS_SKIP_TARGETS" - make -j$(expr $(nproc) + 1) all check-build $MAKE_CHECK_ARGS From 7fcc3f759536b9a6e07d979e73b81ccdc4751c01 Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Wed, 14 Sep 2022 16:59:22 +0100 Subject: [PATCH 02/30] tests/avocado/boot_linux_console: Fix the test_aarch64_xlnx_versal_virt test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The assets that this test tries to download have been removed from the server. Update to a newer version to get it working again. Signed-off-by: Thomas Huth Reviewed-by: Alistair Francis Message-Id: <20220829080940.110831-1-thuth@redhat.com> Signed-off-by: Alex Bennée Message-Id: <20220914155950.804707-3-alex.bennee@linaro.org> --- tests/avocado/boot_linux_console.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/avocado/boot_linux_console.py b/tests/avocado/boot_linux_console.py index 6b1533c17c..f26e036ab5 100644 --- a/tests/avocado/boot_linux_console.py +++ b/tests/avocado/boot_linux_console.py @@ -335,13 +335,13 @@ class BootLinuxConsole(LinuxKernelTest): """ images_url = ('http://ports.ubuntu.com/ubuntu-ports/dists/' 'bionic-updates/main/installer-arm64/' - '20101020ubuntu543.15/images/') + '20101020ubuntu543.19/images/') kernel_url = images_url + 'netboot/ubuntu-installer/arm64/linux' - kernel_hash = '5bfc54cf7ed8157d93f6e5b0241e727b6dc22c50' + kernel_hash = 'e167757620640eb26de0972f578741924abb3a82' kernel_path = self.fetch_asset(kernel_url, asset_hash=kernel_hash) initrd_url = images_url + 'netboot/ubuntu-installer/arm64/initrd.gz' - initrd_hash = 'd385d3e88d53e2004c5d43cbe668b458a094f772' + initrd_hash = 'cab5cb3fcefca8408aa5aae57f24574bfce8bdb9' initrd_path = self.fetch_asset(initrd_url, asset_hash=initrd_hash) self.vm.set_console() From c7f059ac112d5efe939a9d131442a8b019677a50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:23 +0100 Subject: [PATCH 03/30] tests/avocado: add explicit timeout for Aarch64 TCG tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We don't want to rely on the soon to be reduced default time. These tests are still slow for something we want to run in CI though. Signed-off-by: Alex Bennée Reviewed-by: Richard Henderson Message-Id: <20220914155950.804707-4-alex.bennee@linaro.org> --- tests/avocado/boot_linux.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/avocado/boot_linux.py b/tests/avocado/boot_linux.py index ee584d2fdf..67a24fe51c 100644 --- a/tests/avocado/boot_linux.py +++ b/tests/avocado/boot_linux.py @@ -63,6 +63,7 @@ class BootLinuxAarch64(LinuxTest): :avocado: tags=machine:virt :avocado: tags=machine:gic-version=2 """ + timeout = 240 def add_common_args(self): self.vm.add_args('-bios', From 06a4c9cf2447578bea2e464daaf17c80c40f212b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:24 +0100 Subject: [PATCH 04/30] tests/avocado: add explicit timeout for s390 TCG tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We don't want to rely on the soon to be reduced default time. These tests are still slow for something we want to run in CI though. Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20220914155950.804707-5-alex.bennee@linaro.org> --- tests/avocado/boot_linux.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/avocado/boot_linux.py b/tests/avocado/boot_linux.py index 67a24fe51c..4f07c27ac6 100644 --- a/tests/avocado/boot_linux.py +++ b/tests/avocado/boot_linux.py @@ -130,6 +130,8 @@ class BootLinuxS390X(LinuxTest): :avocado: tags=arch:s390x """ + timeout = 240 + @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_s390_ccw_virtio_tcg(self): """ From b247dba067bf2808de6395ff09ff0cb220ed7c95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:25 +0100 Subject: [PATCH 05/30] tests/avocado: add explicit timeout for ppc64le TCG tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We don't want to rely on the soon to be reduced default time. These tests are still slow for something we want to run in CI though. Signed-off-by: Alex Bennée Reviewed-by: Richard Henderson Message-Id: <20220914155950.804707-6-alex.bennee@linaro.org> --- tests/avocado/boot_linux.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/avocado/boot_linux.py b/tests/avocado/boot_linux.py index 4f07c27ac6..b7522ad3a1 100644 --- a/tests/avocado/boot_linux.py +++ b/tests/avocado/boot_linux.py @@ -115,6 +115,8 @@ class BootLinuxPPC64(LinuxTest): :avocado: tags=arch:ppc64 """ + timeout = 180 + def test_pseries_tcg(self): """ :avocado: tags=machine:pseries From 92f8e8e2fe3adcbbfaa6ee93591e7e7a3a896b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:26 +0100 Subject: [PATCH 06/30] tests/avocado: split the AST2x00Machine classes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SDK tests take a lot longer to run and hence need a longer timeout. As they run well over the 60 second maximum for CI lets also disable them for CI as well. I suspect they also suffer from the inability to detect the login prompt due to no newlines being processed. Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Reviewed-by: Richard Henderson Message-Id: <20220914155950.804707-7-alex.bennee@linaro.org> --- tests/avocado/machine_aspeed.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/avocado/machine_aspeed.py b/tests/avocado/machine_aspeed.py index 65d38f4efa..0f64eb636c 100644 --- a/tests/avocado/machine_aspeed.py +++ b/tests/avocado/machine_aspeed.py @@ -6,12 +6,14 @@ # later. See the COPYING file in the top-level directory. import time +import os from avocado_qemu import QemuSystemTest from avocado_qemu import wait_for_console_pattern from avocado_qemu import exec_command from avocado_qemu import exec_command_and_wait_for_pattern from avocado.utils import archive +from avocado import skipIf class AST1030Machine(QemuSystemTest): @@ -176,6 +178,20 @@ class AST2x00Machine(QemuSystemTest): self.do_test_arm_aspeed_buidroot_poweroff() +class AST2x00MachineSDK(QemuSystemTest): + + # FIXME: Although these tests boot a whole distro they are still + # slower than comparable machine models. There may be some + # optimisations which bring down the runtime. In the meantime they + # have generous timeouts and are disable for CI which aims for all + # tests to run in less than 60 seconds. + timeout = 240 + + def wait_for_console_pattern(self, success_message, vm=None): + wait_for_console_pattern(self, success_message, + failure_message='Kernel panic - not syncing', + vm=vm) + def do_test_arm_aspeed_sdk_start(self, image, cpu_id): self.vm.set_console() self.vm.add_args('-drive', 'file=' + image + ',if=mtd,format=raw', @@ -187,6 +203,7 @@ class AST2x00Machine(QemuSystemTest): self.wait_for_console_pattern('Starting kernel ...') self.wait_for_console_pattern('Booting Linux on physical CPU ' + cpu_id) + @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_arm_ast2500_evb_sdk(self): """ :avocado: tags=arch:arm @@ -204,6 +221,7 @@ class AST2x00Machine(QemuSystemTest): self.workdir + '/ast2500-default/image-bmc', '0x0') self.wait_for_console_pattern('ast2500-default login:') + @skipIf(os.getenv('GITLAB_CI'), 'Running on GitLab') def test_arm_ast2600_evb_sdk(self): """ :avocado: tags=arch:arm From 8f58f0c7d9389083695237d8f477771df9c51e48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:27 +0100 Subject: [PATCH 07/30] tests/avocado: reduce the default timeout to 120s MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We should be aiming to keep our tests under 2 minutes so lets reduce the default timeout to that. Tests that we know take longer should explicitly set a longer timeout. Signed-off-by: Alex Bennée Reviewed-by: Richard Henderson Message-Id: <20220914155950.804707-8-alex.bennee@linaro.org> --- tests/avocado/avocado_qemu/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/avocado/avocado_qemu/__init__.py b/tests/avocado/avocado_qemu/__init__.py index 5f1d3e551c..910f3ba1ea 100644 --- a/tests/avocado/avocado_qemu/__init__.py +++ b/tests/avocado/avocado_qemu/__init__.py @@ -229,7 +229,7 @@ def exec_command_and_wait_for_pattern(test, command, class QemuBaseTest(avocado.Test): # default timeout for all tests, can be overridden - timeout = 900 + timeout = 120 def _get_unique_tag_val(self, tag_name): """ From d9df358f53494d121f569abe2e5a0888731315c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:28 +0100 Subject: [PATCH 08/30] tests/docker: update and flatten debian-alpha-cross MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update to the latest stable Debian. While we are at it flatten into a single dockerfile. We really don't need the rest of the stuff from the QEMU base image just to compile test images. Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20220914155950.804707-9-alex.bennee@linaro.org> --- .gitlab-ci.d/container-cross.yml | 1 - tests/docker/Makefile.include | 1 - tests/docker/dockerfiles/debian-alpha-cross.docker | 12 +++++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml index 505b267542..802e332205 100644 --- a/.gitlab-ci.d/container-cross.yml +++ b/.gitlab-ci.d/container-cross.yml @@ -1,7 +1,6 @@ alpha-debian-cross-container: extends: .container_job_template stage: containers - needs: ['amd64-debian10-container'] variables: NAME: debian-alpha-cross diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 9a45e8890b..c565aa5e7b 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -88,7 +88,6 @@ DOCKER_PARTIAL_IMAGES += debian-s390x-cross DOCKER_PARTIAL_IMAGES += fedora endif -docker-image-debian-alpha-cross: docker-image-debian10 docker-image-debian-hppa-cross: docker-image-debian10 docker-image-debian-m68k-cross: docker-image-debian10 docker-image-debian-mips-cross: docker-image-debian10 diff --git a/tests/docker/dockerfiles/debian-alpha-cross.docker b/tests/docker/dockerfiles/debian-alpha-cross.docker index 10fe30df0d..4eeb43c78a 100644 --- a/tests/docker/dockerfiles/debian-alpha-cross.docker +++ b/tests/docker/dockerfiles/debian-alpha-cross.docker @@ -1,12 +1,14 @@ # # Docker cross-compiler target # -# This docker target builds on the debian Buster base image. +# This docker target builds on the Debian Bullseye base image. # -FROM qemu/debian10 +FROM docker.io/library/debian:11-slim -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive eatmydata \ - apt install -y --no-install-recommends \ +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y eatmydata && \ + eatmydata apt-get dist-upgrade -y && \ + eatmydata apt-get install --no-install-recommends -y \ gcc-alpha-linux-gnu \ libc6.1-dev-alpha-cross From 8b034187607f5b9c153b4b42ed45ee0b9b4c9881 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:29 +0100 Subject: [PATCH 09/30] tests/docker: update and flatten debian-hppa-cross MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update to the latest stable Debian. While we are at it flatten into a single dockerfile. We really don't need the rest of the stuff from the QEMU base image just to compile test images. Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20220914155950.804707-10-alex.bennee@linaro.org> --- .gitlab-ci.d/container-cross.yml | 1 - tests/docker/Makefile.include | 1 - tests/docker/dockerfiles/debian-hppa-cross.docker | 12 +++++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml index 802e332205..6c1d765463 100644 --- a/.gitlab-ci.d/container-cross.yml +++ b/.gitlab-ci.d/container-cross.yml @@ -65,7 +65,6 @@ hexagon-cross-container: hppa-debian-cross-container: extends: .container_job_template stage: containers - needs: ['amd64-debian10-container'] variables: NAME: debian-hppa-cross diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index c565aa5e7b..e39597d35c 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -88,7 +88,6 @@ DOCKER_PARTIAL_IMAGES += debian-s390x-cross DOCKER_PARTIAL_IMAGES += fedora endif -docker-image-debian-hppa-cross: docker-image-debian10 docker-image-debian-m68k-cross: docker-image-debian10 docker-image-debian-mips-cross: docker-image-debian10 docker-image-debian-mips64-cross: docker-image-debian10 diff --git a/tests/docker/dockerfiles/debian-hppa-cross.docker b/tests/docker/dockerfiles/debian-hppa-cross.docker index 3d6c65a3ef..af1c8403d8 100644 --- a/tests/docker/dockerfiles/debian-hppa-cross.docker +++ b/tests/docker/dockerfiles/debian-hppa-cross.docker @@ -1,12 +1,14 @@ # # Docker cross-compiler target # -# This docker target builds on the debian Buster base image. +# This docker target builds on the Debian Bullseye base image. # -FROM qemu/debian10 +FROM docker.io/library/debian:11-slim -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive eatmydata \ - apt install -y --no-install-recommends \ +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y eatmydata && \ + eatmydata apt-get dist-upgrade -y && \ + eatmydata apt-get install --no-install-recommends -y \ gcc-hppa-linux-gnu \ libc6-dev-hppa-cross From 77855002369b80d7d31cb85952c2149f7f3f2624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:30 +0100 Subject: [PATCH 10/30] tests/docker: update and flatten debian-m68k-cross MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update to the latest stable Debian. While we are at it flatten into a single dockerfile. We really don't need the rest of the stuff from the QEMU base image just to compile test images. Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20220914155950.804707-11-alex.bennee@linaro.org> --- .gitlab-ci.d/container-cross.yml | 1 - tests/docker/Makefile.include | 1 - tests/docker/dockerfiles/debian-m68k-cross.docker | 12 +++++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml index 6c1d765463..15a5270f6d 100644 --- a/.gitlab-ci.d/container-cross.yml +++ b/.gitlab-ci.d/container-cross.yml @@ -71,7 +71,6 @@ hppa-debian-cross-container: m68k-debian-cross-container: extends: .container_job_template stage: containers - needs: ['amd64-debian10-container'] variables: NAME: debian-m68k-cross diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index e39597d35c..95790e974e 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -88,7 +88,6 @@ DOCKER_PARTIAL_IMAGES += debian-s390x-cross DOCKER_PARTIAL_IMAGES += fedora endif -docker-image-debian-m68k-cross: docker-image-debian10 docker-image-debian-mips-cross: docker-image-debian10 docker-image-debian-mips64-cross: docker-image-debian10 docker-image-debian-sh4-cross: docker-image-debian10 diff --git a/tests/docker/dockerfiles/debian-m68k-cross.docker b/tests/docker/dockerfiles/debian-m68k-cross.docker index fcb10e3534..dded71c5d2 100644 --- a/tests/docker/dockerfiles/debian-m68k-cross.docker +++ b/tests/docker/dockerfiles/debian-m68k-cross.docker @@ -1,12 +1,14 @@ # # Docker cross-compiler target # -# This docker target builds on the debian Buster base image. +# This docker target builds on the Debian Bullseye base image. # -FROM qemu/debian10 +FROM docker.io/library/debian:11-slim -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive eatmydata \ - apt install -y --no-install-recommends \ +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y eatmydata && \ + eatmydata apt-get dist-upgrade -y && \ + eatmydata apt-get install --no-install-recommends -y \ gcc-m68k-linux-gnu \ libc6-dev-m68k-cross From 2d2a154be711162478fccf4c5450080be41f4543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:31 +0100 Subject: [PATCH 11/30] tests/docker: update and flatten debian-mips64-cross MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update to the latest stable Debian. While we are at it flatten into a single dockerfile. We really don't need the rest of the stuff from the QEMU base image just to compile test images. Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20220914155950.804707-12-alex.bennee@linaro.org> --- .gitlab-ci.d/container-cross.yml | 1 - tests/docker/Makefile.include | 1 - tests/docker/dockerfiles/debian-mips64-cross.docker | 12 +++++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml index 15a5270f6d..a3bfa483bf 100644 --- a/.gitlab-ci.d/container-cross.yml +++ b/.gitlab-ci.d/container-cross.yml @@ -77,7 +77,6 @@ m68k-debian-cross-container: mips64-debian-cross-container: extends: .container_job_template stage: containers - needs: ['amd64-debian10-container'] variables: NAME: debian-mips64-cross diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 95790e974e..6c2ee3b175 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -89,7 +89,6 @@ DOCKER_PARTIAL_IMAGES += fedora endif docker-image-debian-mips-cross: docker-image-debian10 -docker-image-debian-mips64-cross: docker-image-debian10 docker-image-debian-sh4-cross: docker-image-debian10 docker-image-debian-sparc64-cross: docker-image-debian10 diff --git a/tests/docker/dockerfiles/debian-mips64-cross.docker b/tests/docker/dockerfiles/debian-mips64-cross.docker index 09c2ba584e..afcff9726f 100644 --- a/tests/docker/dockerfiles/debian-mips64-cross.docker +++ b/tests/docker/dockerfiles/debian-mips64-cross.docker @@ -1,12 +1,14 @@ # # Docker cross-compiler target # -# This docker target builds on the debian Buster base image. +# This docker target builds on the Debian Bullseye base image. # -FROM qemu/debian10 +FROM docker.io/library/debian:11-slim -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive eatmydata \ - apt install -y --no-install-recommends \ +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y eatmydata && \ + eatmydata apt-get dist-upgrade -y && \ + eatmydata apt-get install --no-install-recommends -y \ gcc-mips64-linux-gnuabi64 \ libc6-dev-mips64-cross From 376c4109af99463aab8f33fb68a298d72da2b59b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:32 +0100 Subject: [PATCH 12/30] tests/docker: update and flatten debian-sh4-cross MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update to the latest stable Debian. While we are at it flatten into a single dockerfile. We really don't need the rest of the stuff from the QEMU base image just to compile test images. Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20220914155950.804707-13-alex.bennee@linaro.org> --- .gitlab-ci.d/container-cross.yml | 1 - tests/docker/Makefile.include | 1 - tests/docker/dockerfiles/debian-sh4-cross.docker | 12 +++++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml index a3bfa483bf..8a611fc824 100644 --- a/.gitlab-ci.d/container-cross.yml +++ b/.gitlab-ci.d/container-cross.yml @@ -137,7 +137,6 @@ s390x-debian-cross-container: sh4-debian-cross-container: extends: .container_job_template stage: containers - needs: ['amd64-debian10-container'] variables: NAME: debian-sh4-cross diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 6c2ee3b175..37c4ea913f 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -89,7 +89,6 @@ DOCKER_PARTIAL_IMAGES += fedora endif docker-image-debian-mips-cross: docker-image-debian10 -docker-image-debian-sh4-cross: docker-image-debian10 docker-image-debian-sparc64-cross: docker-image-debian10 # The native build should never use the registry diff --git a/tests/docker/dockerfiles/debian-sh4-cross.docker b/tests/docker/dockerfiles/debian-sh4-cross.docker index fd3af89575..d48ed9065f 100644 --- a/tests/docker/dockerfiles/debian-sh4-cross.docker +++ b/tests/docker/dockerfiles/debian-sh4-cross.docker @@ -1,12 +1,14 @@ # # Docker cross-compiler target # -# This docker target builds on the debian Buster base image. +# This docker target builds on the Debian Bullseye base image. # -FROM qemu/debian10 +FROM docker.io/library/debian:11-slim -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive eatmydata \ - apt install -y --no-install-recommends \ +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y eatmydata && \ + eatmydata apt-get dist-upgrade -y && \ + eatmydata apt-get install --no-install-recommends -y \ gcc-sh4-linux-gnu \ libc6-dev-sh4-cross From 6ede0767bac84d0b722c3fcc37ec102d5418f09b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:33 +0100 Subject: [PATCH 13/30] tests/docker: update and flatten debian-sparc64-cross MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update to the latest stable Debian. While we are at it flatten into a single dockerfile. We really don't need the rest of the stuff from the QEMU base image just to compile test images. Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20220914155950.804707-14-alex.bennee@linaro.org> --- .gitlab-ci.d/container-cross.yml | 1 - tests/docker/Makefile.include | 1 - tests/docker/dockerfiles/debian-sparc64-cross.docker | 12 +++++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml index 8a611fc824..db0ea15d0d 100644 --- a/.gitlab-ci.d/container-cross.yml +++ b/.gitlab-ci.d/container-cross.yml @@ -143,7 +143,6 @@ sh4-debian-cross-container: sparc64-debian-cross-container: extends: .container_job_template stage: containers - needs: ['amd64-debian10-container'] variables: NAME: debian-sparc64-cross diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 37c4ea913f..8828b6b8fa 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -89,7 +89,6 @@ DOCKER_PARTIAL_IMAGES += fedora endif docker-image-debian-mips-cross: docker-image-debian10 -docker-image-debian-sparc64-cross: docker-image-debian10 # The native build should never use the registry docker-image-debian-native: DOCKER_REGISTRY= diff --git a/tests/docker/dockerfiles/debian-sparc64-cross.docker b/tests/docker/dockerfiles/debian-sparc64-cross.docker index f4bb9b561c..8d3d306bc1 100644 --- a/tests/docker/dockerfiles/debian-sparc64-cross.docker +++ b/tests/docker/dockerfiles/debian-sparc64-cross.docker @@ -1,12 +1,14 @@ # # Docker cross-compiler target # -# This docker target builds on the debian Buster base image. +# This docker target builds on the Debian Bullseye base image. # -FROM qemu/debian10 +FROM docker.io/library/debian:11-slim -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive eatmydata \ - apt install -y --no-install-recommends \ +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y eatmydata && \ + eatmydata apt-get dist-upgrade -y && \ + eatmydata apt-get install --no-install-recommends -y \ gcc-sparc64-linux-gnu \ libc6-dev-sparc64-cross From 0fd8f7a2dfcb3abcccd7b20ae18c1924f5fae708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:34 +0100 Subject: [PATCH 14/30] tests/docker: flatten debian-powerpc-test-cross MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flatten into a single dockerfile. We really don't need the rest of the stuff from the QEMU base image just to compile test images. Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20220914155950.804707-15-alex.bennee@linaro.org> --- .gitlab-ci.d/container-cross.yml | 1 - tests/docker/Makefile.include | 1 - .../dockerfiles/debian-powerpc-test-cross.docker | 12 +++++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml index db0ea15d0d..67bbf19a27 100644 --- a/.gitlab-ci.d/container-cross.yml +++ b/.gitlab-ci.d/container-cross.yml @@ -102,7 +102,6 @@ mipsel-debian-cross-container: powerpc-test-cross-container: extends: .container_job_template stage: containers - needs: ['amd64-debian11-container'] variables: NAME: debian-powerpc-test-cross diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 8828b6b8fa..e034eca3af 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -137,7 +137,6 @@ docker-image-debian-all-test-cross: docker-image-debian10 docker-image-debian-loongarch-cross: docker-image-debian11 docker-image-debian-microblaze-cross: docker-image-debian10 docker-image-debian-nios2-cross: docker-image-debian10 -docker-image-debian-powerpc-test-cross: docker-image-debian11 docker-image-debian-riscv64-test-cross: docker-image-debian11 # These images may be good enough for building tests but not for test builds diff --git a/tests/docker/dockerfiles/debian-powerpc-test-cross.docker b/tests/docker/dockerfiles/debian-powerpc-test-cross.docker index 36b336f709..d6b2909cc4 100644 --- a/tests/docker/dockerfiles/debian-powerpc-test-cross.docker +++ b/tests/docker/dockerfiles/debian-powerpc-test-cross.docker @@ -1,13 +1,15 @@ # # Docker powerpc/ppc64/ppc64le cross-compiler target # -# This docker target builds on the debian Bullseye base image. +# This docker target builds on the Debian Bullseye base image. # -FROM qemu/debian11 +FROM docker.io/library/debian:11-slim -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive eatmydata \ - apt install -y --no-install-recommends \ +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y eatmydata && \ + eatmydata apt-get dist-upgrade -y && \ + eatmydata apt-get install --no-install-recommends -y \ gcc-powerpc-linux-gnu \ libc6-dev-powerpc-cross \ gcc-10-powerpc64-linux-gnu \ From 95b0af40ded2aa0b350a9d8cb4256146e3397e48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:35 +0100 Subject: [PATCH 15/30] tests/docker: remove tricore qemu/debian10 dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We missed removing this dependency when we flattened the build. Fixes: 39ce923732 (gitlab: enable a very minimal build with the tricore container) Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20220914155950.804707-16-alex.bennee@linaro.org> --- .gitlab-ci.d/container-cross.yml | 1 - tests/docker/Makefile.include | 1 - 2 files changed, 2 deletions(-) diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml index 67bbf19a27..611c6c0b39 100644 --- a/.gitlab-ci.d/container-cross.yml +++ b/.gitlab-ci.d/container-cross.yml @@ -148,7 +148,6 @@ sparc64-debian-cross-container: tricore-debian-cross-container: extends: .container_job_template stage: containers - needs: ['amd64-debian10-container'] variables: NAME: debian-tricore-cross diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index e034eca3af..5c9398bbc9 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -132,7 +132,6 @@ docker-image-debian-nios2-cross: $(DOCKER_FILES_DIR)/debian-toolchain.docker \ $(call debian-toolchain, $@) # Specialist build images, sometimes very limited tools -docker-image-debian-tricore-cross: docker-image-debian10 docker-image-debian-all-test-cross: docker-image-debian10 docker-image-debian-loongarch-cross: docker-image-debian11 docker-image-debian-microblaze-cross: docker-image-debian10 From 0068cf8bd23738c20e131cb5f89ed3947ca77bd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:36 +0100 Subject: [PATCH 16/30] tests/docker: remove amd64 qemu/debian10 dependency MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We missed removing this dependency when we flattened the build. Fixes 9e19fd7d4a (tests/docker: update debian-amd64 with lcitool) Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20220914155950.804707-17-alex.bennee@linaro.org> --- tests/docker/Makefile.include | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 5c9398bbc9..c3375f89c5 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -73,7 +73,6 @@ docker-binfmt-image-debian-%: $(DOCKER_FILES_DIR)/debian-bootstrap.docker # we don't run tests on intermediate images (used as base by another image) DOCKER_PARTIAL_IMAGES := debian10 debian11 ifeq ($(HOST_ARCH),x86_64) -docker-image-debian-amd64: docker-image-debian10 DOCKER_PARTIAL_IMAGES += debian-amd64-cross else docker-image-debian-amd64-cross: docker-image-debian10 From 3c696fdc596cdf04f8a1b3fa2cecabbb82507bb4 Mon Sep 17 00:00:00 2001 From: Thomas Huth Date: Wed, 14 Sep 2022 16:59:37 +0100 Subject: [PATCH 17/30] tests/vm: Remove obsolete Fedora VM test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's still based on Fedora 30 - which is not supported anymore by QEMU since years. Seems like nobody is using (and refreshing) this, and it's easier to test this via a container anyway, so let's remove this now. Signed-off-by: Thomas Huth Message-Id: <20220822175317.190551-1-thuth@redhat.com> Signed-off-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Message-Id: <20220914155950.804707-18-alex.bennee@linaro.org> --- tests/vm/Makefile.include | 3 +- tests/vm/fedora | 190 -------------------------------------- 2 files changed, 1 insertion(+), 192 deletions(-) delete mode 100755 tests/vm/fedora diff --git a/tests/vm/Makefile.include b/tests/vm/Makefile.include index 8d2a164552..2cc2203d09 100644 --- a/tests/vm/Makefile.include +++ b/tests/vm/Makefile.include @@ -15,7 +15,7 @@ endif EFI_AARCH64 = $(wildcard $(BUILD_DIR)/pc-bios/edk2-aarch64-code.fd) -X86_IMAGES := freebsd netbsd openbsd fedora haiku.x86_64 +X86_IMAGES := freebsd netbsd openbsd haiku.x86_64 ifneq ($(GENISOIMAGE),) X86_IMAGES += centos ifneq ($(EFI_AARCH64),) @@ -45,7 +45,6 @@ vm-help vm-test: @echo " vm-build-freebsd - Build QEMU in FreeBSD VM" @echo " vm-build-netbsd - Build QEMU in NetBSD VM" @echo " vm-build-openbsd - Build QEMU in OpenBSD VM" - @echo " vm-build-fedora - Build QEMU in Fedora VM" ifneq ($(GENISOIMAGE),) @echo " vm-build-centos - Build QEMU in CentOS VM, with Docker" ifneq ($(EFI_AARCH64),) diff --git a/tests/vm/fedora b/tests/vm/fedora deleted file mode 100755 index 12eca919a0..0000000000 --- a/tests/vm/fedora +++ /dev/null @@ -1,190 +0,0 @@ -#!/usr/bin/env python3 -# -# Fedora VM image -# -# Copyright 2019 Red Hat Inc. -# -# Authors: -# Gerd Hoffmann -# -# This code is licensed under the GPL version 2 or later. See -# the COPYING file in the top-level directory. -# - -import os -import re -import sys -import time -import socket -import subprocess -import basevm - -class FedoraVM(basevm.BaseVM): - name = "fedora" - arch = "x86_64" - - base = "https://archives.fedoraproject.org/pub/archive/fedora/linux/releases/30/" - link = base + "Server/x86_64/iso/Fedora-Server-netinst-x86_64-30-1.2.iso" - repo = base + "Server/x86_64/os/" - full = base + "Everything/x86_64/os/" - csum = "5e4eac4566d8c572bfb3bcf54b7d6c82006ec3c6c882a2c9235c6d3494d7b100" - size = "20G" - pkgs = [ - # tools - 'git-core', - 'gcc', 'binutils', 'make', 'ninja-build', - - # perl - 'perl', - - # libs: usb - '"pkgconfig(libusb-1.0)"', - '"pkgconfig(libusbredirparser-0.5)"', - - # libs: crypto - '"pkgconfig(gnutls)"', - - # libs: ui - '"pkgconfig(sdl2)"', - '"pkgconfig(gtk+-3.0)"', - '"pkgconfig(ncursesw)"', - - # libs: audio - '"pkgconfig(libpulse)"', - '"pkgconfig(alsa)"', - - # libs: migration - '"pkgconfig(libzstd)"', -] - - BUILD_SCRIPT = """ - set -e; - rm -rf /home/qemu/qemu-test.* - cd $(mktemp -d /home/qemu/qemu-test.XXXXXX); - mkdir src build; cd src; - tar -xf /dev/vdb; - cd ../build - ../src/configure --python=python3 {configure_opts}; - gmake --output-sync -j{jobs} {target} {verbose}; - """ - - def build_image(self, img): - self.print_step("Downloading install iso") - cimg = self._download_with_cache(self.link, sha256sum=self.csum) - img_tmp = img + ".tmp" - iso = img + ".install.iso" - - self.print_step("Preparing iso and disk image") - subprocess.check_call(["cp", "-f", cimg, iso]) - self.exec_qemu_img("create", "-f", "qcow2", img_tmp, self.size) - self.print_step("Booting installer") - self.boot(img_tmp, extra_args = [ - "-machine", "graphics=off", - "-device", "VGA", - "-cdrom", iso - ]) - self.console_init(300) - self.console_wait("installation process.") - time.sleep(0.3) - self.console_send("\t") - time.sleep(0.3) - self.console_send(" console=ttyS0") - proxy = os.environ.get("http_proxy") - if not proxy is None: - self.console_send(" proxy=%s" % proxy) - self.console_send(" inst.proxy=%s" % proxy) - self.console_send(" inst.repo=%s" % self.repo) - self.console_send("\n") - - self.console_wait_send("2) Use text mode", "2\n") - - self.console_wait_send("5) [!] Installation Dest", "5\n") - self.console_wait_send("1) [x]", "c\n") - self.console_wait_send("2) [ ] Use All Space", "2\n") - self.console_wait_send("2) [x] Use All Space", "c\n") - self.console_wait_send("1) [ ] Standard Part", "1\n") - self.console_wait_send("1) [x] Standard Part", "c\n") - - self.console_wait_send("7) [!] Root password", "7\n") - self.console_wait("Password:") - self.console_send("%s\n" % self._config["root_pass"]) - self.console_wait("Password (confirm):") - self.console_send("%s\n" % self._config["root_pass"]) - - self.console_wait_send("8) [ ] User creation", "8\n") - self.console_wait_send("1) [ ] Create user", "1\n") - self.console_wait_send("3) User name", "3\n") - self.console_wait_send("ENTER:", "%s\n" % self._config["guest_user"]) - self.console_wait_send("4) [ ] Use password", "4\n") - self.console_wait_send("5) Password", "5\n") - self.console_wait("Password:") - self.console_send("%s\n" % self._config["guest_pass"]) - self.console_wait("Password (confirm):") - self.console_send("%s\n" % self._config["guest_pass"]) - self.console_wait_send("7) Groups", "c\n") - - while True: - good = self.console_wait("3) [x] Installation", - "3) [!] Installation") - self.console_send("r\n") - if good: - break - time.sleep(10) - - while True: - good = self.console_wait("4) [x] Software", - "4) [!] Software") - self.console_send("r\n") - if good: - break - time.sleep(10) - self.console_send("r\n" % self._config["guest_pass"]) - - self.console_wait_send("'b' to begin install", "b\n") - - self.print_step("Installation started now, this will take a while") - - self.console_wait_send("Installation complete", "\n") - self.print_step("Installation finished, rebooting") - - # setup qemu user - prompt = " ~]$" - self.console_ssh_init(prompt, self._config["guest_user"], - self._config["guest_pass"]) - self.console_wait_send(prompt, "exit\n") - - # setup root user - prompt = " ~]#" - self.console_ssh_init(prompt, "root", self._config["root_pass"]) - self.console_sshd_config(prompt) - - # setup virtio-blk #1 (tarfile) - self.console_wait(prompt) - self.console_send("echo 'KERNEL==\"vdb\" MODE=\"666\"' >> %s\n" % - "/etc/udev/rules.d/99-qemu.rules") - - self.print_step("Configuration finished, rebooting") - self.console_wait_send(prompt, "reboot\n") - self.console_wait("login:") - self.wait_ssh() - - self.print_step("Installing packages") - self.ssh_root_check("rm -vf /etc/yum.repos.d/fedora*.repo\n") - self.ssh_root_check("echo '[fedora]' >> /etc/yum.repos.d/qemu.repo\n") - self.ssh_root_check("echo 'baseurl=%s' >> /etc/yum.repos.d/qemu.repo\n" % self.full) - self.ssh_root_check("echo 'gpgcheck=0' >> /etc/yum.repos.d/qemu.repo\n") - self.ssh_root_check("dnf install -y %s\n" % " ".join(self.pkgs)) - - # shutdown - self.ssh_root(self.poweroff) - self.console_wait("sleep state S5") - self.wait() - - if os.path.exists(img): - os.remove(img) - os.rename(img_tmp, img) - os.remove(iso) - self.print_step("All done") - -if __name__ == "__main__": - sys.exit(basevm.main(FedoraVM)) From abafb64b6dcbdb0ca0943cd04373bd84353ce7ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:38 +0100 Subject: [PATCH 18/30] configure: explicitly set cflags for --disable-pie MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is working around current limitation of Meson's handling of --disable-pie. Signed-off-by: Alex Bennée Cc: Paolo Bonzini Reviewed-by: Richard Henderson Message-Id: <20220914155950.804707-19-alex.bennee@linaro.org> --- configure | 3 +++ 1 file changed, 3 insertions(+) diff --git a/configure b/configure index 575dde1c1f..0bbf9d28af 100755 --- a/configure +++ b/configure @@ -1382,6 +1382,9 @@ elif test "$pie" = "no"; then if compile_prog "-Werror -fno-pie" "-no-pie"; then CONFIGURE_CFLAGS="-fno-pie $CONFIGURE_CFLAGS" CONFIGURE_LDFLAGS="-no-pie $CONFIGURE_LDFLAGS" + # Meson currently only handles pie as a boolean for now so if we have + # explicitly disabled PIE we need to extend our cflags because it wont. + QEMU_CFLAGS="-fno-pie -no-pie $QEMU_CFLAGS" fi elif compile_prog "-Werror -fPIE -DPIE" "-pie"; then CONFIGURE_CFLAGS="-fPIE -DPIE $CONFIGURE_CFLAGS" From 45e1b746943eefe4c0fb35e29878cd038ac62fd7 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 14 Sep 2022 16:59:39 +0100 Subject: [PATCH 19/30] gitlab-ci/custom-runners: Disable -static-pie for ubuntu-20.04-aarch64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The project has reached the magic size at which we see /usr/aarch64-linux-gnu/lib/libc.a(init-first.o): in function `__libc_init_first': (.text+0x10): relocation truncated to fit: R_AARCH64_LD64_GOTPAGE_LO15 against \ symbol `__environ' defined in .bss section in /usr/aarch64-linux-gnu/lib/libc.a(environ.o) /usr/bin/ld: (.text+0x10): warning: too many GOT entries for -fpic, please recompile with -fPIC The bug has been reported upstream, but in the meantime there is nothing we can do except build a non-pie executable. Signed-off-by: Richard Henderson Signed-off-by: Alex Bennée Message-Id: <20220823210329.1969895-1-richard.henderson@linaro.org> Reviewed-by: Thomas Huth Message-Id: <20220914155950.804707-20-alex.bennee@linaro.org> --- .gitlab-ci.d/custom-runners/ubuntu-20.04-aarch64.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.d/custom-runners/ubuntu-20.04-aarch64.yml b/.gitlab-ci.d/custom-runners/ubuntu-20.04-aarch64.yml index 3d878914e7..3f4dee4f86 100644 --- a/.gitlab-ci.d/custom-runners/ubuntu-20.04-aarch64.yml +++ b/.gitlab-ci.d/custom-runners/ubuntu-20.04-aarch64.yml @@ -16,7 +16,9 @@ ubuntu-20.04-aarch64-all-linux-static: # --disable-glusterfs is needed because there's no static version of those libs in distro supplied packages - mkdir build - cd build - - ../configure --enable-debug --static --disable-system --disable-glusterfs --disable-libssh + # Disable -static-pie due to build error with system libc: + # https://bugs.launchpad.net/ubuntu/+source/glibc/+bug/1987438 + - ../configure --enable-debug --static --disable-system --disable-glusterfs --disable-libssh --disable-pie || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check V=1 From 66dca267489e7b150546297e7c33a04430a15eb4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:40 +0100 Subject: [PATCH 20/30] gitlab-ci: update aarch32/aarch64 custom runner jobs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The custom runner is now using 22.04 so we can drop our hacks to deal with broken libssh and glusterfs. The provisioning scripts will be updated in a separate commit. Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20220914155950.804707-21-alex.bennee@linaro.org> --- .gitlab-ci.d/custom-runners.yml | 4 +-- ...4-aarch32.yml => ubuntu-22.04-aarch32.yml} | 6 ++-- ...4-aarch64.yml => ubuntu-22.04-aarch64.yml} | 36 +++++++++---------- MAINTAINERS | 3 +- 4 files changed, 24 insertions(+), 25 deletions(-) rename .gitlab-ci.d/custom-runners/{ubuntu-20.04-aarch32.yml => ubuntu-22.04-aarch32.yml} (86%) rename .gitlab-ci.d/custom-runners/{ubuntu-20.04-aarch64.yml => ubuntu-22.04-aarch64.yml} (82%) diff --git a/.gitlab-ci.d/custom-runners.yml b/.gitlab-ci.d/custom-runners.yml index 15aaccc481..97f99e29c2 100644 --- a/.gitlab-ci.d/custom-runners.yml +++ b/.gitlab-ci.d/custom-runners.yml @@ -15,6 +15,6 @@ variables: include: - local: '/.gitlab-ci.d/custom-runners/ubuntu-20.04-s390x.yml' - - local: '/.gitlab-ci.d/custom-runners/ubuntu-20.04-aarch64.yml' - - local: '/.gitlab-ci.d/custom-runners/ubuntu-20.04-aarch32.yml' + - local: '/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml' + - local: '/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml' - local: '/.gitlab-ci.d/custom-runners/centos-stream-8-x86_64.yml' diff --git a/.gitlab-ci.d/custom-runners/ubuntu-20.04-aarch32.yml b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml similarity index 86% rename from .gitlab-ci.d/custom-runners/ubuntu-20.04-aarch32.yml rename to .gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml index cbfa9cc164..1a2f9b8dbe 100644 --- a/.gitlab-ci.d/custom-runners/ubuntu-20.04-aarch32.yml +++ b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml @@ -1,12 +1,12 @@ -# All ubuntu-20.04 jobs should run successfully in an environment +# All ubuntu-22.04 jobs should run successfully in an environment # setup by the scripts/ci/setup/qemu/build-environment.yml task # "Install basic packages to build QEMU on Ubuntu 20.04" -ubuntu-20.04-aarch32-all: +ubuntu-22.04-aarch32-all: needs: [] stage: build tags: - - ubuntu_20.04 + - ubuntu_22.04 - aarch32 rules: - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/' diff --git a/.gitlab-ci.d/custom-runners/ubuntu-20.04-aarch64.yml b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml similarity index 82% rename from .gitlab-ci.d/custom-runners/ubuntu-20.04-aarch64.yml rename to .gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml index 3f4dee4f86..ce0b18af6f 100644 --- a/.gitlab-ci.d/custom-runners/ubuntu-20.04-aarch64.yml +++ b/.gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml @@ -2,23 +2,21 @@ # setup by the scripts/ci/setup/qemu/build-environment.yml task # "Install basic packages to build QEMU on Ubuntu 20.04" -ubuntu-20.04-aarch64-all-linux-static: +ubuntu-22.04-aarch64-all-linux-static: needs: [] stage: build tags: - - ubuntu_20.04 + - ubuntu_22.04 - aarch64 rules: - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/' - if: "$AARCH64_RUNNER_AVAILABLE" script: - # --disable-libssh is needed because of https://bugs.launchpad.net/qemu/+bug/1838763 - # --disable-glusterfs is needed because there's no static version of those libs in distro supplied packages - mkdir build - cd build # Disable -static-pie due to build error with system libc: # https://bugs.launchpad.net/ubuntu/+source/glibc/+bug/1987438 - - ../configure --enable-debug --static --disable-system --disable-glusterfs --disable-libssh --disable-pie + - ../configure --enable-debug --static --disable-system --disable-pie || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check V=1 @@ -26,11 +24,11 @@ ubuntu-20.04-aarch64-all-linux-static: - make --output-sync -j`nproc --ignore=40` check-tcg V=1 || { cat meson-logs/testlog.txt; exit 1; } ; -ubuntu-20.04-aarch64-all: +ubuntu-22.04-aarch64-all: needs: [] stage: build tags: - - ubuntu_20.04 + - ubuntu_22.04 - aarch64 rules: - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/' @@ -42,17 +40,17 @@ ubuntu-20.04-aarch64-all: script: - mkdir build - cd build - - ../configure --disable-libssh + - ../configure || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check V=1 || { cat meson-logs/testlog.txt; exit 1; } ; -ubuntu-20.04-aarch64-alldbg: +ubuntu-22.04-aarch64-alldbg: needs: [] stage: build tags: - - ubuntu_20.04 + - ubuntu_22.04 - aarch64 rules: - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/' @@ -60,18 +58,18 @@ ubuntu-20.04-aarch64-alldbg: script: - mkdir build - cd build - - ../configure --enable-debug --disable-libssh + - ../configure --enable-debug || { cat config.log meson-logs/meson-log.txt; exit 1; } - make clean - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check V=1 || { cat meson-logs/testlog.txt; exit 1; } ; -ubuntu-20.04-aarch64-clang: +ubuntu-22.04-aarch64-clang: needs: [] stage: build tags: - - ubuntu_20.04 + - ubuntu_22.04 - aarch64 rules: - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/' @@ -89,11 +87,11 @@ ubuntu-20.04-aarch64-clang: - make --output-sync -j`nproc --ignore=40` check V=1 || { cat meson-logs/testlog.txt; exit 1; } ; -ubuntu-20.04-aarch64-tci: +ubuntu-22.04-aarch64-tci: needs: [] stage: build tags: - - ubuntu_20.04 + - ubuntu_22.04 - aarch64 rules: - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/' @@ -105,15 +103,15 @@ ubuntu-20.04-aarch64-tci: script: - mkdir build - cd build - - ../configure --disable-libssh --enable-tcg-interpreter + - ../configure --enable-tcg-interpreter || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` -ubuntu-20.04-aarch64-notcg: +ubuntu-22.04-aarch64-notcg: needs: [] stage: build tags: - - ubuntu_20.04 + - ubuntu_22.04 - aarch64 rules: - if: '$CI_PROJECT_NAMESPACE == "qemu-project" && $CI_COMMIT_BRANCH =~ /^staging/' @@ -125,7 +123,7 @@ ubuntu-20.04-aarch64-notcg: script: - mkdir build - cd build - - ../configure --disable-libssh --disable-tcg + - ../configure --disable-tcg || { cat config.log meson-logs/meson-log.txt; exit 1; } - make --output-sync -j`nproc --ignore=40` - make --output-sync -j`nproc --ignore=40` check V=1 diff --git a/MAINTAINERS b/MAINTAINERS index 1729c0901c..738c4eb647 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -3724,7 +3724,8 @@ GitLab custom runner (Works On Arm Sponsored) M: Alex Bennée M: Philippe Mathieu-Daudé S: Maintained -F: .gitlab-ci.d/custom-runners/ubuntu-20.04-aarch64.yml +F: .gitlab-ci.d/custom-runners/ubuntu-22.04-aarch64.yml +F: .gitlab-ci.d/custom-runners/ubuntu-22.04-aarch32.yml Documentation ------------- From 54ab3c3feef6e5a6c6cf97c4532ebed642ec2705 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:41 +0100 Subject: [PATCH 21/30] Deprecate 32 bit big-endian MIPS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's becoming harder to maintain a cross-compiler to test this host architecture as the old stable Debian 10 ("Buster") moved into LTS which supports fewer architectures. For now: - mark it's deprecation in the docs - downgrade the containers to build TCG tests only - drop the cross builds from our CI Users with an appropriate toolchain and user-space can still take their chances building it. Signed-off-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Huacai Chen Reviewed-by: Richard Henderson Message-Id: <20220914155950.804707-22-alex.bennee@linaro.org> --- .gitlab-ci.d/container-cross.yml | 1 - .gitlab-ci.d/crossbuilds.yml | 14 -------- docs/about/build-platforms.rst | 2 +- docs/about/deprecated.rst | 13 +++++++ tests/docker/Makefile.include | 5 ++- .../dockerfiles/debian-mips-cross.docker | 36 +++++-------------- 6 files changed, 25 insertions(+), 46 deletions(-) diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml index 611c6c0b39..95d57e1c5d 100644 --- a/.gitlab-ci.d/container-cross.yml +++ b/.gitlab-ci.d/container-cross.yml @@ -89,7 +89,6 @@ mips64el-debian-cross-container: mips-debian-cross-container: extends: .container_job_template stage: containers - needs: ['amd64-debian10-container'] variables: NAME: debian-mips-cross diff --git a/.gitlab-ci.d/crossbuilds.yml b/.gitlab-ci.d/crossbuilds.yml index 4a5fb6ea2a..c4cd96433d 100644 --- a/.gitlab-ci.d/crossbuilds.yml +++ b/.gitlab-ci.d/crossbuilds.yml @@ -70,20 +70,6 @@ cross-i386-tci: EXTRA_CONFIGURE_OPTS: --target-list=i386-softmmu,i386-linux-user,aarch64-softmmu,aarch64-linux-user,ppc-softmmu,ppc-linux-user MAKE_CHECK_ARGS: check check-tcg -cross-mips-system: - extends: .cross_system_build_job - needs: - job: mips-debian-cross-container - variables: - IMAGE: debian-mips-cross - -cross-mips-user: - extends: .cross_user_build_job - needs: - job: mips-debian-cross-container - variables: - IMAGE: debian-mips-cross - cross-mipsel-system: extends: .cross_system_build_job needs: diff --git a/docs/about/build-platforms.rst b/docs/about/build-platforms.rst index a2fee53248..1c1e7b9e11 100644 --- a/docs/about/build-platforms.rst +++ b/docs/about/build-platforms.rst @@ -41,7 +41,7 @@ Those hosts are officially supported, with various accelerators: - Accelerators * - Arm - kvm (64 bit only), tcg, xen - * - MIPS + * - MIPS (little endian only) - kvm, tcg * - PPC - kvm, tcg diff --git a/docs/about/deprecated.rst b/docs/about/deprecated.rst index c75a25daad..0d1fd4469b 100644 --- a/docs/about/deprecated.rst +++ b/docs/about/deprecated.rst @@ -213,6 +213,19 @@ MIPS ``Trap-and-Emul`` KVM support (since 6.0) The MIPS ``Trap-and-Emul`` KVM host and guest support has been removed from Linux upstream kernel, declare it deprecated. +Host Architectures +------------------ + +BE MIPS (since 7.2) +''''''''''''''''''' + +As Debian 10 ("Buster") moved into LTS the big endian 32 bit version of +MIPS moved out of support making it hard to maintain our +cross-compilation CI tests of the architecture. As we no longer have +CI coverage support may bitrot away before the deprecation process +completes. The little endian variants of MIPS (both 32 and 64 bit) are +still a supported host architecture. + QEMU API (QAPI) events ---------------------- diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index c3375f89c5..b1bf56434f 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -81,14 +81,12 @@ endif # For non-x86 hosts not all cross-compilers have been packaged ifneq ($(HOST_ARCH),x86_64) -DOCKER_PARTIAL_IMAGES += debian-mips-cross debian-mipsel-cross debian-mips64el-cross +DOCKER_PARTIAL_IMAGES += debian-mipsel-cross debian-mips64el-cross DOCKER_PARTIAL_IMAGES += debian-ppc64el-cross DOCKER_PARTIAL_IMAGES += debian-s390x-cross DOCKER_PARTIAL_IMAGES += fedora endif -docker-image-debian-mips-cross: docker-image-debian10 - # The native build should never use the registry docker-image-debian-native: DOCKER_REGISTRY= @@ -144,6 +142,7 @@ DOCKER_PARTIAL_IMAGES += debian-hppa-cross DOCKER_PARTIAL_IMAGES += debian-loongarch-cross DOCKER_PARTIAL_IMAGES += debian-m68k-cross debian-mips64-cross DOCKER_PARTIAL_IMAGES += debian-microblaze-cross +DOCKER_PARTIAL_IMAGES += debian-mips-cross DOCKER_PARTIAL_IMAGES += debian-nios2-cross DOCKER_PARTIAL_IMAGES += debian-riscv64-test-cross DOCKER_PARTIAL_IMAGES += debian-sh4-cross debian-sparc64-cross diff --git a/tests/docker/dockerfiles/debian-mips-cross.docker b/tests/docker/dockerfiles/debian-mips-cross.docker index 26c154014d..7b55f0f3b2 100644 --- a/tests/docker/dockerfiles/debian-mips-cross.docker +++ b/tests/docker/dockerfiles/debian-mips-cross.docker @@ -1,32 +1,14 @@ # # Docker mips cross-compiler target # -# This docker target builds on the debian Buster base image. +# This docker target builds on the Debian Bullseye base image. # -FROM qemu/debian10 +FROM docker.io/library/debian:11-slim -MAINTAINER Philippe Mathieu-Daudé - -# Add the foreign architecture we want and install dependencies -RUN dpkg --add-architecture mips -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive eatmydata \ - apt install -y --no-install-recommends \ - gcc-mips-linux-gnu - -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive eatmydata \ - apt build-dep -yy -a mips --arch-only qemu - -# Specify the cross prefix for this image (see tests/docker/common.rc) -ENV QEMU_CONFIGURE_OPTS --cross-prefix=mips-linux-gnu- -ENV DEF_TARGET_LIST mips-softmmu,mipsel-linux-user - -# Install extra libraries to increase code coverage -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive eatmydata \ - apt install -y --no-install-recommends \ - libbz2-dev:mips \ - liblzo2-dev:mips \ - librdmacm-dev:mips \ - libsnappy-dev:mips +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y eatmydata && \ + eatmydata apt-get dist-upgrade -y && \ + eatmydata apt-get install --no-install-recommends -y \ + gcc-mips-linux-gnu \ + libc6-dev-mips-cross From fab08026a2627625a1c4e856285ab8f350b1b3ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:42 +0100 Subject: [PATCH 22/30] tests/docker: flatten debian-riscv64-test-cross MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flatten into a single dockerfile and update to match the rest of the test cross compile dockerfiles. Signed-off-by: Alex Bennée Reviewed-by: Richard Henderson Message-Id: <20220914155950.804707-23-alex.bennee@linaro.org> --- .gitlab-ci.d/container-cross.yml | 1 - tests/docker/Makefile.include | 1 - .../dockerfiles/debian-riscv64-test-cross.docker | 10 ++++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml index 95d57e1c5d..3a8bd75473 100644 --- a/.gitlab-ci.d/container-cross.yml +++ b/.gitlab-ci.d/container-cross.yml @@ -122,7 +122,6 @@ riscv64-debian-cross-container: riscv64-debian-test-cross-container: extends: .container_job_template stage: containers - needs: ['amd64-debian11-container'] variables: NAME: debian-riscv64-test-cross diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index b1bf56434f..24cd44e667 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -133,7 +133,6 @@ docker-image-debian-all-test-cross: docker-image-debian10 docker-image-debian-loongarch-cross: docker-image-debian11 docker-image-debian-microblaze-cross: docker-image-debian10 docker-image-debian-nios2-cross: docker-image-debian10 -docker-image-debian-riscv64-test-cross: docker-image-debian11 # These images may be good enough for building tests but not for test builds DOCKER_PARTIAL_IMAGES += debian-alpha-cross diff --git a/tests/docker/dockerfiles/debian-riscv64-test-cross.docker b/tests/docker/dockerfiles/debian-riscv64-test-cross.docker index 1d90901298..e5f83a5aeb 100644 --- a/tests/docker/dockerfiles/debian-riscv64-test-cross.docker +++ b/tests/docker/dockerfiles/debian-riscv64-test-cross.docker @@ -3,10 +3,12 @@ # # This docker target builds on the Debian Bullseye base image. # -FROM qemu/debian11 +FROM docker.io/library/debian:11-slim -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive eatmydata \ - apt install -y --no-install-recommends \ +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y eatmydata && \ + eatmydata apt-get dist-upgrade -y && \ + eatmydata apt-get install --no-install-recommends -y \ gcc-riscv64-linux-gnu \ libc6-dev-riscv64-cross From b6fafe5b3e5eb555cf13f51cd5d08400c86dac96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:43 +0100 Subject: [PATCH 23/30] tests/docker: update and flatten debian-all-test-cross MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update to the latest stable Debian. While we are at it flatten into a single dockerfile. We also need to ensure we install clang as it is used for those builds as well. It would be nice to port this to lcitool but for now this will do. Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Reviewed-by: Richard Henderson Message-Id: <20220914155950.804707-24-alex.bennee@linaro.org> --- .gitlab-ci.d/container-cross.yml | 1 - tests/docker/Makefile.include | 1 - .../dockerfiles/debian-all-test-cross.docker | 18 +++++++++++++----- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml index 3a8bd75473..091c0d8fcb 100644 --- a/.gitlab-ci.d/container-cross.yml +++ b/.gitlab-ci.d/container-cross.yml @@ -14,7 +14,6 @@ amd64-debian-cross-container: amd64-debian-user-cross-container: extends: .container_job_template stage: containers - needs: ['amd64-debian10-container'] variables: NAME: debian-all-test-cross diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 24cd44e667..ddcc502049 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -129,7 +129,6 @@ docker-image-debian-nios2-cross: $(DOCKER_FILES_DIR)/debian-toolchain.docker \ $(call debian-toolchain, $@) # Specialist build images, sometimes very limited tools -docker-image-debian-all-test-cross: docker-image-debian10 docker-image-debian-loongarch-cross: docker-image-debian11 docker-image-debian-microblaze-cross: docker-image-debian10 docker-image-debian-nios2-cross: docker-image-debian10 diff --git a/tests/docker/dockerfiles/debian-all-test-cross.docker b/tests/docker/dockerfiles/debian-all-test-cross.docker index dedcea58b4..2beb077fb4 100644 --- a/tests/docker/dockerfiles/debian-all-test-cross.docker +++ b/tests/docker/dockerfiles/debian-all-test-cross.docker @@ -6,16 +6,24 @@ # basic compilers for as many targets as possible. We shall use this # to build and run linux-user tests on GitLab # -FROM qemu/debian10 +FROM docker.io/library/debian:11-slim -# What we need to build QEMU itself -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive eatmydata \ +# Duplicate deb line as deb-src +RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.list + +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y eatmydata && \ + eatmydata apt-get dist-upgrade -y && \ apt build-dep -yy qemu -# Add the foreign architecture we want and install dependencies +# Add extra build tools and as many cross compilers as we can for testing RUN DEBIAN_FRONTEND=noninteractive eatmydata \ apt install -y --no-install-recommends \ + ccache \ + clang \ + git \ + ninja-build \ gcc-aarch64-linux-gnu \ libc6-dev-arm64-cross \ gcc-alpha-linux-gnu \ From 8bb499955e26466b2b64f0736d77037a56fcddcb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:44 +0100 Subject: [PATCH 24/30] tests/lcitool: bump to latest version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We need this to be able to cleanly build the x86 cross images. There are a few minor updates triggered by lcitool-refresh including adding "libslirp" to the freebsd vars and opensuse-leap which will help when we finally drop the slirp submodule from QEMU. Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20220914155950.804707-25-alex.bennee@linaro.org> --- .gitlab-ci.d/cirrus/freebsd-12.vars | 2 +- .gitlab-ci.d/cirrus/freebsd-13.vars | 2 +- tests/docker/dockerfiles/opensuse-leap.docker | 3 ++- tests/docker/dockerfiles/ubuntu2004.docker | 2 +- tests/lcitool/libvirt-ci | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.gitlab-ci.d/cirrus/freebsd-12.vars b/.gitlab-ci.d/cirrus/freebsd-12.vars index 8fa5a320e9..1a5959810f 100644 --- a/.gitlab-ci.d/cirrus/freebsd-12.vars +++ b/.gitlab-ci.d/cirrus/freebsd-12.vars @@ -11,6 +11,6 @@ MAKE='/usr/local/bin/gmake' NINJA='/usr/local/bin/ninja' PACKAGING_COMMAND='pkg' PIP3='/usr/local/bin/pip-3.8' -PKGS='alsa-lib bash bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv perl5 pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' +PKGS='alsa-lib bash bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv perl5 pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' PYPI_PKGS='' PYTHON='/usr/local/bin/python3' diff --git a/.gitlab-ci.d/cirrus/freebsd-13.vars b/.gitlab-ci.d/cirrus/freebsd-13.vars index 8ed7e33a77..5e5aafd7e5 100644 --- a/.gitlab-ci.d/cirrus/freebsd-13.vars +++ b/.gitlab-ci.d/cirrus/freebsd-13.vars @@ -11,6 +11,6 @@ MAKE='/usr/local/bin/gmake' NINJA='/usr/local/bin/ninja' PACKAGING_COMMAND='pkg' PIP3='/usr/local/bin/pip-3.8' -PKGS='alsa-lib bash bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv perl5 pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' +PKGS='alsa-lib bash bzip2 ca_root_nss capstone4 ccache cdrkit-genisoimage cmocka ctags curl cyrus-sasl dbus diffutils dtc fusefs-libs3 gettext git glib gmake gnutls gsed gtk3 json-c libepoxy libffi libgcrypt libjpeg-turbo libnfs libslirp libspice-server libssh libtasn1 llvm lzo2 meson ncurses nettle ninja opencv perl5 pixman pkgconf png py39-numpy py39-pillow py39-pip py39-sphinx py39-sphinx_rtd_theme py39-yaml python3 rpm2cpio sdl2 sdl2_image snappy spice-protocol tesseract texinfo usbredir virglrenderer vte3 zstd' PYPI_PKGS='' PYTHON='/usr/local/bin/python3' diff --git a/tests/docker/dockerfiles/opensuse-leap.docker b/tests/docker/dockerfiles/opensuse-leap.docker index 047a435ab5..041cf9c1ff 100644 --- a/tests/docker/dockerfiles/opensuse-leap.docker +++ b/tests/docker/dockerfiles/opensuse-leap.docker @@ -66,6 +66,7 @@ RUN zypper update -y && \ librbd-devel \ libseccomp-devel \ libselinux-devel \ + libslirp-devel \ libspice-server-devel \ libssh-devel \ libtasn1-devel \ @@ -127,7 +128,7 @@ RUN zypper update -y && \ ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/g++ && \ ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/gcc -RUN pip3 install meson==0.56.0 +RUN /usr/bin/pip3 install meson==0.56.0 ENV LANG "en_US.UTF-8" ENV MAKE "/usr/bin/make" diff --git a/tests/docker/dockerfiles/ubuntu2004.docker b/tests/docker/dockerfiles/ubuntu2004.docker index 99803b343b..e1f4ed7c80 100644 --- a/tests/docker/dockerfiles/ubuntu2004.docker +++ b/tests/docker/dockerfiles/ubuntu2004.docker @@ -137,7 +137,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \ ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/g++ && \ ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/gcc -RUN pip3 install meson==0.56.0 +RUN /usr/bin/pip3 install meson==0.56.0 ENV LANG "en_US.UTF-8" ENV MAKE "/usr/bin/make" diff --git a/tests/lcitool/libvirt-ci b/tests/lcitool/libvirt-ci index 324355cf62..e3712b7912 160000 --- a/tests/lcitool/libvirt-ci +++ b/tests/lcitool/libvirt-ci @@ -1 +1 @@ -Subproject commit 324355cf62e86fb551408575afb123bac989ac37 +Subproject commit e3712b79122180fdb3b7a7ea8cbee47ece253f97 From cd150e19e83833d164d69fd159ee95ff3a203e17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:45 +0100 Subject: [PATCH 25/30] tests/docker: update and flatten debian-amd64-cross MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Now lcitool has support for building a x86_64 cross image we can use it for this. Signed-off-by: Alex Bennée Acked-by: Thomas Huth Message-Id: <20220914155950.804707-26-alex.bennee@linaro.org> --- .gitlab-ci.d/container-cross.yml | 1 - tests/docker/Makefile.include | 1 - .../dockerfiles/debian-amd64-cross.docker | 178 ++++++++++++++++-- tests/lcitool/refresh | 7 + 4 files changed, 169 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.d/container-cross.yml b/.gitlab-ci.d/container-cross.yml index 091c0d8fcb..2d560e9764 100644 --- a/.gitlab-ci.d/container-cross.yml +++ b/.gitlab-ci.d/container-cross.yml @@ -7,7 +7,6 @@ alpha-debian-cross-container: amd64-debian-cross-container: extends: .container_job_template stage: containers - needs: ['amd64-debian10-container'] variables: NAME: debian-amd64-cross diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index ddcc502049..a0f5109628 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -75,7 +75,6 @@ DOCKER_PARTIAL_IMAGES := debian10 debian11 ifeq ($(HOST_ARCH),x86_64) DOCKER_PARTIAL_IMAGES += debian-amd64-cross else -docker-image-debian-amd64-cross: docker-image-debian10 DOCKER_PARTIAL_IMAGES += debian-amd64 endif diff --git a/tests/docker/dockerfiles/debian-amd64-cross.docker b/tests/docker/dockerfiles/debian-amd64-cross.docker index 870109ef6a..7d2feb7bf7 100644 --- a/tests/docker/dockerfiles/debian-amd64-cross.docker +++ b/tests/docker/dockerfiles/debian-amd64-cross.docker @@ -1,22 +1,168 @@ +# THIS FILE WAS AUTO-GENERATED # -# Docker x86_64 cross target +# $ lcitool dockerfile --layers all --cross x86_64 debian-11 qemu # -# This docker target is used on non-x86_64 machines which need the -# x86_64 cross compilers installed. -# -FROM qemu/debian10 -MAINTAINER Alex Bennée +# https://gitlab.com/libvirt/libvirt-ci -# Add the foreign architecture we want and install dependencies -RUN dpkg --add-architecture amd64 -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive eatmydata \ - apt install -y --no-install-recommends \ - crossbuild-essential-amd64 -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive eatmydata \ - apt build-dep -yy -a amd64 --arch-only qemu +FROM docker.io/library/debian:11-slim -# Specify the cross prefix for this image (see tests/docker/common.rc) +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get install -y eatmydata && \ + eatmydata apt-get dist-upgrade -y && \ + eatmydata apt-get install --no-install-recommends -y \ + bash \ + bc \ + bsdextrautils \ + bzip2 \ + ca-certificates \ + ccache \ + dbus \ + debianutils \ + diffutils \ + exuberant-ctags \ + findutils \ + gcovr \ + genisoimage \ + gettext \ + git \ + hostname \ + libpcre2-dev \ + libspice-protocol-dev \ + llvm \ + locales \ + make \ + meson \ + ncat \ + ninja-build \ + openssh-client \ + perl-base \ + pkgconf \ + python3 \ + python3-numpy \ + python3-opencv \ + python3-pillow \ + python3-pip \ + python3-sphinx \ + python3-sphinx-rtd-theme \ + python3-venv \ + python3-yaml \ + rpm2cpio \ + sed \ + sparse \ + tar \ + tesseract-ocr \ + tesseract-ocr-eng \ + texinfo && \ + eatmydata apt-get autoremove -y && \ + eatmydata apt-get autoclean -y && \ + sed -Ei 's,^# (en_US\.UTF-8 .*)$,\1,' /etc/locale.gen && \ + dpkg-reconfigure locales + +ENV LANG "en_US.UTF-8" +ENV MAKE "/usr/bin/make" +ENV NINJA "/usr/bin/ninja" +ENV PYTHON "/usr/bin/python3" +ENV CCACHE_WRAPPERSDIR "/usr/libexec/ccache-wrappers" + +RUN export DEBIAN_FRONTEND=noninteractive && \ + dpkg --add-architecture amd64 && \ + eatmydata apt-get update && \ + eatmydata apt-get dist-upgrade -y && \ + eatmydata apt-get install --no-install-recommends -y dpkg-dev && \ + eatmydata apt-get install --no-install-recommends -y \ + g++-x86-64-linux-gnu \ + gcc-x86-64-linux-gnu \ + libaio-dev:amd64 \ + libasan5:amd64 \ + libasound2-dev:amd64 \ + libattr1-dev:amd64 \ + libbpf-dev:amd64 \ + libbrlapi-dev:amd64 \ + libbz2-dev:amd64 \ + libc6-dev:amd64 \ + libcacard-dev:amd64 \ + libcap-ng-dev:amd64 \ + libcapstone-dev:amd64 \ + libcmocka-dev:amd64 \ + libcurl4-gnutls-dev:amd64 \ + libdaxctl-dev:amd64 \ + libdrm-dev:amd64 \ + libepoxy-dev:amd64 \ + libfdt-dev:amd64 \ + libffi-dev:amd64 \ + libfuse3-dev:amd64 \ + libgbm-dev:amd64 \ + libgcrypt20-dev:amd64 \ + libglib2.0-dev:amd64 \ + libglusterfs-dev:amd64 \ + libgnutls28-dev:amd64 \ + libgtk-3-dev:amd64 \ + libibumad-dev:amd64 \ + libibverbs-dev:amd64 \ + libiscsi-dev:amd64 \ + libjemalloc-dev:amd64 \ + libjpeg62-turbo-dev:amd64 \ + libjson-c-dev:amd64 \ + liblttng-ust-dev:amd64 \ + liblzo2-dev:amd64 \ + libncursesw5-dev:amd64 \ + libnfs-dev:amd64 \ + libnuma-dev:amd64 \ + libpam0g-dev:amd64 \ + libpixman-1-dev:amd64 \ + libpmem-dev:amd64 \ + libpng-dev:amd64 \ + libpulse-dev:amd64 \ + librbd-dev:amd64 \ + librdmacm-dev:amd64 \ + libsasl2-dev:amd64 \ + libsdl2-dev:amd64 \ + libsdl2-image-dev:amd64 \ + libseccomp-dev:amd64 \ + libselinux1-dev:amd64 \ + libslirp-dev:amd64 \ + libsnappy-dev:amd64 \ + libspice-server-dev:amd64 \ + libssh-gcrypt-dev:amd64 \ + libsystemd-dev:amd64 \ + libtasn1-6-dev:amd64 \ + libubsan1:amd64 \ + libudev-dev:amd64 \ + liburing-dev:amd64 \ + libusb-1.0-0-dev:amd64 \ + libusbredirhost-dev:amd64 \ + libvdeplug-dev:amd64 \ + libvirglrenderer-dev:amd64 \ + libvte-2.91-dev:amd64 \ + libxen-dev:amd64 \ + libzstd-dev:amd64 \ + nettle-dev:amd64 \ + systemtap-sdt-dev:amd64 \ + xfslibs-dev:amd64 \ + zlib1g-dev:amd64 && \ + eatmydata apt-get autoremove -y && \ + eatmydata apt-get autoclean -y && \ + mkdir -p /usr/local/share/meson/cross && \ + echo "[binaries]\n\ +c = '/usr/bin/x86_64-linux-gnu-gcc'\n\ +ar = '/usr/bin/x86_64-linux-gnu-gcc-ar'\n\ +strip = '/usr/bin/x86_64-linux-gnu-strip'\n\ +pkgconfig = '/usr/bin/x86_64-linux-gnu-pkg-config'\n\ +\n\ +[host_machine]\n\ +system = 'linux'\n\ +cpu_family = 'x86_64'\n\ +cpu = 'x86_64'\n\ +endian = 'little'" > /usr/local/share/meson/cross/x86_64-linux-gnu && \ + dpkg-query --showformat '${Package}_${Version}_${Architecture}\n' --show > /packages.txt && \ + mkdir -p /usr/libexec/ccache-wrappers && \ + ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/x86_64-linux-gnu-c++ && \ + ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/x86_64-linux-gnu-cc && \ + ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/x86_64-linux-gnu-g++ && \ + ln -s /usr/bin/ccache /usr/libexec/ccache-wrappers/x86_64-linux-gnu-gcc + +ENV ABI "x86_64-linux-gnu" +ENV MESON_OPTS "--cross-file=x86_64-linux-gnu" ENV QEMU_CONFIGURE_OPTS --cross-prefix=x86_64-linux-gnu- ENV DEF_TARGET_LIST x86_64-softmmu,x86_64-linux-user,i386-softmmu,i386-linux-user diff --git a/tests/lcitool/refresh b/tests/lcitool/refresh index 2a59150322..fc604d8a21 100755 --- a/tests/lcitool/refresh +++ b/tests/lcitool/refresh @@ -119,6 +119,13 @@ try: # # Cross compiling builds # + generate_dockerfile("debian-amd64-cross", "debian-11", + cross="x86_64", + trailer=debian_cross_build("x86_64-linux-gnu-", + "x86_64-softmmu," + "x86_64-linux-user," + "i386-softmmu,i386-linux-user")) + generate_dockerfile("debian-arm64-cross", "debian-11", cross="aarch64", trailer=debian_cross_build("aarch64-linux-gnu-", From 35782a1b9a2ee71fb037bf1ec6379d5720abf817 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:46 +0100 Subject: [PATCH 26/30] tests/docker: update and flatten debian-loongarch-cross MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update to the latest stable Debian. While we are at it flatten into a single dockerfile. We really don't need the rest of the stuff from the QEMU base image just to compile test images. In this case it is a binary distribution of the toolchain anyway. Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Message-Id: <20220914155950.804707-27-alex.bennee@linaro.org> --- tests/docker/Makefile.include | 1 - tests/docker/dockerfiles/debian-loongarch-cross.docker | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index a0f5109628..1d5a6f1fb4 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -128,7 +128,6 @@ docker-image-debian-nios2-cross: $(DOCKER_FILES_DIR)/debian-toolchain.docker \ $(call debian-toolchain, $@) # Specialist build images, sometimes very limited tools -docker-image-debian-loongarch-cross: docker-image-debian11 docker-image-debian-microblaze-cross: docker-image-debian10 docker-image-debian-nios2-cross: docker-image-debian10 diff --git a/tests/docker/dockerfiles/debian-loongarch-cross.docker b/tests/docker/dockerfiles/debian-loongarch-cross.docker index ca2469d2a8..a8e8e98909 100644 --- a/tests/docker/dockerfiles/debian-loongarch-cross.docker +++ b/tests/docker/dockerfiles/debian-loongarch-cross.docker @@ -1,11 +1,13 @@ # # Docker cross-compiler target # -# This docker target builds on the debian11 base image, -# using a prebuilt toolchains for LoongArch64 from: +# This docker target uses prebuilt toolchains for LoongArch64 from: # https://github.com/loongson/build-tools/releases # -FROM qemu/debian11 +FROM docker.io/library/debian:11-slim + +# Duplicate deb line as deb-src +RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.list RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata && \ From 581cd47fe5a708ebd999bd426a111984074c9ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:47 +0100 Subject: [PATCH 27/30] tests/docker: update and flatten debian-hexagon-cross MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update to the latest stable Debian. While we are at it flatten into a single dockerfile as we do not some of the extraneous packages from the base image to build the toolchain. Signed-off-by: Alex Bennée Reviewed-by: Richard Henderson Message-Id: <20220914155950.804707-28-alex.bennee@linaro.org> --- .../dockerfiles/debian-hexagon-cross.docker | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/docker/dockerfiles/debian-hexagon-cross.docker b/tests/docker/dockerfiles/debian-hexagon-cross.docker index d5dc299dc1..8d219bb81d 100644 --- a/tests/docker/dockerfiles/debian-hexagon-cross.docker +++ b/tests/docker/dockerfiles/debian-hexagon-cross.docker @@ -2,12 +2,10 @@ # Docker Hexagon cross-compiler target # # This docker target is used for building hexagon tests. As it also -# needs to be able to build QEMU itself in CI we include it's -# build-deps. It is also a "stand-alone" image so as not to be -# triggered by re-builds on other base images given it takes a long -# time to build. +# needs to be able to build QEMU itself in CI we include its +# build-deps. # -FROM qemu/debian10 +FROM docker.io/library/debian:11-slim # Install common build utilities RUN apt update && \ @@ -15,11 +13,18 @@ RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ apt install -y --no-install-recommends \ bison \ + ca-certificates \ + clang \ cmake \ flex \ + gcc \ lld \ + make \ + ninja-build \ + python3 \ rsync \ - wget + wget \ + xz-utils ENV TOOLCHAIN_INSTALL /usr/local ENV ROOTFS /usr/local @@ -32,7 +37,7 @@ ADD build-toolchain.sh /root/hexagon-toolchain/build-toolchain.sh RUN cd /root/hexagon-toolchain && ./build-toolchain.sh -FROM debian:buster-slim +FROM docker.io/library/debian:11-slim # Duplicate deb line as deb-src RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.list # Install QEMU build deps for use in CI From 4239162add71101feddeaba216d576c581128975 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:48 +0100 Subject: [PATCH 28/30] tests/docker: update and flatten debian-toolchain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Update to the latest stable Debian. While we are at it flatten into a single dockerfile as we do not need anything from the base image to build the toolchain. This is used to build both the nios and microblaze toolchains. Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Reviewed-by: Richard Henderson Message-Id: <20220914155950.804707-29-alex.bennee@linaro.org> --- tests/docker/Makefile.include | 4 ---- tests/docker/dockerfiles/debian-toolchain.docker | 5 +++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index 1d5a6f1fb4..a3174346da 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -127,10 +127,6 @@ docker-image-debian-nios2-cross: $(DOCKER_FILES_DIR)/debian-toolchain.docker \ $(DOCKER_FILES_DIR)/debian-nios2-cross.d/build-toolchain.sh $(call debian-toolchain, $@) -# Specialist build images, sometimes very limited tools -docker-image-debian-microblaze-cross: docker-image-debian10 -docker-image-debian-nios2-cross: docker-image-debian10 - # These images may be good enough for building tests but not for test builds DOCKER_PARTIAL_IMAGES += debian-alpha-cross DOCKER_PARTIAL_IMAGES += debian-powerpc-test-cross diff --git a/tests/docker/dockerfiles/debian-toolchain.docker b/tests/docker/dockerfiles/debian-toolchain.docker index 738d808aa6..d3d4d3344e 100644 --- a/tests/docker/dockerfiles/debian-toolchain.docker +++ b/tests/docker/dockerfiles/debian-toolchain.docker @@ -4,7 +4,7 @@ # This dockerfile is used for building a cross-compiler toolchain. # The script for building the toolchain is supplied via extra-files. # -FROM qemu/debian10 +FROM docker.io/library/debian:11-slim # Install build utilities for building gcc and glibc. # ??? The build-dep isn't working, missing a number of @@ -15,6 +15,7 @@ RUN apt update && \ DEBIAN_FRONTEND=noninteractive eatmydata \ apt install -y --no-install-recommends \ bison \ + ca-certificates \ flex \ gawk \ libmpc-dev \ @@ -32,5 +33,5 @@ RUN cd /root && ./build-toolchain.sh # Throw away the extra toolchain build deps, the downloaded source, # and the build trees by restoring the original debian10 image, # then copying the built toolchain from stage 0. -FROM qemu/debian10 +FROM docker.io/library/debian:bullseye-slim COPY --from=0 /usr/local /usr/local From 07056db1b504acf12a0a39647846030956445fea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:49 +0100 Subject: [PATCH 29/30] tests/docker: remove FROM qemu/ support from docker.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We want to migrate from docker.py to building our images directly with docker/podman. Before we get there we need to make sure we don't re-introduce our layered builds so bug out if we see FROM qemu/ in a Dockerfile. Signed-off-by: Alex Bennée Acked-by: Thomas Huth Reviewed-by: Richard Henderson Message-Id: <20220914155950.804707-30-alex.bennee@linaro.org> --- tests/docker/docker.py | 38 ++++++++++---------------------------- 1 file changed, 10 insertions(+), 28 deletions(-) diff --git a/tests/docker/docker.py b/tests/docker/docker.py index d0af2861b8..3a1ed7cb18 100755 --- a/tests/docker/docker.py +++ b/tests/docker/docker.py @@ -205,22 +205,17 @@ def _read_qemu_dockerfile(img_name): return _read_dockerfile(df) -def _dockerfile_preprocess(df): - out = "" +def _dockerfile_verify_flat(df): + "Verify we do not include other qemu/ layers" for l in df.splitlines(): if len(l.strip()) == 0 or l.startswith("#"): continue from_pref = "FROM qemu/" if l.startswith(from_pref): - # TODO: Alternatively we could replace this line with "FROM $ID" - # where $ID is the image's hex id obtained with - # $ docker images $IMAGE --format="{{.Id}}" - # but unfortunately that's not supported by RHEL 7. - inlining = _read_qemu_dockerfile(l[len(from_pref):]) - out += _dockerfile_preprocess(inlining) - continue - out += l + "\n" - return out + print("We no longer support multiple QEMU layers.") + print("Dockerfiles should be flat, ideally created by lcitool") + return False + return True class Docker(object): @@ -309,23 +304,10 @@ class Docker(object): if argv is None: argv = [] - # pre-calculate the docker checksum before any - # substitutions we make for caching - checksum = _text_checksum(_dockerfile_preprocess(dockerfile)) + if not _dockerfile_verify_flat(dockerfile): + return -1 - if registry is not None: - sources = re.findall("FROM qemu\/(.*)", dockerfile) - # Fetch any cache layers we can, may fail - for s in sources: - pull_args = ["pull", "%s/qemu/%s" % (registry, s)] - if self._do(pull_args, quiet=quiet) != 0: - registry = None - break - # Make substitutions - if registry is not None: - dockerfile = dockerfile.replace("FROM qemu/", - "FROM %s/qemu/" % - (registry)) + checksum = _text_checksum(dockerfile) tmp_df = tempfile.NamedTemporaryFile(mode="w+t", encoding='utf-8', @@ -371,7 +353,7 @@ class Docker(object): checksum = self.get_image_dockerfile_checksum(tag) except Exception: return False - return checksum == _text_checksum(_dockerfile_preprocess(dockerfile)) + return checksum == _text_checksum(dockerfile) def run(self, cmd, keep, quiet, as_user=False): label = uuid.uuid4().hex From d996f0aeb2fe8f5c5f30ac8a2f633e3196457d6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20Benn=C3=A9e?= Date: Wed, 14 Sep 2022 16:59:50 +0100 Subject: [PATCH 30/30] tests/docker: remove the Debian base images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We no longer use these in any of our images. Clean-up the remaining comments and documentation that reference them and remove from the build. Signed-off-by: Alex Bennée Reviewed-by: Thomas Huth Reviewed-by: Richard Henderson Message-Id: <20220914155950.804707-31-alex.bennee@linaro.org> --- .gitlab-ci.d/container-core.yml | 5 ---- .gitlab-ci.d/containers.yml | 5 ---- docs/devel/testing.rst | 2 +- tests/docker/Makefile.include | 8 +---- tests/docker/dockerfiles/debian10.docker | 38 ------------------------ tests/docker/dockerfiles/debian11.docker | 18 ----------- 6 files changed, 2 insertions(+), 74 deletions(-) delete mode 100644 tests/docker/dockerfiles/debian10.docker delete mode 100644 tests/docker/dockerfiles/debian11.docker diff --git a/.gitlab-ci.d/container-core.yml b/.gitlab-ci.d/container-core.yml index e8dd1f476a..08f8450fa1 100644 --- a/.gitlab-ci.d/container-core.yml +++ b/.gitlab-ci.d/container-core.yml @@ -10,8 +10,3 @@ amd64-fedora-container: extends: .container_job_template variables: NAME: fedora - -amd64-debian10-container: - extends: .container_job_template - variables: - NAME: debian10 diff --git a/.gitlab-ci.d/containers.yml b/.gitlab-ci.d/containers.yml index be34cbc7ba..96d2a3b58b 100644 --- a/.gitlab-ci.d/containers.yml +++ b/.gitlab-ci.d/containers.yml @@ -7,11 +7,6 @@ amd64-alpine-container: variables: NAME: alpine -amd64-debian11-container: - extends: .container_job_template - variables: - NAME: debian11 - amd64-debian-container: extends: .container_job_template stage: containers diff --git a/docs/devel/testing.rst b/docs/devel/testing.rst index f35f117d95..aea5b42356 100644 --- a/docs/devel/testing.rst +++ b/docs/devel/testing.rst @@ -375,7 +375,7 @@ locally by using the ``NOCACHE`` build option: .. code:: - make docker-image-debian10 NOCACHE=1 + make docker-image-debian-arm64-cross NOCACHE=1 Images ~~~~~~ diff --git a/tests/docker/Makefile.include b/tests/docker/Makefile.include index a3174346da..270e99786e 100644 --- a/tests/docker/Makefile.include +++ b/tests/docker/Makefile.include @@ -69,9 +69,7 @@ docker-binfmt-image-debian-%: $(DOCKER_FILES_DIR)/debian-bootstrap.docker { echo "You will need to build $(EXECUTABLE)"; exit 1;},\ "CHECK", "debian-$* exists")) -# Enforce dependencies for composite images -# we don't run tests on intermediate images (used as base by another image) -DOCKER_PARTIAL_IMAGES := debian10 debian11 +# Special case cross-compiling x86_64 on non-x86_64 systems. ifeq ($(HOST_ARCH),x86_64) DOCKER_PARTIAL_IMAGES += debian-amd64-cross else @@ -89,10 +87,6 @@ endif # The native build should never use the registry docker-image-debian-native: DOCKER_REGISTRY= -# base images should not add a local user -docker-image-debian10: NOUSER=1 -docker-image-debian11: NOUSER=1 - # alpine has no adduser docker-image-alpine: NOUSER=1 diff --git a/tests/docker/dockerfiles/debian10.docker b/tests/docker/dockerfiles/debian10.docker deleted file mode 100644 index 03be923066..0000000000 --- a/tests/docker/dockerfiles/debian10.docker +++ /dev/null @@ -1,38 +0,0 @@ -# -# Docker multiarch cross-compiler target -# -# This docker target is builds on Debian cross compiler targets to build distro -# with a selection of cross compilers for building test binaries. -# -# On its own you can't build much but the docker-foo-cross targets -# build on top of the base debian image. -# -FROM docker.io/library/debian:buster-slim - -# Duplicate deb line as deb-src -RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.list - -# Install common build utilities -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata && \ - DEBIAN_FRONTEND=noninteractive eatmydata \ - apt install -y --no-install-recommends \ - bc \ - build-essential \ - ca-certificates \ - ccache \ - clang \ - dbus \ - gdb-multiarch \ - gettext \ - git \ - libffi-dev \ - libncurses5-dev \ - ninja-build \ - pkg-config \ - psmisc \ - python3 \ - python3-sphinx \ - python3-sphinx-rtd-theme \ - python3-venv \ - $(apt-get -s build-dep --arch-only qemu | egrep ^Inst | fgrep '[all]' | cut -d\ -f2) diff --git a/tests/docker/dockerfiles/debian11.docker b/tests/docker/dockerfiles/debian11.docker deleted file mode 100644 index febf884f8f..0000000000 --- a/tests/docker/dockerfiles/debian11.docker +++ /dev/null @@ -1,18 +0,0 @@ -# -# Docker multiarch cross-compiler target -# -# This docker target uses the current development version of Debian as -# a base for cross compilers for building test binaries. We won't -# attempt to build QEMU on it yet given it is still in development. -# -# On its own you can't build much but the docker-foo-cross targets -# build on top of the base debian image. -# -FROM docker.io/library/debian:bullseye-slim - -# Duplicate deb line as deb-src -RUN cat /etc/apt/sources.list | sed "s/^deb\ /deb-src /" >> /etc/apt/sources.list - -# Install common build utilities -RUN apt update && \ - DEBIAN_FRONTEND=noninteractive apt install -yy eatmydata