From 1e4d8b31be35e54b6429fea54f5ecaa0083f91e7 Mon Sep 17 00:00:00 2001 From: John Snow Date: Thu, 20 Jan 2022 19:52:21 -0500 Subject: [PATCH 01/17] python: pin setuptools below v60.0.0 setuptools is a package that replaces the python stdlib 'distutils'. It is generally installed by all venv-creating tools "by default". It isn't actually needed at runtime for the qemu package, so our own setup.cfg does not mention it as a dependency. However, tox will create virtual environments that include it, and will upgrade it to the very latest version. the 'venv' tool will also include whichever version your host system happens to have. Unfortunately, setuptools version 60.0.0 and above include a hack to forcibly overwrite python's built-in distutils. The pylint tool that we use to run code analysis checks on this package relies on distutils and suffers regressions when setuptools >= 60.0.0 is present at all, see https://github.com/PyCQA/pylint/issues/5704 Instruct tox and the 'check-dev' targets to avoid setuptools packages that are too new, for now. Pipenv is unaffected, because setuptools 60 does not offer Python 3.6 support, and our pipenv config is pinned against Python 3.6. Signed-off-by: John Snow Reviewed-by: Beraldo Leal Reviewed-by: Cleber Rosa Tested-by: Cleber Rosa Message-id: 20220121005221.142236-1-jsnow@redhat.com Signed-off-by: John Snow --- python/Makefile | 2 ++ python/setup.cfg | 1 + 2 files changed, 3 insertions(+) diff --git a/python/Makefile b/python/Makefile index 3334311362..949c472624 100644 --- a/python/Makefile +++ b/python/Makefile @@ -68,6 +68,8 @@ $(QEMU_VENV_DIR) $(QEMU_VENV_DIR)/bin/activate: setup.cfg echo "ACTIVATE $(QEMU_VENV_DIR)"; \ . $(QEMU_VENV_DIR)/bin/activate; \ echo "INSTALL qemu[devel] $(QEMU_VENV_DIR)"; \ + pip install --disable-pip-version-check \ + "setuptools<60.0.0" 1>/dev/null; \ make develop 1>/dev/null; \ ) @touch $(QEMU_VENV_DIR) diff --git a/python/setup.cfg b/python/setup.cfg index 417e937839..aa238d8bc9 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -163,6 +163,7 @@ deps = .[devel] .[fuse] # Workaround to trigger tox venv rebuild .[tui] # Workaround to trigger tox venv rebuild + setuptools < 60 # Workaround, please see commit msg. commands = make check From 57a6b4478ce11a6871ea724778ff21f8290ea5d9 Mon Sep 17 00:00:00 2001 From: John Snow Date: Wed, 19 Jan 2022 14:39:13 -0500 Subject: [PATCH 02/17] python: use avocado's "new" runner The old legacy runner no longer seems to work with output logging, so we can't see failure logs when a test case fails. The new runner doesn't (seem to) support Coverage.py yet, but seeing error output is a more important feature. Signed-off-by: John Snow Reviewed-by: Beraldo Leal Message-id: 20220119193916.4138217-3-jsnow@redhat.com Signed-off-by: John Snow --- python/avocado.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/avocado.cfg b/python/avocado.cfg index c7722e7ecd..a460420059 100644 --- a/python/avocado.cfg +++ b/python/avocado.cfg @@ -1,5 +1,5 @@ [run] -test_runner = runner +test_runner = nrunner [simpletests] # Don't show stdout/stderr in the test *summary* From dc6877bd2ea04a38700adcea2359d5d20c1082a6 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Jan 2022 18:28:44 -0500 Subject: [PATCH 03/17] python/aqmp: fix docstring typo Reported-by: Vladimir Sementsov-Ogievskiy Signed-off-by: John Snow Reviewed-by: Beraldo Leal --- python/qemu/aqmp/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/qemu/aqmp/__init__.py b/python/qemu/aqmp/__init__.py index 880d5b6fa7..173556404d 100644 --- a/python/qemu/aqmp/__init__.py +++ b/python/qemu/aqmp/__init__.py @@ -6,7 +6,7 @@ asynchronously with QMP protocol servers, as implemented by QEMU, the QEMU Guest Agent, and the QEMU Storage Daemon. `QMPClient` provides the main functionality of this package. All errors -raised by this library dervive from `AQMPError`, see `aqmp.error` for +raised by this library derive from `AQMPError`, see `aqmp.error` for additional detail. See `aqmp.events` for an in-depth tutorial on managing QMP events. """ From 3bc72e3aed76e0326703db81964b13f1da075cbf Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Jan 2022 18:28:45 -0500 Subject: [PATCH 04/17] python/aqmp: add __del__ method to legacy interface asyncio can complain *very* loudly if you forget to back out of things gracefully before the garbage collector starts destroying objects that contain live references to asyncio Tasks. The usual fix is just to remember to call aqmp.disconnect(), but for the sake of the legacy wrapper and quick, one-off scripts where a graceful shutdown is not necessarily of paramount imporance, add a courtesy cleanup that will trigger prior to seeing screenfuls of confusing asyncio tracebacks. Note that we can't *always* save you from yourself; depending on when the GC runs, you might just seriously be out of luck. The best we can do in this case is to gently remind you to clean up after yourself. (Still much better than multiple pages of incomprehensible python warnings for the crime of forgetting to put your toys away.) Signed-off-by: John Snow Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Beraldo Leal --- python/qemu/aqmp/legacy.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py index 9e7b9fb80b..2ccb136b02 100644 --- a/python/qemu/aqmp/legacy.py +++ b/python/qemu/aqmp/legacy.py @@ -16,6 +16,8 @@ from typing import ( import qemu.qmp from qemu.qmp import QMPMessage, QMPReturnValue, SocketAddrT +from .error import AQMPError +from .protocol import Runstate from .qmp_client import QMPClient @@ -136,3 +138,19 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol): def send_fd_scm(self, fd: int) -> None: self._aqmp.send_fd_scm(fd) + + def __del__(self) -> None: + if self._aqmp.runstate == Runstate.IDLE: + return + + if not self._aloop.is_running(): + self.close() + else: + # Garbage collection ran while the event loop was running. + # Nothing we can do about it now, but if we don't raise our + # own error, the user will be treated to a lot of traceback + # they might not understand. + raise AQMPError( + "QEMUMonitorProtocol.close()" + " was not called before object was garbage collected" + ) From 3b5bf136f5798a4ea2c66875d6337ca3d6b79434 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Jan 2022 18:28:46 -0500 Subject: [PATCH 05/17] python/aqmp: handle asyncio.TimeoutError on execute() This exception can be injected into any await statement. If we are canceled via timeout, we want to clear the pending execution record on our way out. Signed-off-by: John Snow Reviewed-by: Beraldo Leal Reviewed-by: Vladimir Sementsov-Ogievskiy --- python/qemu/aqmp/qmp_client.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/python/qemu/aqmp/qmp_client.py b/python/qemu/aqmp/qmp_client.py index 8105e29fa8..6a985ffe30 100644 --- a/python/qemu/aqmp/qmp_client.py +++ b/python/qemu/aqmp/qmp_client.py @@ -435,7 +435,11 @@ class QMPClient(AsyncProtocol[Message], Events): msg_id = msg['id'] self._pending[msg_id] = asyncio.Queue(maxsize=1) - await self._outgoing.put(msg) + try: + await self._outgoing.put(msg) + except: + del self._pending[msg_id] + raise return msg_id @@ -452,9 +456,9 @@ class QMPClient(AsyncProtocol[Message], Events): was lost, or some other problem. """ queue = self._pending[msg_id] - result = await queue.get() try: + result = await queue.get() if isinstance(result, ExecInterruptedError): raise result return result From 0e6bfd8b96e407db7b0cb5e8c14cc315a7154f53 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Jan 2022 18:28:47 -0500 Subject: [PATCH 06/17] python/aqmp: copy type definitions from qmp Copy the remaining type definitions from QMP into the qemu.aqmp.legacy module. Now, users that require the legacy interface don't need to import anything else but qemu.aqmp.legacy wrapper. Signed-off-by: John Snow Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Beraldo Leal --- python/qemu/aqmp/legacy.py | 22 ++++++++++++++++++++-- python/qemu/aqmp/protocol.py | 16 ++++++++++------ 2 files changed, 30 insertions(+), 8 deletions(-) diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py index 2ccb136b02..9431fe9330 100644 --- a/python/qemu/aqmp/legacy.py +++ b/python/qemu/aqmp/legacy.py @@ -6,7 +6,9 @@ This class pretends to be qemu.qmp.QEMUMonitorProtocol. import asyncio from typing import ( + Any, Awaitable, + Dict, List, Optional, TypeVar, @@ -14,13 +16,29 @@ from typing import ( ) import qemu.qmp -from qemu.qmp import QMPMessage, QMPReturnValue, SocketAddrT from .error import AQMPError -from .protocol import Runstate +from .protocol import Runstate, SocketAddrT from .qmp_client import QMPClient +#: QMPMessage is an entire QMP message of any kind. +QMPMessage = Dict[str, Any] + +#: QMPReturnValue is the 'return' value of a command. +QMPReturnValue = object + +#: QMPObject is any object in a QMP message. +QMPObject = Dict[str, object] + +# QMPMessage can be outgoing commands or incoming events/returns. +# QMPReturnValue is usually a dict/json object, but due to QAPI's +# 'returns-whitelist', it can actually be anything. +# +# {'return': {}} is a QMPMessage, +# {} is the QMPReturnValue. + + # pylint: disable=missing-docstring diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py index c4fbe35a0e..5b4f2f0d0a 100644 --- a/python/qemu/aqmp/protocol.py +++ b/python/qemu/aqmp/protocol.py @@ -46,6 +46,10 @@ T = TypeVar('T') _U = TypeVar('_U') _TaskFN = Callable[[], Awaitable[None]] # aka ``async def func() -> None`` +InternetAddrT = Tuple[str, int] +UnixAddrT = str +SocketAddrT = Union[UnixAddrT, InternetAddrT] + class Runstate(Enum): """Protocol session runstate.""" @@ -257,7 +261,7 @@ class AsyncProtocol(Generic[T]): @upper_half @require(Runstate.IDLE) - async def accept(self, address: Union[str, Tuple[str, int]], + async def accept(self, address: SocketAddrT, ssl: Optional[SSLContext] = None) -> None: """ Accept a connection and begin processing message queues. @@ -275,7 +279,7 @@ class AsyncProtocol(Generic[T]): @upper_half @require(Runstate.IDLE) - async def connect(self, address: Union[str, Tuple[str, int]], + async def connect(self, address: SocketAddrT, ssl: Optional[SSLContext] = None) -> None: """ Connect to the server and begin processing message queues. @@ -337,7 +341,7 @@ class AsyncProtocol(Generic[T]): @upper_half async def _new_session(self, - address: Union[str, Tuple[str, int]], + address: SocketAddrT, ssl: Optional[SSLContext] = None, accept: bool = False) -> None: """ @@ -397,7 +401,7 @@ class AsyncProtocol(Generic[T]): @upper_half async def _establish_connection( self, - address: Union[str, Tuple[str, int]], + address: SocketAddrT, ssl: Optional[SSLContext] = None, accept: bool = False ) -> None: @@ -424,7 +428,7 @@ class AsyncProtocol(Generic[T]): await self._do_connect(address, ssl) @upper_half - async def _do_accept(self, address: Union[str, Tuple[str, int]], + async def _do_accept(self, address: SocketAddrT, ssl: Optional[SSLContext] = None) -> None: """ Acting as the transport server, accept a single connection. @@ -482,7 +486,7 @@ class AsyncProtocol(Generic[T]): self.logger.debug("Connection accepted.") @upper_half - async def _do_connect(self, address: Union[str, Tuple[str, int]], + async def _do_connect(self, address: SocketAddrT, ssl: Optional[SSLContext] = None) -> None: """ Acting as the transport client, initiate a connection to a server. From 728dcac5e356ce5b948943f21c0c72a1b2d96122 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Jan 2022 18:28:48 -0500 Subject: [PATCH 07/17] python/aqmp: add SocketAddrT to package root It's a commonly needed definition, it can be re-exported by the root. Signed-off-by: John Snow Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Beraldo Leal --- python/qemu/aqmp/__init__.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/python/qemu/aqmp/__init__.py b/python/qemu/aqmp/__init__.py index 173556404d..05f467c141 100644 --- a/python/qemu/aqmp/__init__.py +++ b/python/qemu/aqmp/__init__.py @@ -26,7 +26,12 @@ import logging from .error import AQMPError from .events import EventListener from .message import Message -from .protocol import ConnectError, Runstate, StateError +from .protocol import ( + ConnectError, + Runstate, + SocketAddrT, + StateError, +) from .qmp_client import ExecInterruptedError, ExecuteError, QMPClient @@ -48,4 +53,7 @@ __all__ = ( 'ConnectError', 'ExecuteError', 'ExecInterruptedError', + + # Type aliases + 'SocketAddrT', ) From 6e7751dc388df6daf425db0e245d4d3a10859803 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Jan 2022 18:28:49 -0500 Subject: [PATCH 08/17] python/aqmp: rename AQMPError to QMPError This is in preparation for renaming qemu.aqmp to qemu.qmp. I should have done this from this from the very beginning, but it's a convenient time to make sure this churn is taken care of. Signed-off-by: John Snow Reviewed-by: Vladimir Sementsov-Ogievskiy --- python/qemu/aqmp/__init__.py | 6 +++--- python/qemu/aqmp/error.py | 12 ++++++------ python/qemu/aqmp/events.py | 4 ++-- python/qemu/aqmp/legacy.py | 4 ++-- python/qemu/aqmp/protocol.py | 8 ++++---- python/qemu/aqmp/qmp_client.py | 8 ++++---- 6 files changed, 21 insertions(+), 21 deletions(-) diff --git a/python/qemu/aqmp/__init__.py b/python/qemu/aqmp/__init__.py index 05f467c141..4c22c38079 100644 --- a/python/qemu/aqmp/__init__.py +++ b/python/qemu/aqmp/__init__.py @@ -6,7 +6,7 @@ asynchronously with QMP protocol servers, as implemented by QEMU, the QEMU Guest Agent, and the QEMU Storage Daemon. `QMPClient` provides the main functionality of this package. All errors -raised by this library derive from `AQMPError`, see `aqmp.error` for +raised by this library derive from `QMPError`, see `aqmp.error` for additional detail. See `aqmp.events` for an in-depth tutorial on managing QMP events. """ @@ -23,7 +23,7 @@ managing QMP events. import logging -from .error import AQMPError +from .error import QMPError from .events import EventListener from .message import Message from .protocol import ( @@ -48,7 +48,7 @@ __all__ = ( 'Runstate', # Exceptions, most generic to most explicit - 'AQMPError', + 'QMPError', 'StateError', 'ConnectError', 'ExecuteError', diff --git a/python/qemu/aqmp/error.py b/python/qemu/aqmp/error.py index 781f49b008..24ba4d5054 100644 --- a/python/qemu/aqmp/error.py +++ b/python/qemu/aqmp/error.py @@ -1,21 +1,21 @@ """ -AQMP Error Classes +QMP Error Classes This package seeks to provide semantic error classes that are intended to be used directly by clients when they would like to handle particular semantic failures (e.g. "failed to connect") without needing to know the enumeration of possible reasons for that failure. -AQMPError serves as the ancestor for all exceptions raised by this +QMPError serves as the ancestor for all exceptions raised by this package, and is suitable for use in handling semantic errors from this library. In most cases, individual public methods will attempt to catch and re-encapsulate various exceptions to provide a semantic error-handling interface. -.. admonition:: AQMP Exception Hierarchy Reference +.. admonition:: QMP Exception Hierarchy Reference | `Exception` - | +-- `AQMPError` + | +-- `QMPError` | +-- `ConnectError` | +-- `StateError` | +-- `ExecInterruptedError` @@ -31,11 +31,11 @@ error-handling interface. """ -class AQMPError(Exception): +class QMPError(Exception): """Abstract error class for all errors originating from this package.""" -class ProtocolError(AQMPError): +class ProtocolError(QMPError): """ Abstract error class for protocol failures. diff --git a/python/qemu/aqmp/events.py b/python/qemu/aqmp/events.py index 5f7150c78d..f3d4e2b5e8 100644 --- a/python/qemu/aqmp/events.py +++ b/python/qemu/aqmp/events.py @@ -443,7 +443,7 @@ from typing import ( Union, ) -from .error import AQMPError +from .error import QMPError from .message import Message @@ -451,7 +451,7 @@ EventNames = Union[str, Iterable[str], None] EventFilter = Callable[[Message], bool] -class ListenerError(AQMPError): +class ListenerError(QMPError): """ Generic error class for `EventListener`-related problems. """ diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py index 9431fe9330..27df22818a 100644 --- a/python/qemu/aqmp/legacy.py +++ b/python/qemu/aqmp/legacy.py @@ -17,7 +17,7 @@ from typing import ( import qemu.qmp -from .error import AQMPError +from .error import QMPError from .protocol import Runstate, SocketAddrT from .qmp_client import QMPClient @@ -168,7 +168,7 @@ class QEMUMonitorProtocol(qemu.qmp.QEMUMonitorProtocol): # Nothing we can do about it now, but if we don't raise our # own error, the user will be treated to a lot of traceback # they might not understand. - raise AQMPError( + raise QMPError( "QEMUMonitorProtocol.close()" " was not called before object was garbage collected" ) diff --git a/python/qemu/aqmp/protocol.py b/python/qemu/aqmp/protocol.py index 5b4f2f0d0a..50e973c2f2 100644 --- a/python/qemu/aqmp/protocol.py +++ b/python/qemu/aqmp/protocol.py @@ -29,7 +29,7 @@ from typing import ( cast, ) -from .error import AQMPError +from .error import QMPError from .util import ( bottom_half, create_task, @@ -65,7 +65,7 @@ class Runstate(Enum): DISCONNECTING = 3 -class ConnectError(AQMPError): +class ConnectError(QMPError): """ Raised when the initial connection process has failed. @@ -90,7 +90,7 @@ class ConnectError(AQMPError): return f"{self.error_message}: {cause}" -class StateError(AQMPError): +class StateError(QMPError): """ An API command (connect, execute, etc) was issued at an inappropriate time. @@ -363,7 +363,7 @@ class AsyncProtocol(Generic[T]): This exception will wrap a more concrete one. In most cases, the wrapped exception will be `OSError` or `EOFError`. If a protocol-level failure occurs while establishing a new - session, the wrapped error may also be an `AQMPError`. + session, the wrapped error may also be an `QMPError`. """ assert self.runstate == Runstate.IDLE diff --git a/python/qemu/aqmp/qmp_client.py b/python/qemu/aqmp/qmp_client.py index 6a985ffe30..f1a845cc82 100644 --- a/python/qemu/aqmp/qmp_client.py +++ b/python/qemu/aqmp/qmp_client.py @@ -20,7 +20,7 @@ from typing import ( cast, ) -from .error import AQMPError, ProtocolError +from .error import ProtocolError, QMPError from .events import Events from .message import Message from .models import ErrorResponse, Greeting @@ -66,7 +66,7 @@ class NegotiationError(_WrappedProtocolError): """ -class ExecuteError(AQMPError): +class ExecuteError(QMPError): """ Exception raised by `QMPClient.execute()` on RPC failure. @@ -87,7 +87,7 @@ class ExecuteError(AQMPError): self.error_class: str = error_response.error.class_ -class ExecInterruptedError(AQMPError): +class ExecInterruptedError(QMPError): """ Exception raised by `execute()` (et al) when an RPC is interrupted. @@ -641,7 +641,7 @@ class QMPClient(AsyncProtocol[Message], Events): sock = self._writer.transport.get_extra_info('socket') if sock.family != socket.AF_UNIX: - raise AQMPError("Sending file descriptors requires a UNIX socket.") + raise QMPError("Sending file descriptors requires a UNIX socket.") if not hasattr(sock, 'sendmsg'): # We need to void the warranty sticker. From 7017f3853a59cf9d16c6be24dc657417d1c8f0cb Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Jan 2022 18:28:50 -0500 Subject: [PATCH 09/17] python/qemu-ga-client: don't use deprecated CLI syntax in usage comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cleanup related to commit ccd3b3b8112b670f, "qemu-option: warn for short-form boolean options". Signed-off-by: John Snow Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Daniel P. Berrangé --- python/qemu/qmp/qemu_ga_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/qemu/qmp/qemu_ga_client.py b/python/qemu/qmp/qemu_ga_client.py index 67ac0b4211..b3e1d98c9e 100644 --- a/python/qemu/qmp/qemu_ga_client.py +++ b/python/qemu/qmp/qemu_ga_client.py @@ -5,7 +5,7 @@ Usage: Start QEMU with: -# qemu [...] -chardev socket,path=/tmp/qga.sock,server,wait=off,id=qga0 \ +# qemu [...] -chardev socket,path=/tmp/qga.sock,server=on,wait=off,id=qga0 \ -device virtio-serial \ -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0 From 26db07516fea6e264ba3c30651145f3873f7e4a7 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Jan 2022 18:28:51 -0500 Subject: [PATCH 10/17] python/qmp: switch qemu-ga-client to AQMP Async QMP always raises a "ConnectError" on any connection error which houses the cause in a second exception. We can check if this root cause was python's ConnectionError to determine a fairly similar condition to the original error check here. Signed-off-by: John Snow Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Beraldo Leal --- python/qemu/qmp/qemu_ga_client.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/python/qemu/qmp/qemu_ga_client.py b/python/qemu/qmp/qemu_ga_client.py index b3e1d98c9e..15ed430c61 100644 --- a/python/qemu/qmp/qemu_ga_client.py +++ b/python/qemu/qmp/qemu_ga_client.py @@ -37,8 +37,8 @@ See also: https://wiki.qemu.org/Features/QAPI/GuestAgent # the COPYING file in the top-level directory. import argparse +import asyncio import base64 -import errno import os import random import sys @@ -50,8 +50,8 @@ from typing import ( Sequence, ) -from qemu import qmp -from qemu.qmp import SocketAddrT +from qemu.aqmp import ConnectError, SocketAddrT +from qemu.aqmp.legacy import QEMUMonitorProtocol # This script has not seen many patches or careful attention in quite @@ -61,7 +61,7 @@ from qemu.qmp import SocketAddrT # pylint: disable=missing-docstring -class QemuGuestAgent(qmp.QEMUMonitorProtocol): +class QemuGuestAgent(QEMUMonitorProtocol): def __getattr__(self, name: str) -> Callable[..., Any]: def wrapper(**kwds: object) -> object: return self.command('guest-' + name.replace('_', '-'), **kwds) @@ -149,7 +149,7 @@ class QemuGuestAgentClient: self.qga.settimeout(timeout) try: self.qga.ping() - except TimeoutError: + except asyncio.TimeoutError: return False return True @@ -172,7 +172,7 @@ class QemuGuestAgentClient: try: getattr(self.qga, 'suspend' + '_' + mode)() # On error exception will raise - except TimeoutError: + except asyncio.TimeoutError: # On success command will timed out return @@ -182,7 +182,7 @@ class QemuGuestAgentClient: try: self.qga.shutdown(mode=mode) - except TimeoutError: + except asyncio.TimeoutError: pass @@ -277,7 +277,7 @@ commands = [m.replace('_cmd_', '') for m in dir() if '_cmd_' in m] def send_command(address: str, cmd: str, args: Sequence[str]) -> None: if not os.path.exists(address): - print('%s not found' % address) + print(f"'{address}' not found. (Is QEMU running?)") sys.exit(1) if cmd not in commands: @@ -287,10 +287,10 @@ def send_command(address: str, cmd: str, args: Sequence[str]) -> None: try: client = QemuGuestAgentClient(address) - except OSError as err: + except ConnectError as err: print(err) - if err.errno == errno.ECONNREFUSED: - print('Hint: qemu is not running?') + if isinstance(err.exc, ConnectionError): + print('(Is QEMU running?)') sys.exit(1) if cmd == 'fsfreeze' and args[0] == 'freeze': From 8d6cdc5118e8c7d71ccb716ded2a618e6f78a295 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Jan 2022 18:28:52 -0500 Subject: [PATCH 11/17] python/qmp: switch qom tools to AQMP Signed-off-by: John Snow Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Beraldo Leal --- python/qemu/qmp/qom.py | 5 +++-- python/qemu/qmp/qom_common.py | 3 ++- python/qemu/qmp/qom_fuse.py | 11 ++++++----- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/python/qemu/qmp/qom.py b/python/qemu/qmp/qom.py index 8ff28a8343..bb5d1a78f5 100644 --- a/python/qemu/qmp/qom.py +++ b/python/qemu/qmp/qom.py @@ -32,7 +32,8 @@ QOM commands: import argparse -from . import QMPResponseError +from qemu.aqmp import ExecuteError + from .qom_common import QOMCommand @@ -233,7 +234,7 @@ class QOMTree(QOMCommand): rsp = self.qmp.command('qom-get', path=path, property=item.name) print(f" {item.name}: {rsp} ({item.type})") - except QMPResponseError as err: + except ExecuteError as err: print(f" {item.name}: ({item.type})") print('') for item in items: diff --git a/python/qemu/qmp/qom_common.py b/python/qemu/qmp/qom_common.py index 2e4c741f77..e034a6f247 100644 --- a/python/qemu/qmp/qom_common.py +++ b/python/qemu/qmp/qom_common.py @@ -27,7 +27,8 @@ from typing import ( TypeVar, ) -from . import QEMUMonitorProtocol, QMPError +from qemu.aqmp import QMPError +from qemu.aqmp.legacy import QEMUMonitorProtocol class ObjectPropertyInfo: diff --git a/python/qemu/qmp/qom_fuse.py b/python/qemu/qmp/qom_fuse.py index 43f4671fdb..653a76b93b 100644 --- a/python/qemu/qmp/qom_fuse.py +++ b/python/qemu/qmp/qom_fuse.py @@ -48,7 +48,8 @@ from typing import ( import fuse from fuse import FUSE, FuseOSError, Operations -from . import QMPResponseError +from qemu.aqmp import ExecuteError + from .qom_common import QOMCommand @@ -99,7 +100,7 @@ class QOMFuse(QOMCommand, Operations): try: self.qom_list(path) return True - except QMPResponseError: + except ExecuteError: return False def is_property(self, path: str) -> bool: @@ -112,7 +113,7 @@ class QOMFuse(QOMCommand, Operations): if item.name == prop: return True return False - except QMPResponseError: + except ExecuteError: return False def is_link(self, path: str) -> bool: @@ -125,7 +126,7 @@ class QOMFuse(QOMCommand, Operations): if item.name == prop and item.link: return True return False - except QMPResponseError: + except ExecuteError: return False def read(self, path: str, size: int, offset: int, fh: IO[bytes]) -> bytes: @@ -138,7 +139,7 @@ class QOMFuse(QOMCommand, Operations): try: data = str(self.qmp.command('qom-get', path=path, property=prop)) data += '\n' # make values shell friendly - except QMPResponseError as err: + except ExecuteError as err: raise FuseOSError(EPERM) from err if offset > len(data): From f3efd12930f34b9724e15d8fd2ff56a97b67219d Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Jan 2022 18:28:53 -0500 Subject: [PATCH 12/17] python/qmp: switch qmp-shell to AQMP We have a replacement for async QMP, but it doesn't have feature parity yet. For now, then, port the old tool onto the new backend. Signed-off-by: John Snow Reviewed-by: Vladimir Sementsov-Ogievskiy --- python/qemu/aqmp/legacy.py | 3 +++ python/qemu/qmp/qmp_shell.py | 31 +++++++++++++++++-------------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/python/qemu/aqmp/legacy.py b/python/qemu/aqmp/legacy.py index 27df22818a..0890f95b16 100644 --- a/python/qemu/aqmp/legacy.py +++ b/python/qemu/aqmp/legacy.py @@ -22,6 +22,9 @@ from .protocol import Runstate, SocketAddrT from .qmp_client import QMPClient +# (Temporarily) Re-export QMPBadPortError +QMPBadPortError = qemu.qmp.QMPBadPortError + #: QMPMessage is an entire QMP message of any kind. QMPMessage = Dict[str, Any] diff --git a/python/qemu/qmp/qmp_shell.py b/python/qemu/qmp/qmp_shell.py index e7d7eb18f1..d11bf54b00 100644 --- a/python/qemu/qmp/qmp_shell.py +++ b/python/qemu/qmp/qmp_shell.py @@ -95,8 +95,13 @@ from typing import ( Sequence, ) -from qemu import qmp -from qemu.qmp import QMPMessage +from qemu.aqmp import ConnectError, QMPError, SocketAddrT +from qemu.aqmp.legacy import ( + QEMUMonitorProtocol, + QMPBadPortError, + QMPMessage, + QMPObject, +) LOG = logging.getLogger(__name__) @@ -125,7 +130,7 @@ class QMPCompleter: return None -class QMPShellError(qmp.QMPError): +class QMPShellError(QMPError): """ QMP Shell Base error class. """ @@ -153,7 +158,7 @@ class FuzzyJSON(ast.NodeTransformer): return node -class QMPShell(qmp.QEMUMonitorProtocol): +class QMPShell(QEMUMonitorProtocol): """ QMPShell provides a basic readline-based QMP shell. @@ -161,7 +166,7 @@ class QMPShell(qmp.QEMUMonitorProtocol): :param pretty: Pretty-print QMP messages. :param verbose: Echo outgoing QMP messages to console. """ - def __init__(self, address: qmp.SocketAddrT, + def __init__(self, address: SocketAddrT, pretty: bool = False, verbose: bool = False): super().__init__(address) self._greeting: Optional[QMPMessage] = None @@ -237,7 +242,7 @@ class QMPShell(qmp.QEMUMonitorProtocol): def _cli_expr(self, tokens: Sequence[str], - parent: qmp.QMPObject) -> None: + parent: QMPObject) -> None: for arg in tokens: (key, sep, val) = arg.partition('=') if sep != '=': @@ -403,7 +408,7 @@ class HMPShell(QMPShell): :param pretty: Pretty-print QMP messages. :param verbose: Echo outgoing QMP messages to console. """ - def __init__(self, address: qmp.SocketAddrT, + def __init__(self, address: SocketAddrT, pretty: bool = False, verbose: bool = False): super().__init__(address, pretty, verbose) self._cpu_index = 0 @@ -512,19 +517,17 @@ def main() -> None: try: address = shell_class.parse_address(args.qmp_server) - except qmp.QMPBadPortError: + except QMPBadPortError: parser.error(f"Bad port number: {args.qmp_server}") return # pycharm doesn't know error() is noreturn with shell_class(address, args.pretty, args.verbose) as qemu: try: qemu.connect(negotiate=not args.skip_negotiation) - except qmp.QMPConnectError: - die("Didn't get QMP greeting message") - except qmp.QMPCapabilitiesError: - die("Couldn't negotiate capabilities") - except OSError as err: - die(f"Couldn't connect to {args.qmp_server}: {err!s}") + except ConnectError as err: + if isinstance(err.exc, OSError): + die(f"Couldn't connect to {args.qmp_server}: {err!s}") + die(str(err)) for _ in qemu.repl(): pass From 0347c4c4cfed47e54d9dc275ceb28d35b250749f Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Jan 2022 18:28:54 -0500 Subject: [PATCH 13/17] python: move qmp utilities to python/qemu/utils In order to upload a QMP package to PyPI, I want to remove any scripts that I am not 100% confident I want to support upstream, beyond our castle walls. Move most of our QMP utilities into the utils package so we can split them out from the PyPI upload. Signed-off-by: John Snow Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Beraldo Leal --- python/qemu/{qmp => utils}/qemu_ga_client.py | 0 python/qemu/{qmp => utils}/qom.py | 0 python/qemu/{qmp => utils}/qom_common.py | 0 python/qemu/{qmp => utils}/qom_fuse.py | 0 python/setup.cfg | 16 ++++++++-------- scripts/qmp/qemu-ga-client | 2 +- scripts/qmp/qom-fuse | 2 +- scripts/qmp/qom-get | 2 +- scripts/qmp/qom-list | 2 +- scripts/qmp/qom-set | 2 +- scripts/qmp/qom-tree | 2 +- 11 files changed, 14 insertions(+), 14 deletions(-) rename python/qemu/{qmp => utils}/qemu_ga_client.py (100%) rename python/qemu/{qmp => utils}/qom.py (100%) rename python/qemu/{qmp => utils}/qom_common.py (100%) rename python/qemu/{qmp => utils}/qom_fuse.py (100%) diff --git a/python/qemu/qmp/qemu_ga_client.py b/python/qemu/utils/qemu_ga_client.py similarity index 100% rename from python/qemu/qmp/qemu_ga_client.py rename to python/qemu/utils/qemu_ga_client.py diff --git a/python/qemu/qmp/qom.py b/python/qemu/utils/qom.py similarity index 100% rename from python/qemu/qmp/qom.py rename to python/qemu/utils/qom.py diff --git a/python/qemu/qmp/qom_common.py b/python/qemu/utils/qom_common.py similarity index 100% rename from python/qemu/qmp/qom_common.py rename to python/qemu/utils/qom_common.py diff --git a/python/qemu/qmp/qom_fuse.py b/python/qemu/utils/qom_fuse.py similarity index 100% rename from python/qemu/qmp/qom_fuse.py rename to python/qemu/utils/qom_fuse.py diff --git a/python/setup.cfg b/python/setup.cfg index aa238d8bc9..04a41ef1a0 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -60,13 +60,13 @@ tui = [options.entry_points] console_scripts = - qom = qemu.qmp.qom:main - qom-set = qemu.qmp.qom:QOMSet.entry_point - qom-get = qemu.qmp.qom:QOMGet.entry_point - qom-list = qemu.qmp.qom:QOMList.entry_point - qom-tree = qemu.qmp.qom:QOMTree.entry_point - qom-fuse = qemu.qmp.qom_fuse:QOMFuse.entry_point [fuse] - qemu-ga-client = qemu.qmp.qemu_ga_client:main + qom = qemu.utils.qom:main + qom-set = qemu.utils.qom:QOMSet.entry_point + qom-get = qemu.utils.qom:QOMGet.entry_point + qom-list = qemu.utils.qom:QOMList.entry_point + qom-tree = qemu.utils.qom:QOMTree.entry_point + qom-fuse = qemu.utils.qom_fuse:QOMFuse.entry_point [fuse] + qemu-ga-client = qemu.utils.qemu_ga_client:main qmp-shell = qemu.qmp.qmp_shell:main aqmp-tui = qemu.aqmp.aqmp_tui:main [tui] @@ -80,7 +80,7 @@ python_version = 3.6 warn_unused_configs = True namespace_packages = True -[mypy-qemu.qmp.qom_fuse] +[mypy-qemu.utils.qom_fuse] # fusepy has no type stubs: allow_subclassing_any = True diff --git a/scripts/qmp/qemu-ga-client b/scripts/qmp/qemu-ga-client index 102fd2cad9..56edd0234a 100755 --- a/scripts/qmp/qemu-ga-client +++ b/scripts/qmp/qemu-ga-client @@ -4,7 +4,7 @@ import os import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu.qmp import qemu_ga_client +from qemu.utils import qemu_ga_client if __name__ == '__main__': diff --git a/scripts/qmp/qom-fuse b/scripts/qmp/qom-fuse index a58c8ef979..d453807b27 100755 --- a/scripts/qmp/qom-fuse +++ b/scripts/qmp/qom-fuse @@ -4,7 +4,7 @@ import os import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu.qmp.qom_fuse import QOMFuse +from qemu.utils.qom_fuse import QOMFuse if __name__ == '__main__': diff --git a/scripts/qmp/qom-get b/scripts/qmp/qom-get index e4f3e0c013..04ebe052e8 100755 --- a/scripts/qmp/qom-get +++ b/scripts/qmp/qom-get @@ -4,7 +4,7 @@ import os import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu.qmp.qom import QOMGet +from qemu.utils.qom import QOMGet if __name__ == '__main__': diff --git a/scripts/qmp/qom-list b/scripts/qmp/qom-list index 7a071a54e1..853b85a8d3 100755 --- a/scripts/qmp/qom-list +++ b/scripts/qmp/qom-list @@ -4,7 +4,7 @@ import os import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu.qmp.qom import QOMList +from qemu.utils.qom import QOMList if __name__ == '__main__': diff --git a/scripts/qmp/qom-set b/scripts/qmp/qom-set index 9ca9e2ba10..06820feec4 100755 --- a/scripts/qmp/qom-set +++ b/scripts/qmp/qom-set @@ -4,7 +4,7 @@ import os import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu.qmp.qom import QOMSet +from qemu.utils.qom import QOMSet if __name__ == '__main__': diff --git a/scripts/qmp/qom-tree b/scripts/qmp/qom-tree index 7d0ccca3a4..760e172277 100755 --- a/scripts/qmp/qom-tree +++ b/scripts/qmp/qom-tree @@ -4,7 +4,7 @@ import os import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu.qmp.qom import QOMTree +from qemu.utils.qom import QOMTree if __name__ == '__main__': From fd9c3a6219b0470c356c8486188052d353846806 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Jan 2022 18:28:55 -0500 Subject: [PATCH 14/17] python: move qmp-shell under the AQMP package Signed-off-by: John Snow Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Beraldo Leal --- python/README.rst | 2 +- python/qemu/{qmp => aqmp}/qmp_shell.py | 0 python/setup.cfg | 2 +- scripts/qmp/qmp-shell | 2 +- 4 files changed, 3 insertions(+), 3 deletions(-) rename python/qemu/{qmp => aqmp}/qmp_shell.py (100%) diff --git a/python/README.rst b/python/README.rst index 9c1fceaee7..fcf74f69ea 100644 --- a/python/README.rst +++ b/python/README.rst @@ -59,7 +59,7 @@ Package installation also normally provides executable console scripts, so that tools like ``qmp-shell`` are always available via $PATH. To invoke them without installation, you can invoke e.g.: -``> PYTHONPATH=~/src/qemu/python python3 -m qemu.qmp.qmp_shell`` +``> PYTHONPATH=~/src/qemu/python python3 -m qemu.aqmp.qmp_shell`` The mappings between console script name and python module path can be found in ``setup.cfg``. diff --git a/python/qemu/qmp/qmp_shell.py b/python/qemu/aqmp/qmp_shell.py similarity index 100% rename from python/qemu/qmp/qmp_shell.py rename to python/qemu/aqmp/qmp_shell.py diff --git a/python/setup.cfg b/python/setup.cfg index 04a41ef1a0..3fb18f845d 100644 --- a/python/setup.cfg +++ b/python/setup.cfg @@ -67,7 +67,7 @@ console_scripts = qom-tree = qemu.utils.qom:QOMTree.entry_point qom-fuse = qemu.utils.qom_fuse:QOMFuse.entry_point [fuse] qemu-ga-client = qemu.utils.qemu_ga_client:main - qmp-shell = qemu.qmp.qmp_shell:main + qmp-shell = qemu.aqmp.qmp_shell:main aqmp-tui = qemu.aqmp.aqmp_tui:main [tui] [flake8] diff --git a/scripts/qmp/qmp-shell b/scripts/qmp/qmp-shell index 4a20f97db7..31b19d73e2 100755 --- a/scripts/qmp/qmp-shell +++ b/scripts/qmp/qmp-shell @@ -4,7 +4,7 @@ import os import sys sys.path.append(os.path.join(os.path.dirname(__file__), '..', '..', 'python')) -from qemu.qmp import qmp_shell +from qemu.aqmp import qmp_shell if __name__ == '__main__': From 99221256151845f5eb4cedac750b08e7ab14d92d Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Jan 2022 18:28:57 -0500 Subject: [PATCH 15/17] scripts/cpu-x86-uarch-abi: fix CLI parsing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: John Snow Reviewed-by: Daniel P. Berrangé Reviewed-by: Vladimir Sementsov-Ogievskiy --- scripts/cpu-x86-uarch-abi.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/cpu-x86-uarch-abi.py b/scripts/cpu-x86-uarch-abi.py index 08acc52a81..8963d90f0b 100644 --- a/scripts/cpu-x86-uarch-abi.py +++ b/scripts/cpu-x86-uarch-abi.py @@ -9,7 +9,7 @@ from qemu import qmp import sys -if len(sys.argv) != 1: +if len(sys.argv) != 2: print("syntax: %s QMP-SOCK\n\n" % __file__ + "Where QMP-SOCK points to a QEMU process such as\n\n" + " # qemu-system-x86_64 -qmp unix:/tmp/qmp,server,nowait " + @@ -66,7 +66,6 @@ levels = [ sock = sys.argv[1] -cmd = sys.argv[2] shell = qmp.QEMUMonitorProtocol(sock) shell.connect() From 0665410dcf3d3fa8758545668998dd16d3abbaa9 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Jan 2022 18:28:58 -0500 Subject: [PATCH 16/17] scripts/cpu-x86-uarch-abi: switch to AQMP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: John Snow Reviewed-by: Daniel P. Berrangé Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Beraldo Leal --- scripts/cpu-x86-uarch-abi.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/cpu-x86-uarch-abi.py b/scripts/cpu-x86-uarch-abi.py index 8963d90f0b..c262d2f027 100644 --- a/scripts/cpu-x86-uarch-abi.py +++ b/scripts/cpu-x86-uarch-abi.py @@ -6,7 +6,7 @@ # compatibility levels for each CPU model. # -from qemu import qmp +from qemu.aqmp.legacy import QEMUMonitorProtocol import sys if len(sys.argv) != 2: @@ -66,7 +66,7 @@ levels = [ sock = sys.argv[1] -shell = qmp.QEMUMonitorProtocol(sock) +shell = QEMUMonitorProtocol(sock) shell.connect() models = shell.cmd("query-cpu-definitions") From 05908602429cf9d6fce9b60704b8395f6d295441 Mon Sep 17 00:00:00 2001 From: John Snow Date: Mon, 10 Jan 2022 18:28:59 -0500 Subject: [PATCH 17/17] scripts/render-block-graph: switch to AQMP Creating an instance of qemu.aqmp.ExecuteError is too involved here, so just drop the specificity down to a generic QMPError. Signed-off-by: John Snow Reviewed-by: Vladimir Sementsov-Ogievskiy Reviewed-by: Beraldo Leal --- scripts/render_block_graph.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/render_block_graph.py b/scripts/render_block_graph.py index 42288a3cfb..b33fb70d5e 100755 --- a/scripts/render_block_graph.py +++ b/scripts/render_block_graph.py @@ -25,10 +25,8 @@ import json from graphviz import Digraph sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'python')) -from qemu.qmp import ( - QEMUMonitorProtocol, - QMPResponseError, -) +from qemu.aqmp import QMPError +from qemu.aqmp.legacy import QEMUMonitorProtocol def perm(arr): @@ -104,7 +102,7 @@ class LibvirtGuest(): reply = json.loads(subprocess.check_output(ar)) if 'error' in reply: - raise QMPResponseError(reply) + raise QMPError(reply) return reply['return']