From 5e97a28a8b91efca0c731ee19805a1e25cfd26ea Mon Sep 17 00:00:00 2001 From: Mark Cave-Ayland Date: Wed, 30 Nov 2022 10:04:34 +0000 Subject: [PATCH 01/47] tcg: convert tcg/README to rst Convert tcg/README to rst and move it to docs/devel as a new "TCG Intermediate Representation" page. There are a few minor changes to improve the aesthetic of the final output which are as follows: - Rename the title from "Tiny Code Generator - Fabrice Bellard" to "TCG Intermediate Representation" - Remove the section numbering - Add the missing parameters to the ssadd_vec operations in the "Host vector operations" section - Change the path to the Atomic Operations document to use a proper reference - Replace tcg/README in tcg.rst with a proper reference to the new document Signed-off-by: Mark Cave-Ayland Reviewed-by: Fabiano Rosas Message-Id: <20221130100434.64207-2-mark.cave-ayland@ilande.co.uk> Signed-off-by: Richard Henderson --- docs/devel/atomics.rst | 2 + docs/devel/index-tcg.rst | 1 + docs/devel/tcg-ops.rst | 941 +++++++++++++++++++++++++++++++++++++++ docs/devel/tcg.rst | 2 +- tcg/README | 784 -------------------------------- 5 files changed, 945 insertions(+), 785 deletions(-) create mode 100644 docs/devel/tcg-ops.rst delete mode 100644 tcg/README diff --git a/docs/devel/atomics.rst b/docs/devel/atomics.rst index 52baa0736d..7957310071 100644 --- a/docs/devel/atomics.rst +++ b/docs/devel/atomics.rst @@ -1,3 +1,5 @@ +.. _atomics-ref: + ========================= Atomic operations in QEMU ========================= diff --git a/docs/devel/index-tcg.rst b/docs/devel/index-tcg.rst index 7b9760b26f..b44ff8b5a4 100644 --- a/docs/devel/index-tcg.rst +++ b/docs/devel/index-tcg.rst @@ -9,6 +9,7 @@ are only implementing things for HW accelerated hypervisors. :maxdepth: 2 tcg + tcg-ops decodetree multi-thread-tcg tcg-icount diff --git a/docs/devel/tcg-ops.rst b/docs/devel/tcg-ops.rst new file mode 100644 index 0000000000..9adc0c9b6c --- /dev/null +++ b/docs/devel/tcg-ops.rst @@ -0,0 +1,941 @@ +.. _tcg-ops-ref: + +******************************* +TCG Intermediate Representation +******************************* + +Introduction +============ + +TCG (Tiny Code Generator) began as a generic backend for a C +compiler. It was simplified to be used in QEMU. It also has its roots +in the QOP code generator written by Paul Brook. + +Definitions +=========== + +TCG receives RISC-like *TCG ops* and performs some optimizations on them, +including liveness analysis and trivial constant expression +evaluation. TCG ops are then implemented in the host CPU back end, +also known as the TCG target. + +The TCG *target* is the architecture for which we generate the +code. It is of course not the same as the "target" of QEMU which is +the emulated architecture. As TCG started as a generic C backend used +for cross compiling, it is assumed that the TCG target is different +from the host, although it is never the case for QEMU. + +In this document, we use *guest* to specify what architecture we are +emulating; *target* always means the TCG target, the machine on which +we are running QEMU. + +A TCG *function* corresponds to a QEMU Translated Block (TB). + +A TCG *temporary* is a variable only live in a basic block. Temporaries are allocated explicitly in each function. + +A TCG *local temporary* is a variable only live in a function. Local temporaries are allocated explicitly in each function. + +A TCG *global* is a variable which is live in all the functions +(equivalent of a C global variable). They are defined before the +functions defined. A TCG global can be a memory location (e.g. a QEMU +CPU register), a fixed host register (e.g. the QEMU CPU state pointer) +or a memory location which is stored in a register outside QEMU TBs +(not implemented yet). + +A TCG *basic block* corresponds to a list of instructions terminated +by a branch instruction. + +An operation with *undefined behavior* may result in a crash. + +An operation with *unspecified behavior* shall not crash. However, +the result may be one of several possibilities so may be considered +an *undefined result*. + +Intermediate representation +=========================== + +Introduction +------------ + +TCG instructions operate on variables which are temporaries, local +temporaries or globals. TCG instructions and variables are strongly +typed. Two types are supported: 32 bit integers and 64 bit +integers. Pointers are defined as an alias to 32 bit or 64 bit +integers depending on the TCG target word size. + +Each instruction has a fixed number of output variable operands, input +variable operands and always constant operands. + +The notable exception is the call instruction which has a variable +number of outputs and inputs. + +In the textual form, output operands usually come first, followed by +input operands, followed by constant operands. The output type is +included in the instruction name. Constants are prefixed with a '$'. + +.. code-block:: none + + add_i32 t0, t1, t2 /* (t0 <- t1 + t2) */ + + +Assumptions +----------- + +Basic blocks +^^^^^^^^^^^^ + +* Basic blocks end after branches (e.g. brcond_i32 instruction), + goto_tb and exit_tb instructions. + +* Basic blocks start after the end of a previous basic block, or at a + set_label instruction. + +After the end of a basic block, the content of temporaries is +destroyed, but local temporaries and globals are preserved. + +Floating point types +^^^^^^^^^^^^^^^^^^^^ + +* Floating point types are not supported yet + +Pointers +^^^^^^^^ + +* Depending on the TCG target, pointer size is 32 bit or 64 + bit. The type ``TCG_TYPE_PTR`` is an alias to ``TCG_TYPE_I32`` or + ``TCG_TYPE_I64``. + +Helpers +^^^^^^^ + +* Using the tcg_gen_helper_x_y it is possible to call any function + taking i32, i64 or pointer types. By default, before calling a helper, + all globals are stored at their canonical location and it is assumed + that the function can modify them. By default, the helper is allowed to + modify the CPU state or raise an exception. + + This can be overridden using the following function modifiers: + + - ``TCG_CALL_NO_READ_GLOBALS`` means that the helper does not read globals, + either directly or via an exception. They will not be saved to their + canonical locations before calling the helper. + + - ``TCG_CALL_NO_WRITE_GLOBALS`` means that the helper does not modify any globals. + They will only be saved to their canonical location before calling helpers, + but they won't be reloaded afterwards. + + - ``TCG_CALL_NO_SIDE_EFFECTS`` means that the call to the function is removed if + the return value is not used. + + Note that ``TCG_CALL_NO_READ_GLOBALS`` implies ``TCG_CALL_NO_WRITE_GLOBALS``. + + On some TCG targets (e.g. x86), several calling conventions are + supported. + +Branches +^^^^^^^^ + +* Use the instruction 'br' to jump to a label. + +Code Optimizations +------------------ + +When generating instructions, you can count on at least the following +optimizations: + +- Single instructions are simplified, e.g. + + .. code-block:: none + + and_i32 t0, t0, $0xffffffff + + is suppressed. + +- A liveness analysis is done at the basic block level. The + information is used to suppress moves from a dead variable to + another one. It is also used to remove instructions which compute + dead results. The later is especially useful for condition code + optimization in QEMU. + + In the following example: + + .. code-block:: none + + add_i32 t0, t1, t2 + add_i32 t0, t0, $1 + mov_i32 t0, $1 + + only the last instruction is kept. + + +Instruction Reference +===================== + +Function call +------------- + +.. list-table:: + + * - call ** ** ptr + + - | call function 'ptr' (pointer type) + | + | ** optional 32 bit or 64 bit return value + | ** optional 32 bit or 64 bit parameters + +Jumps/Labels +------------ + +.. list-table:: + + * - set_label $label + + - | Define label 'label' at the current program point. + + * - br $label + + - | Jump to label. + + * - brcond_i32/i64 *t0*, *t1*, *cond*, *label* + + - | Conditional jump if *t0* *cond* *t1* is true. *cond* can be: + | + | ``TCG_COND_EQ`` + | ``TCG_COND_NE`` + | ``TCG_COND_LT /* signed */`` + | ``TCG_COND_GE /* signed */`` + | ``TCG_COND_LE /* signed */`` + | ``TCG_COND_GT /* signed */`` + | ``TCG_COND_LTU /* unsigned */`` + | ``TCG_COND_GEU /* unsigned */`` + | ``TCG_COND_LEU /* unsigned */`` + | ``TCG_COND_GTU /* unsigned */`` + +Arithmetic +---------- + +.. list-table:: + + * - add_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* + *t2* + + * - sub_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* - *t2* + + * - neg_i32/i64 *t0*, *t1* + + - | *t0* = -*t1* (two's complement) + + * - mul_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* * *t2* + + * - div_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* / *t2* (signed) + | Undefined behavior if division by zero or overflow. + + * - divu_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* / *t2* (unsigned) + | Undefined behavior if division by zero. + + * - rem_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* % *t2* (signed) + | Undefined behavior if division by zero or overflow. + + * - remu_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* % *t2* (unsigned) + | Undefined behavior if division by zero. + + +Logical +------- + +.. list-table:: + + * - and_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* & *t2* + + * - or_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* | *t2* + + * - xor_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* ^ *t2* + + * - not_i32/i64 *t0*, *t1* + + - | *t0* = ~\ *t1* + + * - andc_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* & ~\ *t2* + + * - eqv_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = ~(*t1* ^ *t2*), or equivalently, *t0* = *t1* ^ ~\ *t2* + + * - nand_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = ~(*t1* & *t2*) + + * - nor_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = ~(*t1* | *t2*) + + * - orc_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* | ~\ *t2* + + * - clz_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* ? clz(*t1*) : *t2* + + * - ctz_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* ? ctz(*t1*) : *t2* + + * - ctpop_i32/i64 *t0*, *t1* + + - | *t0* = number of bits set in *t1* + | + | With *ctpop* short for "count population", matching + | the function name used in ``include/qemu/host-utils.h``. + + +Shifts/Rotates +-------------- + +.. list-table:: + + * - shl_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* << *t2* + | Unspecified behavior if *t2* < 0 or *t2* >= 32 (resp 64) + + * - shr_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* >> *t2* (unsigned) + | Unspecified behavior if *t2* < 0 or *t2* >= 32 (resp 64) + + * - sar_i32/i64 *t0*, *t1*, *t2* + + - | *t0* = *t1* >> *t2* (signed) + | Unspecified behavior if *t2* < 0 or *t2* >= 32 (resp 64) + + * - rotl_i32/i64 *t0*, *t1*, *t2* + + - | Rotation of *t2* bits to the left + | Unspecified behavior if *t2* < 0 or *t2* >= 32 (resp 64) + + * - rotr_i32/i64 *t0*, *t1*, *t2* + + - | Rotation of *t2* bits to the right. + | Unspecified behavior if *t2* < 0 or *t2* >= 32 (resp 64) + + +Misc +---- + +.. list-table:: + + * - mov_i32/i64 *t0*, *t1* + + - | *t0* = *t1* + | Move *t1* to *t0* (both operands must have the same type). + + * - ext8s_i32/i64 *t0*, *t1* + + ext8u_i32/i64 *t0*, *t1* + + ext16s_i32/i64 *t0*, *t1* + + ext16u_i32/i64 *t0*, *t1* + + ext32s_i64 *t0*, *t1* + + ext32u_i64 *t0*, *t1* + + - | 8, 16 or 32 bit sign/zero extension (both operands must have the same type) + + * - bswap16_i32/i64 *t0*, *t1*, *flags* + + - | 16 bit byte swap on the low bits of a 32/64 bit input. + | + | If *flags* & ``TCG_BSWAP_IZ``, then *t1* is known to be zero-extended from bit 15. + | If *flags* & ``TCG_BSWAP_OZ``, then *t0* will be zero-extended from bit 15. + | If *flags* & ``TCG_BSWAP_OS``, then *t0* will be sign-extended from bit 15. + | + | If neither ``TCG_BSWAP_OZ`` nor ``TCG_BSWAP_OS`` are set, then the bits of *t0* above bit 15 may contain any value. + + * - bswap32_i64 *t0*, *t1*, *flags* + + - | 32 bit byte swap on a 64-bit value. The flags are the same as for bswap16, + except they apply from bit 31 instead of bit 15. + + * - bswap32_i32 *t0*, *t1*, *flags* + + bswap64_i64 *t0*, *t1*, *flags* + + - | 32/64 bit byte swap. The flags are ignored, but still present + for consistency with the other bswap opcodes. + + * - discard_i32/i64 *t0* + + - | Indicate that the value of *t0* won't be used later. It is useful to + force dead code elimination. + + * - deposit_i32/i64 *dest*, *t1*, *t2*, *pos*, *len* + + - | Deposit *t2* as a bitfield into *t1*, placing the result in *dest*. + | + | The bitfield is described by *pos*/*len*, which are immediate values: + | + | *len* - the length of the bitfield + | *pos* - the position of the first bit, counting from the LSB + | + | For example, "deposit_i32 dest, t1, t2, 8, 4" indicates a 4-bit field + at bit 8. This operation would be equivalent to + | + | *dest* = (*t1* & ~0x0f00) | ((*t2* << 8) & 0x0f00) + + * - extract_i32/i64 *dest*, *t1*, *pos*, *len* + + sextract_i32/i64 *dest*, *t1*, *pos*, *len* + + - | Extract a bitfield from *t1*, placing the result in *dest*. + | + | The bitfield is described by *pos*/*len*, which are immediate values, + as above for deposit. For extract_*, the result will be extended + to the left with zeros; for sextract_*, the result will be extended + to the left with copies of the bitfield sign bit at *pos* + *len* - 1. + | + | For example, "sextract_i32 dest, t1, 8, 4" indicates a 4-bit field + at bit 8. This operation would be equivalent to + | + | *dest* = (*t1* << 20) >> 28 + | + | (using an arithmetic right shift). + + * - extract2_i32/i64 *dest*, *t1*, *t2*, *pos* + + - | For N = {32,64}, extract an N-bit quantity from the concatenation + of *t2*:*t1*, beginning at *pos*. The tcg_gen_extract2_{i32,i64} expander + accepts 0 <= *pos* <= N as inputs. The backend code generator will + not see either 0 or N as inputs for these opcodes. + + * - extrl_i64_i32 *t0*, *t1* + + - | For 64-bit hosts only, extract the low 32-bits of input *t1* and place it + into 32-bit output *t0*. Depending on the host, this may be a simple move, + or may require additional canonicalization. + + * - extrh_i64_i32 *t0*, *t1* + + - | For 64-bit hosts only, extract the high 32-bits of input *t1* and place it + into 32-bit output *t0*. Depending on the host, this may be a simple shift, + or may require additional canonicalization. + + +Conditional moves +----------------- + +.. list-table:: + + * - setcond_i32/i64 *dest*, *t1*, *t2*, *cond* + + - | *dest* = (*t1* *cond* *t2*) + | + | Set *dest* to 1 if (*t1* *cond* *t2*) is true, otherwise set to 0. + + * - movcond_i32/i64 *dest*, *c1*, *c2*, *v1*, *v2*, *cond* + + - | *dest* = (*c1* *cond* *c2* ? *v1* : *v2*) + | + | Set *dest* to *v1* if (*c1* *cond* *c2*) is true, otherwise set to *v2*. + + +Type conversions +---------------- + +.. list-table:: + + * - ext_i32_i64 *t0*, *t1* + + - | Convert *t1* (32 bit) to *t0* (64 bit) and does sign extension + + * - extu_i32_i64 *t0*, *t1* + + - | Convert *t1* (32 bit) to *t0* (64 bit) and does zero extension + + * - trunc_i64_i32 *t0*, *t1* + + - | Truncate *t1* (64 bit) to *t0* (32 bit) + + * - concat_i32_i64 *t0*, *t1*, *t2* + + - | Construct *t0* (64-bit) taking the low half from *t1* (32 bit) and the high half + from *t2* (32 bit). + + * - concat32_i64 *t0*, *t1*, *t2* + + - | Construct *t0* (64-bit) taking the low half from *t1* (64 bit) and the high half + from *t2* (64 bit). + + +Load/Store +---------- + +.. list-table:: + + * - ld_i32/i64 *t0*, *t1*, *offset* + + ld8s_i32/i64 *t0*, *t1*, *offset* + + ld8u_i32/i64 *t0*, *t1*, *offset* + + ld16s_i32/i64 *t0*, *t1*, *offset* + + ld16u_i32/i64 *t0*, *t1*, *offset* + + ld32s_i64 t0, *t1*, *offset* + + ld32u_i64 t0, *t1*, *offset* + + - | *t0* = read(*t1* + *offset*) + | + | Load 8, 16, 32 or 64 bits with or without sign extension from host memory. + *offset* must be a constant. + + * - st_i32/i64 *t0*, *t1*, *offset* + + st8_i32/i64 *t0*, *t1*, *offset* + + st16_i32/i64 *t0*, *t1*, *offset* + + st32_i64 *t0*, *t1*, *offset* + + - | write(*t0*, *t1* + *offset*) + | + | Write 8, 16, 32 or 64 bits to host memory. + +All this opcodes assume that the pointed host memory doesn't correspond +to a global. In the latter case the behaviour is unpredictable. + + +Multiword arithmetic support +---------------------------- + +.. list-table:: + + * - add2_i32/i64 *t0_low*, *t0_high*, *t1_low*, *t1_high*, *t2_low*, *t2_high* + + sub2_i32/i64 *t0_low*, *t0_high*, *t1_low*, *t1_high*, *t2_low*, *t2_high* + + - | Similar to add/sub, except that the double-word inputs *t1* and *t2* are + formed from two single-word arguments, and the double-word output *t0* + is returned in two single-word outputs. + + * - mulu2_i32/i64 *t0_low*, *t0_high*, *t1*, *t2* + + - | Similar to mul, except two unsigned inputs *t1* and *t2* yielding the full + double-word product *t0*. The latter is returned in two single-word outputs. + + * - muls2_i32/i64 *t0_low*, *t0_high*, *t1*, *t2* + + - | Similar to mulu2, except the two inputs *t1* and *t2* are signed. + + * - mulsh_i32/i64 *t0*, *t1*, *t2* + + muluh_i32/i64 *t0*, *t1*, *t2* + + - | Provide the high part of a signed or unsigned multiply, respectively. + | + | If mulu2/muls2 are not provided by the backend, the tcg-op generator + can obtain the same results by emitting a pair of opcodes, mul + muluh/mulsh. + + +Memory Barrier support +---------------------- + +.. list-table:: + + * - mb *<$arg>* + + - | Generate a target memory barrier instruction to ensure memory ordering + as being enforced by a corresponding guest memory barrier instruction. + | + | The ordering enforced by the backend may be stricter than the ordering + required by the guest. It cannot be weaker. This opcode takes a constant + argument which is required to generate the appropriate barrier + instruction. The backend should take care to emit the target barrier + instruction only when necessary i.e., for SMP guests and when MTTCG is + enabled. + | + | The guest translators should generate this opcode for all guest instructions + which have ordering side effects. + | + | Please see :ref:`atomics-ref` for more information on memory barriers. + + +64-bit guest on 32-bit host support +----------------------------------- + +The following opcodes are internal to TCG. Thus they are to be implemented by +32-bit host code generators, but are not to be emitted by guest translators. +They are emitted as needed by inline functions within ``tcg-op.h``. + +.. list-table:: + + * - brcond2_i32 *t0_low*, *t0_high*, *t1_low*, *t1_high*, *cond*, *label* + + - | Similar to brcond, except that the 64-bit values *t0* and *t1* + are formed from two 32-bit arguments. + + * - setcond2_i32 *dest*, *t1_low*, *t1_high*, *t2_low*, *t2_high*, *cond* + + - | Similar to setcond, except that the 64-bit values *t1* and *t2* are + formed from two 32-bit arguments. The result is a 32-bit value. + + +QEMU specific operations +------------------------ + +.. list-table:: + + * - exit_tb *t0* + + - | Exit the current TB and return the value *t0* (word type). + + * - goto_tb *index* + + - | Exit the current TB and jump to the TB index *index* (constant) if the + current TB was linked to this TB. Otherwise execute the next + instructions. Only indices 0 and 1 are valid and tcg_gen_goto_tb may be issued + at most once with each slot index per TB. + + * - lookup_and_goto_ptr *tb_addr* + + - | Look up a TB address *tb_addr* and jump to it if valid. If not valid, + jump to the TCG epilogue to go back to the exec loop. + | + | This operation is optional. If the TCG backend does not implement the + goto_ptr opcode, emitting this op is equivalent to emitting exit_tb(0). + + * - qemu_ld_i32/i64 *t0*, *t1*, *flags*, *memidx* + + qemu_st_i32/i64 *t0*, *t1*, *flags*, *memidx* + + qemu_st8_i32 *t0*, *t1*, *flags*, *memidx* + + - | Load data at the guest address *t1* into *t0*, or store data in *t0* at guest + address *t1*. The _i32/_i64 size applies to the size of the input/output + register *t0* only. The address *t1* is always sized according to the guest, + and the width of the memory operation is controlled by *flags*. + | + | Both *t0* and *t1* may be split into little-endian ordered pairs of registers + if dealing with 64-bit quantities on a 32-bit host. + | + | The *memidx* selects the qemu tlb index to use (e.g. user or kernel access). + The flags are the MemOp bits, selecting the sign, width, and endianness + of the memory access. + | + | For a 32-bit host, qemu_ld/st_i64 is guaranteed to only be used with a + 64-bit memory access specified in *flags*. + | + | For i386, qemu_st8_i32 is exactly like qemu_st_i32, except the size of + the memory operation is known to be 8-bit. This allows the backend to + provide a different set of register constraints. + + +Host vector operations +---------------------- + +All of the vector ops have two parameters, ``TCGOP_VECL`` & ``TCGOP_VECE``. +The former specifies the length of the vector in log2 64-bit units; the +latter specifies the length of the element (if applicable) in log2 8-bit units. +E.g. VECL = 1 -> 64 << 1 -> v128, and VECE = 2 -> 1 << 2 -> i32. + +.. list-table:: + + * - mov_vec *v0*, *v1* + ld_vec *v0*, *t1* + st_vec *v0*, *t1* + + - | Move, load and store. + + * - dup_vec *v0*, *r1* + + - | Duplicate the low N bits of *r1* into VECL/VECE copies across *v0*. + + * - dupi_vec *v0*, *c* + + - | Similarly, for a constant. + | Smaller values will be replicated to host register size by the expanders. + + * - dup2_vec *v0*, *r1*, *r2* + + - | Duplicate *r2*:*r1* into VECL/64 copies across *v0*. This opcode is + only present for 32-bit hosts. + + * - add_vec *v0*, *v1*, *v2* + + - | *v0* = *v1* + *v2*, in elements across the vector. + + * - sub_vec *v0*, *v1*, *v2* + + - | Similarly, *v0* = *v1* - *v2*. + + * - mul_vec *v0*, *v1*, *v2* + + - | Similarly, *v0* = *v1* * *v2*. + + * - neg_vec *v0*, *v1* + + - | Similarly, *v0* = -*v1*. + + * - abs_vec *v0*, *v1* + + - | Similarly, *v0* = *v1* < 0 ? -*v1* : *v1*, in elements across the vector. + + * - smin_vec *v0*, *v1*, *v2* + + umin_vec *v0*, *v1*, *v2* + + - | Similarly, *v0* = MIN(*v1*, *v2*), for signed and unsigned element types. + + * - smax_vec *v0*, *v1*, *v2* + + umax_vec *v0*, *v1*, *v2* + + - | Similarly, *v0* = MAX(*v1*, *v2*), for signed and unsigned element types. + + * - ssadd_vec *v0*, *v1*, *v2* + + sssub_vec *v0*, *v1*, *v2* + + usadd_vec *v0*, *v1*, *v2* + + ussub_vec *v0*, *v1*, *v2* + + - | Signed and unsigned saturating addition and subtraction. + | + | If the true result is not representable within the element type, the + element is set to the minimum or maximum value for the type. + + * - and_vec *v0*, *v1*, *v2* + + or_vec *v0*, *v1*, *v2* + + xor_vec *v0*, *v1*, *v2* + + andc_vec *v0*, *v1*, *v2* + + orc_vec *v0*, *v1*, *v2* + + not_vec *v0*, *v1* + + - | Similarly, logical operations with and without complement. + | + | Note that VECE is unused. + + * - shli_vec *v0*, *v1*, *i2* + + shls_vec *v0*, *v1*, *s2* + + - | Shift all elements from v1 by a scalar *i2*/*s2*. I.e. + + .. code-block:: c + + for (i = 0; i < VECL/VECE; ++i) { + v0[i] = v1[i] << s2; + } + + * - shri_vec *v0*, *v1*, *i2* + + sari_vec *v0*, *v1*, *i2* + + rotli_vec *v0*, *v1*, *i2* + + shrs_vec *v0*, *v1*, *s2* + + sars_vec *v0*, *v1*, *s2* + + - | Similarly for logical and arithmetic right shift, and left rotate. + + * - shlv_vec *v0*, *v1*, *v2* + + - | Shift elements from *v1* by elements from *v2*. I.e. + + .. code-block:: c + + for (i = 0; i < VECL/VECE; ++i) { + v0[i] = v1[i] << v2[i]; + } + + * - shrv_vec *v0*, *v1*, *v2* + + sarv_vec *v0*, *v1*, *v2* + + rotlv_vec *v0*, *v1*, *v2* + + rotrv_vec *v0*, *v1*, *v2* + + - | Similarly for logical and arithmetic right shift, and rotates. + + * - cmp_vec *v0*, *v1*, *v2*, *cond* + + - | Compare vectors by element, storing -1 for true and 0 for false. + + * - bitsel_vec *v0*, *v1*, *v2*, *v3* + + - | Bitwise select, *v0* = (*v2* & *v1*) | (*v3* & ~\ *v1*), across the entire vector. + + * - cmpsel_vec *v0*, *c1*, *c2*, *v3*, *v4*, *cond* + + - | Select elements based on comparison results: + + .. code-block:: c + + for (i = 0; i < n; ++i) { + v0[i] = (c1[i] cond c2[i]) ? v3[i] : v4[i]. + } + +**Note 1**: Some shortcuts are defined when the last operand is known to be +a constant (e.g. addi for add, movi for mov). + +**Note 2**: When using TCG, the opcodes must never be generated directly +as some of them may not be available as "real" opcodes. Always use the +function tcg_gen_xxx(args). + + +Backend +======= + +``tcg-target.h`` contains the target specific definitions. ``tcg-target.c.inc`` +contains the target specific code; it is #included by ``tcg/tcg.c``, rather +than being a standalone C file. + +Assumptions +----------- + +The target word size (``TCG_TARGET_REG_BITS``) is expected to be 32 bit or +64 bit. It is expected that the pointer has the same size as the word. + +On a 32 bit target, all 64 bit operations are converted to 32 bits. A +few specific operations must be implemented to allow it (see add2_i32, +sub2_i32, brcond2_i32). + +On a 64 bit target, the values are transferred between 32 and 64-bit +registers using the following ops: + +- trunc_shr_i64_i32 +- ext_i32_i64 +- extu_i32_i64 + +They ensure that the values are correctly truncated or extended when +moved from a 32-bit to a 64-bit register or vice-versa. Note that the +trunc_shr_i64_i32 is an optional op. It is not necessary to implement +it if all the following conditions are met: + +- 64-bit registers can hold 32-bit values +- 32-bit values in a 64-bit register do not need to stay zero or + sign extended +- all 32-bit TCG ops ignore the high part of 64-bit registers + +Floating point operations are not supported in this version. A +previous incarnation of the code generator had full support of them, +but it is better to concentrate on integer operations first. + +Constraints +---------------- + +GCC like constraints are used to define the constraints of every +instruction. Memory constraints are not supported in this +version. Aliases are specified in the input operands as for GCC. + +The same register may be used for both an input and an output, even when +they are not explicitly aliased. If an op expands to multiple target +instructions then care must be taken to avoid clobbering input values. +GCC style "early clobber" outputs are supported, with '``&``'. + +A target can define specific register or constant constraints. If an +operation uses a constant input constraint which does not allow all +constants, it must also accept registers in order to have a fallback. +The constraint '``i``' is defined generically to accept any constant. +The constraint '``r``' is not defined generically, but is consistently +used by each backend to indicate all registers. + +The movi_i32 and movi_i64 operations must accept any constants. + +The mov_i32 and mov_i64 operations must accept any registers of the +same type. + +The ld/st/sti instructions must accept signed 32 bit constant offsets. +This can be implemented by reserving a specific register in which to +compute the address if the offset is too big. + +The ld/st instructions must accept any destination (ld) or source (st) +register. + +The sti instruction may fail if it cannot store the given constant. + +Function call assumptions +------------------------- + +- The only supported types for parameters and return value are: 32 and + 64 bit integers and pointer. +- The stack grows downwards. +- The first N parameters are passed in registers. +- The next parameters are passed on the stack by storing them as words. +- Some registers are clobbered during the call. +- The function can return 0 or 1 value in registers. On a 32 bit + target, functions must be able to return 2 values in registers for + 64 bit return type. + + +Recommended coding rules for best performance +============================================= + +- Use globals to represent the parts of the QEMU CPU state which are + often modified, e.g. the integer registers and the condition + codes. TCG will be able to use host registers to store them. + +- Avoid globals stored in fixed registers. They must be used only to + store the pointer to the CPU state and possibly to store a pointer + to a register window. + +- Use temporaries. Use local temporaries only when really needed, + e.g. when you need to use a value after a jump. Local temporaries + introduce a performance hit in the current TCG implementation: their + content is saved to memory at end of each basic block. + +- Free temporaries and local temporaries when they are no longer used + (tcg_temp_free). Since tcg_const_x() also creates a temporary, you + should free it after it is used. Freeing temporaries does not yield + a better generated code, but it reduces the memory usage of TCG and + the speed of the translation. + +- Don't hesitate to use helpers for complicated or seldom used guest + instructions. There is little performance advantage in using TCG to + implement guest instructions taking more than about twenty TCG + instructions. Note that this rule of thumb is more applicable to + helpers doing complex logic or arithmetic, where the C compiler has + scope to do a good job of optimisation; it is less relevant where + the instruction is mostly doing loads and stores, and in those cases + inline TCG may still be faster for longer sequences. + +- The hard limit on the number of TCG instructions you can generate + per guest instruction is set by ``MAX_OP_PER_INSTR`` in ``exec-all.h`` -- + you cannot exceed this without risking a buffer overrun. + +- Use the 'discard' instruction if you know that TCG won't be able to + prove that a given global is "dead" at a given program point. The + x86 guest uses it to improve the condition codes optimisation. diff --git a/docs/devel/tcg.rst b/docs/devel/tcg.rst index a65fb7b1c4..136a7a0d96 100644 --- a/docs/devel/tcg.rst +++ b/docs/devel/tcg.rst @@ -9,7 +9,7 @@ which make it relatively easily portable and simple while achieving good performances. QEMU's dynamic translation backend is called TCG, for "Tiny Code -Generator". For more information, please take a look at ``tcg/README``. +Generator". For more information, please take a look at :ref:`tcg-ops-ref`. The following sections outline some notable features and implementation details of QEMU's dynamic translator. diff --git a/tcg/README b/tcg/README deleted file mode 100644 index bc15cc3b32..0000000000 --- a/tcg/README +++ /dev/null @@ -1,784 +0,0 @@ -Tiny Code Generator - Fabrice Bellard. - -1) Introduction - -TCG (Tiny Code Generator) began as a generic backend for a C -compiler. It was simplified to be used in QEMU. It also has its roots -in the QOP code generator written by Paul Brook. - -2) Definitions - -TCG receives RISC-like "TCG ops" and performs some optimizations on them, -including liveness analysis and trivial constant expression -evaluation. TCG ops are then implemented in the host CPU back end, -also known as the TCG "target". - -The TCG "target" is the architecture for which we generate the -code. It is of course not the same as the "target" of QEMU which is -the emulated architecture. As TCG started as a generic C backend used -for cross compiling, it is assumed that the TCG target is different -from the host, although it is never the case for QEMU. - -In this document, we use "guest" to specify what architecture we are -emulating; "target" always means the TCG target, the machine on which -we are running QEMU. - -A TCG "function" corresponds to a QEMU Translated Block (TB). - -A TCG "temporary" is a variable only live in a basic -block. Temporaries are allocated explicitly in each function. - -A TCG "local temporary" is a variable only live in a function. Local -temporaries are allocated explicitly in each function. - -A TCG "global" is a variable which is live in all the functions -(equivalent of a C global variable). They are defined before the -functions defined. A TCG global can be a memory location (e.g. a QEMU -CPU register), a fixed host register (e.g. the QEMU CPU state pointer) -or a memory location which is stored in a register outside QEMU TBs -(not implemented yet). - -A TCG "basic block" corresponds to a list of instructions terminated -by a branch instruction. - -An operation with "undefined behavior" may result in a crash. - -An operation with "unspecified behavior" shall not crash. However, -the result may be one of several possibilities so may be considered -an "undefined result". - -3) Intermediate representation - -3.1) Introduction - -TCG instructions operate on variables which are temporaries, local -temporaries or globals. TCG instructions and variables are strongly -typed. Two types are supported: 32 bit integers and 64 bit -integers. Pointers are defined as an alias to 32 bit or 64 bit -integers depending on the TCG target word size. - -Each instruction has a fixed number of output variable operands, input -variable operands and always constant operands. - -The notable exception is the call instruction which has a variable -number of outputs and inputs. - -In the textual form, output operands usually come first, followed by -input operands, followed by constant operands. The output type is -included in the instruction name. Constants are prefixed with a '$'. - -add_i32 t0, t1, t2 (t0 <- t1 + t2) - -3.2) Assumptions - -* Basic blocks - -- Basic blocks end after branches (e.g. brcond_i32 instruction), - goto_tb and exit_tb instructions. -- Basic blocks start after the end of a previous basic block, or at a - set_label instruction. - -After the end of a basic block, the content of temporaries is -destroyed, but local temporaries and globals are preserved. - -* Floating point types are not supported yet - -* Pointers: depending on the TCG target, pointer size is 32 bit or 64 - bit. The type TCG_TYPE_PTR is an alias to TCG_TYPE_I32 or - TCG_TYPE_I64. - -* Helpers: - -Using the tcg_gen_helper_x_y it is possible to call any function -taking i32, i64 or pointer types. By default, before calling a helper, -all globals are stored at their canonical location and it is assumed -that the function can modify them. By default, the helper is allowed to -modify the CPU state or raise an exception. - -This can be overridden using the following function modifiers: -- TCG_CALL_NO_READ_GLOBALS means that the helper does not read globals, - either directly or via an exception. They will not be saved to their - canonical locations before calling the helper. -- TCG_CALL_NO_WRITE_GLOBALS means that the helper does not modify any globals. - They will only be saved to their canonical location before calling helpers, - but they won't be reloaded afterwards. -- TCG_CALL_NO_SIDE_EFFECTS means that the call to the function is removed if - the return value is not used. - -Note that TCG_CALL_NO_READ_GLOBALS implies TCG_CALL_NO_WRITE_GLOBALS. - -On some TCG targets (e.g. x86), several calling conventions are -supported. - -* Branches: - -Use the instruction 'br' to jump to a label. - -3.3) Code Optimizations - -When generating instructions, you can count on at least the following -optimizations: - -- Single instructions are simplified, e.g. - - and_i32 t0, t0, $0xffffffff - - is suppressed. - -- A liveness analysis is done at the basic block level. The - information is used to suppress moves from a dead variable to - another one. It is also used to remove instructions which compute - dead results. The later is especially useful for condition code - optimization in QEMU. - - In the following example: - - add_i32 t0, t1, t2 - add_i32 t0, t0, $1 - mov_i32 t0, $1 - - only the last instruction is kept. - -3.4) Instruction Reference - -********* Function call - -* call ptr - -call function 'ptr' (pointer type) - - optional 32 bit or 64 bit return value - optional 32 bit or 64 bit parameters - -********* Jumps/Labels - -* set_label $label - -Define label 'label' at the current program point. - -* br $label - -Jump to label. - -* brcond_i32/i64 t0, t1, cond, label - -Conditional jump if t0 cond t1 is true. cond can be: - TCG_COND_EQ - TCG_COND_NE - TCG_COND_LT /* signed */ - TCG_COND_GE /* signed */ - TCG_COND_LE /* signed */ - TCG_COND_GT /* signed */ - TCG_COND_LTU /* unsigned */ - TCG_COND_GEU /* unsigned */ - TCG_COND_LEU /* unsigned */ - TCG_COND_GTU /* unsigned */ - -********* Arithmetic - -* add_i32/i64 t0, t1, t2 - -t0=t1+t2 - -* sub_i32/i64 t0, t1, t2 - -t0=t1-t2 - -* neg_i32/i64 t0, t1 - -t0=-t1 (two's complement) - -* mul_i32/i64 t0, t1, t2 - -t0=t1*t2 - -* div_i32/i64 t0, t1, t2 - -t0=t1/t2 (signed). Undefined behavior if division by zero or overflow. - -* divu_i32/i64 t0, t1, t2 - -t0=t1/t2 (unsigned). Undefined behavior if division by zero. - -* rem_i32/i64 t0, t1, t2 - -t0=t1%t2 (signed). Undefined behavior if division by zero or overflow. - -* remu_i32/i64 t0, t1, t2 - -t0=t1%t2 (unsigned). Undefined behavior if division by zero. - -********* Logical - -* and_i32/i64 t0, t1, t2 - -t0=t1&t2 - -* or_i32/i64 t0, t1, t2 - -t0=t1|t2 - -* xor_i32/i64 t0, t1, t2 - -t0=t1^t2 - -* not_i32/i64 t0, t1 - -t0=~t1 - -* andc_i32/i64 t0, t1, t2 - -t0=t1&~t2 - -* eqv_i32/i64 t0, t1, t2 - -t0=~(t1^t2), or equivalently, t0=t1^~t2 - -* nand_i32/i64 t0, t1, t2 - -t0=~(t1&t2) - -* nor_i32/i64 t0, t1, t2 - -t0=~(t1|t2) - -* orc_i32/i64 t0, t1, t2 - -t0=t1|~t2 - -* clz_i32/i64 t0, t1, t2 - -t0 = t1 ? clz(t1) : t2 - -* ctz_i32/i64 t0, t1, t2 - -t0 = t1 ? ctz(t1) : t2 - -* ctpop_i32/i64 t0, t1 - -t0 = number of bits set in t1 -With "ctpop" short for "count population", matching -the function name used in include/qemu/host-utils.h. - -********* Shifts/Rotates - -* shl_i32/i64 t0, t1, t2 - -t0=t1 << t2. Unspecified behavior if t2 < 0 or t2 >= 32 (resp 64) - -* shr_i32/i64 t0, t1, t2 - -t0=t1 >> t2 (unsigned). Unspecified behavior if t2 < 0 or t2 >= 32 (resp 64) - -* sar_i32/i64 t0, t1, t2 - -t0=t1 >> t2 (signed). Unspecified behavior if t2 < 0 or t2 >= 32 (resp 64) - -* rotl_i32/i64 t0, t1, t2 - -Rotation of t2 bits to the left. -Unspecified behavior if t2 < 0 or t2 >= 32 (resp 64) - -* rotr_i32/i64 t0, t1, t2 - -Rotation of t2 bits to the right. -Unspecified behavior if t2 < 0 or t2 >= 32 (resp 64) - -********* Misc - -* mov_i32/i64 t0, t1 - -t0 = t1 - -Move t1 to t0 (both operands must have the same type). - -* ext8s_i32/i64 t0, t1 -ext8u_i32/i64 t0, t1 -ext16s_i32/i64 t0, t1 -ext16u_i32/i64 t0, t1 -ext32s_i64 t0, t1 -ext32u_i64 t0, t1 - -8, 16 or 32 bit sign/zero extension (both operands must have the same type) - -* bswap16_i32/i64 t0, t1, flags - -16 bit byte swap on the low bits of a 32/64 bit input. -If flags & TCG_BSWAP_IZ, then t1 is known to be zero-extended from bit 15. -If flags & TCG_BSWAP_OZ, then t0 will be zero-extended from bit 15. -If flags & TCG_BSWAP_OS, then t0 will be sign-extended from bit 15. -If neither TCG_BSWAP_OZ nor TCG_BSWAP_OS are set, then the bits of -t0 above bit 15 may contain any value. - -* bswap32_i64 t0, t1, flags - -32 bit byte swap on a 64-bit value. The flags are the same as for bswap16, -except they apply from bit 31 instead of bit 15. - -* bswap32_i32 t0, t1, flags -* bswap64_i64 t0, t1, flags - -32/64 bit byte swap. The flags are ignored, but still present -for consistency with the other bswap opcodes. - -* discard_i32/i64 t0 - -Indicate that the value of t0 won't be used later. It is useful to -force dead code elimination. - -* deposit_i32/i64 dest, t1, t2, pos, len - -Deposit T2 as a bitfield into T1, placing the result in DEST. -The bitfield is described by POS/LEN, which are immediate values: - - LEN - the length of the bitfield - POS - the position of the first bit, counting from the LSB - -For example, "deposit_i32 dest, t1, t2, 8, 4" indicates a 4-bit field -at bit 8. This operation would be equivalent to - - dest = (t1 & ~0x0f00) | ((t2 << 8) & 0x0f00) - -* extract_i32/i64 dest, t1, pos, len -* sextract_i32/i64 dest, t1, pos, len - -Extract a bitfield from T1, placing the result in DEST. -The bitfield is described by POS/LEN, which are immediate values, -as above for deposit. For extract_*, the result will be extended -to the left with zeros; for sextract_*, the result will be extended -to the left with copies of the bitfield sign bit at pos + len - 1. - -For example, "sextract_i32 dest, t1, 8, 4" indicates a 4-bit field -at bit 8. This operation would be equivalent to - - dest = (t1 << 20) >> 28 - -(using an arithmetic right shift). - -* extract2_i32/i64 dest, t1, t2, pos - -For N = {32,64}, extract an N-bit quantity from the concatenation -of t2:t1, beginning at pos. The tcg_gen_extract2_{i32,i64} expander -accepts 0 <= pos <= N as inputs. The backend code generator will -not see either 0 or N as inputs for these opcodes. - -* extrl_i64_i32 t0, t1 - -For 64-bit hosts only, extract the low 32-bits of input T1 and place it -into 32-bit output T0. Depending on the host, this may be a simple move, -or may require additional canonicalization. - -* extrh_i64_i32 t0, t1 - -For 64-bit hosts only, extract the high 32-bits of input T1 and place it -into 32-bit output T0. Depending on the host, this may be a simple shift, -or may require additional canonicalization. - -********* Conditional moves - -* setcond_i32/i64 dest, t1, t2, cond - -dest = (t1 cond t2) - -Set DEST to 1 if (T1 cond T2) is true, otherwise set to 0. - -* movcond_i32/i64 dest, c1, c2, v1, v2, cond - -dest = (c1 cond c2 ? v1 : v2) - -Set DEST to V1 if (C1 cond C2) is true, otherwise set to V2. - -********* Type conversions - -* ext_i32_i64 t0, t1 -Convert t1 (32 bit) to t0 (64 bit) and does sign extension - -* extu_i32_i64 t0, t1 -Convert t1 (32 bit) to t0 (64 bit) and does zero extension - -* trunc_i64_i32 t0, t1 -Truncate t1 (64 bit) to t0 (32 bit) - -* concat_i32_i64 t0, t1, t2 -Construct t0 (64-bit) taking the low half from t1 (32 bit) and the high half -from t2 (32 bit). - -* concat32_i64 t0, t1, t2 -Construct t0 (64-bit) taking the low half from t1 (64 bit) and the high half -from t2 (64 bit). - -********* Load/Store - -* ld_i32/i64 t0, t1, offset -ld8s_i32/i64 t0, t1, offset -ld8u_i32/i64 t0, t1, offset -ld16s_i32/i64 t0, t1, offset -ld16u_i32/i64 t0, t1, offset -ld32s_i64 t0, t1, offset -ld32u_i64 t0, t1, offset - -t0 = read(t1 + offset) -Load 8, 16, 32 or 64 bits with or without sign extension from host memory. -offset must be a constant. - -* st_i32/i64 t0, t1, offset -st8_i32/i64 t0, t1, offset -st16_i32/i64 t0, t1, offset -st32_i64 t0, t1, offset - -write(t0, t1 + offset) -Write 8, 16, 32 or 64 bits to host memory. - -All this opcodes assume that the pointed host memory doesn't correspond -to a global. In the latter case the behaviour is unpredictable. - -********* Multiword arithmetic support - -* add2_i32/i64 t0_low, t0_high, t1_low, t1_high, t2_low, t2_high -* sub2_i32/i64 t0_low, t0_high, t1_low, t1_high, t2_low, t2_high - -Similar to add/sub, except that the double-word inputs T1 and T2 are -formed from two single-word arguments, and the double-word output T0 -is returned in two single-word outputs. - -* mulu2_i32/i64 t0_low, t0_high, t1, t2 - -Similar to mul, except two unsigned inputs T1 and T2 yielding the full -double-word product T0. The later is returned in two single-word outputs. - -* muls2_i32/i64 t0_low, t0_high, t1, t2 - -Similar to mulu2, except the two inputs T1 and T2 are signed. - -* mulsh_i32/i64 t0, t1, t2 -* muluh_i32/i64 t0, t1, t2 - -Provide the high part of a signed or unsigned multiply, respectively. -If mulu2/muls2 are not provided by the backend, the tcg-op generator -can obtain the same results can be obtained by emitting a pair of -opcodes, mul+muluh/mulsh. - -********* Memory Barrier support - -* mb <$arg> - -Generate a target memory barrier instruction to ensure memory ordering as being -enforced by a corresponding guest memory barrier instruction. The ordering -enforced by the backend may be stricter than the ordering required by the guest. -It cannot be weaker. This opcode takes a constant argument which is required to -generate the appropriate barrier instruction. The backend should take care to -emit the target barrier instruction only when necessary i.e., for SMP guests and -when MTTCG is enabled. - -The guest translators should generate this opcode for all guest instructions -which have ordering side effects. - -Please see docs/devel/atomics.rst for more information on memory barriers. - -********* 64-bit guest on 32-bit host support - -The following opcodes are internal to TCG. Thus they are to be implemented by -32-bit host code generators, but are not to be emitted by guest translators. -They are emitted as needed by inline functions within "tcg-op.h". - -* brcond2_i32 t0_low, t0_high, t1_low, t1_high, cond, label - -Similar to brcond, except that the 64-bit values T0 and T1 -are formed from two 32-bit arguments. - -* setcond2_i32 dest, t1_low, t1_high, t2_low, t2_high, cond - -Similar to setcond, except that the 64-bit values T1 and T2 are -formed from two 32-bit arguments. The result is a 32-bit value. - -********* QEMU specific operations - -* exit_tb t0 - -Exit the current TB and return the value t0 (word type). - -* goto_tb index - -Exit the current TB and jump to the TB index 'index' (constant) if the -current TB was linked to this TB. Otherwise execute the next -instructions. Only indices 0 and 1 are valid and tcg_gen_goto_tb may be issued -at most once with each slot index per TB. - -* lookup_and_goto_ptr tb_addr - -Look up a TB address ('tb_addr') and jump to it if valid. If not valid, -jump to the TCG epilogue to go back to the exec loop. - -This operation is optional. If the TCG backend does not implement the -goto_ptr opcode, emitting this op is equivalent to emitting exit_tb(0). - -* qemu_ld_i32/i64 t0, t1, flags, memidx -* qemu_st_i32/i64 t0, t1, flags, memidx -* qemu_st8_i32 t0, t1, flags, memidx - -Load data at the guest address t1 into t0, or store data in t0 at guest -address t1. The _i32/_i64 size applies to the size of the input/output -register t0 only. The address t1 is always sized according to the guest, -and the width of the memory operation is controlled by flags. - -Both t0 and t1 may be split into little-endian ordered pairs of registers -if dealing with 64-bit quantities on a 32-bit host. - -The memidx selects the qemu tlb index to use (e.g. user or kernel access). -The flags are the MemOp bits, selecting the sign, width, and endianness -of the memory access. - -For a 32-bit host, qemu_ld/st_i64 is guaranteed to only be used with a -64-bit memory access specified in flags. - -For i386, qemu_st8_i32 is exactly like qemu_st_i32, except the size of -the memory operation is known to be 8-bit. This allows the backend to -provide a different set of register constraints. - -********* Host vector operations - -All of the vector ops have two parameters, TCGOP_VECL & TCGOP_VECE. -The former specifies the length of the vector in log2 64-bit units; the -later specifies the length of the element (if applicable) in log2 8-bit units. -E.g. VECL=1 -> 64 << 1 -> v128, and VECE=2 -> 1 << 2 -> i32. - -* mov_vec v0, v1 -* ld_vec v0, t1 -* st_vec v0, t1 - - Move, load and store. - -* dup_vec v0, r1 - - Duplicate the low N bits of R1 into VECL/VECE copies across V0. - -* dupi_vec v0, c - - Similarly, for a constant. - Smaller values will be replicated to host register size by the expanders. - -* dup2_vec v0, r1, r2 - - Duplicate r2:r1 into VECL/64 copies across V0. This opcode is - only present for 32-bit hosts. - -* add_vec v0, v1, v2 - - v0 = v1 + v2, in elements across the vector. - -* sub_vec v0, v1, v2 - - Similarly, v0 = v1 - v2. - -* mul_vec v0, v1, v2 - - Similarly, v0 = v1 * v2. - -* neg_vec v0, v1 - - Similarly, v0 = -v1. - -* abs_vec v0, v1 - - Similarly, v0 = v1 < 0 ? -v1 : v1, in elements across the vector. - -* smin_vec: -* umin_vec: - - Similarly, v0 = MIN(v1, v2), for signed and unsigned element types. - -* smax_vec: -* umax_vec: - - Similarly, v0 = MAX(v1, v2), for signed and unsigned element types. - -* ssadd_vec: -* sssub_vec: -* usadd_vec: -* ussub_vec: - - Signed and unsigned saturating addition and subtraction. If the true - result is not representable within the element type, the element is - set to the minimum or maximum value for the type. - -* and_vec v0, v1, v2 -* or_vec v0, v1, v2 -* xor_vec v0, v1, v2 -* andc_vec v0, v1, v2 -* orc_vec v0, v1, v2 -* not_vec v0, v1 - - Similarly, logical operations with and without complement. - Note that VECE is unused. - -* shli_vec v0, v1, i2 -* shls_vec v0, v1, s2 - - Shift all elements from v1 by a scalar i2/s2. I.e. - - for (i = 0; i < VECL/VECE; ++i) { - v0[i] = v1[i] << s2; - } - -* shri_vec v0, v1, i2 -* sari_vec v0, v1, i2 -* rotli_vec v0, v1, i2 -* shrs_vec v0, v1, s2 -* sars_vec v0, v1, s2 - - Similarly for logical and arithmetic right shift, and left rotate. - -* shlv_vec v0, v1, v2 - - Shift elements from v1 by elements from v2. I.e. - - for (i = 0; i < VECL/VECE; ++i) { - v0[i] = v1[i] << v2[i]; - } - -* shrv_vec v0, v1, v2 -* sarv_vec v0, v1, v2 -* rotlv_vec v0, v1, v2 -* rotrv_vec v0, v1, v2 - - Similarly for logical and arithmetic right shift, and rotates. - -* cmp_vec v0, v1, v2, cond - - Compare vectors by element, storing -1 for true and 0 for false. - -* bitsel_vec v0, v1, v2, v3 - - Bitwise select, v0 = (v2 & v1) | (v3 & ~v1), across the entire vector. - -* cmpsel_vec v0, c1, c2, v3, v4, cond - - Select elements based on comparison results: - for (i = 0; i < n; ++i) { - v0[i] = (c1[i] cond c2[i]) ? v3[i] : v4[i]. - } - -********* - -Note 1: Some shortcuts are defined when the last operand is known to be -a constant (e.g. addi for add, movi for mov). - -Note 2: When using TCG, the opcodes must never be generated directly -as some of them may not be available as "real" opcodes. Always use the -function tcg_gen_xxx(args). - -4) Backend - -tcg-target.h contains the target specific definitions. tcg-target.c.inc -contains the target specific code; it is #included by tcg/tcg.c, rather -than being a standalone C file. - -4.1) Assumptions - -The target word size (TCG_TARGET_REG_BITS) is expected to be 32 bit or -64 bit. It is expected that the pointer has the same size as the word. - -On a 32 bit target, all 64 bit operations are converted to 32 bits. A -few specific operations must be implemented to allow it (see add2_i32, -sub2_i32, brcond2_i32). - -On a 64 bit target, the values are transferred between 32 and 64-bit -registers using the following ops: -- trunc_shr_i64_i32 -- ext_i32_i64 -- extu_i32_i64 - -They ensure that the values are correctly truncated or extended when -moved from a 32-bit to a 64-bit register or vice-versa. Note that the -trunc_shr_i64_i32 is an optional op. It is not necessary to implement -it if all the following conditions are met: -- 64-bit registers can hold 32-bit values -- 32-bit values in a 64-bit register do not need to stay zero or - sign extended -- all 32-bit TCG ops ignore the high part of 64-bit registers - -Floating point operations are not supported in this version. A -previous incarnation of the code generator had full support of them, -but it is better to concentrate on integer operations first. - -4.2) Constraints - -GCC like constraints are used to define the constraints of every -instruction. Memory constraints are not supported in this -version. Aliases are specified in the input operands as for GCC. - -The same register may be used for both an input and an output, even when -they are not explicitly aliased. If an op expands to multiple target -instructions then care must be taken to avoid clobbering input values. -GCC style "early clobber" outputs are supported, with '&'. - -A target can define specific register or constant constraints. If an -operation uses a constant input constraint which does not allow all -constants, it must also accept registers in order to have a fallback. -The constraint 'i' is defined generically to accept any constant. -The constraint 'r' is not defined generically, but is consistently -used by each backend to indicate all registers. - -The movi_i32 and movi_i64 operations must accept any constants. - -The mov_i32 and mov_i64 operations must accept any registers of the -same type. - -The ld/st/sti instructions must accept signed 32 bit constant offsets. -This can be implemented by reserving a specific register in which to -compute the address if the offset is too big. - -The ld/st instructions must accept any destination (ld) or source (st) -register. - -The sti instruction may fail if it cannot store the given constant. - -4.3) Function call assumptions - -- The only supported types for parameters and return value are: 32 and - 64 bit integers and pointer. -- The stack grows downwards. -- The first N parameters are passed in registers. -- The next parameters are passed on the stack by storing them as words. -- Some registers are clobbered during the call. -- The function can return 0 or 1 value in registers. On a 32 bit - target, functions must be able to return 2 values in registers for - 64 bit return type. - -5) Recommended coding rules for best performance - -- Use globals to represent the parts of the QEMU CPU state which are - often modified, e.g. the integer registers and the condition - codes. TCG will be able to use host registers to store them. - -- Avoid globals stored in fixed registers. They must be used only to - store the pointer to the CPU state and possibly to store a pointer - to a register window. - -- Use temporaries. Use local temporaries only when really needed, - e.g. when you need to use a value after a jump. Local temporaries - introduce a performance hit in the current TCG implementation: their - content is saved to memory at end of each basic block. - -- Free temporaries and local temporaries when they are no longer used - (tcg_temp_free). Since tcg_const_x() also creates a temporary, you - should free it after it is used. Freeing temporaries does not yield - a better generated code, but it reduces the memory usage of TCG and - the speed of the translation. - -- Don't hesitate to use helpers for complicated or seldom used guest - instructions. There is little performance advantage in using TCG to - implement guest instructions taking more than about twenty TCG - instructions. Note that this rule of thumb is more applicable to - helpers doing complex logic or arithmetic, where the C compiler has - scope to do a good job of optimisation; it is less relevant where - the instruction is mostly doing loads and stores, and in those cases - inline TCG may still be faster for longer sequences. - -- The hard limit on the number of TCG instructions you can generate - per guest instruction is set by MAX_OP_PER_INSTR in exec-all.h -- - you cannot exceed this without risking a buffer overrun. - -- Use the 'discard' instruction if you know that TCG won't be able to - prove that a given global is "dead" at a given program point. The - x86 guest uses it to improve the condition codes optimisation. From ed145144c465f8e1ba7d5b692450c135c45af3db Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 17 Oct 2022 16:11:53 +1000 Subject: [PATCH 02/47] meson: Move CONFIG_TCG_INTERPRETER to config_host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Like CONFIG_TCG, the enabled method of execution is a host property not a guest property. This exposes the define to compile-once files. Acked-by: Paolo Bonzini Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- meson.build | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/meson.build b/meson.build index 4c6f8a674a..14e3eba227 100644 --- a/meson.build +++ b/meson.build @@ -469,6 +469,7 @@ if get_option('tcg').allowed() endif if get_option('tcg_interpreter') tcg_arch = 'tci' + config_host += { 'CONFIG_TCG_INTERPRETER': 'y' } elif host_arch == 'x86_64' tcg_arch = 'i386' elif host_arch == 'ppc64' @@ -2545,9 +2546,6 @@ foreach target : target_dirs if sym == 'CONFIG_TCG' or target in accelerator_targets.get(sym, []) config_target += { sym: 'y' } config_all += { sym: 'y' } - if sym == 'CONFIG_TCG' and tcg_arch == 'tci' - config_target += { 'CONFIG_TCG_INTERPRETER': 'y' } - endif if target in modular_tcg config_target += { 'CONFIG_TCG_MODULAR': 'y' } else From dba6a9ebc1466cc6d495f2206ba8339e96e2abd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Wed, 30 Nov 2022 14:26:52 +0100 Subject: [PATCH 03/47] tcg/s390x: Fix coding style MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We are going to modify this code, so fix its style first to avoid: ERROR: spaces required around that '*' (ctx:VxV) #281: FILE: tcg/s390x/tcg-target.c.inc:1224: + uintptr_t mask = ~(0xffffull << i*16); ^ Reviewed-by: Wilfred Mallawa Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20221130132654.76369-2-philmd@linaro.org> Signed-off-by: Richard Henderson --- tcg/s390x/tcg-target.c.inc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index 33becd7694..f1d3907cd8 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -802,9 +802,9 @@ static bool maybe_out_small_movi(TCGContext *s, TCGType type, } for (i = 0; i < 4; i++) { - tcg_target_long mask = 0xffffull << i*16; + tcg_target_long mask = 0xffffull << i * 16; if ((uval & mask) == uval) { - tcg_out_insn_RI(s, lli_insns[i], ret, uval >> i*16); + tcg_out_insn_RI(s, lli_insns[i], ret, uval >> i * 16); return true; } } @@ -1221,9 +1221,9 @@ static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val) /* Try all 32-bit insns that can perform it in one go. */ for (i = 0; i < 4; i++) { - tcg_target_ulong mask = ~(0xffffull << i*16); + tcg_target_ulong mask = ~(0xffffull << i * 16); if (((val | ~valid) & mask) == mask) { - tcg_out_insn_RI(s, ni_insns[i], dest, val >> i*16); + tcg_out_insn_RI(s, ni_insns[i], dest, val >> i * 16); return; } } @@ -1231,9 +1231,9 @@ static void tgen_andi(TCGContext *s, TCGType type, TCGReg dest, uint64_t val) /* Try all 48-bit insns that can perform it in one go. */ if (HAVE_FACILITY(EXT_IMM)) { for (i = 0; i < 2; i++) { - tcg_target_ulong mask = ~(0xffffffffull << i*32); + tcg_target_ulong mask = ~(0xffffffffull << i * 32); if (((val | ~valid) & mask) == mask) { - tcg_out_insn_RIL(s, nif_insns[i], dest, val >> i*32); + tcg_out_insn_RIL(s, nif_insns[i], dest, val >> i * 32); return; } } @@ -1279,9 +1279,9 @@ static void tgen_ori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val) /* Try all 32-bit insns that can perform it in one go. */ for (i = 0; i < 4; i++) { - tcg_target_ulong mask = (0xffffull << i*16); + tcg_target_ulong mask = (0xffffull << i * 16); if ((val & mask) != 0 && (val & ~mask) == 0) { - tcg_out_insn_RI(s, oi_insns[i], dest, val >> i*16); + tcg_out_insn_RI(s, oi_insns[i], dest, val >> i * 16); return; } } @@ -1289,9 +1289,9 @@ static void tgen_ori(TCGContext *s, TCGType type, TCGReg dest, uint64_t val) /* Try all 48-bit insns that can perform it in one go. */ if (HAVE_FACILITY(EXT_IMM)) { for (i = 0; i < 2; i++) { - tcg_target_ulong mask = (0xffffffffull << i*32); + tcg_target_ulong mask = (0xffffffffull << i * 32); if ((val & mask) != 0 && (val & ~mask) == 0) { - tcg_out_insn_RIL(s, oif_insns[i], dest, val >> i*32); + tcg_out_insn_RIL(s, oif_insns[i], dest, val >> i * 32); return; } } From a813e36f2bdf254a6a41e702fe85453105ad91da Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 30 Nov 2022 22:38:25 -0800 Subject: [PATCH 04/47] tcg: Cleanup trailing whitespace MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove whitespace at end of line, plus one place this also highlights some missing braces. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/ppc/tcg-target.c.inc | 2 +- tcg/tcg.c | 33 +++++++++++++++++---------------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index e3dba47697..9e34df94ba 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -42,7 +42,7 @@ # else # error "Unknown ABI" # endif -#endif +#endif #ifdef _CALL_SYSV # define TCG_TARGET_CALL_ALIGN_ARGS 1 diff --git a/tcg/tcg.c b/tcg/tcg.c index 436fcf6ebd..db64799e03 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -496,7 +496,7 @@ void *tcg_malloc_internal(TCGContext *s, int size) { TCGPool *p; int pool_size; - + if (size > TCG_POOL_CHUNK_SIZE) { /* big malloc: insert a new pool (XXX: could optimize) */ p = g_malloc(sizeof(TCGPool) + size); @@ -517,10 +517,11 @@ void *tcg_malloc_internal(TCGContext *s, int size) p = g_malloc(sizeof(TCGPool) + pool_size); p->size = pool_size; p->next = NULL; - if (s->pool_current) + if (s->pool_current) { s->pool_current->next = p; - else + } else { s->pool_first = p; + } } else { p = p->next; } @@ -2949,8 +2950,8 @@ static void dump_regs(TCGContext *s) for(i = 0; i < TCG_TARGET_NB_REGS; i++) { if (s->reg_to_temp[i] != NULL) { - printf("%s: %s\n", - tcg_target_reg_names[i], + printf("%s: %s\n", + tcg_target_reg_names[i], tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i])); } } @@ -2967,7 +2968,7 @@ static void check_regs(TCGContext *s) ts = s->reg_to_temp[reg]; if (ts != NULL) { if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) { - printf("Inconsistency for register %s:\n", + printf("Inconsistency for register %s:\n", tcg_target_reg_names[reg]); goto fail; } @@ -3597,14 +3598,14 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) nb_iargs = def->nb_iargs; /* copy constants */ - memcpy(new_args + nb_oargs + nb_iargs, + memcpy(new_args + nb_oargs + nb_iargs, op->args + nb_oargs + nb_iargs, sizeof(TCGArg) * def->nb_cargs); i_allocated_regs = s->reserved_regs; o_allocated_regs = s->reserved_regs; - /* satisfy input constraints */ + /* satisfy input constraints */ for (k = 0; k < nb_iargs; k++) { TCGRegSet i_preferred_regs, o_preferred_regs; @@ -3678,7 +3679,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) const_args[i] = 0; tcg_regset_set_reg(i_allocated_regs, reg); } - + /* mark dead temporaries and free the associated registers */ for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) { if (IS_DEAD_ARG(i)) { @@ -3692,7 +3693,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) tcg_reg_alloc_bb_end(s, i_allocated_regs); } else { if (def->flags & TCG_OPF_CALL_CLOBBER) { - /* XXX: permit generic clobber register list ? */ + /* XXX: permit generic clobber register list ? */ for (i = 0; i < TCG_TARGET_NB_REGS; i++) { if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { tcg_reg_free(s, i, i_allocated_regs); @@ -3704,7 +3705,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) an exception. */ sync_globals(s, i_allocated_regs); } - + /* satisfy the output constraints */ for(k = 0; k < nb_oargs; k++) { i = def->args_ct[k].sort_index; @@ -3889,7 +3890,7 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) /* assign stack slots first */ call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long); - call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & + call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & ~(TCG_TARGET_STACK_ALIGN - 1); allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE); if (allocate_args) { @@ -3914,7 +3915,7 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) stack_offset += sizeof(tcg_target_long); #endif } - + /* assign input registers */ allocated_regs = s->reserved_regs; for (i = 0; i < nb_regs; i++) { @@ -3947,14 +3948,14 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) tcg_regset_set_reg(allocated_regs, reg); } } - + /* mark dead temporaries and free the associated registers */ for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { if (IS_DEAD_ARG(i)) { temp_dead(s, arg_temp(op->args[i])); } } - + /* clobber call registers */ for (i = 0; i < TCG_TARGET_NB_REGS; i++) { if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { @@ -4395,7 +4396,7 @@ void tcg_dump_info(GString *buf) (double)s->code_out_len / tb_div_count); g_string_append_printf(buf, "avg search data/TB %0.1f\n", (double)s->search_out_len / tb_div_count); - + g_string_append_printf(buf, "cycles/op %0.1f\n", s->op_count ? (double)tot / s->op_count : 0); g_string_append_printf(buf, "cycles/in byte %0.1f\n", From 0aa5d47ac58d914cd1e0dbd69d5a21c89d1b8079 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 17 Nov 2022 18:51:43 -0800 Subject: [PATCH 05/47] qemu/main-loop: Introduce QEMU_IOTHREAD_LOCK_GUARD MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create a wrapper for locking/unlocking the iothread lock. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- include/qemu/main-loop.h | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/include/qemu/main-loop.h b/include/qemu/main-loop.h index 3c9a9a982d..c25f390696 100644 --- a/include/qemu/main-loop.h +++ b/include/qemu/main-loop.h @@ -343,6 +343,35 @@ void qemu_mutex_lock_iothread_impl(const char *file, int line); */ void qemu_mutex_unlock_iothread(void); +/** + * QEMU_IOTHREAD_LOCK_GUARD + * + * Wrap a block of code in a conditional qemu_mutex_{lock,unlock}_iothread. + */ +typedef struct IOThreadLockAuto IOThreadLockAuto; + +static inline IOThreadLockAuto *qemu_iothread_auto_lock(const char *file, + int line) +{ + if (qemu_mutex_iothread_locked()) { + return NULL; + } + qemu_mutex_lock_iothread_impl(file, line); + /* Anything non-NULL causes the cleanup function to be called */ + return (IOThreadLockAuto *)(uintptr_t)1; +} + +static inline void qemu_iothread_auto_unlock(IOThreadLockAuto *l) +{ + qemu_mutex_unlock_iothread(); +} + +G_DEFINE_AUTOPTR_CLEANUP_FUNC(IOThreadLockAuto, qemu_iothread_auto_unlock) + +#define QEMU_IOTHREAD_LOCK_GUARD() \ + g_autoptr(IOThreadLockAuto) _iothread_lock_auto __attribute__((unused)) \ + = qemu_iothread_auto_lock(__FILE__, __LINE__) + /* * qemu_cond_wait_iothread: Wait on condition for the main loop mutex * From 6fa8c46e55f1ef141b188563e914c46957ce163c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 17 Nov 2022 19:00:22 -0800 Subject: [PATCH 06/47] hw/mips: Use QEMU_IOTHREAD_LOCK_GUARD in cpu_mips_irq_request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- hw/mips/mips_int.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/hw/mips/mips_int.c b/hw/mips/mips_int.c index 2db5e10fe0..73437cd90f 100644 --- a/hw/mips/mips_int.c +++ b/hw/mips/mips_int.c @@ -32,17 +32,12 @@ static void cpu_mips_irq_request(void *opaque, int irq, int level) MIPSCPU *cpu = opaque; CPUMIPSState *env = &cpu->env; CPUState *cs = CPU(cpu); - bool locked = false; if (irq < 0 || irq > 7) { return; } - /* Make sure locking works even if BQL is already held by the caller */ - if (!qemu_mutex_iothread_locked()) { - locked = true; - qemu_mutex_lock_iothread(); - } + QEMU_IOTHREAD_LOCK_GUARD(); if (level) { env->CP0_Cause |= 1 << (irq + CP0Ca_IP); @@ -59,10 +54,6 @@ static void cpu_mips_irq_request(void *opaque, int irq, int level) } else { cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD); } - - if (locked) { - qemu_mutex_unlock_iothread(); - } } void cpu_mips_irq_init_cpu(MIPSCPU *cpu) From 2fc4f9f3ff2362d4436476087fb13ad26fd2814a Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 17 Nov 2022 19:07:11 -0800 Subject: [PATCH 07/47] target/ppc: Use QEMU_IOTHREAD_LOCK_GUARD in ppc_maybe_interrupt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Daniel Henrique Barboza Signed-off-by: Richard Henderson --- target/ppc/excp_helper.c | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/target/ppc/excp_helper.c b/target/ppc/excp_helper.c index add4d54ae7..287659c74d 100644 --- a/target/ppc/excp_helper.c +++ b/target/ppc/excp_helper.c @@ -2163,22 +2163,13 @@ static int ppc_next_unmasked_interrupt(CPUPPCState *env) void ppc_maybe_interrupt(CPUPPCState *env) { CPUState *cs = env_cpu(env); - bool locked = false; - - if (!qemu_mutex_iothread_locked()) { - locked = true; - qemu_mutex_lock_iothread(); - } + QEMU_IOTHREAD_LOCK_GUARD(); if (ppc_next_unmasked_interrupt(env)) { cpu_interrupt(cs, CPU_INTERRUPT_HARD); } else { cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD); } - - if (locked) { - qemu_mutex_unlock_iothread(); - } } #if defined(TARGET_PPC64) From 03ac0a0cfdfd8dd0708c2fb1fdec09319c1add3c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 17 Nov 2022 19:11:26 -0800 Subject: [PATCH 08/47] target/ppc: Use QEMU_IOTHREAD_LOCK_GUARD in cpu_interrupt_exittb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In addition, use tcg_enabled instead of !kvm_enabled. Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Daniel Henrique Barboza Signed-off-by: Richard Henderson --- target/ppc/helper_regs.c | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/target/ppc/helper_regs.c b/target/ppc/helper_regs.c index c0aee5855b..779e7db513 100644 --- a/target/ppc/helper_regs.c +++ b/target/ppc/helper_regs.c @@ -22,6 +22,7 @@ #include "qemu/main-loop.h" #include "exec/exec-all.h" #include "sysemu/kvm.h" +#include "sysemu/tcg.h" #include "helper_regs.h" #include "power8-pmu.h" #include "cpu-models.h" @@ -203,17 +204,10 @@ void cpu_interrupt_exittb(CPUState *cs) { /* * We don't need to worry about translation blocks - * when running with KVM. + * unless running with TCG. */ - if (kvm_enabled()) { - return; - } - - if (!qemu_mutex_iothread_locked()) { - qemu_mutex_lock_iothread(); - cpu_interrupt(cs, CPU_INTERRUPT_EXITTB); - qemu_mutex_unlock_iothread(); - } else { + if (tcg_enabled()) { + QEMU_IOTHREAD_LOCK_GUARD(); cpu_interrupt(cs, CPU_INTERRUPT_EXITTB); } } From b3eb5b861a9ad5d885c15760cc3570fa3678b836 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 17 Nov 2022 19:15:31 -0800 Subject: [PATCH 09/47] target/riscv: Use QEMU_IOTHREAD_LOCK_GUARD in riscv_cpu_update_mip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Alistair Francis Signed-off-by: Richard Henderson --- target/riscv/cpu_helper.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/target/riscv/cpu_helper.c b/target/riscv/cpu_helper.c index 278d163803..241d06bab8 100644 --- a/target/riscv/cpu_helper.c +++ b/target/riscv/cpu_helper.c @@ -610,7 +610,6 @@ uint64_t riscv_cpu_update_mip(RISCVCPU *cpu, uint64_t mask, uint64_t value) CPURISCVState *env = &cpu->env; CPUState *cs = CPU(cpu); uint64_t gein, vsgein = 0, vstip = 0, old = env->mip; - bool locked = false; if (riscv_cpu_virt_enabled(env)) { gein = get_field(env->hstatus, HSTATUS_VGEIN); @@ -621,10 +620,7 @@ uint64_t riscv_cpu_update_mip(RISCVCPU *cpu, uint64_t mask, uint64_t value) mask = ((mask == MIP_VSTIP) && env->vstime_irq) ? 0 : mask; vstip = env->vstime_irq ? MIP_VSTIP : 0; - if (!qemu_mutex_iothread_locked()) { - locked = true; - qemu_mutex_lock_iothread(); - } + QEMU_IOTHREAD_LOCK_GUARD(); env->mip = (env->mip & ~mask) | (value & mask); @@ -634,10 +630,6 @@ uint64_t riscv_cpu_update_mip(RISCVCPU *cpu, uint64_t mask, uint64_t value) cpu_reset_interrupt(cs, CPU_INTERRUPT_HARD); } - if (locked) { - qemu_mutex_unlock_iothread(); - } - return old; } From 50c9c512ac15e8eeb0ec63f6a92ba8e017e7676e Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 17 Nov 2022 19:17:18 -0800 Subject: [PATCH 10/47] hw/ppc: Use QEMU_IOTHREAD_LOCK_GUARD in ppc_set_irq MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewed-by: Philippe Mathieu-Daudé Reviewed-by: Daniel Henrique Barboza Signed-off-by: Richard Henderson --- hw/ppc/ppc.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/hw/ppc/ppc.c b/hw/ppc/ppc.c index dc86c1c7db..4e816c68c7 100644 --- a/hw/ppc/ppc.c +++ b/hw/ppc/ppc.c @@ -44,13 +44,9 @@ void ppc_set_irq(PowerPCCPU *cpu, int irq, int level) { CPUPPCState *env = &cpu->env; unsigned int old_pending; - bool locked = false; /* We may already have the BQL if coming from the reset path */ - if (!qemu_mutex_iothread_locked()) { - locked = true; - qemu_mutex_lock_iothread(); - } + QEMU_IOTHREAD_LOCK_GUARD(); old_pending = env->pending_interrupts; @@ -67,10 +63,6 @@ void ppc_set_irq(PowerPCCPU *cpu, int irq, int level) trace_ppc_irq_set_exit(env, irq, level, env->pending_interrupts, CPU(cpu)->interrupt_request); - - if (locked) { - qemu_mutex_unlock_iothread(); - } } /* PowerPC 6xx / 7xx internal IRQ controller */ From 61b59fb2691f749e1b647a512121d6813b23ff44 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 17 Nov 2022 19:59:16 -0800 Subject: [PATCH 11/47] accel/tcg: Use QEMU_IOTHREAD_LOCK_GUARD in io_readx/io_writex MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Narrow the scope of the lock to the actual read/write, moving the cpu_transation_failed call outside the lock. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- accel/tcg/cputlb.c | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/accel/tcg/cputlb.c b/accel/tcg/cputlb.c index 03674d598f..4948729917 100644 --- a/accel/tcg/cputlb.c +++ b/accel/tcg/cputlb.c @@ -1356,7 +1356,6 @@ static uint64_t io_readx(CPUArchState *env, CPUTLBEntryFull *full, MemoryRegionSection *section; MemoryRegion *mr; uint64_t val; - bool locked = false; MemTxResult r; section = iotlb_to_section(cpu, full->xlat_section, full->attrs); @@ -1367,11 +1366,11 @@ static uint64_t io_readx(CPUArchState *env, CPUTLBEntryFull *full, cpu_io_recompile(cpu, retaddr); } - if (!qemu_mutex_iothread_locked()) { - qemu_mutex_lock_iothread(); - locked = true; + { + QEMU_IOTHREAD_LOCK_GUARD(); + r = memory_region_dispatch_read(mr, mr_offset, &val, op, full->attrs); } - r = memory_region_dispatch_read(mr, mr_offset, &val, op, full->attrs); + if (r != MEMTX_OK) { hwaddr physaddr = mr_offset + section->offset_within_address_space - @@ -1380,10 +1379,6 @@ static uint64_t io_readx(CPUArchState *env, CPUTLBEntryFull *full, cpu_transaction_failed(cpu, physaddr, addr, memop_size(op), access_type, mmu_idx, full->attrs, r, retaddr); } - if (locked) { - qemu_mutex_unlock_iothread(); - } - return val; } @@ -1410,7 +1405,6 @@ static void io_writex(CPUArchState *env, CPUTLBEntryFull *full, hwaddr mr_offset; MemoryRegionSection *section; MemoryRegion *mr; - bool locked = false; MemTxResult r; section = iotlb_to_section(cpu, full->xlat_section, full->attrs); @@ -1427,11 +1421,11 @@ static void io_writex(CPUArchState *env, CPUTLBEntryFull *full, */ save_iotlb_data(cpu, section, mr_offset); - if (!qemu_mutex_iothread_locked()) { - qemu_mutex_lock_iothread(); - locked = true; + { + QEMU_IOTHREAD_LOCK_GUARD(); + r = memory_region_dispatch_write(mr, mr_offset, val, op, full->attrs); } - r = memory_region_dispatch_write(mr, mr_offset, val, op, full->attrs); + if (r != MEMTX_OK) { hwaddr physaddr = mr_offset + section->offset_within_address_space - @@ -1441,9 +1435,6 @@ static void io_writex(CPUArchState *env, CPUTLBEntryFull *full, MMU_DATA_STORE, mmu_idx, full->attrs, r, retaddr); } - if (locked) { - qemu_mutex_unlock_iothread(); - } } static inline target_ulong tlb_read_ofs(CPUTLBEntry *entry, size_t ofs) From 1c1824dca404155b1116023d794ec227ae71a424 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 9 Oct 2022 21:06:31 -0700 Subject: [PATCH 12/47] tcg: Tidy tcg_reg_alloc_op MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace goto allocate_in_reg with a boolean. Remove o_preferred_regs which isn't used, except to copy. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/tcg.c | 45 +++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index db64799e03..215ddf2db5 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -3607,7 +3607,8 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) /* satisfy input constraints */ for (k = 0; k < nb_iargs; k++) { - TCGRegSet i_preferred_regs, o_preferred_regs; + TCGRegSet i_preferred_regs; + bool allocate_new_reg; i = def->args_ct[nb_oargs + k].sort_index; arg = op->args[i]; @@ -3622,9 +3623,12 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) continue; } - i_preferred_regs = o_preferred_regs = 0; + reg = ts->reg; + i_preferred_regs = 0; + allocate_new_reg = false; + if (arg_ct->ialias) { - o_preferred_regs = op->output_pref[arg_ct->alias_index]; + i_preferred_regs = op->output_pref[arg_ct->alias_index]; /* * If the input is readonly, then it cannot also be an @@ -3633,30 +3637,23 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) * register and move it. */ if (temp_readonly(ts) || !IS_DEAD_ARG(i)) { - goto allocate_in_reg; + allocate_new_reg = true; + } else if (ts->val_type == TEMP_VAL_REG) { + /* + * Check if the current register has already been + * allocated for another input. + */ + allocate_new_reg = tcg_regset_test_reg(i_allocated_regs, reg); } - - /* - * Check if the current register has already been allocated - * for another input aliased to an output. - */ - if (ts->val_type == TEMP_VAL_REG) { - reg = ts->reg; - for (int k2 = 0; k2 < k; k2++) { - int i2 = def->args_ct[nb_oargs + k2].sort_index; - if (def->args_ct[i2].ialias && reg == new_args[i2]) { - goto allocate_in_reg; - } - } - } - i_preferred_regs = o_preferred_regs; } - temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs); - reg = ts->reg; + if (!allocate_new_reg) { + temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs); + reg = ts->reg; + allocate_new_reg = !tcg_regset_test_reg(arg_ct->regs, reg); + } - if (!tcg_regset_test_reg(arg_ct->regs, reg)) { - allocate_in_reg: + if (allocate_new_reg) { /* * Allocate a new register matching the constraint * and move the temporary register into it. @@ -3664,7 +3661,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) temp_load(s, ts, tcg_target_available_regs[ts->type], i_allocated_regs, 0); reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs, - o_preferred_regs, ts->indirect_base); + i_preferred_regs, ts->indirect_base); if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { /* * Cross register class move not supported. Sync the From 8d21de51b93e439cd026cb9edcea192550345a6c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 16 Oct 2022 22:05:14 +1000 Subject: [PATCH 13/47] tcg: Remove TCG_TARGET_STACK_GROWSUP MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hppa host code has been removed since 2013; this should have been deleted at the same time. Fixes: 802b5081233a ("tcg-hppa: Remove tcg backend") Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/aarch64/tcg-target.h | 1 - tcg/arm/tcg-target.h | 1 - tcg/tcg.c | 32 ++------------------------------ 3 files changed, 2 insertions(+), 32 deletions(-) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index 485f685bd2..e145d50fef 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -16,7 +16,6 @@ #define TCG_TARGET_INSN_UNIT_SIZE 4 #define TCG_TARGET_TLB_DISPLACEMENT_BITS 24 #define MAX_CODE_GEN_BUFFER_SIZE (2 * GiB) -#undef TCG_TARGET_STACK_GROWSUP typedef enum { TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3, diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index 7e96495392..56c1ac4586 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -30,7 +30,6 @@ extern int arm_arch; #define use_armv7_instructions (__ARM_ARCH >= 7 || arm_arch >= 7) -#undef TCG_TARGET_STACK_GROWSUP #define TCG_TARGET_INSN_UNIT_SIZE 4 #define TCG_TARGET_TLB_DISPLACEMENT_BITS 16 #define MAX_CODE_GEN_BUFFER_SIZE UINT32_MAX diff --git a/tcg/tcg.c b/tcg/tcg.c index 215ddf2db5..05d2b70ab7 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -1552,25 +1552,8 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) } if (TCG_TARGET_REG_BITS < 64 && is_64bit) { - /* - * If stack grows up, then we will be placing successive - * arguments at lower addresses, which means we need to - * reverse the order compared to how we would normally - * treat either big or little-endian. For those arguments - * that will wind up in registers, this still works for - * HPPA (the only current STACK_GROWSUP target) since the - * argument registers are *also* allocated in decreasing - * order. If another such target is added, this logic may - * have to get more complicated to differentiate between - * stack arguments and register arguments. - */ -#if HOST_BIG_ENDIAN != defined(TCG_TARGET_STACK_GROWSUP) - op->args[pi++] = temp_arg(args[i] + 1); - op->args[pi++] = temp_arg(args[i]); -#else - op->args[pi++] = temp_arg(args[i]); - op->args[pi++] = temp_arg(args[i] + 1); -#endif + op->args[pi++] = temp_arg(args[i] + HOST_BIG_ENDIAN); + op->args[pi++] = temp_arg(args[i] + !HOST_BIG_ENDIAN); real_args += 2; continue; } @@ -3854,12 +3837,6 @@ static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) return true; } -#ifdef TCG_TARGET_STACK_GROWSUP -#define STACK_DIR(x) (-(x)) -#else -#define STACK_DIR(x) (x) -#endif - static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) { const int nb_oargs = TCGOP_CALLO(op); @@ -3899,18 +3876,13 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) stack_offset = TCG_TARGET_CALL_STACK_OFFSET; for (i = nb_regs; i < nb_iargs; i++) { arg = op->args[nb_oargs + i]; -#ifdef TCG_TARGET_STACK_GROWSUP - stack_offset -= sizeof(tcg_target_long); -#endif if (arg != TCG_CALL_DUMMY_ARG) { ts = arg_temp(arg); temp_load(s, ts, tcg_target_available_regs[ts->type], s->reserved_regs, 0); tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); } -#ifndef TCG_TARGET_STACK_GROWSUP stack_offset += sizeof(tcg_target_long); -#endif } /* assign input registers */ From 7789b16d110af4fbf73310fac8fa4012925928ab Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 10 Nov 2022 23:29:47 +1000 Subject: [PATCH 14/47] tci: MAX_OPC_PARAM_IARGS is no longer used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unused since commit 7b7d8b2d9a ("tcg/tci: Use ffi for calls"). Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/tci.c | 1 - tcg/tci/tcg-target.c.inc | 4 ---- 2 files changed, 5 deletions(-) diff --git a/tcg/tci.c b/tcg/tci.c index bdfac83492..05a24163d3 100644 --- a/tcg/tci.c +++ b/tcg/tci.c @@ -18,7 +18,6 @@ */ #include "qemu/osdep.h" -#include "tcg/tcg.h" /* MAX_OPC_PARAM_IARGS */ #include "exec/cpu_ldst.h" #include "tcg/tcg-op.h" #include "tcg/tcg-ldst.h" diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index f3d7441e06..c1acaa943e 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -197,10 +197,6 @@ static const int tcg_target_reg_alloc_order[] = { TCG_REG_R0, }; -#if MAX_OPC_PARAM_IARGS != 7 -# error Fix needed, number of supported input arguments changed! -#endif - /* No call arguments via registers. All will be stored on the "stack". */ static const int tcg_target_call_iarg_regs[] = { }; From 36f5539cfd3f09444657e4c1f6b6b461cea2235f Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 1 Dec 2022 00:44:13 -0800 Subject: [PATCH 15/47] tcg: Fix tcg_reg_alloc_dup* The assignment to mem_coherent should be done with any modification, not simply with a newly allocated register. Signed-off-by: Richard Henderson --- tcg/tcg.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index 05d2b70ab7..371908b34b 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -3498,7 +3498,6 @@ static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, op->output_pref[0], ots->indirect_base); ots->val_type = TEMP_VAL_REG; - ots->mem_coherent = 0; s->reg_to_temp[ots->reg] = ots; } @@ -3552,6 +3551,7 @@ static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) tcg_debug_assert(ok); done: + ots->mem_coherent = 0; if (IS_DEAD_ARG(1)) { temp_dead(s, its); } @@ -3779,7 +3779,6 @@ static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, op->output_pref[0], ots->indirect_base); ots->val_type = TEMP_VAL_REG; - ots->mem_coherent = 0; s->reg_to_temp[ots->reg] = ots; } @@ -3823,6 +3822,7 @@ static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) return false; done: + ots->mem_coherent = 0; if (IS_DEAD_ARG(1)) { temp_dead(s, itsl); } From 098859f108af3e7f96efe6eeb1418f693e5e64ce Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 1 Dec 2022 01:05:05 -0800 Subject: [PATCH 16/47] tcg: Centralize updates to reg_to_temp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Create two new functions, set_temp_val_{reg,nonreg}. Assert that the reg_to_temp mapping is correct before any changes are made. Reviewed-by: Alex Bennée Signed-off-by: Richard Henderson --- tcg/tcg.c | 159 +++++++++++++++++++++++++++++------------------------- 1 file changed, 85 insertions(+), 74 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index 371908b34b..a1ae761a58 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -3019,6 +3019,35 @@ static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) ts->mem_allocated = 1; } +/* Assign @reg to @ts, and update reg_to_temp[]. */ +static void set_temp_val_reg(TCGContext *s, TCGTemp *ts, TCGReg reg) +{ + if (ts->val_type == TEMP_VAL_REG) { + TCGReg old = ts->reg; + tcg_debug_assert(s->reg_to_temp[old] == ts); + if (old == reg) { + return; + } + s->reg_to_temp[old] = NULL; + } + tcg_debug_assert(s->reg_to_temp[reg] == NULL); + s->reg_to_temp[reg] = ts; + ts->val_type = TEMP_VAL_REG; + ts->reg = reg; +} + +/* Assign a non-register value type to @ts, and update reg_to_temp[]. */ +static void set_temp_val_nonreg(TCGContext *s, TCGTemp *ts, TCGTempVal type) +{ + tcg_debug_assert(type != TEMP_VAL_REG); + if (ts->val_type == TEMP_VAL_REG) { + TCGReg reg = ts->reg; + tcg_debug_assert(s->reg_to_temp[reg] == ts); + s->reg_to_temp[reg] = NULL; + } + ts->val_type = type; +} + static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet); /* Mark a temporary as free or dead. If 'free_or_dead' is negative, @@ -3044,10 +3073,7 @@ static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead) default: g_assert_not_reached(); } - if (ts->val_type == TEMP_VAL_REG) { - s->reg_to_temp[ts->reg] = NULL; - } - ts->val_type = new_type; + set_temp_val_nonreg(s, ts, new_type); } /* Mark a temporary as dead. */ @@ -3227,9 +3253,7 @@ static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, default: tcg_abort(); } - ts->reg = reg; - ts->val_type = TEMP_VAL_REG; - s->reg_to_temp[reg] = ts; + set_temp_val_reg(s, ts, reg); } /* Save a temporary to memory. 'allocated_regs' is used in case a @@ -3341,10 +3365,7 @@ static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots, tcg_debug_assert(!temp_readonly(ots)); /* The movi is not explicitly generated here. */ - if (ots->val_type == TEMP_VAL_REG) { - s->reg_to_temp[ots->reg] = NULL; - } - ots->val_type = TEMP_VAL_CONST; + set_temp_val_nonreg(s, ots, TEMP_VAL_CONST); ots->val = val; ots->mem_coherent = 0; if (NEED_SYNC_ARG(0)) { @@ -3363,6 +3384,7 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) TCGRegSet allocated_regs, preferred_regs; TCGTemp *ts, *ots; TCGType otype, itype; + TCGReg oreg, ireg; allocated_regs = s->reserved_regs; preferred_regs = op->output_pref[0]; @@ -3394,8 +3416,9 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) temp_load(s, ts, tcg_target_available_regs[itype], allocated_regs, preferred_regs); } - tcg_debug_assert(ts->val_type == TEMP_VAL_REG); + ireg = ts->reg; + if (IS_DEAD_ARG(0)) { /* mov to a non-saved dead register makes no sense (even with liveness analysis disabled). */ @@ -3403,52 +3426,53 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) if (!ots->mem_allocated) { temp_allocate_frame(s, ots); } - tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset); + tcg_out_st(s, otype, ireg, ots->mem_base->reg, ots->mem_offset); if (IS_DEAD_ARG(1)) { temp_dead(s, ts); } temp_dead(s, ots); + return; + } + + if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { + /* + * The mov can be suppressed. Kill input first, so that it + * is unlinked from reg_to_temp, then set the output to the + * reg that we saved from the input. + */ + temp_dead(s, ts); + oreg = ireg; } else { - if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) { - /* the mov can be suppressed */ - if (ots->val_type == TEMP_VAL_REG) { - s->reg_to_temp[ots->reg] = NULL; - } - ots->reg = ts->reg; - temp_dead(s, ts); + if (ots->val_type == TEMP_VAL_REG) { + oreg = ots->reg; } else { - if (ots->val_type != TEMP_VAL_REG) { - /* When allocating a new register, make sure to not spill the - input one. */ - tcg_regset_set_reg(allocated_regs, ts->reg); - ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype], - allocated_regs, preferred_regs, - ots->indirect_base); - } - if (!tcg_out_mov(s, otype, ots->reg, ts->reg)) { - /* - * Cross register class move not supported. - * Store the source register into the destination slot - * and leave the destination temp as TEMP_VAL_MEM. - */ - assert(!temp_readonly(ots)); - if (!ts->mem_allocated) { - temp_allocate_frame(s, ots); - } - tcg_out_st(s, ts->type, ts->reg, - ots->mem_base->reg, ots->mem_offset); - ots->mem_coherent = 1; - temp_free_or_dead(s, ots, -1); - return; - } + /* Make sure to not spill the input register during allocation. */ + oreg = tcg_reg_alloc(s, tcg_target_available_regs[otype], + allocated_regs | ((TCGRegSet)1 << ireg), + preferred_regs, ots->indirect_base); } - ots->val_type = TEMP_VAL_REG; - ots->mem_coherent = 0; - s->reg_to_temp[ots->reg] = ots; - if (NEED_SYNC_ARG(0)) { - temp_sync(s, ots, allocated_regs, 0, 0); + if (!tcg_out_mov(s, otype, oreg, ireg)) { + /* + * Cross register class move not supported. + * Store the source register into the destination slot + * and leave the destination temp as TEMP_VAL_MEM. + */ + assert(!temp_readonly(ots)); + if (!ts->mem_allocated) { + temp_allocate_frame(s, ots); + } + tcg_out_st(s, ts->type, ireg, ots->mem_base->reg, ots->mem_offset); + set_temp_val_nonreg(s, ts, TEMP_VAL_MEM); + ots->mem_coherent = 1; + return; } } + set_temp_val_reg(s, ots, oreg); + ots->mem_coherent = 0; + + if (NEED_SYNC_ARG(0)) { + temp_sync(s, ots, allocated_regs, 0, 0); + } } /* @@ -3490,15 +3514,15 @@ static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) /* Allocate the output register now. */ if (ots->val_type != TEMP_VAL_REG) { TCGRegSet allocated_regs = s->reserved_regs; + TCGReg oreg; if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) { /* Make sure to not spill the input register. */ tcg_regset_set_reg(allocated_regs, its->reg); } - ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, - op->output_pref[0], ots->indirect_base); - ots->val_type = TEMP_VAL_REG; - s->reg_to_temp[ots->reg] = ots; + oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, + op->output_pref[0], ots->indirect_base); + set_temp_val_reg(s, ots, oreg); } switch (its->val_type) { @@ -3535,10 +3559,12 @@ static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) #else endian_fixup = 0; #endif + /* Attempt to dup directly from the input memory slot. */ if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, its->mem_offset + endian_fixup)) { goto done; } + /* Load the input into the destination vector register. */ tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset); break; @@ -3707,17 +3733,8 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) op->output_pref[k], ts->indirect_base); } tcg_regset_set_reg(o_allocated_regs, reg); - if (ts->val_type == TEMP_VAL_REG) { - s->reg_to_temp[ts->reg] = NULL; - } - ts->val_type = TEMP_VAL_REG; - ts->reg = reg; - /* - * Temp value is modified, so the value kept in memory is - * potentially not the same. - */ + set_temp_val_reg(s, ts, reg); ts->mem_coherent = 0; - s->reg_to_temp[reg] = ts; new_args[i] = reg; } } @@ -3767,6 +3784,7 @@ static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) TCGRegSet allocated_regs = s->reserved_regs; TCGRegSet dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs; + TCGReg oreg; /* Make sure to not spill the input registers. */ if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) { @@ -3776,10 +3794,9 @@ static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) tcg_regset_set_reg(allocated_regs, itsh->reg); } - ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, - op->output_pref[0], ots->indirect_base); - ots->val_type = TEMP_VAL_REG; - s->reg_to_temp[ots->reg] = ots; + oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, + op->output_pref[0], ots->indirect_base); + set_temp_val_reg(s, ots, oreg); } /* Promote dup2 of immediates to dupi_vec. */ @@ -3962,14 +3979,8 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) tcg_debug_assert(!temp_readonly(ts)); reg = tcg_target_call_oarg_regs[i]; - tcg_debug_assert(s->reg_to_temp[reg] == NULL); - if (ts->val_type == TEMP_VAL_REG) { - s->reg_to_temp[ts->reg] = NULL; - } - ts->val_type = TEMP_VAL_REG; - ts->reg = reg; + set_temp_val_reg(s, ts, reg); ts->mem_coherent = 0; - s->reg_to_temp[reg] = ts; if (NEED_SYNC_ARG(i)) { temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i)); } else if (IS_DEAD_ARG(i)) { From 568e010b897783bd1f44c9ea989582098e737fc9 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 1 Dec 2022 09:12:36 -0800 Subject: [PATCH 17/47] tcg: Remove check_regs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We now check the consistency of reg_to_temp[] with each update, so the utility of checking consistency at the end of each opcode is minimal. In addition, the form of this check is quite expensive, consuming 10% of a checking-enabled build. Reviewed-by: Alex Bennée Signed-off-by: Richard Henderson --- tcg/tcg.c | 76 ------------------------------------------------------- 1 file changed, 76 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index a1ae761a58..c330d114bc 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -2900,79 +2900,6 @@ static bool liveness_pass_2(TCGContext *s) return changes; } -#ifdef CONFIG_DEBUG_TCG -static void dump_regs(TCGContext *s) -{ - TCGTemp *ts; - int i; - char buf[64]; - - for(i = 0; i < s->nb_temps; i++) { - ts = &s->temps[i]; - printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); - switch(ts->val_type) { - case TEMP_VAL_REG: - printf("%s", tcg_target_reg_names[ts->reg]); - break; - case TEMP_VAL_MEM: - printf("%d(%s)", (int)ts->mem_offset, - tcg_target_reg_names[ts->mem_base->reg]); - break; - case TEMP_VAL_CONST: - printf("$0x%" PRIx64, ts->val); - break; - case TEMP_VAL_DEAD: - printf("D"); - break; - default: - printf("???"); - break; - } - printf("\n"); - } - - for(i = 0; i < TCG_TARGET_NB_REGS; i++) { - if (s->reg_to_temp[i] != NULL) { - printf("%s: %s\n", - tcg_target_reg_names[i], - tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i])); - } - } -} - -static void check_regs(TCGContext *s) -{ - int reg; - int k; - TCGTemp *ts; - char buf[64]; - - for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) { - ts = s->reg_to_temp[reg]; - if (ts != NULL) { - if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) { - printf("Inconsistency for register %s:\n", - tcg_target_reg_names[reg]); - goto fail; - } - } - } - for (k = 0; k < s->nb_temps; k++) { - ts = &s->temps[k]; - if (ts->val_type == TEMP_VAL_REG - && ts->kind != TEMP_FIXED - && s->reg_to_temp[ts->reg] != ts) { - printf("Inconsistency for temp %s:\n", - tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts)); - fail: - printf("reg state:\n"); - dump_regs(s); - tcg_abort(); - } - } -} -#endif - static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) { intptr_t off, size, align; @@ -4297,9 +4224,6 @@ int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start) tcg_reg_alloc_op(s, op); break; } -#ifdef CONFIG_DEBUG_TCG - check_regs(s); -#endif /* Test for (pending) buffer overflow. The assumption is that any one operation beginning below the high water mark cannot overrun the buffer completely. Thus we can test for overflow after From 8940ea0d326252cdbfd0fb5d6092caa3647a3b94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Mon, 19 Dec 2022 23:09:23 +0100 Subject: [PATCH 18/47] tcg: Massage process_op_defs() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In preparation of introducing paired registers, massage a bit process_op_defs()'s switch case. Signed-off-by: Richard Henderson [PMD: Split from bigger patch, 1/3] Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20221219220925.79218-2-philmd@linaro.org> --- tcg/tcg.c | 61 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 34 insertions(+), 27 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index c330d114bc..92141bd79a 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -2012,7 +2012,7 @@ static void process_op_defs(TCGContext *s) for (op = 0; op < NB_OPS; op++) { TCGOpDef *def = &tcg_op_defs[op]; const TCGTargetOpDef *tdefs; - int i, nb_args; + int i, o, nb_args; if (def->flags & TCG_OPF_NOT_PRESENT) { continue; @@ -2034,53 +2034,60 @@ static void process_op_defs(TCGContext *s) for (i = 0; i < nb_args; i++) { const char *ct_str = tdefs->args_ct_str[i]; + bool input_p = i >= def->nb_oargs; + /* Incomplete TCGTargetOpDef entry. */ tcg_debug_assert(ct_str != NULL); - while (*ct_str != '\0') { - switch(*ct_str) { - case '0' ... '9': - { - int oarg = *ct_str - '0'; - tcg_debug_assert(ct_str == tdefs->args_ct_str[i]); - tcg_debug_assert(oarg < def->nb_oargs); - tcg_debug_assert(def->args_ct[oarg].regs != 0); - def->args_ct[i] = def->args_ct[oarg]; - /* The output sets oalias. */ - def->args_ct[oarg].oalias = true; - def->args_ct[oarg].alias_index = i; - /* The input sets ialias. */ - def->args_ct[i].ialias = true; - def->args_ct[i].alias_index = oarg; - } - ct_str++; - break; - case '&': - def->args_ct[i].newreg = true; - ct_str++; - break; + switch (*ct_str) { + case '0' ... '9': + o = *ct_str - '0'; + tcg_debug_assert(input_p); + tcg_debug_assert(o < def->nb_oargs); + tcg_debug_assert(def->args_ct[o].regs != 0); + tcg_debug_assert(!def->args_ct[o].oalias); + def->args_ct[i] = def->args_ct[o]; + /* The output sets oalias. */ + def->args_ct[o].oalias = 1; + def->args_ct[o].alias_index = i; + /* The input sets ialias. */ + def->args_ct[i].ialias = 1; + def->args_ct[i].alias_index = o; + tcg_debug_assert(ct_str[1] == '\0'); + continue; + + case '&': + tcg_debug_assert(!input_p); + def->args_ct[i].newreg = true; + ct_str++; + break; + } + + do { + switch (*ct_str) { case 'i': def->args_ct[i].ct |= TCG_CT_CONST; - ct_str++; break; /* Include all of the target-specific constraints. */ #undef CONST #define CONST(CASE, MASK) \ - case CASE: def->args_ct[i].ct |= MASK; ct_str++; break; + case CASE: def->args_ct[i].ct |= MASK; break; #define REGS(CASE, MASK) \ - case CASE: def->args_ct[i].regs |= MASK; ct_str++; break; + case CASE: def->args_ct[i].regs |= MASK; break; #include "tcg-target-con-str.h" #undef REGS #undef CONST default: + case '0' ... '9': + case '&': /* Typo in TCGTargetOpDef constraint. */ g_assert_not_reached(); } - } + } while (*++ct_str != '\0'); } /* TCGTargetOpDef entry with too much information? */ From 29f5e92502effeadde242500a63dfd87a275ab46 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 14 Oct 2022 07:37:38 +1100 Subject: [PATCH 19/47] tcg: Introduce paired register allocation There are several instances where we need to be able to allocate a pair of registers to related inputs/outputs. Add 'p' and 'm' register constraints for this, in order to be able to allocate the even/odd register first or second. Signed-off-by: Richard Henderson --- include/tcg/tcg.h | 2 + tcg/tcg.c | 424 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 378 insertions(+), 48 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index d84bae6e3f..5c2254ce9f 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -951,6 +951,8 @@ typedef struct TCGArgConstraint { unsigned ct : 16; unsigned alias_index : 4; unsigned sort_index : 4; + unsigned pair_index : 4; + unsigned pair : 2; /* 0: none, 1: first, 2: second, 3: second alias */ bool oalias : 1; bool ialias : 1; bool newreg : 1; diff --git a/tcg/tcg.c b/tcg/tcg.c index 92141bd79a..2cf24b4453 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -1969,15 +1969,32 @@ static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) static int get_constraint_priority(const TCGOpDef *def, int k) { const TCGArgConstraint *arg_ct = &def->args_ct[k]; - int n; + int n = ctpop64(arg_ct->regs); - if (arg_ct->oalias) { - /* an alias is equivalent to a single register */ - n = 1; - } else { - n = ctpop64(arg_ct->regs); + /* + * Sort constraints of a single register first, which includes output + * aliases (which must exactly match the input already allocated). + */ + if (n == 1 || arg_ct->oalias) { + return INT_MAX; } - return TCG_TARGET_NB_REGS - n + 1; + + /* + * Sort register pairs next, first then second immediately after. + * Arbitrarily sort multiple pairs by the index of the first reg; + * there shouldn't be many pairs. + */ + switch (arg_ct->pair) { + case 1: + case 3: + return (k + 1) * 2; + case 2: + return (arg_ct->pair_index + 1) * 2 - 1; + } + + /* Finally, sort by decreasing register count. */ + assert(n > 1); + return -n; } /* sort from highest priority to lowest */ @@ -2012,7 +2029,8 @@ static void process_op_defs(TCGContext *s) for (op = 0; op < NB_OPS; op++) { TCGOpDef *def = &tcg_op_defs[op]; const TCGTargetOpDef *tdefs; - int i, o, nb_args; + bool saw_alias_pair = false; + int i, o, i2, o2, nb_args; if (def->flags & TCG_OPF_NOT_PRESENT) { continue; @@ -2053,6 +2071,9 @@ static void process_op_defs(TCGContext *s) /* The input sets ialias. */ def->args_ct[i].ialias = 1; def->args_ct[i].alias_index = o; + if (def->args_ct[i].pair) { + saw_alias_pair = true; + } tcg_debug_assert(ct_str[1] == '\0'); continue; @@ -2061,6 +2082,38 @@ static void process_op_defs(TCGContext *s) def->args_ct[i].newreg = true; ct_str++; break; + + case 'p': /* plus */ + /* Allocate to the register after the previous. */ + tcg_debug_assert(i > (input_p ? def->nb_oargs : 0)); + o = i - 1; + tcg_debug_assert(!def->args_ct[o].pair); + tcg_debug_assert(!def->args_ct[o].ct); + def->args_ct[i] = (TCGArgConstraint){ + .pair = 2, + .pair_index = o, + .regs = def->args_ct[o].regs << 1, + }; + def->args_ct[o].pair = 1; + def->args_ct[o].pair_index = i; + tcg_debug_assert(ct_str[1] == '\0'); + continue; + + case 'm': /* minus */ + /* Allocate to the register before the previous. */ + tcg_debug_assert(i > (input_p ? def->nb_oargs : 0)); + o = i - 1; + tcg_debug_assert(!def->args_ct[o].pair); + tcg_debug_assert(!def->args_ct[o].ct); + def->args_ct[i] = (TCGArgConstraint){ + .pair = 1, + .pair_index = o, + .regs = def->args_ct[o].regs >> 1, + }; + def->args_ct[o].pair = 2; + def->args_ct[o].pair_index = i; + tcg_debug_assert(ct_str[1] == '\0'); + continue; } do { @@ -2084,6 +2137,8 @@ static void process_op_defs(TCGContext *s) default: case '0' ... '9': case '&': + case 'p': + case 'm': /* Typo in TCGTargetOpDef constraint. */ g_assert_not_reached(); } @@ -2093,6 +2148,79 @@ static void process_op_defs(TCGContext *s) /* TCGTargetOpDef entry with too much information? */ tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL); + /* + * Fix up output pairs that are aliased with inputs. + * When we created the alias, we copied pair from the output. + * There are three cases: + * (1a) Pairs of inputs alias pairs of outputs. + * (1b) One input aliases the first of a pair of outputs. + * (2) One input aliases the second of a pair of outputs. + * + * Case 1a is handled by making sure that the pair_index'es are + * properly updated so that they appear the same as a pair of inputs. + * + * Case 1b is handled by setting the pair_index of the input to + * itself, simply so it doesn't point to an unrelated argument. + * Since we don't encounter the "second" during the input allocation + * phase, nothing happens with the second half of the input pair. + * + * Case 2 is handled by setting the second input to pair=3, the + * first output to pair=3, and the pair_index'es to match. + */ + if (saw_alias_pair) { + for (i = def->nb_oargs; i < nb_args; i++) { + /* + * Since [0-9pm] must be alone in the constraint string, + * the only way they can both be set is if the pair comes + * from the output alias. + */ + if (!def->args_ct[i].ialias) { + continue; + } + switch (def->args_ct[i].pair) { + case 0: + break; + case 1: + o = def->args_ct[i].alias_index; + o2 = def->args_ct[o].pair_index; + tcg_debug_assert(def->args_ct[o].pair == 1); + tcg_debug_assert(def->args_ct[o2].pair == 2); + if (def->args_ct[o2].oalias) { + /* Case 1a */ + i2 = def->args_ct[o2].alias_index; + tcg_debug_assert(def->args_ct[i2].pair == 2); + def->args_ct[i2].pair_index = i; + def->args_ct[i].pair_index = i2; + } else { + /* Case 1b */ + def->args_ct[i].pair_index = i; + } + break; + case 2: + o = def->args_ct[i].alias_index; + o2 = def->args_ct[o].pair_index; + tcg_debug_assert(def->args_ct[o].pair == 2); + tcg_debug_assert(def->args_ct[o2].pair == 1); + if (def->args_ct[o2].oalias) { + /* Case 1a */ + i2 = def->args_ct[o2].alias_index; + tcg_debug_assert(def->args_ct[i2].pair == 1); + def->args_ct[i2].pair_index = i; + def->args_ct[i].pair_index = i2; + } else { + /* Case 2 */ + def->args_ct[i].pair = 3; + def->args_ct[o2].pair = 3; + def->args_ct[i].pair_index = o2; + def->args_ct[o2].pair_index = i; + } + break; + default: + g_assert_not_reached(); + } + } + } + /* sort the constraints (XXX: this is just an heuristic) */ sort_constraints(def, 0, def->nb_oargs); sort_constraints(def, def->nb_oargs, def->nb_iargs); @@ -3141,6 +3269,52 @@ static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs, tcg_abort(); } +static TCGReg tcg_reg_alloc_pair(TCGContext *s, TCGRegSet required_regs, + TCGRegSet allocated_regs, + TCGRegSet preferred_regs, bool rev) +{ + int i, j, k, fmin, n = ARRAY_SIZE(tcg_target_reg_alloc_order); + TCGRegSet reg_ct[2]; + const int *order; + + /* Ensure that if I is not in allocated_regs, I+1 is not either. */ + reg_ct[1] = required_regs & ~(allocated_regs | (allocated_regs >> 1)); + tcg_debug_assert(reg_ct[1] != 0); + reg_ct[0] = reg_ct[1] & preferred_regs; + + order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order; + + /* + * Skip the preferred_regs option if it cannot be satisfied, + * or if the preference made no difference. + */ + k = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1]; + + /* + * Minimize the number of flushes by looking for 2 free registers first, + * then a single flush, then two flushes. + */ + for (fmin = 2; fmin >= 0; fmin--) { + for (j = k; j < 2; j++) { + TCGRegSet set = reg_ct[j]; + + for (i = 0; i < n; i++) { + TCGReg reg = order[i]; + + if (tcg_regset_test_reg(set, reg)) { + int f = !s->reg_to_temp[reg] + !s->reg_to_temp[reg + 1]; + if (f >= fmin) { + tcg_reg_free(s, reg, allocated_regs); + tcg_reg_free(s, reg + 1, allocated_regs); + return reg; + } + } + } + } + } + tcg_abort(); +} + /* Make sure the temporary is in a register. If needed, allocate the register from DESIRED while avoiding ALLOCATED. */ static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs, @@ -3550,8 +3724,10 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) /* satisfy input constraints */ for (k = 0; k < nb_iargs; k++) { - TCGRegSet i_preferred_regs; - bool allocate_new_reg; + TCGRegSet i_preferred_regs, i_required_regs; + bool allocate_new_reg, copyto_new_reg; + TCGTemp *ts2; + int i1, i2; i = def->args_ct[nb_oargs + k].sort_index; arg = op->args[i]; @@ -3568,43 +3744,164 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) reg = ts->reg; i_preferred_regs = 0; + i_required_regs = arg_ct->regs; allocate_new_reg = false; + copyto_new_reg = false; - if (arg_ct->ialias) { + switch (arg_ct->pair) { + case 0: /* not paired */ + if (arg_ct->ialias) { + i_preferred_regs = op->output_pref[arg_ct->alias_index]; + + /* + * If the input is readonly, then it cannot also be an + * output and aliased to itself. If the input is not + * dead after the instruction, we must allocate a new + * register and move it. + */ + if (temp_readonly(ts) || !IS_DEAD_ARG(i)) { + allocate_new_reg = true; + } else if (ts->val_type == TEMP_VAL_REG) { + /* + * Check if the current register has already been + * allocated for another input. + */ + allocate_new_reg = + tcg_regset_test_reg(i_allocated_regs, reg); + } + } + if (!allocate_new_reg) { + temp_load(s, ts, i_required_regs, i_allocated_regs, + i_preferred_regs); + reg = ts->reg; + allocate_new_reg = !tcg_regset_test_reg(i_required_regs, reg); + } + if (allocate_new_reg) { + /* + * Allocate a new register matching the constraint + * and move the temporary register into it. + */ + temp_load(s, ts, tcg_target_available_regs[ts->type], + i_allocated_regs, 0); + reg = tcg_reg_alloc(s, i_required_regs, i_allocated_regs, + i_preferred_regs, ts->indirect_base); + copyto_new_reg = true; + } + break; + + case 1: + /* First of an input pair; if i1 == i2, the second is an output. */ + i1 = i; + i2 = arg_ct->pair_index; + ts2 = i1 != i2 ? arg_temp(op->args[i2]) : NULL; + + /* + * It is easier to default to allocating a new pair + * and to identify a few cases where it's not required. + */ + if (arg_ct->ialias) { + i_preferred_regs = op->output_pref[arg_ct->alias_index]; + if (IS_DEAD_ARG(i1) && + IS_DEAD_ARG(i2) && + !temp_readonly(ts) && + ts->val_type == TEMP_VAL_REG && + ts->reg < TCG_TARGET_NB_REGS - 1 && + tcg_regset_test_reg(i_required_regs, reg) && + !tcg_regset_test_reg(i_allocated_regs, reg) && + !tcg_regset_test_reg(i_allocated_regs, reg + 1) && + (ts2 + ? ts2->val_type == TEMP_VAL_REG && + ts2->reg == reg + 1 && + !temp_readonly(ts2) + : s->reg_to_temp[reg + 1] == NULL)) { + break; + } + } else { + /* Without aliasing, the pair must also be an input. */ + tcg_debug_assert(ts2); + if (ts->val_type == TEMP_VAL_REG && + ts2->val_type == TEMP_VAL_REG && + ts2->reg == reg + 1 && + tcg_regset_test_reg(i_required_regs, reg)) { + break; + } + } + reg = tcg_reg_alloc_pair(s, i_required_regs, i_allocated_regs, + 0, ts->indirect_base); + goto do_pair; + + case 2: /* pair second */ + reg = new_args[arg_ct->pair_index] + 1; + goto do_pair; + + case 3: /* ialias with second output, no first input */ + tcg_debug_assert(arg_ct->ialias); i_preferred_regs = op->output_pref[arg_ct->alias_index]; - /* - * If the input is readonly, then it cannot also be an - * output and aliased to itself. If the input is not - * dead after the instruction, we must allocate a new - * register and move it. - */ - if (temp_readonly(ts) || !IS_DEAD_ARG(i)) { - allocate_new_reg = true; - } else if (ts->val_type == TEMP_VAL_REG) { - /* - * Check if the current register has already been - * allocated for another input. - */ - allocate_new_reg = tcg_regset_test_reg(i_allocated_regs, reg); + if (IS_DEAD_ARG(i) && + !temp_readonly(ts) && + ts->val_type == TEMP_VAL_REG && + reg > 0 && + s->reg_to_temp[reg - 1] == NULL && + tcg_regset_test_reg(i_required_regs, reg) && + !tcg_regset_test_reg(i_allocated_regs, reg) && + !tcg_regset_test_reg(i_allocated_regs, reg - 1)) { + tcg_regset_set_reg(i_allocated_regs, reg - 1); + break; } - } + reg = tcg_reg_alloc_pair(s, i_required_regs >> 1, + i_allocated_regs, 0, + ts->indirect_base); + tcg_regset_set_reg(i_allocated_regs, reg); + reg += 1; + goto do_pair; - if (!allocate_new_reg) { - temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs); - reg = ts->reg; - allocate_new_reg = !tcg_regset_test_reg(arg_ct->regs, reg); - } - - if (allocate_new_reg) { + do_pair: /* - * Allocate a new register matching the constraint - * and move the temporary register into it. + * If an aliased input is not dead after the instruction, + * we must allocate a new register and move it. */ - temp_load(s, ts, tcg_target_available_regs[ts->type], - i_allocated_regs, 0); - reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs, - i_preferred_regs, ts->indirect_base); + if (arg_ct->ialias && (!IS_DEAD_ARG(i) || temp_readonly(ts))) { + TCGRegSet t_allocated_regs = i_allocated_regs; + + /* + * Because of the alias, and the continued life, make sure + * that the temp is somewhere *other* than the reg pair, + * and we get a copy in reg. + */ + tcg_regset_set_reg(t_allocated_regs, reg); + tcg_regset_set_reg(t_allocated_regs, reg + 1); + if (ts->val_type == TEMP_VAL_REG && ts->reg == reg) { + /* If ts was already in reg, copy it somewhere else. */ + TCGReg nr; + bool ok; + + tcg_debug_assert(ts->kind != TEMP_FIXED); + nr = tcg_reg_alloc(s, tcg_target_available_regs[ts->type], + t_allocated_regs, 0, ts->indirect_base); + ok = tcg_out_mov(s, ts->type, nr, reg); + tcg_debug_assert(ok); + + set_temp_val_reg(s, ts, nr); + } else { + temp_load(s, ts, tcg_target_available_regs[ts->type], + t_allocated_regs, 0); + copyto_new_reg = true; + } + } else { + /* Preferably allocate to reg, otherwise copy. */ + i_required_regs = (TCGRegSet)1 << reg; + temp_load(s, ts, i_required_regs, i_allocated_regs, + i_preferred_regs); + copyto_new_reg = ts->reg != reg; + } + break; + + default: + g_assert_not_reached(); + } + + if (copyto_new_reg) { if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { /* * Cross register class move not supported. Sync the @@ -3656,15 +3953,46 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) /* ENV should not be modified. */ tcg_debug_assert(!temp_readonly(ts)); - if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { - reg = new_args[arg_ct->alias_index]; - } else if (arg_ct->newreg) { - reg = tcg_reg_alloc(s, arg_ct->regs, - i_allocated_regs | o_allocated_regs, - op->output_pref[k], ts->indirect_base); - } else { - reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, - op->output_pref[k], ts->indirect_base); + switch (arg_ct->pair) { + case 0: /* not paired */ + if (arg_ct->oalias && !const_args[arg_ct->alias_index]) { + reg = new_args[arg_ct->alias_index]; + } else if (arg_ct->newreg) { + reg = tcg_reg_alloc(s, arg_ct->regs, + i_allocated_regs | o_allocated_regs, + op->output_pref[k], ts->indirect_base); + } else { + reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, + op->output_pref[k], ts->indirect_base); + } + break; + + case 1: /* first of pair */ + tcg_debug_assert(!arg_ct->newreg); + if (arg_ct->oalias) { + reg = new_args[arg_ct->alias_index]; + break; + } + reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs, + op->output_pref[k], ts->indirect_base); + break; + + case 2: /* second of pair */ + tcg_debug_assert(!arg_ct->newreg); + if (arg_ct->oalias) { + reg = new_args[arg_ct->alias_index]; + } else { + reg = new_args[arg_ct->pair_index] + 1; + } + break; + + case 3: /* first of pair, aliasing with a second input */ + tcg_debug_assert(!arg_ct->newreg); + reg = new_args[arg_ct->pair_index] - 1; + break; + + default: + g_assert_not_reached(); } tcg_regset_set_reg(o_allocated_regs, reg); set_temp_val_reg(s, ts, reg); From c8cc6879f61cd589a9ed211729f12bc3b46beed4 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 19 Oct 2022 20:33:28 +1000 Subject: [PATCH 20/47] accel/tcg: Set cflags_next_tb in cpu_common_initfn MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit While we initialize this value in cpu_common_reset, that isn't called during startup, so set it as well in init. This fixes -singlestep versus the very first TB. Fixes: 04f5b647ed07 ("accel/tcg: Handle -singlestep in curr_cflags") Reviewed-by: Alex Bennée Signed-off-by: Richard Henderson --- hw/core/cpu-common.c | 1 + 1 file changed, 1 insertion(+) diff --git a/hw/core/cpu-common.c b/hw/core/cpu-common.c index 78b5f350a0..b177e761f0 100644 --- a/hw/core/cpu-common.c +++ b/hw/core/cpu-common.c @@ -235,6 +235,7 @@ static void cpu_common_initfn(Object *obj) /* the default value is changed by qemu_init_vcpu() for softmmu */ cpu->nr_cores = 1; cpu->nr_threads = 1; + cpu->cflags_next_tb = -1; qemu_mutex_init(&cpu->work_mutex); QSIMPLEQ_INIT(&cpu->work_list); From 8e7bbc7575f85a350a3e6650a551983036a2698d Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 17 Oct 2022 10:57:03 +1000 Subject: [PATCH 21/47] target/sparc: Avoid TCGV_{LOW,HIGH} MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the official extend/extract functions instead of routines that will shortly be internal to tcg. Cc: Mark Cave-Ayland Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- target/sparc/translate.c | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/target/sparc/translate.c b/target/sparc/translate.c index 34858eb95f..150aeecd14 100644 --- a/target/sparc/translate.c +++ b/target/sparc/translate.c @@ -163,13 +163,6 @@ static inline void gen_update_fprs_dirty(DisasContext *dc, int rd) /* floating point registers moves */ static TCGv_i32 gen_load_fpr_F(DisasContext *dc, unsigned int src) { -#if TCG_TARGET_REG_BITS == 32 - if (src & 1) { - return TCGV_LOW(cpu_fpr[src / 2]); - } else { - return TCGV_HIGH(cpu_fpr[src / 2]); - } -#else TCGv_i32 ret = get_temp_i32(dc); if (src & 1) { tcg_gen_extrl_i64_i32(ret, cpu_fpr[src / 2]); @@ -177,22 +170,16 @@ static TCGv_i32 gen_load_fpr_F(DisasContext *dc, unsigned int src) tcg_gen_extrh_i64_i32(ret, cpu_fpr[src / 2]); } return ret; -#endif } static void gen_store_fpr_F(DisasContext *dc, unsigned int dst, TCGv_i32 v) { -#if TCG_TARGET_REG_BITS == 32 - if (dst & 1) { - tcg_gen_mov_i32(TCGV_LOW(cpu_fpr[dst / 2]), v); - } else { - tcg_gen_mov_i32(TCGV_HIGH(cpu_fpr[dst / 2]), v); - } -#else - TCGv_i64 t = (TCGv_i64)v; + TCGv_i64 t = tcg_temp_new_i64(); + + tcg_gen_extu_i32_i64(t, v); tcg_gen_deposit_i64(cpu_fpr[dst / 2], cpu_fpr[dst / 2], t, (dst & 1 ? 0 : 32), 32); -#endif + tcg_temp_free_i64(t); gen_update_fprs_dirty(dc, dst); } From d56fea79f9d24c62b0a8c3a80924147942409258 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 17 Oct 2022 11:07:39 +1000 Subject: [PATCH 22/47] tcg: Move TCG_{LOW,HIGH} to tcg-internal.h MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the error-generating fallback from tcg-op.c, and replace "_link_error" with modern QEMU_ERROR markup. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- include/tcg/tcg-op.h | 33 +++++---------------------------- include/tcg/tcg.h | 12 ------------ tcg/tcg-internal.h | 14 ++++++++++++++ tcg/tcg-op-vec.c | 2 ++ tcg/tcg-op.c | 37 ++++++++++++++++++++++++++++--------- 5 files changed, 49 insertions(+), 49 deletions(-) diff --git a/include/tcg/tcg-op.h b/include/tcg/tcg-op.h index 209e168305..8176f194cb 100644 --- a/include/tcg/tcg-op.h +++ b/include/tcg/tcg-op.h @@ -667,35 +667,12 @@ static inline void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) tcg_gen_op3_i64(INDEX_op_mul_i64, ret, arg1, arg2); } #else /* TCG_TARGET_REG_BITS == 32 */ -static inline void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset); -} +void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset); +void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset); +void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset); -static inline void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset); -} - -static inline void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, - tcg_target_long offset) -{ - tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); -} - -static inline void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) -{ - tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), - TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); -} - -static inline void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) -{ - tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), - TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); -} +void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); +void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2); void tcg_gen_discard_i64(TCGv_i64 arg); void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg); diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index 5c2254ce9f..d207bc47be 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -737,18 +737,6 @@ static inline TCGv_vec temp_tcgv_vec(TCGTemp *t) return (TCGv_vec)temp_tcgv_i32(t); } -#if TCG_TARGET_REG_BITS == 32 -static inline TCGv_i32 TCGV_LOW(TCGv_i64 t) -{ - return temp_tcgv_i32(tcgv_i64_temp(t)); -} - -static inline TCGv_i32 TCGV_HIGH(TCGv_i64 t) -{ - return temp_tcgv_i32(tcgv_i64_temp(t) + 1); -} -#endif - static inline TCGArg tcg_get_insn_param(TCGOp *op, int arg) { return op->args[arg]; diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index cc82088d52..a9ea27f67a 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -59,4 +59,18 @@ static inline unsigned tcg_call_flags(TCGOp *op) return tcg_call_info(op)->flags; } +#if TCG_TARGET_REG_BITS == 32 +static inline TCGv_i32 TCGV_LOW(TCGv_i64 t) +{ + return temp_tcgv_i32(tcgv_i64_temp(t)); +} +static inline TCGv_i32 TCGV_HIGH(TCGv_i64 t) +{ + return temp_tcgv_i32(tcgv_i64_temp(t) + 1); +} +#else +extern TCGv_i32 TCGV_LOW(TCGv_i64) QEMU_ERROR("32-bit code path is reachable"); +extern TCGv_i32 TCGV_HIGH(TCGv_i64) QEMU_ERROR("32-bit code path is reachable"); +#endif + #endif /* TCG_INTERNAL_H */ diff --git a/tcg/tcg-op-vec.c b/tcg/tcg-op-vec.c index 463dabf515..5bf100ea7d 100644 --- a/tcg/tcg-op-vec.c +++ b/tcg/tcg-op-vec.c @@ -21,6 +21,8 @@ #include "tcg/tcg.h" #include "tcg/tcg-op.h" #include "tcg/tcg-mo.h" +#include "tcg-internal.h" + /* Reduce the number of ifdefs below. This assumes that all uses of TCGV_HIGH and TCGV_LOW are properly protected by a conditional that diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index 019fab00cc..6168327030 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -28,16 +28,8 @@ #include "tcg/tcg-op.h" #include "tcg/tcg-mo.h" #include "exec/plugin-gen.h" +#include "tcg-internal.h" -/* Reduce the number of ifdefs below. This assumes that all uses of - TCGV_HIGH and TCGV_LOW are properly protected by a conditional that - the compiler can eliminate. */ -#if TCG_TARGET_REG_BITS == 64 -extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64); -extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64); -#define TCGV_LOW TCGV_LOW_link_error -#define TCGV_HIGH TCGV_HIGH_link_error -#endif void tcg_gen_op1(TCGOpcode opc, TCGArg a1) { @@ -1171,6 +1163,21 @@ void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset) #endif } +void tcg_gen_st8_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) +{ + tcg_gen_st8_i32(TCGV_LOW(arg1), arg2, offset); +} + +void tcg_gen_st16_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) +{ + tcg_gen_st16_i32(TCGV_LOW(arg1), arg2, offset); +} + +void tcg_gen_st32_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) +{ + tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset); +} + void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) { #if HOST_BIG_ENDIAN @@ -1182,6 +1189,18 @@ void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset) #endif } +void tcg_gen_add_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) +{ + tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), + TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); +} + +void tcg_gen_sub_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) +{ + tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret), TCGV_LOW(arg1), + TCGV_HIGH(arg1), TCGV_LOW(arg2), TCGV_HIGH(arg2)); +} + void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2) { tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2)); From fac87bd2a49bf16edeb1d2823a993ad7c9ed073b Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 19 Oct 2022 11:26:37 +1000 Subject: [PATCH 23/47] tcg: Add temp_subindex to TCGTemp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Record the location of a TCGTemp within a larger object. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- include/tcg/tcg.h | 1 + tcg/tcg.c | 3 +++ 2 files changed, 4 insertions(+) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index d207bc47be..afa18986b1 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -456,6 +456,7 @@ typedef struct TCGTemp { unsigned int mem_coherent:1; unsigned int mem_allocated:1; unsigned int temp_allocated:1; + unsigned int temp_subindex:1; int64_t val; struct TCGTemp *mem_base; diff --git a/tcg/tcg.c b/tcg/tcg.c index 2cf24b4453..8b2193409a 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -928,6 +928,7 @@ TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, ts2->mem_allocated = 1; ts2->mem_base = base_ts; ts2->mem_offset = offset + (1 - bigendian) * 4; + ts2->temp_subindex = 1; pstrcpy(buf, sizeof(buf), name); pstrcat(buf, sizeof(buf), "_1"); ts2->name = strdup(buf); @@ -974,6 +975,7 @@ TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local) ts2->base_type = TCG_TYPE_I64; ts2->type = TCG_TYPE_I32; ts2->temp_allocated = 1; + ts2->temp_subindex = 1; ts2->kind = kind; } else { ts->base_type = type; @@ -1092,6 +1094,7 @@ TCGTemp *tcg_constant_internal(TCGType type, int64_t val) ts2->type = TCG_TYPE_I32; ts2->kind = TEMP_CONST; ts2->temp_allocated = 1; + ts2->temp_subindex = 1; ts2->val = val >> 32; } else { ts->base_type = type; From f01847c251c24beaf3a0a41764d331355d08ab54 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 21 Oct 2022 11:00:35 +1000 Subject: [PATCH 24/47] tcg: Simplify calls to temp_sync vs mem_coherent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first thing that temp_sync does is check mem_coherent, so there's no need for the caller to do so. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/tcg.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index 8b2193409a..034d1b6604 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -4083,12 +4083,8 @@ static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) /* If the two inputs form one 64-bit value, try dupm_vec. */ if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) { - if (!itsl->mem_coherent) { - temp_sync(s, itsl, s->reserved_regs, 0, 0); - } - if (!itsh->mem_coherent) { - temp_sync(s, itsh, s->reserved_regs, 0, 0); - } + temp_sync(s, itsl, s->reserved_regs, 0, 0); + temp_sync(s, itsh, s->reserved_regs, 0, 0); #if HOST_BIG_ENDIAN TCGTemp *its = itsh; #else From aef8540290f46b40f225c9f6d993c77c0697ee7c Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 19 Oct 2022 11:53:27 +1000 Subject: [PATCH 25/47] tcg: Allocate TCGTemp pairs in host memory order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allocate the first of a pair at the lower address, and the second of a pair at the higher address. This will make it easier to find the beginning of the larger memory block. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/tcg-internal.h | 4 ++-- tcg/tcg.c | 58 ++++++++++++++++++++++------------------------ 2 files changed, 30 insertions(+), 32 deletions(-) diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index a9ea27f67a..2c06b5116a 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -62,11 +62,11 @@ static inline unsigned tcg_call_flags(TCGOp *op) #if TCG_TARGET_REG_BITS == 32 static inline TCGv_i32 TCGV_LOW(TCGv_i64 t) { - return temp_tcgv_i32(tcgv_i64_temp(t)); + return temp_tcgv_i32(tcgv_i64_temp(t) + HOST_BIG_ENDIAN); } static inline TCGv_i32 TCGV_HIGH(TCGv_i64 t) { - return temp_tcgv_i32(tcgv_i64_temp(t) + 1); + return temp_tcgv_i32(tcgv_i64_temp(t) + !HOST_BIG_ENDIAN); } #else extern TCGv_i32 TCGV_LOW(TCGv_i64) QEMU_ERROR("32-bit code path is reachable"); diff --git a/tcg/tcg.c b/tcg/tcg.c index 034d1b6604..a02086c482 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -887,10 +887,7 @@ TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, TCGContext *s = tcg_ctx; TCGTemp *base_ts = tcgv_ptr_temp(base); TCGTemp *ts = tcg_global_alloc(s); - int indirect_reg = 0, bigendian = 0; -#if HOST_BIG_ENDIAN - bigendian = 1; -#endif + int indirect_reg = 0; switch (base_ts->kind) { case TEMP_FIXED: @@ -916,7 +913,7 @@ TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, ts->indirect_reg = indirect_reg; ts->mem_allocated = 1; ts->mem_base = base_ts; - ts->mem_offset = offset + bigendian * 4; + ts->mem_offset = offset; pstrcpy(buf, sizeof(buf), name); pstrcat(buf, sizeof(buf), "_0"); ts->name = strdup(buf); @@ -927,7 +924,7 @@ TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base, ts2->indirect_reg = indirect_reg; ts2->mem_allocated = 1; ts2->mem_base = base_ts; - ts2->mem_offset = offset + (1 - bigendian) * 4; + ts2->mem_offset = offset + 4; ts2->temp_subindex = 1; pstrcpy(buf, sizeof(buf), name); pstrcat(buf, sizeof(buf), "_1"); @@ -1073,37 +1070,43 @@ TCGTemp *tcg_constant_internal(TCGType type, int64_t val) ts = g_hash_table_lookup(h, &val); if (ts == NULL) { + int64_t *val_ptr; + ts = tcg_temp_alloc(s); if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { TCGTemp *ts2 = tcg_temp_alloc(s); + tcg_debug_assert(ts2 == ts + 1); + ts->base_type = TCG_TYPE_I64; ts->type = TCG_TYPE_I32; ts->kind = TEMP_CONST; ts->temp_allocated = 1; - /* - * Retain the full value of the 64-bit constant in the low - * part, so that the hash table works. Actual uses will - * truncate the value to the low part. - */ - ts->val = val; - tcg_debug_assert(ts2 == ts + 1); ts2->base_type = TCG_TYPE_I64; ts2->type = TCG_TYPE_I32; ts2->kind = TEMP_CONST; ts2->temp_allocated = 1; ts2->temp_subindex = 1; - ts2->val = val >> 32; + + /* + * Retain the full value of the 64-bit constant in the low + * part, so that the hash table works. Actual uses will + * truncate the value to the low part. + */ + ts[HOST_BIG_ENDIAN].val = val; + ts[!HOST_BIG_ENDIAN].val = val >> 32; + val_ptr = &ts[HOST_BIG_ENDIAN].val; } else { ts->base_type = type; ts->type = type; ts->kind = TEMP_CONST; ts->temp_allocated = 1; ts->val = val; + val_ptr = &ts->val; } - g_hash_table_insert(h, &ts->val, ts); + g_hash_table_insert(h, val_ptr, ts); } return ts; @@ -1515,13 +1518,8 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) pi = 0; if (ret != NULL) { if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) { -#if HOST_BIG_ENDIAN - op->args[pi++] = temp_arg(ret + 1); - op->args[pi++] = temp_arg(ret); -#else op->args[pi++] = temp_arg(ret); op->args[pi++] = temp_arg(ret + 1); -#endif nb_rets = 2; } else { op->args[pi++] = temp_arg(ret); @@ -1555,8 +1553,8 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) } if (TCG_TARGET_REG_BITS < 64 && is_64bit) { - op->args[pi++] = temp_arg(args[i] + HOST_BIG_ENDIAN); - op->args[pi++] = temp_arg(args[i] + !HOST_BIG_ENDIAN); + op->args[pi++] = temp_arg(args[i]); + op->args[pi++] = temp_arg(args[i] + 1); real_args += 2; continue; } @@ -4082,14 +4080,14 @@ static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) } /* If the two inputs form one 64-bit value, try dupm_vec. */ - if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) { - temp_sync(s, itsl, s->reserved_regs, 0, 0); - temp_sync(s, itsh, s->reserved_regs, 0, 0); -#if HOST_BIG_ENDIAN - TCGTemp *its = itsh; -#else - TCGTemp *its = itsl; -#endif + if (itsl->temp_subindex == HOST_BIG_ENDIAN && + itsh->temp_subindex == !HOST_BIG_ENDIAN && + itsl == itsh + (HOST_BIG_ENDIAN ? 1 : -1)) { + TCGTemp *its = itsl - HOST_BIG_ENDIAN; + + temp_sync(s, its + 0, s->reserved_regs, 0, 0); + temp_sync(s, its + 1, s->reserved_regs, 0, 0); + if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg, its->mem_base->reg, its->mem_offset)) { goto done; From 89496a85b4696b797e904a65a10b0600a95a12ec Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 19 Oct 2022 12:24:37 +1000 Subject: [PATCH 26/47] tcg: Move TCG_TYPE_COUNT outside enum MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The count is not itself an enumerator. Move it outside to prevent the compiler from considering it with -Wswitch-enum. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- include/tcg/tcg.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index afa18986b1..f2da340bb9 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -294,7 +294,8 @@ typedef enum TCGType { TCG_TYPE_V128, TCG_TYPE_V256, - TCG_TYPE_COUNT, /* number of different types */ + /* Number of different types (integer not enum) */ +#define TCG_TYPE_COUNT (TCG_TYPE_V256 + 1) /* An alias for the size of the host register. */ #if TCG_TARGET_REG_BITS == 32 From 31c96417465b0ff32d6ec1ee8ef271c6e49ab5a3 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Wed, 19 Oct 2022 12:41:15 +1000 Subject: [PATCH 27/47] tcg: Introduce tcg_type_size MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a helper function for computing the size of a type. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- include/tcg/tcg.h | 16 ++++++++++++++++ tcg/tcg.c | 27 ++++++++++++--------------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index f2da340bb9..8bcd60d0ed 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -319,6 +319,22 @@ typedef enum TCGType { #endif } TCGType; +/** + * tcg_type_size + * @t: type + * + * Return the size of the type in bytes. + */ +static inline int tcg_type_size(TCGType t) +{ + unsigned i = t; + if (i >= TCG_TYPE_V64) { + tcg_debug_assert(i < TCG_TYPE_COUNT); + i -= TCG_TYPE_V64 - 1; + } + return 4 << i; +} + /** * get_alignment_bits * @memop: MemOp value diff --git a/tcg/tcg.c b/tcg/tcg.c index a02086c482..f74507ba66 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -3038,22 +3038,22 @@ static bool liveness_pass_2(TCGContext *s) static void temp_allocate_frame(TCGContext *s, TCGTemp *ts) { - intptr_t off, size, align; + int size = tcg_type_size(ts->type); + int align; + intptr_t off; switch (ts->type) { case TCG_TYPE_I32: - size = align = 4; + align = 4; break; case TCG_TYPE_I64: case TCG_TYPE_V64: - size = align = 8; + align = 8; break; case TCG_TYPE_V128: - size = align = 16; - break; case TCG_TYPE_V256: /* Note that we do not require aligned storage for V256. */ - size = 32, align = 16; + align = 16; break; default: g_assert_not_reached(); @@ -3593,8 +3593,8 @@ static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) TCGRegSet dup_out_regs, dup_in_regs; TCGTemp *its, *ots; TCGType itype, vtype; - intptr_t endian_fixup; unsigned vece; + int lowpart_ofs; bool ok; ots = arg_temp(op->args[0]); @@ -3662,15 +3662,12 @@ static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) /* fall through */ case TEMP_VAL_MEM: -#if HOST_BIG_ENDIAN - endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8; - endian_fixup -= 1 << vece; -#else - endian_fixup = 0; -#endif - /* Attempt to dup directly from the input memory slot. */ + lowpart_ofs = 0; + if (HOST_BIG_ENDIAN) { + lowpart_ofs = tcg_type_size(itype) - (1 << vece); + } if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg, - its->mem_offset + endian_fixup)) { + its->mem_offset + lowpart_ofs)) { goto done; } /* Load the input into the destination vector register. */ From 18ff36ab71bdb302028adc263e0505f09b573f73 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 16 Oct 2022 13:47:00 +1100 Subject: [PATCH 28/47] tcg: Introduce TCGCallReturnKind and TCGCallArgumentKind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prepare to replace a bunch of separate ifdefs with a consistent way to describe the ABI of a function call. Reviewed-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/tcg-internal.h | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index 2c06b5116a..f574743ff8 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -27,6 +27,21 @@ #define TCG_HIGHWATER 1024 +/* + * Describe the calling convention of a given argument type. + */ +typedef enum { + TCG_CALL_RET_NORMAL, /* by registers */ +} TCGCallReturnKind; + +typedef enum { + TCG_CALL_ARG_NORMAL, /* by registers (continuing onto stack) */ + TCG_CALL_ARG_EVEN, /* like normal, but skipping odd slots */ + TCG_CALL_ARG_EXTEND, /* for i32, as a sign/zero-extended i64 */ + TCG_CALL_ARG_EXTEND_U, /* ... as a zero-extended i64 */ + TCG_CALL_ARG_EXTEND_S, /* ... as a sign-extended i64 */ +} TCGCallArgumentKind; + typedef struct TCGHelperInfo { void *func; const char *name; From c8eef960460743b6eed98036e51546b0259c63ec Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 16 Oct 2022 13:48:48 +1100 Subject: [PATCH 29/47] tcg: Replace TCG_TARGET_CALL_ALIGN_ARGS with TCG_TARGET_CALL_ARG_I64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For 32-bit hosts when TCG_TARGET_CALL_ALIGN_ARGS was set, use TCG_CALL_ARG_EVEN. For 64-bit hosts, TCG_TARGET_CALL_ALIGN_ARGS was silently ignored, so always use TCG_CALL_ARG_NORMAL. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/aarch64/tcg-target.h | 2 +- tcg/arm/tcg-target.h | 2 +- tcg/i386/tcg-target.h | 1 + tcg/loongarch64/tcg-target.h | 2 +- tcg/mips/tcg-target.h | 3 ++- tcg/ppc/tcg-target.c.inc | 21 ++++++++------------- tcg/riscv/tcg-target.h | 6 +++++- tcg/s390x/tcg-target.h | 1 + tcg/sparc64/tcg-target.h | 1 + tcg/tcg.c | 6 ++++-- tcg/tci/tcg-target.h | 5 +++++ 11 files changed, 30 insertions(+), 20 deletions(-) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index e145d50fef..d9dd777caa 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -51,8 +51,8 @@ typedef enum { /* used for function call generation */ #define TCG_REG_CALL_STACK TCG_REG_SP #define TCG_TARGET_STACK_ALIGN 16 -#define TCG_TARGET_CALL_ALIGN_ARGS 1 #define TCG_TARGET_CALL_STACK_OFFSET 0 +#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL /* optional instructions */ #define TCG_TARGET_HAS_div_i32 1 diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index 56c1ac4586..09dd0550aa 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -88,8 +88,8 @@ extern bool use_neon_instructions; /* used for function call generation */ #define TCG_TARGET_STACK_ALIGN 8 -#define TCG_TARGET_CALL_ALIGN_ARGS 1 #define TCG_TARGET_CALL_STACK_OFFSET 0 +#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_EVEN /* optional instructions */ #define TCG_TARGET_HAS_ext8s_i32 1 diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index 00fcbe297d..42628a2791 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -98,6 +98,7 @@ typedef enum { #else #define TCG_TARGET_CALL_STACK_OFFSET 0 #endif +#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL extern bool have_bmi1; extern bool have_popcnt; diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h index a659c8d6fd..19d4c07170 100644 --- a/tcg/loongarch64/tcg-target.h +++ b/tcg/loongarch64/tcg-target.h @@ -92,8 +92,8 @@ typedef enum { /* used for function call generation */ #define TCG_REG_CALL_STACK TCG_REG_SP #define TCG_TARGET_STACK_ALIGN 16 -#define TCG_TARGET_CALL_ALIGN_ARGS 1 #define TCG_TARGET_CALL_STACK_OFFSET 0 +#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL /* optional instructions */ #define TCG_TARGET_HAS_movcond_i32 0 diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index 7669213175..bb7312aed4 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -83,10 +83,11 @@ typedef enum { #define TCG_TARGET_STACK_ALIGN 16 #if _MIPS_SIM == _ABIO32 # define TCG_TARGET_CALL_STACK_OFFSET 16 +# define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_EVEN #else # define TCG_TARGET_CALL_STACK_OFFSET 0 +# define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL #endif -#define TCG_TARGET_CALL_ALIGN_ARGS 1 /* MOVN/MOVZ instructions detection */ #if (defined(__mips_isa_rev) && (__mips_isa_rev >= 1)) || \ diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index 9e34df94ba..c2e6bc3296 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -45,7 +45,9 @@ #endif #ifdef _CALL_SYSV -# define TCG_TARGET_CALL_ALIGN_ARGS 1 +# define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_EVEN +#else +# define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL #endif /* For some memory operations, we need a scratch that isn't R0. For the AIX @@ -2202,9 +2204,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) lo = lb->addrlo_reg; hi = lb->addrhi_reg; if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { -#ifdef TCG_TARGET_CALL_ALIGN_ARGS - arg |= 1; -#endif + arg |= (TCG_TARGET_CALL_ARG_I64 == TCG_CALL_ARG_EVEN); tcg_out_mov(s, TCG_TYPE_I32, arg++, hi); tcg_out_mov(s, TCG_TYPE_I32, arg++, lo); } else { @@ -2250,9 +2250,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) lo = lb->addrlo_reg; hi = lb->addrhi_reg; if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { -#ifdef TCG_TARGET_CALL_ALIGN_ARGS - arg |= 1; -#endif + arg |= (TCG_TARGET_CALL_ARG_I64 == TCG_CALL_ARG_EVEN); tcg_out_mov(s, TCG_TYPE_I32, arg++, hi); tcg_out_mov(s, TCG_TYPE_I32, arg++, lo); } else { @@ -2266,9 +2264,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) if (TCG_TARGET_REG_BITS == 32) { switch (s_bits) { case MO_64: -#ifdef TCG_TARGET_CALL_ALIGN_ARGS - arg |= 1; -#endif + arg |= (TCG_TARGET_CALL_ARG_I64 == TCG_CALL_ARG_EVEN); tcg_out_mov(s, TCG_TYPE_I32, arg++, hi); /* FALLTHRU */ case MO_32: @@ -2324,9 +2320,8 @@ static bool tcg_out_fail_alignment(TCGContext *s, TCGLabelQemuLdst *l) if (TCG_TARGET_REG_BITS < TARGET_LONG_BITS) { TCGReg arg = TCG_REG_R4; -#ifdef TCG_TARGET_CALL_ALIGN_ARGS - arg |= 1; -#endif + + arg |= (TCG_TARGET_CALL_ARG_I64 == TCG_CALL_ARG_EVEN); if (l->addrlo_reg != arg) { tcg_out_mov(s, TCG_TYPE_I32, arg, l->addrhi_reg); tcg_out_mov(s, TCG_TYPE_I32, arg + 1, l->addrlo_reg); diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index 11c9b3e4f4..2ab4b8d04a 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -81,8 +81,12 @@ typedef enum { /* used for function call generation */ #define TCG_REG_CALL_STACK TCG_REG_SP #define TCG_TARGET_STACK_ALIGN 16 -#define TCG_TARGET_CALL_ALIGN_ARGS 1 #define TCG_TARGET_CALL_STACK_OFFSET 0 +#if TCG_TARGET_REG_BITS == 32 +#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_EVEN +#else +#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL +#endif /* optional instructions */ #define TCG_TARGET_HAS_movcond_i32 0 diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index 23e2063667..3f77fcf5b3 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -166,6 +166,7 @@ extern uint64_t s390_facilities[3]; /* used for function call generation */ #define TCG_TARGET_STACK_ALIGN 8 #define TCG_TARGET_CALL_STACK_OFFSET 160 +#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL #define TCG_TARGET_EXTEND_ARGS 1 #define TCG_TARGET_HAS_MEMORY_BSWAP 1 diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h index 8655acdbe5..44ac164b31 100644 --- a/tcg/sparc64/tcg-target.h +++ b/tcg/sparc64/tcg-target.h @@ -72,6 +72,7 @@ typedef enum { #define TCG_TARGET_STACK_ALIGN 16 #define TCG_TARGET_CALL_STACK_OFFSET (128 + 6*8 + TCG_TARGET_STACK_BIAS) #define TCG_TARGET_EXTEND_ARGS 1 +#define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL #if defined(__VIS__) && __VIS__ >= 0x300 #define use_vis3_instructions 1 diff --git a/tcg/tcg.c b/tcg/tcg.c index f74507ba66..e869243885 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -1542,9 +1542,11 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) * for passing off to ffi_call. */ want_align = true; -#elif defined(TCG_TARGET_CALL_ALIGN_ARGS) +#else /* Some targets want aligned 64 bit args */ - want_align = is_64bit; + if (is_64bit) { + want_align = TCG_TARGET_CALL_ARG_I64 == TCG_CALL_ARG_EVEN; + } #endif if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) { diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index ceb36c4f7a..e11c293906 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -158,6 +158,11 @@ typedef enum { /* Used for function call generation. */ #define TCG_TARGET_CALL_STACK_OFFSET 0 #define TCG_TARGET_STACK_ALIGN 8 +#if TCG_TARGET_REG_BITS == 32 +# define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_EVEN +#else +# define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL +#endif #define HAVE_TCG_QEMU_TB_EXEC #define TCG_TARGET_NEED_POOL_LABELS From eb8b0224fc542120e4071f260d031278ac197155 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 16 Oct 2022 20:07:48 +1000 Subject: [PATCH 30/47] tcg: Replace TCG_TARGET_EXTEND_ARGS with TCG_TARGET_CALL_ARG_I32 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For 64-bit hosts that had TCG_TARGET_EXTEND_ARGS, set TCG_TARGET_CALL_ARG_I32 to TCG_CALL_ARG_EXTEND. Otherwise, use TCG_CALL_ARG_NORMAL. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/aarch64/tcg-target.h | 1 + tcg/arm/tcg-target.h | 1 + tcg/i386/tcg-target.h | 1 + tcg/loongarch64/tcg-target.h | 1 + tcg/mips/tcg-target.h | 1 + tcg/ppc/tcg-target.c.inc | 6 +++++- tcg/riscv/tcg-target.h | 1 + tcg/s390x/tcg-target.h | 2 +- tcg/sparc64/tcg-target.h | 2 +- tcg/tcg.c | 42 ++++++++++++++++++------------------ tcg/tci/tcg-target.h | 1 + 11 files changed, 35 insertions(+), 24 deletions(-) diff --git a/tcg/aarch64/tcg-target.h b/tcg/aarch64/tcg-target.h index d9dd777caa..413a5410c5 100644 --- a/tcg/aarch64/tcg-target.h +++ b/tcg/aarch64/tcg-target.h @@ -52,6 +52,7 @@ typedef enum { #define TCG_REG_CALL_STACK TCG_REG_SP #define TCG_TARGET_STACK_ALIGN 16 #define TCG_TARGET_CALL_STACK_OFFSET 0 +#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL #define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL /* optional instructions */ diff --git a/tcg/arm/tcg-target.h b/tcg/arm/tcg-target.h index 09dd0550aa..b7843d2d54 100644 --- a/tcg/arm/tcg-target.h +++ b/tcg/arm/tcg-target.h @@ -89,6 +89,7 @@ extern bool use_neon_instructions; /* used for function call generation */ #define TCG_TARGET_STACK_ALIGN 8 #define TCG_TARGET_CALL_STACK_OFFSET 0 +#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL #define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_EVEN /* optional instructions */ diff --git a/tcg/i386/tcg-target.h b/tcg/i386/tcg-target.h index 42628a2791..7edb7f1d9a 100644 --- a/tcg/i386/tcg-target.h +++ b/tcg/i386/tcg-target.h @@ -98,6 +98,7 @@ typedef enum { #else #define TCG_TARGET_CALL_STACK_OFFSET 0 #endif +#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL #define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL extern bool have_bmi1; diff --git a/tcg/loongarch64/tcg-target.h b/tcg/loongarch64/tcg-target.h index 19d4c07170..e5f7a1f09d 100644 --- a/tcg/loongarch64/tcg-target.h +++ b/tcg/loongarch64/tcg-target.h @@ -93,6 +93,7 @@ typedef enum { #define TCG_REG_CALL_STACK TCG_REG_SP #define TCG_TARGET_STACK_ALIGN 16 #define TCG_TARGET_CALL_STACK_OFFSET 0 +#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL #define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL /* optional instructions */ diff --git a/tcg/mips/tcg-target.h b/tcg/mips/tcg-target.h index bb7312aed4..15721c3e42 100644 --- a/tcg/mips/tcg-target.h +++ b/tcg/mips/tcg-target.h @@ -88,6 +88,7 @@ typedef enum { # define TCG_TARGET_CALL_STACK_OFFSET 0 # define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL #endif +#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL /* MOVN/MOVZ instructions detection */ #if (defined(__mips_isa_rev) && (__mips_isa_rev >= 1)) || \ diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index c2e6bc3296..38ee9974cd 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -44,6 +44,11 @@ # endif #endif +#if TCG_TARGET_REG_BITS == 64 +# define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_EXTEND +#else +# define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL +#endif #ifdef _CALL_SYSV # define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_EVEN #else @@ -2520,7 +2525,6 @@ static void tcg_out_nop_fill(tcg_insn_unit *p, int count) /* Parameters for function call generation, used in tcg.c. */ #define TCG_TARGET_STACK_ALIGN 16 -#define TCG_TARGET_EXTEND_ARGS 1 #ifdef _CALL_AIX # define LINK_AREA_SIZE (6 * SZR) diff --git a/tcg/riscv/tcg-target.h b/tcg/riscv/tcg-target.h index 2ab4b8d04a..232537ccea 100644 --- a/tcg/riscv/tcg-target.h +++ b/tcg/riscv/tcg-target.h @@ -82,6 +82,7 @@ typedef enum { #define TCG_REG_CALL_STACK TCG_REG_SP #define TCG_TARGET_STACK_ALIGN 16 #define TCG_TARGET_CALL_STACK_OFFSET 0 +#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL #if TCG_TARGET_REG_BITS == 32 #define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_EVEN #else diff --git a/tcg/s390x/tcg-target.h b/tcg/s390x/tcg-target.h index 3f77fcf5b3..22d70d431b 100644 --- a/tcg/s390x/tcg-target.h +++ b/tcg/s390x/tcg-target.h @@ -166,9 +166,9 @@ extern uint64_t s390_facilities[3]; /* used for function call generation */ #define TCG_TARGET_STACK_ALIGN 8 #define TCG_TARGET_CALL_STACK_OFFSET 160 +#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_EXTEND #define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL -#define TCG_TARGET_EXTEND_ARGS 1 #define TCG_TARGET_HAS_MEMORY_BSWAP 1 #define TCG_TARGET_DEFAULT_MO (TCG_MO_ALL & ~TCG_MO_ST_LD) diff --git a/tcg/sparc64/tcg-target.h b/tcg/sparc64/tcg-target.h index 44ac164b31..0044ac8d78 100644 --- a/tcg/sparc64/tcg-target.h +++ b/tcg/sparc64/tcg-target.h @@ -71,7 +71,7 @@ typedef enum { #define TCG_TARGET_STACK_BIAS 2047 #define TCG_TARGET_STACK_ALIGN 16 #define TCG_TARGET_CALL_STACK_OFFSET (128 + 6*8 + TCG_TARGET_STACK_BIAS) -#define TCG_TARGET_EXTEND_ARGS 1 +#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_EXTEND #define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL #if defined(__VIS__) && __VIS__ >= 0x300 diff --git a/tcg/tcg.c b/tcg/tcg.c index e869243885..a62ee980d0 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -1494,24 +1494,24 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) } #endif -#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 - for (i = 0; i < nargs; ++i) { - int argtype = extract32(typemask, (i + 1) * 3, 3); - bool is_32bit = (argtype & ~1) == dh_typecode_i32; - bool is_signed = argtype & 1; + if (TCG_TARGET_CALL_ARG_I32 == TCG_CALL_ARG_EXTEND) { + for (i = 0; i < nargs; ++i) { + int argtype = extract32(typemask, (i + 1) * 3, 3); + bool is_32bit = (argtype & ~1) == dh_typecode_i32; + bool is_signed = argtype & 1; - if (is_32bit) { - TCGv_i64 temp = tcg_temp_new_i64(); - TCGv_i32 orig = temp_tcgv_i32(args[i]); - if (is_signed) { - tcg_gen_ext_i32_i64(temp, orig); - } else { - tcg_gen_extu_i32_i64(temp, orig); + if (is_32bit) { + TCGv_i64 temp = tcg_temp_new_i64(); + TCGv_i32 orig = temp_tcgv_i32(args[i]); + if (is_signed) { + tcg_gen_ext_i32_i64(temp, orig); + } else { + tcg_gen_extu_i32_i64(temp, orig); + } + args[i] = tcgv_i64_temp(temp); } - args[i] = tcgv_i64_temp(temp); } } -#endif /* TCG_TARGET_EXTEND_ARGS */ op = tcg_emit_op(INDEX_op_call); @@ -1572,16 +1572,16 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) tcg_debug_assert(TCGOP_CALLI(op) == real_args); tcg_debug_assert(pi <= ARRAY_SIZE(op->args)); -#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 - for (i = 0; i < nargs; ++i) { - int argtype = extract32(typemask, (i + 1) * 3, 3); - bool is_32bit = (argtype & ~1) == dh_typecode_i32; + if (TCG_TARGET_CALL_ARG_I32 == TCG_CALL_ARG_EXTEND) { + for (i = 0; i < nargs; ++i) { + int argtype = extract32(typemask, (i + 1) * 3, 3); + bool is_32bit = (argtype & ~1) == dh_typecode_i32; - if (is_32bit) { - tcg_temp_free_internal(args[i]); + if (is_32bit) { + tcg_temp_free_internal(args[i]); + } } } -#endif /* TCG_TARGET_EXTEND_ARGS */ } static void tcg_reg_alloc_start(TCGContext *s) diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index e11c293906..d6e0450ed8 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -158,6 +158,7 @@ typedef enum { /* Used for function call generation. */ #define TCG_TARGET_CALL_STACK_OFFSET 0 #define TCG_TARGET_STACK_ALIGN 8 +#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL #if TCG_TARGET_REG_BITS == 32 # define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_EVEN #else From e2a9dd6b6b50f678b13cfdf55d3997f16c7f96b0 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Mon, 17 Oct 2022 15:55:56 +1000 Subject: [PATCH 31/47] tcg: Use TCG_CALL_ARG_EVEN for TCI special case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Change 32-bit tci TCG_TARGET_CALL_ARG_I32 to TCG_CALL_ARG_EVEN, to force 32-bit values to be aligned to 64-bit. With a small reorg to the argument processing loop, this neatly replaces an ifdef for CONFIG_TCG_INTERPRETER. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/tcg.c | 70 ++++++++++++++++++++++++++++---------------- tcg/tci/tcg-target.h | 3 +- 2 files changed, 47 insertions(+), 26 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index a62ee980d0..0b29c1b722 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -1533,36 +1533,56 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) real_args = 0; for (i = 0; i < nargs; i++) { int argtype = extract32(typemask, (i + 1) * 3, 3); - bool is_64bit = (argtype & ~1) == dh_typecode_i64; - bool want_align = false; + TCGCallArgumentKind kind; + TCGType type; -#if defined(CONFIG_TCG_INTERPRETER) - /* - * Align all arguments, so that they land in predictable places - * for passing off to ffi_call. - */ - want_align = true; -#else - /* Some targets want aligned 64 bit args */ - if (is_64bit) { - want_align = TCG_TARGET_CALL_ARG_I64 == TCG_CALL_ARG_EVEN; - } -#endif - - if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) { - op->args[pi++] = TCG_CALL_DUMMY_ARG; - real_args++; + switch (argtype) { + case dh_typecode_i32: + case dh_typecode_s32: + type = TCG_TYPE_I32; + break; + case dh_typecode_i64: + case dh_typecode_s64: + type = TCG_TYPE_I64; + break; + case dh_typecode_ptr: + type = TCG_TYPE_PTR; + break; + default: + g_assert_not_reached(); } - if (TCG_TARGET_REG_BITS < 64 && is_64bit) { + switch (type) { + case TCG_TYPE_I32: + kind = TCG_TARGET_CALL_ARG_I32; + break; + case TCG_TYPE_I64: + kind = TCG_TARGET_CALL_ARG_I64; + break; + default: + g_assert_not_reached(); + } + + switch (kind) { + case TCG_CALL_ARG_EVEN: + if (real_args & 1) { + op->args[pi++] = TCG_CALL_DUMMY_ARG; + real_args++; + } + /* fall through */ + case TCG_CALL_ARG_NORMAL: + if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { + op->args[pi++] = temp_arg(args[i]); + op->args[pi++] = temp_arg(args[i] + 1); + real_args += 2; + break; + } op->args[pi++] = temp_arg(args[i]); - op->args[pi++] = temp_arg(args[i] + 1); - real_args += 2; - continue; + real_args++; + break; + default: + g_assert_not_reached(); } - - op->args[pi++] = temp_arg(args[i]); - real_args++; } op->args[pi++] = (uintptr_t)func; op->args[pi++] = (uintptr_t)info; diff --git a/tcg/tci/tcg-target.h b/tcg/tci/tcg-target.h index d6e0450ed8..94ec541b4e 100644 --- a/tcg/tci/tcg-target.h +++ b/tcg/tci/tcg-target.h @@ -158,10 +158,11 @@ typedef enum { /* Used for function call generation. */ #define TCG_TARGET_CALL_STACK_OFFSET 0 #define TCG_TARGET_STACK_ALIGN 8 -#define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL #if TCG_TARGET_REG_BITS == 32 +# define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_EVEN # define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_EVEN #else +# define TCG_TARGET_CALL_ARG_I32 TCG_CALL_ARG_NORMAL # define TCG_TARGET_CALL_ARG_I64 TCG_CALL_ARG_NORMAL #endif From 05d019abab15882ae726da22c3334e0f4b1797e3 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 10 Nov 2022 22:58:57 +1000 Subject: [PATCH 32/47] accel/tcg/plugin: Don't search for the function pointer index MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The function pointer is immediately after the output and input operands; no need to search. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- accel/tcg/plugin-gen.c | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index 80dff68934..9e359c006a 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -381,32 +381,25 @@ static TCGOp *copy_st_ptr(TCGOp **begin_op, TCGOp *op) static TCGOp *copy_call(TCGOp **begin_op, TCGOp *op, void *empty_func, void *func, int *cb_idx) { + TCGOp *old_op; + int func_idx; + /* copy all ops until the call */ do { op = copy_op_nocheck(begin_op, op); } while (op->opc != INDEX_op_call); /* fill in the op call */ - op->param1 = (*begin_op)->param1; - op->param2 = (*begin_op)->param2; + old_op = *begin_op; + TCGOP_CALLI(op) = TCGOP_CALLI(old_op); + TCGOP_CALLO(op) = TCGOP_CALLO(old_op); tcg_debug_assert(op->life == 0); - if (*cb_idx == -1) { - int i; - /* - * Instead of working out the position of the callback in args[], just - * look for @empty_func, since it should be a unique pointer. - */ - for (i = 0; i < MAX_OPC_PARAM_ARGS; i++) { - if ((uintptr_t)(*begin_op)->args[i] == (uintptr_t)empty_func) { - *cb_idx = i; - break; - } - } - tcg_debug_assert(i < MAX_OPC_PARAM_ARGS); - } - op->args[*cb_idx] = (uintptr_t)func; - op->args[*cb_idx + 1] = (*begin_op)->args[*cb_idx + 1]; + func_idx = TCGOP_CALLO(op) + TCGOP_CALLI(op); + *cb_idx = func_idx; + + op->args[func_idx] = (uintptr_t)func; + op->args[func_idx + 1] = old_op->args[func_idx + 1]; return op; } From ada4cb0c1cba89df3b00b6a92a6c1523c30f4396 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 10 Nov 2022 23:16:36 +1000 Subject: [PATCH 33/47] accel/tcg/plugin: Avoid duplicate copy in copy_call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We copied all of the arguments in copy_op_nocheck. We only need to replace the one argument that we change. Reviewed-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- accel/tcg/plugin-gen.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index 9e359c006a..77e6823d6b 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -397,9 +397,7 @@ static TCGOp *copy_call(TCGOp **begin_op, TCGOp *op, void *empty_func, func_idx = TCGOP_CALLO(op) + TCGOP_CALLI(op); *cb_idx = func_idx; - op->args[func_idx] = (uintptr_t)func; - op->args[func_idx + 1] = old_op->args[func_idx + 1]; return op; } From f266bec890ead3864f1f5f9805112e0fd19c5066 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 10 Nov 2022 23:18:52 +1000 Subject: [PATCH 34/47] accel/tcg/plugin: Use copy_op in append_{udata,mem}_cb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Better to re-use the existing function for copying ops. Acked-by: Alex Bennée Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- accel/tcg/plugin-gen.c | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index 77e6823d6b..a6aaacd053 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -415,11 +415,11 @@ static TCGOp *append_udata_cb(const struct qemu_plugin_dyn_cb *cb, op = copy_const_ptr(&begin_op, op, cb->userp); /* copy the ld_i32, but note that we only have to copy it once */ - begin_op = QTAILQ_NEXT(begin_op, link); - tcg_debug_assert(begin_op && begin_op->opc == INDEX_op_ld_i32); if (*cb_idx == -1) { - op = tcg_op_insert_after(tcg_ctx, op, INDEX_op_ld_i32); - memcpy(op->args, begin_op->args, sizeof(op->args)); + op = copy_op(&begin_op, op, INDEX_op_ld_i32); + } else { + begin_op = QTAILQ_NEXT(begin_op, link); + tcg_debug_assert(begin_op && begin_op->opc == INDEX_op_ld_i32); } /* call */ @@ -462,11 +462,11 @@ static TCGOp *append_mem_cb(const struct qemu_plugin_dyn_cb *cb, op = copy_const_ptr(&begin_op, op, cb->userp); /* copy the ld_i32, but note that we only have to copy it once */ - begin_op = QTAILQ_NEXT(begin_op, link); - tcg_debug_assert(begin_op && begin_op->opc == INDEX_op_ld_i32); if (*cb_idx == -1) { - op = tcg_op_insert_after(tcg_ctx, op, INDEX_op_ld_i32); - memcpy(op->args, begin_op->args, sizeof(op->args)); + op = copy_op(&begin_op, op, INDEX_op_ld_i32); + } else { + begin_op = QTAILQ_NEXT(begin_op, link); + tcg_debug_assert(begin_op && begin_op->opc == INDEX_op_ld_i32); } /* extu_tl_i64 */ From d44789434bbf51bb4d4a3402066d281fa0efc88c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Sun, 18 Dec 2022 22:18:31 +0100 Subject: [PATCH 35/47] tcg: Pass number of arguments to tcg_emit_op() / tcg_op_insert_*() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In order to have variable size allocated TCGOp, pass the number of arguments we use (and would allocate) up to tcg_op_alloc(). This alters tcg_emit_op(), tcg_op_insert_before() and tcg_op_insert_after() prototypes. In tcg_op_alloc() ensure the number of arguments is in range. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson [PMD: Extracted from bigger patch] Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20221218211832.73312-2-philmd@linaro.org> --- accel/tcg/plugin-gen.c | 5 ++++- include/tcg/tcg-op.h | 2 +- include/tcg/tcg.h | 8 +++++--- tcg/optimize.c | 4 ++-- tcg/tcg-op-vec.c | 8 ++++---- tcg/tcg-op.c | 12 ++++++------ tcg/tcg.c | 30 +++++++++++++++++------------- 7 files changed, 39 insertions(+), 30 deletions(-) diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index a6aaacd053..62e775d34d 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -258,10 +258,13 @@ static TCGOp *rm_ops(TCGOp *op) static TCGOp *copy_op_nocheck(TCGOp **begin_op, TCGOp *op) { + unsigned nargs = ARRAY_SIZE(op->args); + *begin_op = QTAILQ_NEXT(*begin_op, link); tcg_debug_assert(*begin_op); - op = tcg_op_insert_after(tcg_ctx, op, (*begin_op)->opc); + op = tcg_op_insert_after(tcg_ctx, op, (*begin_op)->opc, nargs); memcpy(op->args, (*begin_op)->args, sizeof(op->args)); + return op; } diff --git a/include/tcg/tcg-op.h b/include/tcg/tcg-op.h index 8176f194cb..79b1cf786f 100644 --- a/include/tcg/tcg-op.h +++ b/include/tcg/tcg-op.h @@ -818,7 +818,7 @@ static inline void tcg_gen_plugin_cb_start(unsigned from, unsigned type, static inline void tcg_gen_plugin_cb_end(void) { - tcg_emit_op(INDEX_op_plugin_cb_end); + tcg_emit_op(INDEX_op_plugin_cb_end, 0); } #if TARGET_LONG_BITS == 32 diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index 8bcd60d0ed..c55fa21a89 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -1014,10 +1014,12 @@ bool tcg_op_supported(TCGOpcode op); void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args); -TCGOp *tcg_emit_op(TCGOpcode opc); +TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs); void tcg_op_remove(TCGContext *s, TCGOp *op); -TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *op, TCGOpcode opc); -TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *op, TCGOpcode opc); +TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *op, + TCGOpcode opc, unsigned nargs); +TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *op, + TCGOpcode opc, unsigned nargs); /** * tcg_remove_ops_after: diff --git a/tcg/optimize.c b/tcg/optimize.c index ae081ab29c..1afd50175b 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -962,7 +962,7 @@ static bool fold_addsub2(OptContext *ctx, TCGOp *op, bool add) rh = op->args[1]; /* The proper opcode is supplied by tcg_opt_gen_mov. */ - op2 = tcg_op_insert_before(ctx->tcg, op, 0); + op2 = tcg_op_insert_before(ctx->tcg, op, 0, 2); tcg_opt_gen_movi(ctx, op, rl, al); tcg_opt_gen_movi(ctx, op2, rh, ah); @@ -1613,7 +1613,7 @@ static bool fold_multiply2(OptContext *ctx, TCGOp *op) rh = op->args[1]; /* The proper opcode is supplied by tcg_opt_gen_mov. */ - op2 = tcg_op_insert_before(ctx->tcg, op, 0); + op2 = tcg_op_insert_before(ctx->tcg, op, 0, 2); tcg_opt_gen_movi(ctx, op, rl, l); tcg_opt_gen_movi(ctx, op2, rh, h); diff --git a/tcg/tcg-op-vec.c b/tcg/tcg-op-vec.c index 5bf100ea7d..966d41d65a 100644 --- a/tcg/tcg-op-vec.c +++ b/tcg/tcg-op-vec.c @@ -152,7 +152,7 @@ bool tcg_can_emit_vecop_list(const TCGOpcode *list, void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a) { - TCGOp *op = tcg_emit_op(opc); + TCGOp *op = tcg_emit_op(opc, 2); TCGOP_VECL(op) = type - TCG_TYPE_V64; TCGOP_VECE(op) = vece; op->args[0] = r; @@ -162,7 +162,7 @@ void vec_gen_2(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a) void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a, TCGArg b) { - TCGOp *op = tcg_emit_op(opc); + TCGOp *op = tcg_emit_op(opc, 3); TCGOP_VECL(op) = type - TCG_TYPE_V64; TCGOP_VECE(op) = vece; op->args[0] = r; @@ -173,7 +173,7 @@ void vec_gen_3(TCGOpcode opc, TCGType type, unsigned vece, void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a, TCGArg b, TCGArg c) { - TCGOp *op = tcg_emit_op(opc); + TCGOp *op = tcg_emit_op(opc, 4); TCGOP_VECL(op) = type - TCG_TYPE_V64; TCGOP_VECE(op) = vece; op->args[0] = r; @@ -185,7 +185,7 @@ void vec_gen_4(TCGOpcode opc, TCGType type, unsigned vece, static void vec_gen_6(TCGOpcode opc, TCGType type, unsigned vece, TCGArg r, TCGArg a, TCGArg b, TCGArg c, TCGArg d, TCGArg e) { - TCGOp *op = tcg_emit_op(opc); + TCGOp *op = tcg_emit_op(opc, 6); TCGOP_VECL(op) = type - TCG_TYPE_V64; TCGOP_VECE(op) = vece; op->args[0] = r; diff --git a/tcg/tcg-op.c b/tcg/tcg-op.c index 6168327030..cd1cd4e736 100644 --- a/tcg/tcg-op.c +++ b/tcg/tcg-op.c @@ -33,20 +33,20 @@ void tcg_gen_op1(TCGOpcode opc, TCGArg a1) { - TCGOp *op = tcg_emit_op(opc); + TCGOp *op = tcg_emit_op(opc, 1); op->args[0] = a1; } void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2) { - TCGOp *op = tcg_emit_op(opc); + TCGOp *op = tcg_emit_op(opc, 2); op->args[0] = a1; op->args[1] = a2; } void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3) { - TCGOp *op = tcg_emit_op(opc); + TCGOp *op = tcg_emit_op(opc, 3); op->args[0] = a1; op->args[1] = a2; op->args[2] = a3; @@ -54,7 +54,7 @@ void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3) void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4) { - TCGOp *op = tcg_emit_op(opc); + TCGOp *op = tcg_emit_op(opc, 4); op->args[0] = a1; op->args[1] = a2; op->args[2] = a3; @@ -64,7 +64,7 @@ void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4) void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4, TCGArg a5) { - TCGOp *op = tcg_emit_op(opc); + TCGOp *op = tcg_emit_op(opc, 5); op->args[0] = a1; op->args[1] = a2; op->args[2] = a3; @@ -75,7 +75,7 @@ void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4, TCGArg a5, TCGArg a6) { - TCGOp *op = tcg_emit_op(opc); + TCGOp *op = tcg_emit_op(opc, 6); op->args[0] = a1; op->args[1] = a2; op->args[2] = a3; diff --git a/tcg/tcg.c b/tcg/tcg.c index 0b29c1b722..ecbbe9907d 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -1479,7 +1479,7 @@ bool tcg_op_supported(TCGOpcode op) and endian swap in tcg_reg_alloc_call(). */ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) { - int i, real_args, nb_rets, pi; + int i, real_args, nb_rets, pi, max_args; unsigned typemask; const TCGHelperInfo *info; TCGOp *op; @@ -1513,7 +1513,8 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) } } - op = tcg_emit_op(INDEX_op_call); + max_args = ARRAY_SIZE(op->args); + op = tcg_emit_op(INDEX_op_call, max_args); pi = 0; if (ret != NULL) { @@ -1590,7 +1591,7 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) /* Make sure the fields didn't overflow. */ tcg_debug_assert(TCGOP_CALLI(op) == real_args); - tcg_debug_assert(pi <= ARRAY_SIZE(op->args)); + tcg_debug_assert(pi <= max_args); if (TCG_TARGET_CALL_ARG_I32 == TCG_CALL_ARG_EXTEND) { for (i = 0; i < nargs; ++i) { @@ -2294,11 +2295,12 @@ void tcg_remove_ops_after(TCGOp *op) } } -static TCGOp *tcg_op_alloc(TCGOpcode opc) +static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs) { TCGContext *s = tcg_ctx; TCGOp *op; + assert(nargs < ARRAY_SIZE(op->args)); if (likely(QTAILQ_EMPTY(&s->free_ops))) { op = tcg_malloc(sizeof(TCGOp)); } else { @@ -2312,23 +2314,25 @@ static TCGOp *tcg_op_alloc(TCGOpcode opc) return op; } -TCGOp *tcg_emit_op(TCGOpcode opc) +TCGOp *tcg_emit_op(TCGOpcode opc, unsigned nargs) { - TCGOp *op = tcg_op_alloc(opc); + TCGOp *op = tcg_op_alloc(opc, nargs); QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); return op; } -TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc) +TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, + TCGOpcode opc, unsigned nargs) { - TCGOp *new_op = tcg_op_alloc(opc); + TCGOp *new_op = tcg_op_alloc(opc, nargs); QTAILQ_INSERT_BEFORE(old_op, new_op, link); return new_op; } -TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc) +TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, + TCGOpcode opc, unsigned nargs) { - TCGOp *new_op = tcg_op_alloc(opc); + TCGOp *new_op = tcg_op_alloc(opc, nargs); QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link); return new_op; } @@ -2937,7 +2941,7 @@ static bool liveness_pass_2(TCGContext *s) TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32 ? INDEX_op_ld_i32 : INDEX_op_ld_i64); - TCGOp *lop = tcg_op_insert_before(s, op, lopc); + TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3); lop->args[0] = temp_arg(dir_ts); lop->args[1] = temp_arg(arg_ts->mem_base); @@ -3003,7 +3007,7 @@ static bool liveness_pass_2(TCGContext *s) TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 ? INDEX_op_st_i32 : INDEX_op_st_i64); - TCGOp *sop = tcg_op_insert_after(s, op, sopc); + TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); TCGTemp *out_ts = dir_ts; if (IS_DEAD_ARG(0)) { @@ -3039,7 +3043,7 @@ static bool liveness_pass_2(TCGContext *s) TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32 ? INDEX_op_st_i32 : INDEX_op_st_i64); - TCGOp *sop = tcg_op_insert_after(s, op, sopc); + TCGOp *sop = tcg_op_insert_after(s, op, sopc, 3); sop->args[0] = temp_arg(dir_ts); sop->args[1] = temp_arg(arg_ts->mem_base); From cb10bc63b70737eafaceac1bf1d97730ce6d3393 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sun, 18 Dec 2022 22:18:32 +0100 Subject: [PATCH 36/47] tcg: Vary the allocation size for TCGOp MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We have been allocating a worst case number of arguments to support calls. Instead, allow the size to vary. By default leave space for 4 args, to maximize reuse, but allow calls to increase the number of args to 32. Signed-off-by: Richard Henderson [PMD: Split patch in two] Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20221218211832.73312-3-philmd@linaro.org> --- accel/tcg/plugin-gen.c | 10 ++++----- include/exec/helper-head.h | 2 -- include/tcg/tcg.h | 46 +++++++++++++------------------------- tcg/tcg.c | 35 +++++++++++++++++++++-------- 4 files changed, 47 insertions(+), 46 deletions(-) diff --git a/accel/tcg/plugin-gen.c b/accel/tcg/plugin-gen.c index 62e775d34d..c7d6514840 100644 --- a/accel/tcg/plugin-gen.c +++ b/accel/tcg/plugin-gen.c @@ -258,12 +258,12 @@ static TCGOp *rm_ops(TCGOp *op) static TCGOp *copy_op_nocheck(TCGOp **begin_op, TCGOp *op) { - unsigned nargs = ARRAY_SIZE(op->args); + TCGOp *old_op = QTAILQ_NEXT(*begin_op, link); + unsigned nargs = old_op->nargs; - *begin_op = QTAILQ_NEXT(*begin_op, link); - tcg_debug_assert(*begin_op); - op = tcg_op_insert_after(tcg_ctx, op, (*begin_op)->opc, nargs); - memcpy(op->args, (*begin_op)->args, sizeof(op->args)); + *begin_op = old_op; + op = tcg_op_insert_after(tcg_ctx, op, old_op->opc, nargs); + memcpy(op->args, old_op->args, sizeof(op->args[0]) * nargs); return op; } diff --git a/include/exec/helper-head.h b/include/exec/helper-head.h index e242fed46e..8bdf0f6ea2 100644 --- a/include/exec/helper-head.h +++ b/include/exec/helper-head.h @@ -133,6 +133,4 @@ #define DEF_HELPER_7(name, ret, t1, t2, t3, t4, t5, t6, t7) \ DEF_HELPER_FLAGS_7(name, 0, ret, t1, t2, t3, t4, t5, t6, t7) -/* MAX_OPC_PARAM_IARGS must be set to n if last entry is DEF_HELPER_FLAGS_n. */ - #endif /* EXEC_HELPER_HEAD_H */ diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index c55fa21a89..d430ea10c8 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -38,20 +38,6 @@ /* XXX: make safe guess about sizes */ #define MAX_OP_PER_INSTR 266 -#if HOST_LONG_BITS == 32 -#define MAX_OPC_PARAM_PER_ARG 2 -#else -#define MAX_OPC_PARAM_PER_ARG 1 -#endif -#define MAX_OPC_PARAM_IARGS 7 -#define MAX_OPC_PARAM_OARGS 1 -#define MAX_OPC_PARAM_ARGS (MAX_OPC_PARAM_IARGS + MAX_OPC_PARAM_OARGS) - -/* A Call op needs up to 4 + 2N parameters on 32-bit archs, - * and up to 4 + N parameters on 64-bit archs - * (N = number of input arguments + output arguments). */ -#define MAX_OPC_PARAM (4 + (MAX_OPC_PARAM_PER_ARG * MAX_OPC_PARAM_ARGS)) - #define CPU_TEMP_BUF_NLONGS 128 #define TCG_STATIC_FRAME_SIZE (CPU_TEMP_BUF_NLONGS * sizeof(long)) @@ -493,34 +479,34 @@ typedef struct TCGTempSet { unsigned long l[BITS_TO_LONGS(TCG_MAX_TEMPS)]; } TCGTempSet; -/* While we limit helpers to 6 arguments, for 32-bit hosts, with padding, - this imples a max of 6*2 (64-bit in) + 2 (64-bit out) = 14 operands. - There are never more than 2 outputs, which means that we can store all - dead + sync data within 16 bits. */ -#define DEAD_ARG 4 -#define SYNC_ARG 1 -typedef uint16_t TCGLifeData; +/* + * With 1 128-bit output, a 32-bit host requires 4 output parameters, + * which leaves a maximum of 28 other slots. Which is enough for 7 + * 128-bit operands. + */ +#define DEAD_ARG (1 << 4) +#define SYNC_ARG (1 << 0) +typedef uint32_t TCGLifeData; -/* The layout here is designed to avoid a bitfield crossing of - a 32-bit boundary, which would cause GCC to add extra padding. */ typedef struct TCGOp { - TCGOpcode opc : 8; /* 8 */ + TCGOpcode opc : 8; + unsigned nargs : 8; /* Parameters for this opcode. See below. */ - unsigned param1 : 4; /* 12 */ - unsigned param2 : 4; /* 16 */ + unsigned param1 : 8; + unsigned param2 : 8; /* Lifetime data of the operands. */ - unsigned life : 16; /* 32 */ + TCGLifeData life; /* Next and previous opcodes. */ QTAILQ_ENTRY(TCGOp) link; - /* Arguments for the opcode. */ - TCGArg args[MAX_OPC_PARAM]; - /* Register preferences for the output(s). */ TCGRegSet output_pref[2]; + + /* Arguments for the opcode. */ + TCGArg args[]; } TCGOp; #define TCGOP_CALLI(X) (X)->param1 diff --git a/tcg/tcg.c b/tcg/tcg.c index ecbbe9907d..be55e14245 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -1513,7 +1513,12 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) } } - max_args = ARRAY_SIZE(op->args); + /* + * A Call op needs up to 4 + 2N parameters on 32-bit archs, + * and up to 4 + N parameters on 64-bit archs + * (N = number of input arguments + output arguments). + */ + max_args = (64 / TCG_TARGET_REG_BITS) * nargs + 4; op = tcg_emit_op(INDEX_op_call, max_args); pi = 0; @@ -2298,19 +2303,31 @@ void tcg_remove_ops_after(TCGOp *op) static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs) { TCGContext *s = tcg_ctx; - TCGOp *op; + TCGOp *op = NULL; - assert(nargs < ARRAY_SIZE(op->args)); - if (likely(QTAILQ_EMPTY(&s->free_ops))) { - op = tcg_malloc(sizeof(TCGOp)); - } else { - op = QTAILQ_FIRST(&s->free_ops); - QTAILQ_REMOVE(&s->free_ops, op, link); + if (unlikely(!QTAILQ_EMPTY(&s->free_ops))) { + QTAILQ_FOREACH(op, &s->free_ops, link) { + if (nargs <= op->nargs) { + QTAILQ_REMOVE(&s->free_ops, op, link); + nargs = op->nargs; + goto found; + } + } } + + /* Most opcodes have 3 or 4 operands: reduce fragmentation. */ + nargs = MAX(4, nargs); + op = tcg_malloc(sizeof(TCGOp) + sizeof(TCGArg) * nargs); + + found: memset(op, 0, offsetof(TCGOp, link)); op->opc = opc; - s->nb_ops++; + op->nargs = nargs; + /* Check for bitfield overflow. */ + tcg_debug_assert(op->nargs == nargs); + + s->nb_ops++; return op; } From 31fd884b2e53dc50328dd616667c745fc4808fd1 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 11 Nov 2022 15:10:51 +1000 Subject: [PATCH 37/47] tcg: Use output_pref wrapper function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We will shortly have the possibility of more that two outputs, though only for calls (for which preferences are moot). Avoid direct references to op->output_pref[] when possible. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- include/tcg/tcg.h | 5 +++++ tcg/tcg.c | 34 ++++++++++++++++++---------------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index d430ea10c8..a6310b898f 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -518,6 +518,11 @@ typedef struct TCGOp { /* Make sure operands fit in the bitfields above. */ QEMU_BUILD_BUG_ON(NB_OPS > (1 << 8)); +static inline TCGRegSet output_pref(const TCGOp *op, unsigned i) +{ + return i < ARRAY_SIZE(op->output_pref) ? op->output_pref[i] : 0; +} + typedef struct TCGProfile { int64_t cpu_exec_time; int64_t tb_count1; diff --git a/tcg/tcg.c b/tcg/tcg.c index be55e14245..1c154ccbd9 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -1966,7 +1966,7 @@ static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) if (have_prefs) { for (i = 0; i < nb_oargs; ++i) { - TCGRegSet set = op->output_pref[i]; + TCGRegSet set = output_pref(op, i); if (i == 0) { ne_fprintf(f, " pref="); @@ -2636,11 +2636,11 @@ static void liveness_pass_1(TCGContext *s) } ts->state = TS_DEAD; la_reset_pref(ts); - - /* Not used -- it will be tcg_target_call_oarg_regs[i]. */ - op->output_pref[i] = 0; } + /* Not used -- it will be tcg_target_call_oarg_reg(). */ + memset(op->output_pref, 0, sizeof(op->output_pref)); + if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS | TCG_CALL_NO_READ_GLOBALS))) { la_global_kill(s, nb_globals); @@ -2802,7 +2802,9 @@ static void liveness_pass_1(TCGContext *s) ts = arg_temp(op->args[i]); /* Remember the preference of the uses that followed. */ - op->output_pref[i] = *la_temp_pref(ts); + if (i < ARRAY_SIZE(op->output_pref)) { + op->output_pref[i] = *la_temp_pref(ts); + } /* Output args are dead. */ if (ts->state & TS_DEAD) { @@ -2872,7 +2874,7 @@ static void liveness_pass_1(TCGContext *s) set &= ct->regs; if (ct->ialias) { - set &= op->output_pref[ct->alias_index]; + set &= output_pref(op, ct->alias_index); } /* If the combination is not possible, restart. */ if (set == 0) { @@ -3539,7 +3541,7 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op) TCGReg oreg, ireg; allocated_regs = s->reserved_regs; - preferred_regs = op->output_pref[0]; + preferred_regs = output_pref(op, 0); ots = arg_temp(op->args[0]); ts = arg_temp(op->args[1]); @@ -3656,7 +3658,7 @@ static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) if (IS_DEAD_ARG(1)) { temp_dead(s, its); } - tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]); + tcg_reg_alloc_do_movi(s, ots, val, arg_life, output_pref(op, 0)); return; } @@ -3673,7 +3675,7 @@ static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op) tcg_regset_set_reg(allocated_regs, its->reg); } oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, - op->output_pref[0], ots->indirect_base); + output_pref(op, 0), ots->indirect_base); set_temp_val_reg(s, ots, oreg); } @@ -3792,7 +3794,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) switch (arg_ct->pair) { case 0: /* not paired */ if (arg_ct->ialias) { - i_preferred_regs = op->output_pref[arg_ct->alias_index]; + i_preferred_regs = output_pref(op, arg_ct->alias_index); /* * If the input is readonly, then it cannot also be an @@ -3841,7 +3843,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) * and to identify a few cases where it's not required. */ if (arg_ct->ialias) { - i_preferred_regs = op->output_pref[arg_ct->alias_index]; + i_preferred_regs = output_pref(op, arg_ct->alias_index); if (IS_DEAD_ARG(i1) && IS_DEAD_ARG(i2) && !temp_readonly(ts) && @@ -3877,7 +3879,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) case 3: /* ialias with second output, no first input */ tcg_debug_assert(arg_ct->ialias); - i_preferred_regs = op->output_pref[arg_ct->alias_index]; + i_preferred_regs = output_pref(op, arg_ct->alias_index); if (IS_DEAD_ARG(i) && !temp_readonly(ts) && @@ -4001,10 +4003,10 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) } else if (arg_ct->newreg) { reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs | o_allocated_regs, - op->output_pref[k], ts->indirect_base); + output_pref(op, k), ts->indirect_base); } else { reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs, - op->output_pref[k], ts->indirect_base); + output_pref(op, k), ts->indirect_base); } break; @@ -4015,7 +4017,7 @@ static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op) break; } reg = tcg_reg_alloc_pair(s, arg_ct->regs, o_allocated_regs, - op->output_pref[k], ts->indirect_base); + output_pref(op, k), ts->indirect_base); break; case 2: /* second of pair */ @@ -4098,7 +4100,7 @@ static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) } oreg = tcg_reg_alloc(s, dup_out_regs, allocated_regs, - op->output_pref[0], ots->indirect_base); + output_pref(op, 0), ots->indirect_base); set_temp_val_reg(s, ots, oreg); } From 39004a71d8f6b61501e41be21cc874272c78212f Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Fri, 11 Nov 2022 10:09:37 +1000 Subject: [PATCH 38/47] tcg: Reorg function calls Pre-compute the function call layout for each helper at startup. Drop TCG_CALL_DUMMY_ARG, as we no longer need to leave gaps in the op->args[] array. This allows several places to stop checking for NULL TCGTemp, to which TCG_CALL_DUMMY_ARG mapped. For tcg_gen_callN, loop over the arguments once. Allocate the TCGOp for the call early but delay emitting it, collecting arguments first. This allows the argument processing loop to emit code for extensions and have them sequenced before the call. For tcg_reg_alloc_call, loop over the arguments in reverse order, which allows stack slots to be filled first naturally. Signed-off-by: Richard Henderson --- include/exec/helper-head.h | 2 + include/tcg/tcg.h | 5 +- tcg/optimize.c | 6 +- tcg/tcg-internal.h | 22 +- tcg/tcg.c | 609 ++++++++++++++++++++++--------------- 5 files changed, 394 insertions(+), 250 deletions(-) diff --git a/include/exec/helper-head.h b/include/exec/helper-head.h index 8bdf0f6ea2..bc6698b19f 100644 --- a/include/exec/helper-head.h +++ b/include/exec/helper-head.h @@ -133,4 +133,6 @@ #define DEF_HELPER_7(name, ret, t1, t2, t3, t4, t5, t6, t7) \ DEF_HELPER_FLAGS_7(name, 0, ret, t1, t2, t3, t4, t5, t6, t7) +/* MAX_CALL_IARGS must be set to n if last entry is DEF_HELPER_FLAGS_n. */ + #endif /* EXEC_HELPER_HEAD_H */ diff --git a/include/tcg/tcg.h b/include/tcg/tcg.h index a6310b898f..b949d75fdd 100644 --- a/include/tcg/tcg.h +++ b/include/tcg/tcg.h @@ -38,6 +38,8 @@ /* XXX: make safe guess about sizes */ #define MAX_OP_PER_INSTR 266 +#define MAX_CALL_IARGS 7 + #define CPU_TEMP_BUF_NLONGS 128 #define TCG_STATIC_FRAME_SIZE (CPU_TEMP_BUF_NLONGS * sizeof(long)) @@ -411,9 +413,6 @@ typedef TCGv_ptr TCGv_env; #define TCG_CALL_NO_RWG_SE (TCG_CALL_NO_RWG | TCG_CALL_NO_SE) #define TCG_CALL_NO_WG_SE (TCG_CALL_NO_WG | TCG_CALL_NO_SE) -/* Used to align parameters. See the comment before tcgv_i32_temp. */ -#define TCG_CALL_DUMMY_ARG ((TCGArg)0) - /* * Flags for the bswap opcodes. * If IZ, the input is zero-extended, otherwise unknown. diff --git a/tcg/optimize.c b/tcg/optimize.c index 1afd50175b..763bca9ea6 100644 --- a/tcg/optimize.c +++ b/tcg/optimize.c @@ -667,9 +667,7 @@ static void init_arguments(OptContext *ctx, TCGOp *op, int nb_args) { for (int i = 0; i < nb_args; i++) { TCGTemp *ts = arg_temp(op->args[i]); - if (ts) { - init_ts_info(ctx, ts); - } + init_ts_info(ctx, ts); } } @@ -680,7 +678,7 @@ static void copy_propagate(OptContext *ctx, TCGOp *op, for (int i = nb_oargs; i < nb_oargs + nb_iargs; i++) { TCGTemp *ts = arg_temp(op->args[i]); - if (ts && ts_is_copy(ts)) { + if (ts_is_copy(ts)) { op->args[i] = temp_arg(find_better_copy(s, ts)); } } diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index f574743ff8..c7e87e193d 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -42,11 +42,29 @@ typedef enum { TCG_CALL_ARG_EXTEND_S, /* ... as a sign-extended i64 */ } TCGCallArgumentKind; +typedef struct TCGCallArgumentLoc { + TCGCallArgumentKind kind : 8; + unsigned arg_slot : 8; + unsigned ref_slot : 8; + unsigned arg_idx : 4; + unsigned tmp_subindex : 2; +} TCGCallArgumentLoc; + +/* Avoid "unsigned < 0 is always false" Werror, when iarg_regs is empty. */ +#define REG_P(L) \ + ((int)(L)->arg_slot < (int)ARRAY_SIZE(tcg_target_call_iarg_regs)) + typedef struct TCGHelperInfo { void *func; const char *name; - unsigned flags; - unsigned typemask; + unsigned typemask : 32; + unsigned flags : 8; + unsigned nr_in : 8; + unsigned nr_out : 8; + TCGCallReturnKind out_kind : 8; + + /* Maximum physical arguments are constrained by TCG_TYPE_I128. */ + TCGCallArgumentLoc in[MAX_CALL_IARGS * (128 / TCG_TARGET_REG_BITS)]; } TCGHelperInfo; extern TCGContext tcg_init_ctx; diff --git a/tcg/tcg.c b/tcg/tcg.c index 1c154ccbd9..0ac270fc7f 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -547,7 +547,7 @@ void tcg_pool_reset(TCGContext *s) #include "exec/helper-proto.h" -static const TCGHelperInfo all_helpers[] = { +static TCGHelperInfo all_helpers[] = { #include "exec/helper-tcg.h" }; static GHashTable *helper_table; @@ -565,6 +565,154 @@ static ffi_type * const typecode_to_ffi[8] = { }; #endif +typedef struct TCGCumulativeArgs { + int arg_idx; /* tcg_gen_callN args[] */ + int info_in_idx; /* TCGHelperInfo in[] */ + int arg_slot; /* regs+stack slot */ + int ref_slot; /* stack slots for references */ +} TCGCumulativeArgs; + +static void layout_arg_even(TCGCumulativeArgs *cum) +{ + cum->arg_slot += cum->arg_slot & 1; +} + +static void layout_arg_1(TCGCumulativeArgs *cum, TCGHelperInfo *info, + TCGCallArgumentKind kind) +{ + TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; + + *loc = (TCGCallArgumentLoc){ + .kind = kind, + .arg_idx = cum->arg_idx, + .arg_slot = cum->arg_slot, + }; + cum->info_in_idx++; + cum->arg_slot++; +} + +static void layout_arg_normal_n(TCGCumulativeArgs *cum, + TCGHelperInfo *info, int n) +{ + TCGCallArgumentLoc *loc = &info->in[cum->info_in_idx]; + + for (int i = 0; i < n; ++i) { + /* Layout all using the same arg_idx, adjusting the subindex. */ + loc[i] = (TCGCallArgumentLoc){ + .kind = TCG_CALL_ARG_NORMAL, + .arg_idx = cum->arg_idx, + .tmp_subindex = i, + .arg_slot = cum->arg_slot + i, + }; + } + cum->info_in_idx += n; + cum->arg_slot += n; +} + +static void init_call_layout(TCGHelperInfo *info) +{ + int max_reg_slots = ARRAY_SIZE(tcg_target_call_iarg_regs); + int max_stk_slots = TCG_STATIC_CALL_ARGS_SIZE / sizeof(tcg_target_long); + unsigned typemask = info->typemask; + unsigned typecode; + TCGCumulativeArgs cum = { }; + + /* + * Parse and place any function return value. + */ + typecode = typemask & 7; + switch (typecode) { + case dh_typecode_void: + info->nr_out = 0; + break; + case dh_typecode_i32: + case dh_typecode_s32: + case dh_typecode_ptr: + info->nr_out = 1; + info->out_kind = TCG_CALL_RET_NORMAL; + break; + case dh_typecode_i64: + case dh_typecode_s64: + info->nr_out = 64 / TCG_TARGET_REG_BITS; + info->out_kind = TCG_CALL_RET_NORMAL; + break; + default: + g_assert_not_reached(); + } + assert(info->nr_out <= ARRAY_SIZE(tcg_target_call_oarg_regs)); + + /* + * Parse and place function arguments. + */ + for (typemask >>= 3; typemask; typemask >>= 3, cum.arg_idx++) { + TCGCallArgumentKind kind; + TCGType type; + + typecode = typemask & 7; + switch (typecode) { + case dh_typecode_i32: + case dh_typecode_s32: + type = TCG_TYPE_I32; + break; + case dh_typecode_i64: + case dh_typecode_s64: + type = TCG_TYPE_I64; + break; + case dh_typecode_ptr: + type = TCG_TYPE_PTR; + break; + default: + g_assert_not_reached(); + } + + switch (type) { + case TCG_TYPE_I32: + switch (TCG_TARGET_CALL_ARG_I32) { + case TCG_CALL_ARG_EVEN: + layout_arg_even(&cum); + /* fall through */ + case TCG_CALL_ARG_NORMAL: + layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); + break; + case TCG_CALL_ARG_EXTEND: + kind = TCG_CALL_ARG_EXTEND_U + (typecode & 1); + layout_arg_1(&cum, info, kind); + break; + default: + qemu_build_not_reached(); + } + break; + + case TCG_TYPE_I64: + switch (TCG_TARGET_CALL_ARG_I64) { + case TCG_CALL_ARG_EVEN: + layout_arg_even(&cum); + /* fall through */ + case TCG_CALL_ARG_NORMAL: + if (TCG_TARGET_REG_BITS == 32) { + layout_arg_normal_n(&cum, info, 2); + } else { + layout_arg_1(&cum, info, TCG_CALL_ARG_NORMAL); + } + break; + default: + qemu_build_not_reached(); + } + break; + + default: + g_assert_not_reached(); + } + } + info->nr_in = cum.info_in_idx; + + /* Validate that we didn't overrun the input array. */ + assert(cum.info_in_idx <= ARRAY_SIZE(info->in)); + /* Validate the backend has enough argument space. */ + assert(cum.arg_slot <= max_reg_slots + max_stk_slots); + assert(cum.ref_slot <= max_stk_slots); +} + static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)]; static void process_op_defs(TCGContext *s); static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type, @@ -604,6 +752,7 @@ static void tcg_context_init(unsigned max_cpus) helper_table = g_hash_table_new(NULL, NULL); for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { + init_call_layout(&all_helpers[i]); g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, (gpointer)&all_helpers[i]); } @@ -1474,18 +1623,19 @@ bool tcg_op_supported(TCGOpcode op) } } -/* Note: we convert the 64 bit args to 32 bit and do some alignment - and endian swap. Maybe it would be better to do the alignment - and endian swap in tcg_reg_alloc_call(). */ +static TCGOp *tcg_op_alloc(TCGOpcode opc, unsigned nargs); + void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) { - int i, real_args, nb_rets, pi, max_args; - unsigned typemask; const TCGHelperInfo *info; + TCGv_i64 extend_free[MAX_CALL_IARGS]; + int n_extend = 0; TCGOp *op; + int i, n, pi = 0, total_args; info = g_hash_table_lookup(helper_table, (gpointer)func); - typemask = info->typemask; + total_args = info->nr_out + info->nr_in + 2; + op = tcg_op_alloc(INDEX_op_call, total_args); #ifdef CONFIG_PLUGIN /* detect non-plugin helpers */ @@ -1494,119 +1644,65 @@ void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args) } #endif - if (TCG_TARGET_CALL_ARG_I32 == TCG_CALL_ARG_EXTEND) { - for (i = 0; i < nargs; ++i) { - int argtype = extract32(typemask, (i + 1) * 3, 3); - bool is_32bit = (argtype & ~1) == dh_typecode_i32; - bool is_signed = argtype & 1; + TCGOP_CALLO(op) = n = info->nr_out; + switch (n) { + case 0: + tcg_debug_assert(ret == NULL); + break; + case 1: + tcg_debug_assert(ret != NULL); + op->args[pi++] = temp_arg(ret); + break; + case 2: + tcg_debug_assert(ret != NULL); + tcg_debug_assert(ret->base_type == ret->type + 1); + tcg_debug_assert(ret->temp_subindex == 0); + op->args[pi++] = temp_arg(ret); + op->args[pi++] = temp_arg(ret + 1); + break; + default: + g_assert_not_reached(); + } - if (is_32bit) { + TCGOP_CALLI(op) = n = info->nr_in; + for (i = 0; i < n; i++) { + const TCGCallArgumentLoc *loc = &info->in[i]; + TCGTemp *ts = args[loc->arg_idx] + loc->tmp_subindex; + + switch (loc->kind) { + case TCG_CALL_ARG_NORMAL: + op->args[pi++] = temp_arg(ts); + break; + + case TCG_CALL_ARG_EXTEND_U: + case TCG_CALL_ARG_EXTEND_S: + { TCGv_i64 temp = tcg_temp_new_i64(); - TCGv_i32 orig = temp_tcgv_i32(args[i]); - if (is_signed) { + TCGv_i32 orig = temp_tcgv_i32(ts); + + if (loc->kind == TCG_CALL_ARG_EXTEND_S) { tcg_gen_ext_i32_i64(temp, orig); } else { tcg_gen_extu_i32_i64(temp, orig); } - args[i] = tcgv_i64_temp(temp); + op->args[pi++] = tcgv_i64_arg(temp); + extend_free[n_extend++] = temp; } - } - } - - /* - * A Call op needs up to 4 + 2N parameters on 32-bit archs, - * and up to 4 + N parameters on 64-bit archs - * (N = number of input arguments + output arguments). - */ - max_args = (64 / TCG_TARGET_REG_BITS) * nargs + 4; - op = tcg_emit_op(INDEX_op_call, max_args); - - pi = 0; - if (ret != NULL) { - if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) { - op->args[pi++] = temp_arg(ret); - op->args[pi++] = temp_arg(ret + 1); - nb_rets = 2; - } else { - op->args[pi++] = temp_arg(ret); - nb_rets = 1; - } - } else { - nb_rets = 0; - } - TCGOP_CALLO(op) = nb_rets; - - real_args = 0; - for (i = 0; i < nargs; i++) { - int argtype = extract32(typemask, (i + 1) * 3, 3); - TCGCallArgumentKind kind; - TCGType type; - - switch (argtype) { - case dh_typecode_i32: - case dh_typecode_s32: - type = TCG_TYPE_I32; break; - case dh_typecode_i64: - case dh_typecode_s64: - type = TCG_TYPE_I64; - break; - case dh_typecode_ptr: - type = TCG_TYPE_PTR; - break; - default: - g_assert_not_reached(); - } - switch (type) { - case TCG_TYPE_I32: - kind = TCG_TARGET_CALL_ARG_I32; - break; - case TCG_TYPE_I64: - kind = TCG_TARGET_CALL_ARG_I64; - break; - default: - g_assert_not_reached(); - } - - switch (kind) { - case TCG_CALL_ARG_EVEN: - if (real_args & 1) { - op->args[pi++] = TCG_CALL_DUMMY_ARG; - real_args++; - } - /* fall through */ - case TCG_CALL_ARG_NORMAL: - if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) { - op->args[pi++] = temp_arg(args[i]); - op->args[pi++] = temp_arg(args[i] + 1); - real_args += 2; - break; - } - op->args[pi++] = temp_arg(args[i]); - real_args++; - break; default: g_assert_not_reached(); } } op->args[pi++] = (uintptr_t)func; op->args[pi++] = (uintptr_t)info; - TCGOP_CALLI(op) = real_args; + tcg_debug_assert(pi == total_args); - /* Make sure the fields didn't overflow. */ - tcg_debug_assert(TCGOP_CALLI(op) == real_args); - tcg_debug_assert(pi <= max_args); + QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link); - if (TCG_TARGET_CALL_ARG_I32 == TCG_CALL_ARG_EXTEND) { - for (i = 0; i < nargs; ++i) { - int argtype = extract32(typemask, (i + 1) * 3, 3); - bool is_32bit = (argtype & ~1) == dh_typecode_i32; - - if (is_32bit) { - tcg_temp_free_internal(args[i]); - } - } + tcg_debug_assert(n_extend < ARRAY_SIZE(extend_free)); + for (i = 0; i < n_extend; ++i) { + tcg_temp_free_i64(extend_free[i]); } } @@ -1822,10 +1918,7 @@ static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs) } for (i = 0; i < nb_iargs; i++) { TCGArg arg = op->args[nb_oargs + i]; - const char *t = ""; - if (arg != TCG_CALL_DUMMY_ARG) { - t = tcg_get_arg_str(s, buf, sizeof(buf), arg); - } + const char *t = tcg_get_arg_str(s, buf, sizeof(buf), arg); col += ne_fprintf(f, ",%s", t); } } else { @@ -2606,12 +2699,11 @@ static void liveness_pass_1(TCGContext *s) switch (opc) { case INDEX_op_call: { - int call_flags; - int nb_call_regs; + const TCGHelperInfo *info = tcg_call_info(op); + int call_flags = tcg_call_flags(op); nb_oargs = TCGOP_CALLO(op); nb_iargs = TCGOP_CALLI(op); - call_flags = tcg_call_flags(op); /* pure functions can be removed if their result is unused */ if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { @@ -2651,7 +2743,7 @@ static void liveness_pass_1(TCGContext *s) /* Record arguments that die in this helper. */ for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { ts = arg_temp(op->args[i]); - if (ts && ts->state & TS_DEAD) { + if (ts->state & TS_DEAD) { arg_life |= DEAD_ARG << i; } } @@ -2659,31 +2751,59 @@ static void liveness_pass_1(TCGContext *s) /* For all live registers, remove call-clobbered prefs. */ la_cross_call(s, nb_temps); - nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); + /* + * Input arguments are live for preceding opcodes. + * + * For those arguments that die, and will be allocated in + * registers, clear the register set for that arg, to be + * filled in below. For args that will be on the stack, + * reset to any available reg. Process arguments in reverse + * order so that if a temp is used more than once, the stack + * reset to max happens before the register reset to 0. + */ + for (i = nb_iargs - 1; i >= 0; i--) { + const TCGCallArgumentLoc *loc = &info->in[i]; + ts = arg_temp(op->args[nb_oargs + i]); - /* Input arguments are live for preceding opcodes. */ - for (i = 0; i < nb_iargs; i++) { - ts = arg_temp(op->args[i + nb_oargs]); - if (ts && ts->state & TS_DEAD) { - /* For those arguments that die, and will be allocated - * in registers, clear the register set for that arg, - * to be filled in below. For args that will be on - * the stack, reset to any available reg. - */ - *la_temp_pref(ts) - = (i < nb_call_regs ? 0 : - tcg_target_available_regs[ts->type]); + if (ts->state & TS_DEAD) { + switch (loc->kind) { + case TCG_CALL_ARG_NORMAL: + case TCG_CALL_ARG_EXTEND_U: + case TCG_CALL_ARG_EXTEND_S: + if (REG_P(loc)) { + *la_temp_pref(ts) = 0; + break; + } + /* fall through */ + default: + *la_temp_pref(ts) = + tcg_target_available_regs[ts->type]; + break; + } ts->state &= ~TS_DEAD; } } - /* For each input argument, add its input register to prefs. - If a temp is used once, this produces a single set bit. */ - for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) { - ts = arg_temp(op->args[i + nb_oargs]); - if (ts) { - tcg_regset_set_reg(*la_temp_pref(ts), - tcg_target_call_iarg_regs[i]); + /* + * For each input argument, add its input register to prefs. + * If a temp is used once, this produces a single set bit; + * if a temp is used multiple times, this produces a set. + */ + for (i = 0; i < nb_iargs; i++) { + const TCGCallArgumentLoc *loc = &info->in[i]; + ts = arg_temp(op->args[nb_oargs + i]); + + switch (loc->kind) { + case TCG_CALL_ARG_NORMAL: + case TCG_CALL_ARG_EXTEND_U: + case TCG_CALL_ARG_EXTEND_S: + if (REG_P(loc)) { + tcg_regset_set_reg(*la_temp_pref(ts), + tcg_target_call_iarg_regs[loc->arg_slot]); + } + break; + default: + break; } } } @@ -2954,21 +3074,19 @@ static bool liveness_pass_2(TCGContext *s) /* Make sure that input arguments are available. */ for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { arg_ts = arg_temp(op->args[i]); - if (arg_ts) { - dir_ts = arg_ts->state_ptr; - if (dir_ts && arg_ts->state == TS_DEAD) { - TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32 - ? INDEX_op_ld_i32 - : INDEX_op_ld_i64); - TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3); + dir_ts = arg_ts->state_ptr; + if (dir_ts && arg_ts->state == TS_DEAD) { + TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32 + ? INDEX_op_ld_i32 + : INDEX_op_ld_i64); + TCGOp *lop = tcg_op_insert_before(s, op, lopc, 3); - lop->args[0] = temp_arg(dir_ts); - lop->args[1] = temp_arg(arg_ts->mem_base); - lop->args[2] = arg_ts->mem_offset; + lop->args[0] = temp_arg(dir_ts); + lop->args[1] = temp_arg(arg_ts->mem_base); + lop->args[2] = arg_ts->mem_offset; - /* Loaded, but synced with memory. */ - arg_ts->state = TS_MEM; - } + /* Loaded, but synced with memory. */ + arg_ts->state = TS_MEM; } } @@ -2977,14 +3095,12 @@ static bool liveness_pass_2(TCGContext *s) so that we reload when needed. */ for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { arg_ts = arg_temp(op->args[i]); - if (arg_ts) { - dir_ts = arg_ts->state_ptr; - if (dir_ts) { - op->args[i] = temp_arg(dir_ts); - changes = true; - if (IS_DEAD_ARG(i)) { - arg_ts->state = TS_DEAD; - } + dir_ts = arg_ts->state_ptr; + if (dir_ts) { + op->args[i] = temp_arg(dir_ts); + changes = true; + if (IS_DEAD_ARG(i)) { + arg_ts->state = TS_DEAD; } } } @@ -4155,106 +4271,107 @@ static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op) return true; } +static void load_arg_reg(TCGContext *s, TCGReg reg, TCGTemp *ts, + TCGRegSet allocated_regs) +{ + if (ts->val_type == TEMP_VAL_REG) { + if (ts->reg != reg) { + tcg_reg_free(s, reg, allocated_regs); + if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { + /* + * Cross register class move not supported. Sync the + * temp back to its slot and load from there. + */ + temp_sync(s, ts, allocated_regs, 0, 0); + tcg_out_ld(s, ts->type, reg, + ts->mem_base->reg, ts->mem_offset); + } + } + } else { + TCGRegSet arg_set = 0; + + tcg_reg_free(s, reg, allocated_regs); + tcg_regset_set_reg(arg_set, reg); + temp_load(s, ts, arg_set, allocated_regs, 0); + } +} + +static void load_arg_stk(TCGContext *s, int stk_slot, TCGTemp *ts, + TCGRegSet allocated_regs) +{ + /* + * When the destination is on the stack, load up the temp and store. + * If there are many call-saved registers, the temp might live to + * see another use; otherwise it'll be discarded. + */ + temp_load(s, ts, tcg_target_available_regs[ts->type], allocated_regs, 0); + tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, + TCG_TARGET_CALL_STACK_OFFSET + + stk_slot * sizeof(tcg_target_long)); +} + +static void load_arg_normal(TCGContext *s, const TCGCallArgumentLoc *l, + TCGTemp *ts, TCGRegSet *allocated_regs) +{ + if (REG_P(l)) { + TCGReg reg = tcg_target_call_iarg_regs[l->arg_slot]; + load_arg_reg(s, reg, ts, *allocated_regs); + tcg_regset_set_reg(*allocated_regs, reg); + } else { + load_arg_stk(s, l->arg_slot - ARRAY_SIZE(tcg_target_call_iarg_regs), + ts, *allocated_regs); + } +} + static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) { const int nb_oargs = TCGOP_CALLO(op); const int nb_iargs = TCGOP_CALLI(op); const TCGLifeData arg_life = op->life; - const TCGHelperInfo *info; - int flags, nb_regs, i; - TCGReg reg; - TCGArg arg; - TCGTemp *ts; - intptr_t stack_offset; - size_t call_stack_size; - tcg_insn_unit *func_addr; - int allocate_args; - TCGRegSet allocated_regs; + const TCGHelperInfo *info = tcg_call_info(op); + TCGRegSet allocated_regs = s->reserved_regs; + int i; - func_addr = tcg_call_func(op); - info = tcg_call_info(op); - flags = info->flags; + /* + * Move inputs into place in reverse order, + * so that we place stacked arguments first. + */ + for (i = nb_iargs - 1; i >= 0; --i) { + const TCGCallArgumentLoc *loc = &info->in[i]; + TCGTemp *ts = arg_temp(op->args[nb_oargs + i]); - nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); - if (nb_regs > nb_iargs) { - nb_regs = nb_iargs; - } - - /* assign stack slots first */ - call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long); - call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) & - ~(TCG_TARGET_STACK_ALIGN - 1); - allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE); - if (allocate_args) { - /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed, - preallocate call stack */ - tcg_abort(); - } - - stack_offset = TCG_TARGET_CALL_STACK_OFFSET; - for (i = nb_regs; i < nb_iargs; i++) { - arg = op->args[nb_oargs + i]; - if (arg != TCG_CALL_DUMMY_ARG) { - ts = arg_temp(arg); - temp_load(s, ts, tcg_target_available_regs[ts->type], - s->reserved_regs, 0); - tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset); - } - stack_offset += sizeof(tcg_target_long); - } - - /* assign input registers */ - allocated_regs = s->reserved_regs; - for (i = 0; i < nb_regs; i++) { - arg = op->args[nb_oargs + i]; - if (arg != TCG_CALL_DUMMY_ARG) { - ts = arg_temp(arg); - reg = tcg_target_call_iarg_regs[i]; - - if (ts->val_type == TEMP_VAL_REG) { - if (ts->reg != reg) { - tcg_reg_free(s, reg, allocated_regs); - if (!tcg_out_mov(s, ts->type, reg, ts->reg)) { - /* - * Cross register class move not supported. Sync the - * temp back to its slot and load from there. - */ - temp_sync(s, ts, allocated_regs, 0, 0); - tcg_out_ld(s, ts->type, reg, - ts->mem_base->reg, ts->mem_offset); - } - } - } else { - TCGRegSet arg_set = 0; - - tcg_reg_free(s, reg, allocated_regs); - tcg_regset_set_reg(arg_set, reg); - temp_load(s, ts, arg_set, allocated_regs, 0); - } - - tcg_regset_set_reg(allocated_regs, reg); + switch (loc->kind) { + case TCG_CALL_ARG_NORMAL: + case TCG_CALL_ARG_EXTEND_U: + case TCG_CALL_ARG_EXTEND_S: + load_arg_normal(s, loc, ts, &allocated_regs); + break; + default: + g_assert_not_reached(); } } - /* mark dead temporaries and free the associated registers */ + /* Mark dead temporaries and free the associated registers. */ for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { if (IS_DEAD_ARG(i)) { temp_dead(s, arg_temp(op->args[i])); } } - /* clobber call registers */ + /* Clobber call registers. */ for (i = 0; i < TCG_TARGET_NB_REGS; i++) { if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) { tcg_reg_free(s, i, allocated_regs); } } - /* Save globals if they might be written by the helper, sync them if - they might be read. */ - if (flags & TCG_CALL_NO_READ_GLOBALS) { + /* + * Save globals if they might be written by the helper, + * sync them if they might be read. + */ + if (info->flags & TCG_CALL_NO_READ_GLOBALS) { /* Nothing to do */ - } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) { + } else if (info->flags & TCG_CALL_NO_WRITE_GLOBALS) { sync_globals(s, allocated_regs); } else { save_globals(s, allocated_regs); @@ -4265,25 +4382,35 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) gpointer hash = (gpointer)(uintptr_t)info->typemask; ffi_cif *cif = g_hash_table_lookup(ffi_table, hash); assert(cif != NULL); - tcg_out_call(s, func_addr, cif); + tcg_out_call(s, tcg_call_func(op), cif); } #else - tcg_out_call(s, func_addr); + tcg_out_call(s, tcg_call_func(op)); #endif - /* assign output registers and emit moves if needed */ - for(i = 0; i < nb_oargs; i++) { - arg = op->args[i]; - ts = arg_temp(arg); + /* Assign output registers and emit moves if needed. */ + switch (info->out_kind) { + case TCG_CALL_RET_NORMAL: + for (i = 0; i < nb_oargs; i++) { + TCGTemp *ts = arg_temp(op->args[i]); + TCGReg reg = tcg_target_call_oarg_regs[i]; - /* ENV should not be modified. */ - tcg_debug_assert(!temp_readonly(ts)); + /* ENV should not be modified. */ + tcg_debug_assert(!temp_readonly(ts)); - reg = tcg_target_call_oarg_regs[i]; - set_temp_val_reg(s, ts, reg); - ts->mem_coherent = 0; + set_temp_val_reg(s, ts, reg); + ts->mem_coherent = 0; + } + break; + default: + g_assert_not_reached(); + } + + /* Flush or discard output registers as needed. */ + for (i = 0; i < nb_oargs; i++) { + TCGTemp *ts = arg_temp(op->args[i]); if (NEED_SYNC_ARG(i)) { - temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i)); + temp_sync(s, ts, s->reserved_regs, 0, IS_DEAD_ARG(i)); } else if (IS_DEAD_ARG(i)) { temp_dead(s, ts); } From c6ef8c7b350bdbb24cca9206f5b9aa577e5abafa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 22 Nov 2022 19:08:02 +0100 Subject: [PATCH 39/47] tcg: Convert typecode_to_ffi from array to function MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In the unlikely case of invalid typecode mask, the function will abort instead of returning a NULL pointer. Signed-off-by: Richard Henderson Message-Id: <20221111074101.2069454-27-richard.henderson@linaro.org> [PMD: Split from bigger patch] Reviewed-by: Richard Henderson Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20221122180804.938-2-philmd@linaro.org> --- tcg/tcg.c | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index 0ac270fc7f..992ce3b33d 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -555,14 +555,24 @@ static GHashTable *helper_table; #ifdef CONFIG_TCG_INTERPRETER static GHashTable *ffi_table; -static ffi_type * const typecode_to_ffi[8] = { - [dh_typecode_void] = &ffi_type_void, - [dh_typecode_i32] = &ffi_type_uint32, - [dh_typecode_s32] = &ffi_type_sint32, - [dh_typecode_i64] = &ffi_type_uint64, - [dh_typecode_s64] = &ffi_type_sint64, - [dh_typecode_ptr] = &ffi_type_pointer, -}; +static ffi_type *typecode_to_ffi(int argmask) +{ + switch (argmask) { + case dh_typecode_void: + return &ffi_type_void; + case dh_typecode_i32: + return &ffi_type_uint32; + case dh_typecode_s32: + return &ffi_type_sint32; + case dh_typecode_i64: + return &ffi_type_uint64; + case dh_typecode_s64: + return &ffi_type_sint64; + case dh_typecode_ptr: + return &ffi_type_pointer; + } + g_assert_not_reached(); +} #endif typedef struct TCGCumulativeArgs { @@ -779,14 +789,14 @@ static void tcg_context_init(unsigned max_cpus) nargs = DIV_ROUND_UP(nargs, 3); ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); - ca->cif.rtype = typecode_to_ffi[typemask & 7]; + ca->cif.rtype = typecode_to_ffi(typemask & 7); ca->cif.nargs = nargs; if (nargs != 0) { ca->cif.arg_types = ca->args; for (int j = 0; j < nargs; ++j) { int typecode = extract32(typemask, (j + 1) * 3, 3); - ca->args[j] = typecode_to_ffi[typecode]; + ca->args[j] = typecode_to_ffi(typecode); } } From 0c22e17658a583c3d37cd928653caebf6406abd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Philippe=20Mathieu-Daud=C3=A9?= Date: Tue, 22 Nov 2022 19:08:03 +0100 Subject: [PATCH 40/47] tcg: Factor init_ffi_layouts() out of tcg_context_init() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Richard Henderson Message-Id: <20221111074101.2069454-27-richard.henderson@linaro.org> [PMD: Split from bigger patch] Reviewed-by: Richard Henderson Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20221122180804.938-3-philmd@linaro.org> --- tcg/tcg.c | 83 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 44 insertions(+), 39 deletions(-) diff --git a/tcg/tcg.c b/tcg/tcg.c index 992ce3b33d..33ff16d4e8 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -573,7 +573,49 @@ static ffi_type *typecode_to_ffi(int argmask) } g_assert_not_reached(); } -#endif + +static void init_ffi_layouts(void) +{ + /* g_direct_hash/equal for direct comparisons on uint32_t. */ + ffi_table = g_hash_table_new(NULL, NULL); + for (int i = 0; i < ARRAY_SIZE(all_helpers); ++i) { + uint32_t typemask = all_helpers[i].typemask; + gpointer hash = (gpointer)(uintptr_t)typemask; + struct { + ffi_cif cif; + ffi_type *args[]; + } *ca; + ffi_status status; + int nargs; + + if (g_hash_table_lookup(ffi_table, hash)) { + continue; + } + + /* Ignoring the return type, find the last non-zero field. */ + nargs = 32 - clz32(typemask >> 3); + nargs = DIV_ROUND_UP(nargs, 3); + + ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); + ca->cif.rtype = typecode_to_ffi(typemask & 7); + ca->cif.nargs = nargs; + + if (nargs != 0) { + ca->cif.arg_types = ca->args; + for (int j = 0; j < nargs; ++j) { + int typecode = extract32(typemask, (j + 1) * 3, 3); + ca->args[j] = typecode_to_ffi(typecode); + } + } + + status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, + ca->cif.rtype, ca->cif.arg_types); + assert(status == FFI_OK); + + g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif); + } +} +#endif /* CONFIG_TCG_INTERPRETER */ typedef struct TCGCumulativeArgs { int arg_idx; /* tcg_gen_callN args[] */ @@ -768,44 +810,7 @@ static void tcg_context_init(unsigned max_cpus) } #ifdef CONFIG_TCG_INTERPRETER - /* g_direct_hash/equal for direct comparisons on uint32_t. */ - ffi_table = g_hash_table_new(NULL, NULL); - for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { - struct { - ffi_cif cif; - ffi_type *args[]; - } *ca; - uint32_t typemask = all_helpers[i].typemask; - gpointer hash = (gpointer)(uintptr_t)typemask; - ffi_status status; - int nargs; - - if (g_hash_table_lookup(ffi_table, hash)) { - continue; - } - - /* Ignoring the return type, find the last non-zero field. */ - nargs = 32 - clz32(typemask >> 3); - nargs = DIV_ROUND_UP(nargs, 3); - - ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *)); - ca->cif.rtype = typecode_to_ffi(typemask & 7); - ca->cif.nargs = nargs; - - if (nargs != 0) { - ca->cif.arg_types = ca->args; - for (int j = 0; j < nargs; ++j) { - int typecode = extract32(typemask, (j + 1) * 3, 3); - ca->args[j] = typecode_to_ffi(typecode); - } - } - - status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs, - ca->cif.rtype, ca->cif.arg_types); - assert(status == FFI_OK); - - g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif); - } + init_ffi_layouts(); #endif tcg_target_init(s); From f9c4bb804d4bf485b892f3c6523d18025a38550e Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 22 Nov 2022 19:08:04 +0100 Subject: [PATCH 41/47] tcg: Move ffi_cif pointer into TCGHelperInfo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Instead of requiring a separate hash table lookup, put a pointer to the CIF into TCGHelperInfo. Signed-off-by: Richard Henderson Message-Id: <20221111074101.2069454-27-richard.henderson@linaro.org> [PMD: Split from bigger patch] Reviewed-by: Richard Henderson Signed-off-by: Philippe Mathieu-Daudé Message-Id: <20221122180804.938-4-philmd@linaro.org> --- tcg/tcg-internal.h | 7 +++++++ tcg/tcg.c | 30 ++++++++++++++---------------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/tcg/tcg-internal.h b/tcg/tcg-internal.h index c7e87e193d..6e50aeba3a 100644 --- a/tcg/tcg-internal.h +++ b/tcg/tcg-internal.h @@ -25,6 +25,10 @@ #ifndef TCG_INTERNAL_H #define TCG_INTERNAL_H +#ifdef CONFIG_TCG_INTERPRETER +#include +#endif + #define TCG_HIGHWATER 1024 /* @@ -57,6 +61,9 @@ typedef struct TCGCallArgumentLoc { typedef struct TCGHelperInfo { void *func; const char *name; +#ifdef CONFIG_TCG_INTERPRETER + ffi_cif *cif; +#endif unsigned typemask : 32; unsigned flags : 8; unsigned nr_in : 8; diff --git a/tcg/tcg.c b/tcg/tcg.c index 33ff16d4e8..00161e1dd5 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -62,10 +62,6 @@ #include "tcg/tcg-ldst.h" #include "tcg-internal.h" -#ifdef CONFIG_TCG_INTERPRETER -#include -#endif - /* Forward declarations for functions declared in tcg-target.c.inc and used here. */ static void tcg_target_init(TCGContext *s); @@ -553,8 +549,6 @@ static TCGHelperInfo all_helpers[] = { static GHashTable *helper_table; #ifdef CONFIG_TCG_INTERPRETER -static GHashTable *ffi_table; - static ffi_type *typecode_to_ffi(int argmask) { switch (argmask) { @@ -577,9 +571,11 @@ static ffi_type *typecode_to_ffi(int argmask) static void init_ffi_layouts(void) { /* g_direct_hash/equal for direct comparisons on uint32_t. */ - ffi_table = g_hash_table_new(NULL, NULL); + GHashTable *ffi_table = g_hash_table_new(NULL, NULL); + for (int i = 0; i < ARRAY_SIZE(all_helpers); ++i) { - uint32_t typemask = all_helpers[i].typemask; + TCGHelperInfo *info = &all_helpers[i]; + unsigned typemask = info->typemask; gpointer hash = (gpointer)(uintptr_t)typemask; struct { ffi_cif cif; @@ -587,8 +583,11 @@ static void init_ffi_layouts(void) } *ca; ffi_status status; int nargs; + ffi_cif *cif; - if (g_hash_table_lookup(ffi_table, hash)) { + cif = g_hash_table_lookup(ffi_table, hash); + if (cif) { + info->cif = cif; continue; } @@ -612,8 +611,12 @@ static void init_ffi_layouts(void) ca->cif.rtype, ca->cif.arg_types); assert(status == FFI_OK); - g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif); + cif = &ca->cif; + info->cif = cif; + g_hash_table_insert(ffi_table, hash, (gpointer)cif); } + + g_hash_table_destroy(ffi_table); } #endif /* CONFIG_TCG_INTERPRETER */ @@ -4393,12 +4396,7 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) } #ifdef CONFIG_TCG_INTERPRETER - { - gpointer hash = (gpointer)(uintptr_t)info->typemask; - ffi_cif *cif = g_hash_table_lookup(ffi_table, hash); - assert(cif != NULL); - tcg_out_call(s, tcg_call_func(op), cif); - } + tcg_out_call(s, tcg_call_func(op), info->cif); #else tcg_out_call(s, tcg_call_func(op)); #endif From fa3cb9f9ffc9298d28b1a932946564aaefe101d1 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Thu, 3 Nov 2022 05:39:56 +0000 Subject: [PATCH 42/47] tcg/aarch64: Merge tcg_out_callr into tcg_out_call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There is only one use, and BLR is perhaps even more self-documentary than CALLR. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/aarch64/tcg-target.c.inc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index 344b63e20f..1af879e6f5 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -1336,11 +1336,6 @@ static void tcg_out_goto_long(TCGContext *s, const tcg_insn_unit *target) } } -static inline void tcg_out_callr(TCGContext *s, TCGReg reg) -{ - tcg_out_insn(s, 3207, BLR, reg); -} - static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) { ptrdiff_t offset = tcg_pcrel_diff(s, target) >> 2; @@ -1348,7 +1343,7 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) tcg_out_insn(s, 3206, BL, offset); } else { tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, (intptr_t)target); - tcg_out_callr(s, TCG_REG_TMP); + tcg_out_insn(s, 3207, BLR, TCG_REG_TMP); } } From cee44b037b7e40c74401f7a87bef32f6680483c7 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Tue, 18 Oct 2022 17:51:41 +1000 Subject: [PATCH 43/47] tcg: Add TCGHelperInfo argument to tcg_out_call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This eliminates an ifdef for TCI, and will be required for expanding the call for TCGv_i128. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- tcg/aarch64/tcg-target.c.inc | 12 +++++++++--- tcg/arm/tcg-target.c.inc | 10 ++++++++-- tcg/i386/tcg-target.c.inc | 5 +++-- tcg/loongarch64/tcg-target.c.inc | 7 ++++--- tcg/mips/tcg-target.c.inc | 3 ++- tcg/ppc/tcg-target.c.inc | 7 ++++--- tcg/riscv/tcg-target.c.inc | 7 ++++--- tcg/s390x/tcg-target.c.inc | 12 +++++++++--- tcg/sparc64/tcg-target.c.inc | 3 ++- tcg/tcg.c | 12 ++---------- tcg/tci/tcg-target.c.inc | 3 ++- 11 files changed, 49 insertions(+), 32 deletions(-) diff --git a/tcg/aarch64/tcg-target.c.inc b/tcg/aarch64/tcg-target.c.inc index 1af879e6f5..ad1816e32d 100644 --- a/tcg/aarch64/tcg-target.c.inc +++ b/tcg/aarch64/tcg-target.c.inc @@ -1336,7 +1336,7 @@ static void tcg_out_goto_long(TCGContext *s, const tcg_insn_unit *target) } } -static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) +static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *target) { ptrdiff_t offset = tcg_pcrel_diff(s, target) >> 2; if (offset == sextract64(offset, 0, 26)) { @@ -1347,6 +1347,12 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) } } +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, + const TCGHelperInfo *info) +{ + tcg_out_call_int(s, target); +} + void tb_target_set_jmp_target(uintptr_t tc_ptr, uintptr_t jmp_rx, uintptr_t jmp_rw, uintptr_t addr) { @@ -1594,7 +1600,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) tcg_out_mov(s, TARGET_LONG_BITS == 64, TCG_REG_X1, lb->addrlo_reg); tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X2, oi); tcg_out_adr(s, TCG_REG_X3, lb->raddr); - tcg_out_call(s, qemu_ld_helpers[opc & MO_SIZE]); + tcg_out_call_int(s, qemu_ld_helpers[opc & MO_SIZE]); if (opc & MO_SIGN) { tcg_out_sxt(s, lb->type, size, lb->datalo_reg, TCG_REG_X0); } else { @@ -1620,7 +1626,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) tcg_out_mov(s, size == MO_64, TCG_REG_X2, lb->datalo_reg); tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X3, oi); tcg_out_adr(s, TCG_REG_X4, lb->raddr); - tcg_out_call(s, qemu_st_helpers[opc & MO_SIZE]); + tcg_out_call_int(s, qemu_st_helpers[opc & MO_SIZE]); tcg_out_goto(s, lb->raddr); return true; } diff --git a/tcg/arm/tcg-target.c.inc b/tcg/arm/tcg-target.c.inc index 2c6c353eea..9245ea86d0 100644 --- a/tcg/arm/tcg-target.c.inc +++ b/tcg/arm/tcg-target.c.inc @@ -1131,7 +1131,7 @@ static void tcg_out_goto(TCGContext *s, ARMCond cond, const tcg_insn_unit *addr) * The call case is mostly used for helpers - so it's not unreasonable * for them to be beyond branch range. */ -static void tcg_out_call(TCGContext *s, const tcg_insn_unit *addr) +static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *addr) { intptr_t addri = (intptr_t)addr; ptrdiff_t disp = tcg_pcrel_diff(s, addr); @@ -1150,6 +1150,12 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *addr) tcg_out_blx_reg(s, COND_AL, TCG_REG_TMP); } +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *addr, + const TCGHelperInfo *info) +{ + tcg_out_call_int(s, addr); +} + static void tcg_out_goto_label(TCGContext *s, ARMCond cond, TCGLabel *l) { if (l->has_value) { @@ -1515,7 +1521,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) argreg = tcg_out_arg_reg32(s, argreg, TCG_REG_R14); /* Use the canonical unsigned helpers and minimize icache usage. */ - tcg_out_call(s, qemu_ld_helpers[opc & MO_SIZE]); + tcg_out_call_int(s, qemu_ld_helpers[opc & MO_SIZE]); datalo = lb->datalo_reg; datahi = lb->datahi_reg; diff --git a/tcg/i386/tcg-target.c.inc b/tcg/i386/tcg-target.c.inc index cb04e4b3ad..58bd5873f5 100644 --- a/tcg/i386/tcg-target.c.inc +++ b/tcg/i386/tcg-target.c.inc @@ -1661,7 +1661,8 @@ static void tcg_out_branch(TCGContext *s, int call, const tcg_insn_unit *dest) } } -static inline void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest) +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest, + const TCGHelperInfo *info) { tcg_out_branch(s, 1, dest); } @@ -1885,7 +1886,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) (uintptr_t)l->raddr); } - tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]); + tcg_out_branch(s, 1, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]); data_reg = l->datalo_reg; switch (opc & MO_SSIZE) { diff --git a/tcg/loongarch64/tcg-target.c.inc b/tcg/loongarch64/tcg-target.c.inc index d326e28740..c9e99e8ec3 100644 --- a/tcg/loongarch64/tcg-target.c.inc +++ b/tcg/loongarch64/tcg-target.c.inc @@ -567,7 +567,8 @@ static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) } } -static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg) +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg, + const TCGHelperInfo *info) { tcg_out_call_int(s, arg, false); } @@ -760,7 +761,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A2, oi); tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A3, (tcg_target_long)l->raddr); - tcg_out_call(s, qemu_ld_helpers[size]); + tcg_out_call_int(s, qemu_ld_helpers[size], false); switch (opc & MO_SSIZE) { case MO_SB: @@ -821,7 +822,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A3, oi); tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_A4, (tcg_target_long)l->raddr); - tcg_out_call(s, qemu_st_helpers[size]); + tcg_out_call_int(s, qemu_st_helpers[size], false); return tcg_out_goto(s, l->raddr); } diff --git a/tcg/mips/tcg-target.c.inc b/tcg/mips/tcg-target.c.inc index bd76f0c97f..292e490b5c 100644 --- a/tcg/mips/tcg-target.c.inc +++ b/tcg/mips/tcg-target.c.inc @@ -1020,7 +1020,8 @@ static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) } } -static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg) +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg, + const TCGHelperInfo *info) { tcg_out_call_int(s, arg, false); tcg_out_nop(s); diff --git a/tcg/ppc/tcg-target.c.inc b/tcg/ppc/tcg-target.c.inc index 38ee9974cd..e0621463f6 100644 --- a/tcg/ppc/tcg-target.c.inc +++ b/tcg/ppc/tcg-target.c.inc @@ -2002,7 +2002,8 @@ static void tcg_out_call_int(TCGContext *s, int lk, #endif } -static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target) +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, + const TCGHelperInfo *info) { tcg_out_call_int(s, LK, target); } @@ -2221,7 +2222,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) tcg_out_movi(s, TCG_TYPE_I32, arg++, oi); tcg_out32(s, MFSPR | RT(arg) | LR); - tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]); + tcg_out_call_int(s, LK, qemu_ld_helpers[opc & (MO_BSWAP | MO_SIZE)]); lo = lb->datalo_reg; hi = lb->datahi_reg; @@ -2290,7 +2291,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) tcg_out_movi(s, TCG_TYPE_I32, arg++, oi); tcg_out32(s, MFSPR | RT(arg) | LR); - tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]); + tcg_out_call_int(s, LK, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]); tcg_out_b(s, 0, lb->raddr); return true; diff --git a/tcg/riscv/tcg-target.c.inc b/tcg/riscv/tcg-target.c.inc index 81a83e45b1..aa017d665a 100644 --- a/tcg/riscv/tcg-target.c.inc +++ b/tcg/riscv/tcg-target.c.inc @@ -819,7 +819,8 @@ static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *arg, bool tail) } } -static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg) +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *arg, + const TCGHelperInfo *info) { tcg_out_call_int(s, arg, false); } @@ -1002,7 +1003,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *l) tcg_out_movi(s, TCG_TYPE_PTR, a2, oi); tcg_out_movi(s, TCG_TYPE_PTR, a3, (tcg_target_long)l->raddr); - tcg_out_call(s, qemu_ld_helpers[opc & MO_SSIZE]); + tcg_out_call_int(s, qemu_ld_helpers[opc & MO_SSIZE], false); tcg_out_mov(s, (opc & MO_SIZE) == MO_64, l->datalo_reg, a0); tcg_out_goto(s, l->raddr); @@ -1047,7 +1048,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *l) tcg_out_movi(s, TCG_TYPE_PTR, a3, oi); tcg_out_movi(s, TCG_TYPE_PTR, a4, (tcg_target_long)l->raddr); - tcg_out_call(s, qemu_st_helpers[opc & MO_SIZE]); + tcg_out_call_int(s, qemu_st_helpers[opc & MO_SIZE], false); tcg_out_goto(s, l->raddr); return true; diff --git a/tcg/s390x/tcg-target.c.inc b/tcg/s390x/tcg-target.c.inc index f1d3907cd8..b9ba7b605e 100644 --- a/tcg/s390x/tcg-target.c.inc +++ b/tcg/s390x/tcg-target.c.inc @@ -1691,7 +1691,7 @@ static void tgen_brcond(TCGContext *s, TCGType type, TCGCond c, tgen_branch(s, cc, l); } -static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest) +static void tcg_out_call_int(TCGContext *s, const tcg_insn_unit *dest) { ptrdiff_t off = tcg_pcrel_diff(s, dest) >> 1; if (off == (int32_t)off) { @@ -1702,6 +1702,12 @@ static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest) } } +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest, + const TCGHelperInfo *info) +{ + tcg_out_call_int(s, dest); +} + static void tcg_out_qemu_ld_direct(TCGContext *s, MemOp opc, TCGReg data, TCGReg base, TCGReg index, int disp) { @@ -1897,7 +1903,7 @@ static bool tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) } tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R4, oi); tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R5, (uintptr_t)lb->raddr); - tcg_out_call(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SSIZE)]); + tcg_out_call_int(s, qemu_ld_helpers[opc & (MO_BSWAP | MO_SSIZE)]); tcg_out_mov(s, TCG_TYPE_I64, data_reg, TCG_REG_R2); tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr); @@ -1938,7 +1944,7 @@ static bool tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb) } tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_R5, oi); tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_R6, (uintptr_t)lb->raddr); - tcg_out_call(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]); + tcg_out_call_int(s, qemu_st_helpers[opc & (MO_BSWAP | MO_SIZE)]); tgen_gotoi(s, S390_CC_ALWAYS, lb->raddr); return true; diff --git a/tcg/sparc64/tcg-target.c.inc b/tcg/sparc64/tcg-target.c.inc index cb9453efdd..eb913f33c8 100644 --- a/tcg/sparc64/tcg-target.c.inc +++ b/tcg/sparc64/tcg-target.c.inc @@ -859,7 +859,8 @@ static void tcg_out_call_nodelay(TCGContext *s, const tcg_insn_unit *dest, } } -static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest) +static void tcg_out_call(TCGContext *s, const tcg_insn_unit *dest, + const TCGHelperInfo *info) { tcg_out_call_nodelay(s, dest, false); tcg_out_nop(s); diff --git a/tcg/tcg.c b/tcg/tcg.c index 00161e1dd5..da91779890 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -145,12 +145,8 @@ static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, intptr_t arg2); static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val, TCGReg base, intptr_t ofs); -#ifdef CONFIG_TCG_INTERPRETER static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target, - ffi_cif *cif); -#else -static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target); -#endif + const TCGHelperInfo *info); static bool tcg_target_const_match(int64_t val, TCGType type, int ct); #ifdef TCG_TARGET_NEED_LDST_LABELS static int tcg_out_ldst_finalize(TCGContext *s); @@ -4395,11 +4391,7 @@ static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op) save_globals(s, allocated_regs); } -#ifdef CONFIG_TCG_INTERPRETER - tcg_out_call(s, tcg_call_func(op), info->cif); -#else - tcg_out_call(s, tcg_call_func(op)); -#endif + tcg_out_call(s, tcg_call_func(op), info); /* Assign output registers and emit moves if needed. */ switch (info->out_kind) { diff --git a/tcg/tci/tcg-target.c.inc b/tcg/tci/tcg-target.c.inc index c1acaa943e..d36a7ebdd1 100644 --- a/tcg/tci/tcg-target.c.inc +++ b/tcg/tci/tcg-target.c.inc @@ -558,8 +558,9 @@ static void tcg_out_movi(TCGContext *s, TCGType type, } static void tcg_out_call(TCGContext *s, const tcg_insn_unit *func, - ffi_cif *cif) + const TCGHelperInfo *info) { + ffi_cif *cif = info->cif; tcg_insn_unit insn = 0; uint8_t which; From 1b660f42efd721e4bcb5548907ba8d1370053318 Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 24 Dec 2022 04:35:22 -0800 Subject: [PATCH 44/47] accel/tcg: Fix tb_invalidate_phys_page_unwind MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When called from syscall(), we are not within a TB and pc == 0. We can skip the check for invalidating the current TB. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- accel/tcg/tb-maint.c | 78 ++++++++++++++++++++++++-------------------- 1 file changed, 43 insertions(+), 35 deletions(-) diff --git a/accel/tcg/tb-maint.c b/accel/tcg/tb-maint.c index 1b8e860647..b3d6529ae2 100644 --- a/accel/tcg/tb-maint.c +++ b/accel/tcg/tb-maint.c @@ -1024,43 +1024,51 @@ void tb_invalidate_phys_page(tb_page_addr_t addr) */ bool tb_invalidate_phys_page_unwind(tb_page_addr_t addr, uintptr_t pc) { - assert(pc != 0); -#ifdef TARGET_HAS_PRECISE_SMC - assert_memory_lock(); - { - TranslationBlock *current_tb = tcg_tb_lookup(pc); - bool current_tb_modified = false; - TranslationBlock *tb; - PageForEachNext n; + TranslationBlock *current_tb; + bool current_tb_modified; + TranslationBlock *tb; + PageForEachNext n; - addr &= TARGET_PAGE_MASK; - - PAGE_FOR_EACH_TB(addr, addr + TARGET_PAGE_SIZE, unused, tb, n) { - if (current_tb == tb && - (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) { - /* - * If we are modifying the current TB, we must stop its - * execution. We could be more precise by checking that - * the modification is after the current PC, but it would - * require a specialized function to partially restore - * the CPU state. - */ - current_tb_modified = true; - cpu_restore_state_from_tb(current_cpu, current_tb, pc); - } - tb_phys_invalidate__locked(tb); - } - - if (current_tb_modified) { - /* Force execution of one insn next time. */ - CPUState *cpu = current_cpu; - cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(current_cpu); - return true; - } + /* + * Without precise smc semantics, or when outside of a TB, + * we can skip to invalidate. + */ +#ifndef TARGET_HAS_PRECISE_SMC + pc = 0; +#endif + if (!pc) { + tb_invalidate_phys_page(addr); + return false; + } + + assert_memory_lock(); + current_tb = tcg_tb_lookup(pc); + + addr &= TARGET_PAGE_MASK; + current_tb_modified = false; + + PAGE_FOR_EACH_TB(addr, addr + TARGET_PAGE_SIZE, unused, tb, n) { + if (current_tb == tb && + (tb_cflags(current_tb) & CF_COUNT_MASK) != 1) { + /* + * If we are modifying the current TB, we must stop its + * execution. We could be more precise by checking that + * the modification is after the current PC, but it would + * require a specialized function to partially restore + * the CPU state. + */ + current_tb_modified = true; + cpu_restore_state_from_tb(current_cpu, current_tb, pc); + } + tb_phys_invalidate__locked(tb); + } + + if (current_tb_modified) { + /* Force execution of one insn next time. */ + CPUState *cpu = current_cpu; + cpu->cflags_next_tb = 1 | CF_NOIRQ | curr_cflags(current_cpu); + return true; } -#else - tb_invalidate_phys_page(addr); -#endif /* TARGET_HAS_PRECISE_SMC */ return false; } #else From 177a8cb83bcf2c8eb013b997173a25339a7ce86f Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 24 Dec 2022 05:06:29 -0800 Subject: [PATCH 45/47] accel/tcg: Use g_free_rcu for user-exec interval trees MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Because we allow lockless lookups, we have to be careful when it is freed. Use rcu to delay the free until safe. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- accel/tcg/user-exec.c | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index a3cecda405..2c5c10d2e6 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -22,6 +22,7 @@ #include "exec/exec-all.h" #include "tcg/tcg.h" #include "qemu/bitops.h" +#include "qemu/rcu.h" #include "exec/cpu_ldst.h" #include "exec/translate-all.h" #include "exec/helper-proto.h" @@ -136,6 +137,7 @@ bool handle_sigsegv_accerr_write(CPUState *cpu, sigset_t *old_set, } typedef struct PageFlagsNode { + struct rcu_head rcu; IntervalTreeNode itree; int flags; } PageFlagsNode; @@ -266,7 +268,7 @@ static bool pageflags_unset(target_ulong start, target_ulong last) } } else if (p_last <= last) { /* Range completely covers node -- remove it. */ - g_free(p); + g_free_rcu(p, rcu); } else { /* Truncate the node from the start. */ p->itree.start = last + 1; @@ -311,7 +313,7 @@ static void pageflags_create_merge(target_ulong start, target_ulong last, if (prev) { if (next) { prev->itree.last = next->itree.last; - g_free(next); + g_free_rcu(next, rcu); } else { prev->itree.last = last; } @@ -376,7 +378,7 @@ static bool pageflags_set_clear(target_ulong start, target_ulong last, p->flags = merge_flags; } else { interval_tree_remove(&p->itree, &pageflags_root); - g_free(p); + g_free_rcu(p, rcu); } goto done; } @@ -421,7 +423,7 @@ static bool pageflags_set_clear(target_ulong start, target_ulong last, p->flags = merge_flags; } else { interval_tree_remove(&p->itree, &pageflags_root); - g_free(p); + g_free_rcu(p, rcu); } if (p_last < last) { start = p_last + 1; @@ -462,7 +464,7 @@ static bool pageflags_set_clear(target_ulong start, target_ulong last, p->itree.start = last + 1; interval_tree_insert(&p->itree, &pageflags_root); } else { - g_free(p); + g_free_rcu(p, rcu); goto restart; } if (set_flags) { @@ -779,6 +781,7 @@ tb_page_addr_t get_page_addr_code_hostp(CPUArchState *env, target_ulong addr, #define TBD_MASK (TARGET_PAGE_MASK * TPD_PAGES) typedef struct TargetPageDataNode { + struct rcu_head rcu; IntervalTreeNode itree; char data[TPD_PAGES][TARGET_PAGE_DATA_SIZE] __attribute__((aligned)); } TargetPageDataNode; @@ -801,11 +804,11 @@ void page_reset_target_data(target_ulong start, target_ulong end) n = next, next = next ? interval_tree_iter_next(n, start, last) : NULL) { target_ulong n_start, n_last, p_ofs, p_len; - TargetPageDataNode *t; + TargetPageDataNode *t = container_of(n, TargetPageDataNode, itree); if (n->start >= start && n->last <= last) { interval_tree_remove(n, &targetdata_root); - g_free(n); + g_free_rcu(t, rcu); continue; } @@ -819,7 +822,6 @@ void page_reset_target_data(target_ulong start, target_ulong end) n_last = MIN(last, n->last); p_len = (n_last + 1 - n_start) >> TARGET_PAGE_BITS; - t = container_of(n, TargetPageDataNode, itree); memset(t->data[p_ofs], 0, p_len * TARGET_PAGE_DATA_SIZE); } } From e630c0126c561b7db26790e8288c4fe9b61ae8dc Mon Sep 17 00:00:00 2001 From: Richard Henderson Date: Sat, 24 Dec 2022 06:37:56 -0800 Subject: [PATCH 46/47] accel/tcg: Handle false negative lookup in page_check_range MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As in page_get_flags, we need to try again with the mmap lock held if we fail a page lookup. Reviewed-by: Philippe Mathieu-Daudé Signed-off-by: Richard Henderson --- accel/tcg/user-exec.c | 41 ++++++++++++++++++++++++++++++++++------- 1 file changed, 34 insertions(+), 7 deletions(-) diff --git a/accel/tcg/user-exec.c b/accel/tcg/user-exec.c index 2c5c10d2e6..a8eb63ab96 100644 --- a/accel/tcg/user-exec.c +++ b/accel/tcg/user-exec.c @@ -525,6 +525,8 @@ void page_set_flags(target_ulong start, target_ulong end, int flags) int page_check_range(target_ulong start, target_ulong len, int flags) { target_ulong last; + int locked; /* tri-state: =0: unlocked, +1: global, -1: local */ + int ret; if (len == 0) { return 0; /* trivial length */ @@ -535,42 +537,67 @@ int page_check_range(target_ulong start, target_ulong len, int flags) return -1; /* wrap around */ } + locked = have_mmap_lock(); while (true) { PageFlagsNode *p = pageflags_find(start, last); int missing; if (!p) { - return -1; /* entire region invalid */ + if (!locked) { + /* + * Lockless lookups have false negatives. + * Retry with the lock held. + */ + mmap_lock(); + locked = -1; + p = pageflags_find(start, last); + } + if (!p) { + ret = -1; /* entire region invalid */ + break; + } } if (start < p->itree.start) { - return -1; /* initial bytes invalid */ + ret = -1; /* initial bytes invalid */ + break; } missing = flags & ~p->flags; if (missing & PAGE_READ) { - return -1; /* page not readable */ + ret = -1; /* page not readable */ + break; } if (missing & PAGE_WRITE) { if (!(p->flags & PAGE_WRITE_ORG)) { - return -1; /* page not writable */ + ret = -1; /* page not writable */ + break; } /* Asking about writable, but has been protected: undo. */ if (!page_unprotect(start, 0)) { - return -1; + ret = -1; + break; } /* TODO: page_unprotect should take a range, not a single page. */ if (last - start < TARGET_PAGE_SIZE) { - return 0; /* ok */ + ret = 0; /* ok */ + break; } start += TARGET_PAGE_SIZE; continue; } if (last <= p->itree.last) { - return 0; /* ok */ + ret = 0; /* ok */ + break; } start = p->itree.last + 1; } + + /* Release the lock if acquired locally. */ + if (locked < 0) { + mmap_unlock(); + } + return ret; } void page_protect(tb_page_addr_t address) From d4846c33ebe04d2141dcc613b5558d2f1d8077af Mon Sep 17 00:00:00 2001 From: Ilya Leoshkevich Date: Fri, 23 Dec 2022 13:02:52 +0100 Subject: [PATCH 47/47] tests/tcg/multiarch: add vma-pthread.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a test that locklessly changes and exercises page protection bits from various threads. This helps catch race conditions in the VMA handling. Acked-by: Alex Bennée Signed-off-by: Ilya Leoshkevich Message-Id: <20221223120252.513319-1-iii@linux.ibm.com> Signed-off-by: Richard Henderson --- tests/tcg/multiarch/Makefile.target | 3 + tests/tcg/multiarch/munmap-pthread.c | 16 +-- tests/tcg/multiarch/nop_func.h | 25 ++++ tests/tcg/multiarch/vma-pthread.c | 207 +++++++++++++++++++++++++++ 4 files changed, 236 insertions(+), 15 deletions(-) create mode 100644 tests/tcg/multiarch/nop_func.h create mode 100644 tests/tcg/multiarch/vma-pthread.c diff --git a/tests/tcg/multiarch/Makefile.target b/tests/tcg/multiarch/Makefile.target index 5f0fee1aad..e7213af492 100644 --- a/tests/tcg/multiarch/Makefile.target +++ b/tests/tcg/multiarch/Makefile.target @@ -39,6 +39,9 @@ signals: LDFLAGS+=-lrt -lpthread munmap-pthread: CFLAGS+=-pthread munmap-pthread: LDFLAGS+=-pthread +vma-pthread: CFLAGS+=-pthread +vma-pthread: LDFLAGS+=-pthread + # We define the runner for test-mmap after the individual # architectures have defined their supported pages sizes. If no # additional page sizes are defined we only run the default test. diff --git a/tests/tcg/multiarch/munmap-pthread.c b/tests/tcg/multiarch/munmap-pthread.c index d7143b00d5..1c79005846 100644 --- a/tests/tcg/multiarch/munmap-pthread.c +++ b/tests/tcg/multiarch/munmap-pthread.c @@ -7,21 +7,7 @@ #include #include -static const char nop_func[] = { -#if defined(__aarch64__) - 0xc0, 0x03, 0x5f, 0xd6, /* ret */ -#elif defined(__alpha__) - 0x01, 0x80, 0xFA, 0x6B, /* ret */ -#elif defined(__arm__) - 0x1e, 0xff, 0x2f, 0xe1, /* bx lr */ -#elif defined(__riscv) - 0x67, 0x80, 0x00, 0x00, /* ret */ -#elif defined(__s390__) - 0x07, 0xfe, /* br %r14 */ -#elif defined(__i386__) || defined(__x86_64__) - 0xc3, /* ret */ -#endif -}; +#include "nop_func.h" static void *thread_mmap_munmap(void *arg) { diff --git a/tests/tcg/multiarch/nop_func.h b/tests/tcg/multiarch/nop_func.h new file mode 100644 index 0000000000..f714d21000 --- /dev/null +++ b/tests/tcg/multiarch/nop_func.h @@ -0,0 +1,25 @@ +/* + * No-op functions that can be safely copied. + * + * SPDX-License-Identifier: GPL-2.0-or-later + */ +#ifndef NOP_FUNC_H +#define NOP_FUNC_H + +static const char nop_func[] = { +#if defined(__aarch64__) + 0xc0, 0x03, 0x5f, 0xd6, /* ret */ +#elif defined(__alpha__) + 0x01, 0x80, 0xFA, 0x6B, /* ret */ +#elif defined(__arm__) + 0x1e, 0xff, 0x2f, 0xe1, /* bx lr */ +#elif defined(__riscv) + 0x67, 0x80, 0x00, 0x00, /* ret */ +#elif defined(__s390__) + 0x07, 0xfe, /* br %r14 */ +#elif defined(__i386__) || defined(__x86_64__) + 0xc3, /* ret */ +#endif +}; + +#endif diff --git a/tests/tcg/multiarch/vma-pthread.c b/tests/tcg/multiarch/vma-pthread.c new file mode 100644 index 0000000000..7045da08fc --- /dev/null +++ b/tests/tcg/multiarch/vma-pthread.c @@ -0,0 +1,207 @@ +/* + * Test that VMA updates do not race. + * + * SPDX-License-Identifier: GPL-2.0-or-later + * + * Map a contiguous chunk of RWX memory. Split it into 8 equally sized + * regions, each of which is guaranteed to have a certain combination of + * protection bits set. + * + * Reader, writer and executor threads perform the respective operations on + * pages, which are guaranteed to have the respective protection bit set. + * Two mutator threads change the non-fixed protection bits randomly. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "nop_func.h" + +#define PAGE_IDX_BITS 10 +#define PAGE_COUNT (1 << PAGE_IDX_BITS) +#define PAGE_IDX_MASK (PAGE_COUNT - 1) +#define REGION_IDX_BITS 3 +#define PAGE_IDX_R_MASK (1 << 7) +#define PAGE_IDX_W_MASK (1 << 8) +#define PAGE_IDX_X_MASK (1 << 9) +#define REGION_MASK (PAGE_IDX_R_MASK | PAGE_IDX_W_MASK | PAGE_IDX_X_MASK) +#define PAGES_PER_REGION (1 << (PAGE_IDX_BITS - REGION_IDX_BITS)) + +struct context { + int pagesize; + char *ptr; + int dev_null_fd; + volatile int mutator_count; +}; + +static void *thread_read(void *arg) +{ + struct context *ctx = arg; + ssize_t sret; + size_t i, j; + int ret; + + for (i = 0; ctx->mutator_count; i++) { + char *p; + + j = (i & PAGE_IDX_MASK) | PAGE_IDX_R_MASK; + p = &ctx->ptr[j * ctx->pagesize]; + + /* Read directly. */ + ret = memcmp(p, nop_func, sizeof(nop_func)); + if (ret != 0) { + fprintf(stderr, "fail direct read %p\n", p); + abort(); + } + + /* Read indirectly. */ + sret = write(ctx->dev_null_fd, p, 1); + if (sret != 1) { + if (sret < 0) { + fprintf(stderr, "fail indirect read %p (%m)\n", p); + } else { + fprintf(stderr, "fail indirect read %p (%zd)\n", p, sret); + } + abort(); + } + } + + return NULL; +} + +static void *thread_write(void *arg) +{ + struct context *ctx = arg; + struct timespec *ts; + size_t i, j; + int ret; + + for (i = 0; ctx->mutator_count; i++) { + j = (i & PAGE_IDX_MASK) | PAGE_IDX_W_MASK; + + /* Write directly. */ + memcpy(&ctx->ptr[j * ctx->pagesize], nop_func, sizeof(nop_func)); + + /* Write using a syscall. */ + ts = (struct timespec *)(&ctx->ptr[(j + 1) * ctx->pagesize] - + sizeof(struct timespec)); + ret = clock_gettime(CLOCK_REALTIME, ts); + if (ret != 0) { + fprintf(stderr, "fail indirect write %p (%m)\n", ts); + abort(); + } + } + + return NULL; +} + +static void *thread_execute(void *arg) +{ + struct context *ctx = arg; + size_t i, j; + + for (i = 0; ctx->mutator_count; i++) { + j = (i & PAGE_IDX_MASK) | PAGE_IDX_X_MASK; + ((void(*)(void))&ctx->ptr[j * ctx->pagesize])(); + } + + return NULL; +} + +static void *thread_mutate(void *arg) +{ + size_t i, start_idx, end_idx, page_idx, tmp; + struct context *ctx = arg; + unsigned int seed; + int prot, ret; + + seed = (unsigned int)time(NULL); + for (i = 0; i < 10000; i++) { + start_idx = rand_r(&seed) & PAGE_IDX_MASK; + end_idx = rand_r(&seed) & PAGE_IDX_MASK; + if (start_idx > end_idx) { + tmp = start_idx; + start_idx = end_idx; + end_idx = tmp; + } + prot = rand_r(&seed) & (PROT_READ | PROT_WRITE | PROT_EXEC); + for (page_idx = start_idx & REGION_MASK; page_idx <= end_idx; + page_idx += PAGES_PER_REGION) { + if (page_idx & PAGE_IDX_R_MASK) { + prot |= PROT_READ; + } + if (page_idx & PAGE_IDX_W_MASK) { + /* FIXME: qemu syscalls check for both read+write. */ + prot |= PROT_WRITE | PROT_READ; + } + if (page_idx & PAGE_IDX_X_MASK) { + prot |= PROT_EXEC; + } + } + ret = mprotect(&ctx->ptr[start_idx * ctx->pagesize], + (end_idx - start_idx + 1) * ctx->pagesize, prot); + assert(ret == 0); + } + + __atomic_fetch_sub(&ctx->mutator_count, 1, __ATOMIC_SEQ_CST); + + return NULL; +} + +int main(void) +{ + pthread_t threads[5]; + struct context ctx; + size_t i; + int ret; + + /* Without a template, nothing to test. */ + if (sizeof(nop_func) == 0) { + return EXIT_SUCCESS; + } + + /* Initialize memory chunk. */ + ctx.pagesize = getpagesize(); + ctx.ptr = mmap(NULL, PAGE_COUNT * ctx.pagesize, + PROT_READ | PROT_WRITE | PROT_EXEC, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + assert(ctx.ptr != MAP_FAILED); + for (i = 0; i < PAGE_COUNT; i++) { + memcpy(&ctx.ptr[i * ctx.pagesize], nop_func, sizeof(nop_func)); + } + ctx.dev_null_fd = open("/dev/null", O_WRONLY); + assert(ctx.dev_null_fd >= 0); + ctx.mutator_count = 2; + + /* Start threads. */ + ret = pthread_create(&threads[0], NULL, thread_read, &ctx); + assert(ret == 0); + ret = pthread_create(&threads[1], NULL, thread_write, &ctx); + assert(ret == 0); + ret = pthread_create(&threads[2], NULL, thread_execute, &ctx); + assert(ret == 0); + for (i = 3; i <= 4; i++) { + ret = pthread_create(&threads[i], NULL, thread_mutate, &ctx); + assert(ret == 0); + } + + /* Wait for threads to stop. */ + for (i = 0; i < sizeof(threads) / sizeof(threads[0]); i++) { + ret = pthread_join(threads[i], NULL); + assert(ret == 0); + } + + /* Destroy memory chunk. */ + ret = close(ctx.dev_null_fd); + assert(ret == 0); + ret = munmap(ctx.ptr, PAGE_COUNT * ctx.pagesize); + assert(ret == 0); + + return EXIT_SUCCESS; +}