//===-- ARMHazardRecognizer.cpp - ARM postra hazard recognizer ------------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// #include "ARMHazardRecognizer.h" #include "ARMBaseInstrInfo.h" #include "ARMBaseRegisterInfo.h" #include "ARMSubtarget.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/CodeGen/MachineInstr.h" #include "llvm/CodeGen/ScheduleDAG.h" #include "llvm/CodeGen/TargetRegisterInfo.h" #include "llvm/Support/CommandLine.h" using namespace llvm; static cl::opt DataBankMask("arm-data-bank-mask", cl::init(-1), cl::Hidden); static cl::opt AssumeITCMConflict("arm-assume-itcm-bankconflict", cl::init(false), cl::Hidden); static bool hasRAWHazard(MachineInstr *DefMI, MachineInstr *MI, const TargetRegisterInfo &TRI) { // FIXME: Detect integer instructions properly. const MCInstrDesc &MCID = MI->getDesc(); unsigned Domain = MCID.TSFlags & ARMII::DomainMask; if (MI->mayStore()) return false; unsigned Opcode = MCID.getOpcode(); if (Opcode == ARM::VMOVRS || Opcode == ARM::VMOVRRD) return false; if ((Domain & ARMII::DomainVFP) || (Domain & ARMII::DomainNEON)) return MI->readsRegister(DefMI->getOperand(0).getReg(), &TRI); return false; } ScheduleHazardRecognizer::HazardType ARMHazardRecognizerFPMLx::getHazardType(SUnit *SU, int Stalls) { assert(Stalls == 0 && "ARM hazards don't support scoreboard lookahead"); MachineInstr *MI = SU->getInstr(); if (!MI->isDebugInstr()) { // Look for special VMLA / VMLS hazards. A VMUL / VADD / VSUB following // a VMLA / VMLS will cause 4 cycle stall. const MCInstrDesc &MCID = MI->getDesc(); if (LastMI && (MCID.TSFlags & ARMII::DomainMask) != ARMII::DomainGeneral) { MachineInstr *DefMI = LastMI; const MCInstrDesc &LastMCID = LastMI->getDesc(); const MachineFunction *MF = MI->getParent()->getParent(); const ARMBaseInstrInfo &TII = *static_cast( MF->getSubtarget().getInstrInfo()); // Skip over one non-VFP / NEON instruction. if (!LastMI->isBarrier() && !(TII.getSubtarget().hasMuxedUnits() && LastMI->mayLoadOrStore()) && (LastMCID.TSFlags & ARMII::DomainMask) == ARMII::DomainGeneral) { MachineBasicBlock::iterator I = LastMI; if (I != LastMI->getParent()->begin()) { I = std::prev(I); DefMI = &*I; } } if (TII.isFpMLxInstruction(DefMI->getOpcode()) && (TII.canCauseFpMLxStall(MI->getOpcode()) || hasRAWHazard(DefMI, MI, TII.getRegisterInfo()))) { // Try to schedule another instruction for the next 4 cycles. if (FpMLxStalls == 0) FpMLxStalls = 4; return Hazard; } } } return NoHazard; } void ARMHazardRecognizerFPMLx::Reset() { LastMI = nullptr; FpMLxStalls = 0; } void ARMHazardRecognizerFPMLx::EmitInstruction(SUnit *SU) { MachineInstr *MI = SU->getInstr(); if (!MI->isDebugInstr()) { LastMI = MI; FpMLxStalls = 0; } } void ARMHazardRecognizerFPMLx::AdvanceCycle() { if (FpMLxStalls && --FpMLxStalls == 0) // Stalled for 4 cycles but still can't schedule any other instructions. LastMI = nullptr; } void ARMHazardRecognizerFPMLx::RecedeCycle() { llvm_unreachable("reverse ARM hazard checking unsupported"); } ///////// Bank conflicts handled as hazards ////////////// static bool getBaseOffset(const MachineInstr &MI, const MachineOperand *&BaseOp, int64_t &Offset) { uint64_t TSFlags = MI.getDesc().TSFlags; unsigned AddrMode = (TSFlags & ARMII::AddrModeMask); unsigned IndexMode = (TSFlags & ARMII::IndexModeMask) >> ARMII::IndexModeShift; // Address mode tells us what we want to know about operands for T2 // instructions (but not size). It tells us size (but not about operands) // for T1 instructions. switch (AddrMode) { default: return false; case ARMII::AddrModeT2_i8: // t2LDRBT, t2LDRB_POST, t2LDRB_PRE, t2LDRBi8, // t2LDRHT, t2LDRH_POST, t2LDRH_PRE, t2LDRHi8, // t2LDRSBT, t2LDRSB_POST, t2LDRSB_PRE, t2LDRSBi8, // t2LDRSHT, t2LDRSH_POST, t2LDRSH_PRE, t2LDRSHi8, // t2LDRT, t2LDR_POST, t2LDR_PRE, t2LDRi8 BaseOp = &MI.getOperand(1); Offset = (IndexMode == ARMII::IndexModePost) ? 0 : (IndexMode == ARMII::IndexModePre || IndexMode == ARMII::IndexModeUpd) ? MI.getOperand(3).getImm() : MI.getOperand(2).getImm(); return true; case ARMII::AddrModeT2_i12: // t2LDRBi12, t2LDRHi12 // t2LDRSBi12, t2LDRSHi12 // t2LDRi12 BaseOp = &MI.getOperand(1); Offset = MI.getOperand(2).getImm(); return true; case ARMII::AddrModeT2_i8s4: // t2LDRD_POST, t2LDRD_PRE, t2LDRDi8 BaseOp = &MI.getOperand(2); Offset = (IndexMode == ARMII::IndexModePost) ? 0 : (IndexMode == ARMII::IndexModePre || IndexMode == ARMII::IndexModeUpd) ? MI.getOperand(4).getImm() : MI.getOperand(3).getImm(); return true; case ARMII::AddrModeT1_1: // tLDRBi, tLDRBr (watch out!), TLDRSB case ARMII::AddrModeT1_2: // tLDRHi, tLDRHr (watch out!), TLDRSH case ARMII::AddrModeT1_4: // tLDRi, tLDRr (watch out!) BaseOp = &MI.getOperand(1); Offset = MI.getOperand(2).isImm() ? MI.getOperand(2).getImm() : 0; return MI.getOperand(2).isImm(); } return false; } ARMBankConflictHazardRecognizer::ARMBankConflictHazardRecognizer( const ScheduleDAG *DAG, int64_t CPUBankMask, bool CPUAssumeITCMConflict) : ScheduleHazardRecognizer(), MF(DAG->MF), DL(DAG->MF.getDataLayout()), DataMask(DataBankMask.getNumOccurrences() ? int64_t(DataBankMask) : CPUBankMask), AssumeITCMBankConflict(AssumeITCMConflict.getNumOccurrences() ? AssumeITCMConflict : CPUAssumeITCMConflict) { MaxLookAhead = 1; } ScheduleHazardRecognizer::HazardType ARMBankConflictHazardRecognizer::CheckOffsets(unsigned O0, unsigned O1) { return (((O0 ^ O1) & DataMask) != 0) ? NoHazard : Hazard; } ScheduleHazardRecognizer::HazardType ARMBankConflictHazardRecognizer::getHazardType(SUnit *SU, int Stalls) { MachineInstr &L0 = *SU->getInstr(); if (!L0.mayLoad() || L0.mayStore() || L0.getNumMemOperands() != 1) return NoHazard; auto MO0 = *L0.memoperands().begin(); auto BaseVal0 = MO0->getValue(); auto BasePseudoVal0 = MO0->getPseudoValue(); int64_t Offset0 = 0; if (MO0->getSize() > 4) return NoHazard; bool SPvalid = false; const MachineOperand *SP = nullptr; int64_t SPOffset0 = 0; for (auto L1 : Accesses) { auto MO1 = *L1->memoperands().begin(); auto BaseVal1 = MO1->getValue(); auto BasePseudoVal1 = MO1->getPseudoValue(); int64_t Offset1 = 0; // Pointers to the same object if (BaseVal0 && BaseVal1) { const Value *Ptr0, *Ptr1; Ptr0 = GetPointerBaseWithConstantOffset(BaseVal0, Offset0, DL, true); Ptr1 = GetPointerBaseWithConstantOffset(BaseVal1, Offset1, DL, true); if (Ptr0 == Ptr1 && Ptr0) return CheckOffsets(Offset0, Offset1); } if (BasePseudoVal0 && BasePseudoVal1 && BasePseudoVal0->kind() == BasePseudoVal1->kind() && BasePseudoVal0->kind() == PseudoSourceValue::FixedStack) { // Spills/fills auto FS0 = cast(BasePseudoVal0); auto FS1 = cast(BasePseudoVal1); Offset0 = MF.getFrameInfo().getObjectOffset(FS0->getFrameIndex()); Offset1 = MF.getFrameInfo().getObjectOffset(FS1->getFrameIndex()); return CheckOffsets(Offset0, Offset1); } // Constant pools (likely in ITCM) if (BasePseudoVal0 && BasePseudoVal1 && BasePseudoVal0->kind() == BasePseudoVal1->kind() && BasePseudoVal0->isConstantPool() && AssumeITCMBankConflict) return Hazard; // Is this a stack pointer-relative access? We could in general try to // use "is this the same register and is it unchanged?", but the // memory operand tracking is highly likely to have already found that. // What we're after here is bank conflicts between different objects in // the stack frame. if (!SPvalid) { // set up SP if (!getBaseOffset(L0, SP, SPOffset0) || SP->getReg().id() != ARM::SP) SP = nullptr; SPvalid = true; } if (SP) { int64_t SPOffset1; const MachineOperand *SP1; if (getBaseOffset(*L1, SP1, SPOffset1) && SP1->getReg().id() == ARM::SP) return CheckOffsets(SPOffset0, SPOffset1); } } return NoHazard; } void ARMBankConflictHazardRecognizer::Reset() { Accesses.clear(); } void ARMBankConflictHazardRecognizer::EmitInstruction(SUnit *SU) { MachineInstr &MI = *SU->getInstr(); if (!MI.mayLoad() || MI.mayStore() || MI.getNumMemOperands() != 1) return; auto MO = *MI.memoperands().begin(); uint64_t Size1 = MO->getSize(); if (Size1 > 4) return; Accesses.push_back(&MI); } void ARMBankConflictHazardRecognizer::AdvanceCycle() { Accesses.clear(); } void ARMBankConflictHazardRecognizer::RecedeCycle() { Accesses.clear(); }