AArch64InstrInfo.cpp File Reference (original) (raw)

Go to the source code of this file.

Macros
#define	GET_INSTRINFO_CTOR_DTOR
#define	GET_INSTRINFO_HELPERS
#define	GET_INSTRMAP_INFO

Enumerations
enum	AccessKind { AK_Write = 0x01 , AK_Read = 0x10 , AK_All = 0x11 }
enum class	FMAInstKind { Default, Indexed, Accumulator }
enum	MachineOutlinerClass { MachineOutlinerDefault, MachineOutlinerTailCall, MachineOutlinerNoLRSave, MachineOutlinerThunk, MachineOutlinerRegSave }
	Constants defining how certain sequences should be outlined. More...
enum	MachineOutlinerMBBFlags { LRUnavailableSomewhere = 0x2 , HasCalls = 0x4 , UnsafeRegsDead = 0x8 }

Functions
static void	parseCondBranch (MachineInstr LastInst, MachineBasicBlock &Target, SmallVectorImpl< MachineOperand > &Cond)
static unsigned	getBranchDisplacementBits (unsigned Opc)
static unsigned	removeCopies (const MachineRegisterInfo &MRI, unsigned VReg)
static unsigned	canFoldIntoCSel (const MachineRegisterInfo &MRI, unsigned VReg, unsigned *NewReg=nullptr)
static bool	isCheapImmediate (const MachineInstr &MI, unsigned BitSize)
static bool	isCheapCopy (const MachineInstr &MI, const AArch64RegisterInfo &RI)
static bool	UpdateOperandRegClass (MachineInstr &Instr)
static unsigned	convertToNonFlagSettingOpc (const MachineInstr &MI)
	Return the opcode that does not set flags when possible - otherwise return the original opcode.
static bool	areCFlagsAccessedBetweenInstrs (MachineBasicBlock::iterator From, MachineBasicBlock::iterator To, const TargetRegisterInfo *TRI, const AccessKind AccessToCheck=AK_All)
	True when condition flags are accessed (either by writing or reading) on the instruction trace starting at From and ending at To.
static unsigned	sForm (MachineInstr &Instr)
	Get opcode of S version of Instr.
static bool	areCFlagsAliveInSuccessors (const MachineBasicBlock *MBB)
	Check if AArch64::NZCV should be alive in successors of MBB.
static int	findCondCodeUseOperandIdxForBranchOrSelect (const MachineInstr &Instr)
static AArch64CC::CondCode	findCondCodeUsedByInstr (const MachineInstr &Instr)
	Find a condition code used by the instruction.
static UsedNZCV	getUsedNZCV (AArch64CC::CondCode CC)
static bool	isADDSRegImm (unsigned Opcode)
static bool	isSUBSRegImm (unsigned Opcode)
static bool	isANDOpcode (MachineInstr &MI)
static bool	canInstrSubstituteCmpInstr (MachineInstr &MI, MachineInstr &CmpInstr, const TargetRegisterInfo &TRI)
	Check if CmpInstr can be substituted by MI.
static bool	canCmpInstrBeRemoved (MachineInstr &MI, MachineInstr &CmpInstr, int CmpValue, const TargetRegisterInfo &TRI, SmallVectorImpl< MachineInstr * > &CCUseInstrs, bool &IsInvertCC)
static bool	isFrameLoadOpcode (int Opcode)
static bool	isFrameStoreOpcode (int Opcode)
static unsigned	regOffsetOpcode (unsigned Opcode)
unsigned	scaledOffsetOpcode (unsigned Opcode, unsigned &Scale)
unsigned	unscaledOffsetOpcode (unsigned Opcode)
static unsigned	offsetExtendOpcode (unsigned Opcode)
static bool	isPostIndexLdStOpcode (unsigned Opcode)
	Return true if the opcode is a post-index ld/st instruction, which really loads from base+0.
static const TargetRegisterClass *	getRegClass (const MachineInstr &MI, Register Reg)
static bool	scaleOffset (unsigned Opc, int64_t &Offset)
static bool	canPairLdStOpc (unsigned FirstOpc, unsigned SecondOpc)
static bool	shouldClusterFI (const MachineFrameInfo &MFI, int FI1, int64_t Offset1, unsigned Opcode1, int FI2, int64_t Offset2, unsigned Opcode2)
static const MachineInstrBuilder &	AddSubReg (const MachineInstrBuilder &MIB, MCRegister Reg, unsigned SubIdx, unsigned State, const TargetRegisterInfo *TRI)
static bool	forwardCopyWillClobberTuple (unsigned DestReg, unsigned SrcReg, unsigned NumRegs)
static void	storeRegPairToStackSlot (const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register SrcReg, bool IsKill, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
static void	loadRegPairFromStackSlot (const TargetRegisterInfo &TRI, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertBefore, const MCInstrDesc &MCID, Register DestReg, unsigned SubIdx0, unsigned SubIdx1, int FI, MachineMemOperand *MMO)
static void	appendConstantExpr (SmallVectorImpl< char > &Expr, int64_t Constant, dwarf::LocationAtom Operation)
static void	appendReadRegExpr (SmallVectorImpl< char > &Expr, unsigned RegNum)
static void	appendLoadRegExpr (SmallVectorImpl< char > &Expr, int64_t OffsetFromDefCFA)
static void	appendOffsetComment (int NumBytes, llvm::raw_string_ostream &Comment, StringRef RegScale={})
static MCCFIInstruction	createDefCFAExpression (const TargetRegisterInfo &TRI, unsigned Reg, const StackOffset &Offset)
static void	emitFrameOffsetAdj (MachineBasicBlock &MBB, MachineBasicBlock::iterator MBBI, const DebugLoc &DL, unsigned DestReg, unsigned SrcReg, int64_t Offset, unsigned Opc, const TargetInstrInfo TII, MachineInstr::MIFlag Flag, bool NeedsWinCFI, bool HasWinCFI, bool EmitCFAOffset, StackOffset CFAOffset, unsigned FrameReg)
static bool	isCombineInstrSettingFlag (unsigned Opc)
static bool	isCombineInstrCandidate32 (unsigned Opc)
static bool	isCombineInstrCandidate64 (unsigned Opc)
static bool	isCombineInstrCandidateFP (const MachineInstr &Inst)
static bool	isCombineInstrCandidate (unsigned Opc)
static bool	canCombine (MachineBasicBlock &MBB, MachineOperand &MO, unsigned CombineOpc, unsigned ZeroReg=0, bool CheckZeroReg=false)
static bool	canCombineWithMUL (MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc, unsigned ZeroReg)
static bool	canCombineWithFMUL (MachineBasicBlock &MBB, MachineOperand &MO, unsigned MulOpc)
static bool	getMaddPatterns (MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
	Find instructions that can be turned into madd.
static bool	getFMAPatterns (MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
	Floating-Point Support.
static bool	getFMULPatterns (MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static bool	getFNEGPatterns (MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
static bool	getMiscPatterns (MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
	Find other MI combine patterns.
static bool	getGatherLanePattern (MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns, unsigned LoadLaneOpCode, unsigned NumLanes)
	Check if the given instruction forms a gather load pattern that can be optimized for better Memory-Level Parallelism (MLP).
static bool	getLoadPatterns (MachineInstr &Root, SmallVectorImpl< unsigned > &Patterns)
	Search for patterns of LD instructions we can optimize.
static void	generateGatherLanePattern (MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned Pattern, unsigned NumLanes)
	Generate optimized instruction sequence for gather load patterns to improve Memory-Level Parallelism (MLP).
static MachineInstr *	genFusedMultiply (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo TII, MachineInstr &Root, SmallVectorImpl< MachineInstr > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass RC, FMAInstKind kind=FMAInstKind::Default, const Register ReplacedAddend=nullptr)
	genFusedMultiply - Generate fused multiply instructions.
static MachineInstr *	genFNegatedMAD (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo TII, MachineInstr &Root, SmallVectorImpl< MachineInstr > &InsInstrs)
static MachineInstr *	genIndexedMultiply (MachineInstr &Root, SmallVectorImpl< MachineInstr * > &InsInstrs, unsigned IdxDupOp, unsigned MulOpc, const TargetRegisterClass *RC, MachineRegisterInfo &MRI)
	Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)
static MachineInstr *	genFusedMultiplyAcc (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo TII, MachineInstr &Root, SmallVectorImpl< MachineInstr > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
	genFusedMultiplyAcc - Helper to generate fused multiply accumulate instructions.
static Register	genNeg (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo TII, MachineInstr &Root, SmallVectorImpl< MachineInstr > &InsInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned MnegOpc, const TargetRegisterClass *RC)
	genNeg - Helper to generate an intermediate negation of the second operand of Root
static MachineInstr *	genFusedMultiplyAccNeg (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo TII, MachineInstr &Root, SmallVectorImpl< MachineInstr > &InsInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
	genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional negation of the accumulator
static MachineInstr *	genFusedMultiplyIdx (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo TII, MachineInstr &Root, SmallVectorImpl< MachineInstr > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, const TargetRegisterClass *RC)
	genFusedMultiplyIdx - Helper to generate fused multiply accumulate instructions.
static MachineInstr *	genFusedMultiplyIdxNeg (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo TII, MachineInstr &Root, SmallVectorImpl< MachineInstr > &InsInstrs, DenseMap< Register, unsigned > &InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass *RC)
	genFusedMultiplyAccNeg - Helper to generate fused multiply accumulate instructions with an additional negation of the accumulator
static MachineInstr *	genMaddR (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo TII, MachineInstr &Root, SmallVectorImpl< MachineInstr > &InsInstrs, unsigned IdxMulOpd, unsigned MaddOpc, unsigned VR, const TargetRegisterClass *RC)
	genMaddR - Generate madd instruction and combine mul and add using an extra virtual register Example - an ADD intermediate needs to be stored in a register: MUL I=A,B,0 ADD R,I,Imm ==> ORR V, ZR, Imm ==> MADD R,A,B,V
static void	genSubAdd2SubSub (MachineFunction &MF, MachineRegisterInfo &MRI, const TargetInstrInfo TII, MachineInstr &Root, SmallVectorImpl< MachineInstr > &InsInstrs, SmallVectorImpl< MachineInstr * > &DelInstrs, unsigned IdxOpd1, DenseMap< Register, unsigned > &InstrIdxForVirtReg)
	Do the following transformation A - (B + C) ==> (A - B) - C A - (B + C) ==> (A - C) - B.
static bool	outliningCandidatesSigningScopeConsensus (const outliner::Candidate &a, const outliner::Candidate &b)
static bool	outliningCandidatesSigningKeyConsensus (const outliner::Candidate &a, const outliner::Candidate &b)
static bool	outliningCandidatesV8_3OpsConsensus (const outliner::Candidate &a, const outliner::Candidate &b)
static void	signOutlinedFunction (MachineFunction &MF, MachineBasicBlock &MBB, const AArch64InstrInfo *TII, bool ShouldSignReturnAddr)
static std::optional< ParamLoadedValue >	describeORRLoadedValue (const MachineInstr &MI, Register DescribedReg, const TargetInstrInfo TII, const TargetRegisterInfo TRI)
	If the given ORR instruction is a copy, and DescribedReg overlaps with the destination register then, if possible, describe the value in terms of the source register.
static Register	cloneInstr (const MachineInstr *MI, unsigned ReplaceOprNum, Register ReplaceReg, MachineBasicBlock &MBB, MachineBasicBlock::iterator InsertTo)
	Clone an instruction from MI.
static void	extractPhiReg (const MachineInstr &Phi, const MachineBasicBlock *MBB, Register &RegMBB, Register &RegOther)
static bool	isDefinedOutside (Register Reg, const MachineBasicBlock *BB)
static bool	getIndVarInfo (Register Reg, const MachineBasicBlock LoopBB, MachineInstr &UpdateInst, unsigned &UpdateCounterOprNum, Register &InitReg, bool &IsUpdatePriorComp)
	If Reg is an induction variable, return true and set some parameters.

Variables
static cl::opt< unsigned >	CBDisplacementBits ("aarch64-cb-offset-bits", cl::Hidden, cl::init(9), cl::desc("Restrict range of CB instructions (DEBUG)"))
static cl::opt< unsigned >	TBZDisplacementBits ("aarch64-tbz-offset-bits", cl::Hidden, cl::init(14), cl::desc("Restrict range of TB[N]Z instructions (DEBUG)"))
static cl::opt< unsigned >	CBZDisplacementBits ("aarch64-cbz-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of CB[N]Z instructions (DEBUG)"))
static cl::opt< unsigned >	BCCDisplacementBits ("aarch64-bcc-offset-bits", cl::Hidden, cl::init(19), cl::desc("Restrict range of Bcc instructions (DEBUG)"))
static cl::opt< unsigned >	BDisplacementBits ("aarch64-b-offset-bits", cl::Hidden, cl::init(26), cl::desc("Restrict range of B instructions (DEBUG)"))
static cl::opt< unsigned >	GatherOptSearchLimit ("aarch64-search-limit", cl::Hidden, cl::init(2048), cl::desc("Restrict range of instructions to search for the " "machine-combiner gather pattern optimization"))

◆ GET_INSTRINFO_CTOR_DTOR

#define GET_INSTRINFO_CTOR_DTOR

◆ GET_INSTRINFO_HELPERS

#define GET_INSTRINFO_HELPERS

◆ GET_INSTRMAP_INFO

#define GET_INSTRMAP_INFO

◆ AccessKind

◆ FMAInstKind

◆ MachineOutlinerClass

Constants defining how certain sequences should be outlined.

This encompasses how an outlined function should be called, and what kind of frame should be emitted for that outlined function.

MachineOutlinerDefault implies that the function should be called with a save and restore of LR to the stack.

That is,

I1 Save LR OUTLINED_FUNCTION: I2 --> BL OUTLINED_FUNCTION I1 I3 Restore LR I2 I3 RET

Call construction overhead: 3 (save + BL + restore)
Frame construction overhead: 1 (ret)
Requires stack fixups? Yes

MachineOutlinerTailCall implies that the function is being created from a sequence of instructions ending in a return.

That is,

I1 OUTLINED_FUNCTION: I2 --> B OUTLINED_FUNCTION I1 RET I2 RET

Call construction overhead: 1 (B)
Frame construction overhead: 0 (Return included in sequence)
Requires stack fixups? No

MachineOutlinerNoLRSave implies that the function should be called using a BL instruction, but doesn't require LR to be saved and restored. This happens when LR is known to be dead.

That is,

I1 OUTLINED_FUNCTION: I2 --> BL OUTLINED_FUNCTION I1 I3 I2 I3 RET

Call construction overhead: 1 (BL)
Frame construction overhead: 1 (RET)
Requires stack fixups? No

MachineOutlinerThunk implies that the function is being created from a sequence of instructions ending in a call. The outlined function is called with a BL instruction, and the outlined function tail-calls the original call destination.

That is,

I1 OUTLINED_FUNCTION: I2 --> BL OUTLINED_FUNCTION I1 BL f I2 B f

Call construction overhead: 1 (BL)
Frame construction overhead: 0
Requires stack fixups? No

MachineOutlinerRegSave implies that the function should be called with a save and restore of LR to an available register. This allows us to avoid stack fixups. Note that this outlining variant is compatible with the NoLRSave case.

That is,

I1 Save LR OUTLINED_FUNCTION: I2 --> BL OUTLINED_FUNCTION I1 I3 Restore LR I2 I3 RET

Call construction overhead: 3 (save + BL + restore)
Frame construction overhead: 1 (ret)
Requires stack fixups? No

Enumerator
MachineOutlinerDefault
MachineOutlinerTailCall	Emit a save, restore, call, and return.
MachineOutlinerNoLRSave	Only emit a branch.
MachineOutlinerThunk	Emit a call and return.
MachineOutlinerRegSave	Emit a call and tail-call. Same as default, but save to a register.

Definition at line 9759 of file AArch64InstrInfo.cpp.

◆ MachineOutlinerMBBFlags

◆ AddSubReg()

◆ appendConstantExpr()

◆ appendLoadRegExpr()

◆ appendOffsetComment()

◆ appendReadRegExpr()

◆ areCFlagsAccessedBetweenInstrs()

True when condition flags are accessed (either by writing or reading) on the instruction trace starting at From and ending at To.

Note: If From and To are from different blocks it's assumed CC are accessed on the path.

Definition at line 1617 of file AArch64InstrInfo.cpp.

References AK_All, AK_Read, AK_Write, assert(), llvm::MachineInstrBundleIterator< Ty, IsReverse >::getReverse(), llvm::instructionsWithoutDebug(), MI, and TRI.

Referenced by canCmpInstrBeRemoved(), canInstrSubstituteCmpInstr(), and llvm::AArch64InstrInfo::optimizeCondBranch().

◆ areCFlagsAliveInSuccessors()

◆ canCmpInstrBeRemoved()

Returns

True if CmpInstr can be removed.

IsInvertCC is true if, after removing CmpInstr, condition codes used in CCUseInstrs must be inverted.

Definition at line 2215 of file AArch64InstrInfo.cpp.

References AK_Write, areCFlagsAccessedBetweenInstrs(), assert(), llvm::UsedNZCV::C, llvm::AArch64CC::EQ, llvm::examineCFlagsUse(), findCondCodeUsedByInstr(), llvm::MachineInstr::getOpcode(), getUsedNZCV(), llvm::AArch64CC::Invalid, isADDSRegImm(), isSUBSRegImm(), MI, llvm::UsedNZCV::N, llvm::AArch64CC::NE, llvm::AArch64CC::PL, TRI, llvm::UsedNZCV::V, and llvm::UsedNZCV::Z.

◆ canCombine()

◆ canCombineWithFMUL()

◆ canCombineWithMUL()

◆ canFoldIntoCSel()

◆ canInstrSubstituteCmpInstr()

Check if CmpInstr can be substituted by MI.

CmpInstr can be substituted:

CmpInstr is either 'ADDS vreg, 0' or 'SUBS vreg, 0'
and, MI and CmpInstr are from the same MachineBB
and, condition flags are not alive in successors of the CmpInstr parent
and, if MI opcode is the S form there must be no defs of flags between MI and CmpInstr or if MI opcode is not the S form there must be neither defs of flags nor uses of flags between MI and CmpInstr.
and, if C/V flags are not used after CmpInstr or if N flag is used but MI produces poison value if signed overflow occurs.

Definition at line 2146 of file AArch64InstrInfo.cpp.

References AK_All, AK_Write, areCFlagsAccessedBetweenInstrs(), assert(), llvm::examineCFlagsUse(), llvm::MachineOperand::getImm(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), isADDSRegImm(), isANDOpcode(), llvm::MachineOperand::isImm(), isSUBSRegImm(), MI, llvm::MachineInstr::NoSWrap, sForm(), and TRI.

◆ canPairLdStOpc()

◆ cloneInstr()

Clone an instruction from MI.

The register of ReplaceOprNum-th operand is replaced by ReplaceReg. The output register is newly created. The other operands are unchanged from MI.

Definition at line 11257 of file AArch64InstrInfo.cpp.

References llvm::MachineInstr::getDesc(), llvm::MachineInstr::getNumOperands(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), I, llvm::Register::isVirtual(), MBB, MI, MRI, llvm::MachineOperand::setReg(), and TII.

◆ convertToNonFlagSettingOpc()

◆ createDefCFAExpression()

Definition at line 6295 of file AArch64InstrInfo.cpp.

References llvm::SmallString< InternalLen >::append(), appendConstantExpr(), llvm::appendLEB128(), appendOffsetComment(), appendReadRegExpr(), assert(), llvm::MCCFIInstruction::createEscape(), llvm::Offset, llvm::printReg(), llvm::SmallVectorTemplateBase< T, bool >::push_back(), Reg, llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::SmallString< InternalLen >::str(), and TRI.

Referenced by llvm::createDefCFA(), and llvm::RISCVFrameLowering::emitPrologue().

◆ describeORRLoadedValue()

◆ emitFrameOffsetAdj()

void emitFrameOffsetAdj ( MachineBasicBlock & MBB, MachineBasicBlock::iterator MBBI, const DebugLoc & DL, unsigned DestReg, unsigned SrcReg, int64_t Offset, unsigned Opc, const TargetInstrInfo * TII, MachineInstr::MIFlag Flag, bool NeedsWinCFI, bool * HasWinCFI, bool EmitCFAOffset, StackOffset CFAOffset, unsigned FrameReg )	static

Definition at line 6403 of file AArch64InstrInfo.cpp.

References llvm::MachineInstrBuilder::addCFIIndex(), llvm::MachineFunction::addFrameInst(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), llvm::createDefCFA(), DL, llvm::StackOffset::getFixed(), llvm::TargetSubtargetInfo::getRegisterInfo(), llvm::StackOffset::getScalable(), llvm::AArch64_AM::getShifterImm(), llvm::MachineFunction::getSubtarget(), llvm_unreachable, llvm::AArch64_AM::LSL, MBB, MBBI, llvm::Offset, Opc, llvm::MachineInstrBuilder::setMIFlag(), llvm::MachineInstrBuilder::setMIFlags(), TII, and TRI.

Referenced by llvm::emitFrameOffset().

◆ extractPhiReg()

◆ findCondCodeUsedByInstr()

◆ findCondCodeUseOperandIdxForBranchOrSelect()

◆ forwardCopyWillClobberTuple()

◆ generateGatherLanePattern()

Generate optimized instruction sequence for gather load patterns to improve Memory-Level Parallelism (MLP).

This function transforms a chain of sequential NEON lane loads into parallel vector loads that can execute concurrently.

Definition at line 7963 of file AArch64InstrInfo.cpp.

References A(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), B(), llvm::BuildMI(), llvm::enumerate(), llvm::TargetSubtargetInfo::getInstrInfo(), llvm::getKillRegState(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::MachineBasicBlock::getParent(), llvm::MachineInstr::getParent(), llvm::MachineOperand::getReg(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::insert(), llvm::MachineOperand::isKill(), llvm_unreachable, llvm::make_range(), MRI, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::reverse(), llvm::SmallVectorTemplateCommon< T, typename >::size(), llvm::sort(), and TII.

◆ genFNegatedMAD()

Definition at line 8254 of file AArch64InstrInfo.cpp.

References llvm::MachineInstrBuilder::addReg(), llvm::BuildMI(), llvm::getKillRegState(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), llvm::MachineOperand::isKill(), llvm::Register::isVirtual(), MRI, Opc, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and TII.

◆ genFusedMultiply()

genFusedMultiply - Generate fused multiply instructions.

This function supports both integer and floating point instructions. A typical example: F|MUL I=A,B,0 F|ADD R,I,C ==> F|MADD R,A,B,C

Parameters

MF	Containing MachineFunction
MRI	Register information
TII	Target information
Root	is the F\|ADD instruction
[out]	InsInstrs	is a vector of machine instructions and will contain the generated madd instruction
IdxMulOpd	is index of operand in Root that is the result of the F\|MUL. In the example above IdxMulOpd is 1.
MaddOpc	the opcode fo the f\|madd instruction
RC	Register class of operands
kind	of fma instruction (addressing mode) to be generated
ReplacedAddend	is the result register from the instruction replacing the non-combined operand, if any.

Definition at line 8193 of file AArch64InstrInfo.cpp.

References Accumulator, llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), Default, llvm::getKillRegState(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), Indexed, llvm::MachineOperand::isKill(), llvm::Register::isVirtual(), MRI, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and TII.

Referenced by genFusedMultiplyAcc(), genFusedMultiplyAccNeg(), genFusedMultiplyIdx(), and genFusedMultiplyIdxNeg().

◆ genFusedMultiplyAcc()

◆ genFusedMultiplyAccNeg()

MachineInstr * genFusedMultiplyAccNeg ( MachineFunction & MF, MachineRegisterInfo & MRI, const TargetInstrInfo * TII, MachineInstr & Root, SmallVectorImpl< MachineInstr * > & InsInstrs, DenseMap< Register, unsigned > & InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass * RC )	static

◆ genFusedMultiplyIdx()

◆ genFusedMultiplyIdxNeg()

MachineInstr * genFusedMultiplyIdxNeg ( MachineFunction & MF, MachineRegisterInfo & MRI, const TargetInstrInfo * TII, MachineInstr & Root, SmallVectorImpl< MachineInstr * > & InsInstrs, DenseMap< Register, unsigned > & InstrIdxForVirtReg, unsigned IdxMulOpd, unsigned MaddOpc, unsigned MnegOpc, const TargetRegisterClass * RC )	static

◆ genIndexedMultiply()

Fold (FMUL x (DUP y lane)) into (FMUL_indexed x y lane)

Definition at line 8296 of file AArch64InstrInfo.cpp.

References llvm::MachineInstrBuilder::add(), llvm::MachineInstrBuilder::addImm(), llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), llvm::MachineOperand::getImm(), llvm::TargetSubtargetInfo::getInstrInfo(), llvm::MachineInstr::getMF(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), llvm::MachineFunction::getRegInfo(), llvm::MachineFunction::getSubtarget(), llvm::MachineRegisterInfo::getUniqueVRegDef(), MRI, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and TII.

◆ genMaddR()

genMaddR - Generate madd instruction and combine mul and add using an extra virtual register Example - an ADD intermediate needs to be stored in a register: MUL I=A,B,0 ADD R,I,Imm ==> ORR V, ZR, Imm ==> MADD R,A,B,V

Parameters

MF	Containing MachineFunction
MRI	Register information
TII	Target information
Root	is the ADD instruction
[out]	InsInstrs	is a vector of machine instructions and will contain the generated madd instruction
IdxMulOpd	is index of operand in Root that is the result of the MUL. In the example above IdxMulOpd is 1.
MaddOpc	the opcode fo the madd instruction
VR	is a virtual register that holds the value of an ADD operand (V in the example above).
RC	Register class of operands

Definition at line 8426 of file AArch64InstrInfo.cpp.

References llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), llvm::getKillRegState(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), llvm::Register::isVirtual(), llvm::Register::isVirtualRegister(), MRI, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and TII.

◆ genNeg()

genNeg - Helper to generate an intermediate negation of the second operand of Root

Definition at line 8347 of file AArch64InstrInfo.cpp.

References llvm::MachineInstrBuilder::add(), assert(), llvm::BuildMI(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::empty(), llvm::MachineInstr::getOperand(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::insert(), MRI, llvm::SmallVectorTemplateBase< T, bool >::push_back(), and TII.

Referenced by genFusedMultiplyAccNeg(), and genFusedMultiplyIdxNeg().

◆ genSubAdd2SubSub()

Do the following transformation A - (B + C) ==> (A - B) - C A - (B + C) ==> (A - C) - B.

Definition at line 8462 of file AArch64InstrInfo.cpp.

References llvm::MachineInstrBuilder::addReg(), assert(), llvm::BuildMI(), llvm::getKillRegState(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::MachineOperand::getReg(), llvm::DenseMapBase< DerivedT, KeyT, ValueT, KeyInfoT, BucketT >::insert(), llvm::MachineOperand::isKill(), llvm::MachineInstr::mergeFlagsWith(), MRI, llvm::SmallVectorTemplateBase< T, bool >::push_back(), llvm::MachineInstrBuilder::setMIFlags(), and TII.

◆ getBranchDisplacementBits()

◆ getFMAPatterns()

◆ getFMULPatterns()

Definition at line 7585 of file AArch64InstrInfo.cpp.

References llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::MachineInstr::getParent(), llvm::MachineOperand::getReg(), llvm::MachineOperand::isReg(), llvm::Register::isVirtual(), MBB, MI, MRI, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

◆ getFNEGPatterns()

Definition at line 7637 of file AArch64InstrInfo.cpp.

References llvm::MachineInstr::FmContract, llvm::MachineInstr::FmNsz, llvm::FNMADD, llvm::MachineInstr::getFlag(), llvm::MachineInstr::getOpcode(), llvm::MachineInstr::getOperand(), llvm::MachineInstr::getParent(), llvm::MachineOperand::getReg(), MBB, MI, MRI, Opc, and llvm::SmallVectorTemplateBase< T, bool >::push_back().

◆ getGatherLanePattern()

Check if the given instruction forms a gather load pattern that can be optimized for better Memory-Level Parallelism (MLP).

This function identifies chains of NEON lane load instructions that load data from different memory addresses into individual lanes of a 128-bit vector register, then attempts to split the pattern into parallel loads to break the serial dependency between instructions.

Pattern Matched: Initial scalar load -> SUBREG_TO_REG (lane 0) -> LD1i* (lane 1) -> LD1i* (lane 2) -> ... -> LD1i* (lane N-1, Root)

Transformed Into: Two parallel vector loads using fewer lanes each, followed by ZIP1v2i64 to combine the results, enabling better memory-level parallelism.

Supported Element Types: