AArch64SelectionDAGInfo.cpp Source File (original) (raw)

16#define GET_SDNODE_DESC

17#include "AArch64GenSDNodeInfo.inc"

18#undef GET_SDNODE_DESC

20using namespace llvm;

22#define DEBUG_TYPE "aarch64-selectiondag-info"

26 cl::desc("Enable AArch64 SME memory operations "

27 "to lower to librt functions"),

35 switch (N->getOpcode()) {

36 case AArch64ISD::WrapperLarge:

38 return;

39 }

43#ifndef NDEBUG

45 switch (N->getOpcode()) {

46 case AArch64ISD::SADDWT:

47 case AArch64ISD::SADDWB:

48 case AArch64ISD::UADDWT:

49 case AArch64ISD::UADDWB: {

50 EVT VT = N->getValueType(0);

51 EVT Op0VT = N->getOperand(0).getValueType();

52 EVT Op1VT = N->getOperand(1).getValueType();

55 "Expected integer vectors!");

57 "Expected result and first input to have the same type!");

59 "Expected vectors of equal size!");

61 "Expected result vector and first input vector to have half the "

62 "lanes of the second input vector!");

63 break;

64 }

65 case AArch64ISD::SUNPKLO:

66 case AArch64ISD::SUNPKHI:

67 case AArch64ISD::UUNPKLO:

68 case AArch64ISD::UUNPKHI: {

69 EVT VT = N->getValueType(0);

70 EVT OpVT = N->getOperand(0).getValueType();

72 VT.isInteger() && "Expected integer vectors!");

74 "Expected vectors of equal size!");

76 "Expected result vector with half the lanes of its input!");

77 break;

78 }

79 case AArch64ISD::TRN1:

80 case AArch64ISD::TRN2:

81 case AArch64ISD::UZP1:

82 case AArch64ISD::UZP2:

83 case AArch64ISD::ZIP1:

84 case AArch64ISD::ZIP2: {

85 EVT VT = N->getValueType(0);

86 EVT Op0VT = N->getOperand(0).getValueType();

87 EVT Op1VT = N->getOperand(1).getValueType();

89 "Expected vectors!");

90 assert(VT == Op0VT && VT == Op1VT && "Expected matching vectors!");

91 break;

92 }

93 case AArch64ISD::RSHRNB_I: {

94 EVT VT = N->getValueType(0);

95 EVT Op0VT = N->getOperand(0).getValueType();

97 "Expected integer vector result type!");

99 "Expected first operand to be an integer vector!");

101 "Expected vectors of equal size!");

103 "Expected input vector with half the lanes of its result!");

105 "Expected second operand to be a constant!");

106 break;

107 }

108 }

109#endif

110}

111

116 bool isVolatile,

119

120

123 ConstSize = C->getZExtValue();

124

125 const bool IsSet = Opcode == AArch64::MOPSMemorySetPseudo ||

126 Opcode == AArch64::MOPSMemorySetTaggingPseudo;

127

129

130 auto Vol =

135

136 if (IsSet) {

137

141 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::Other};

145 } else {

147 const EVT ResultTys[] = {MVT::i64, MVT::i64, MVT::i64, MVT::Other};

149

155 }

156}

157

166

167 bool UsesResult = false;

168 RTLIB::Libcall NewLC;

169 switch (LC) {

170 case RTLIB::MEMCPY: {

171 NewLC = RTLIB::SC_MEMCPY;

173 break;

174 }

175 case RTLIB::MEMMOVE: {

176 NewLC = RTLIB::SC_MEMMOVE;

178 break;

179 }

180 case RTLIB::MEMSET: {

181 NewLC = RTLIB::SC_MEMSET;

184 break;

185 }

186 case RTLIB::MEMCHR: {

187 UsesResult = true;

188 NewLC = RTLIB::SC_MEMCHR;

191 break;

192 }

193 default:

195 }

196

200

204 TLI->getLibcallCallingConv(NewLC), RetTy, Symbol, std::move(Args));

205

206 auto [Result, ChainOut] = TLI->LowerCallTo(CLI);

207 return UsesResult ? DAG.getMergeValues({Result, ChainOut}, DL) : ChainOut;

208}

209

212 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,

216

217 if (STI.hasMOPS())

218 return EmitMOPS(AArch64::MOPSMemoryCopyPseudo, DAG, DL, Chain, Dst, Src,

219 Size, Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);

220

222 SMEAttrs Attrs = AFI->getSMEFnAttrs();

225 RTLIB::MEMCPY);

227}

228

231 SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline,

235

236 if (STI.hasMOPS())

237 return EmitMOPS(AArch64::MOPSMemorySetPseudo, DAG, dl, Chain, Dst, Src,

238 Size, Alignment, isVolatile, DstPtrInfo,

240

242 SMEAttrs Attrs = AFI->getSMEFnAttrs();

245 RTLIB::MEMSET);

247}

248

255

256 if (STI.hasMOPS())

257 return EmitMOPS(AArch64::MOPSMemoryMovePseudo, DAG, dl, Chain, Dst, Src,

258 Size, Alignment, isVolatile, DstPtrInfo, SrcPtrInfo);

259

261 SMEAttrs Attrs = AFI->getSMEFnAttrs();

264 RTLIB::MEMMOVE);

266}

267

272 SMEAttrs Attrs = AFI->getSMEFnAttrs();

273 if (LowerToSMERoutines && !Attrs.hasNonStreamingInterfaceAndBody()) {

275 DAG, dl, Chain, Src, Char, Length, RTLIB::MEMCHR);

276 return std::make_pair(Result.getValue(0), Result.getValue(1));

277 }

279}

280

282

286 bool ZeroData) {

288 unsigned ObjSizeScaled = ObjSize / 16;

289

294

295

296 TagSrc = DAG.getRegister(AArch64::SP, MVT::i64);

297 }

298

299 const unsigned OpCode1 = ZeroData ? AArch64ISD::STZG : AArch64ISD::STG;

300 const unsigned OpCode2 = ZeroData ? AArch64ISD::STZ2G : AArch64ISD::ST2G;

301

303 unsigned OffsetScaled = 0;

304 while (OffsetScaled < ObjSizeScaled) {

305 if (ObjSizeScaled - OffsetScaled >= 2) {

309 OpCode2, dl, DAG.getVTList(MVT::Other),

310 {Chain, TagSrc, AddrNode},

311 MVT::v4i64,

313 OffsetScaled += 2;

315 continue;

316 }

317

318 if (ObjSizeScaled - OffsetScaled > 0) {

322 OpCode1, dl, DAG.getVTList(MVT::Other),

323 {Chain, TagSrc, AddrNode},

324 MVT::v2i64,

326 OffsetScaled += 1;

328 }

329 }

330

332 return Res;

333}

334

339 assert(ObjSize % 16 == 0);

340

344

345 bool UseSetTagRangeLoop =

347 if (!UseSetTagRangeLoop)

348 return EmitUnrolledSetTag(DAG, dl, Chain, Addr, ObjSize, BaseMemOperand,

349 ZeroData);

350

351 const EVT ResTys[] = {MVT::i64, MVT::i64, MVT::Other};

352

353 unsigned Opcode;

357 Opcode = ZeroData ? AArch64::STZGloop : AArch64::STGloop;

358 } else {

359 Opcode = ZeroData ? AArch64::STZGloop_wback : AArch64::STGloop_wback;

360 }

363

366}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static SDValue EmitUnrolledSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Ptr, uint64_t ObjSize, const MachineMemOperand *BaseMemOperand, bool ZeroData)

Definition AArch64SelectionDAGInfo.cpp:283

static cl::opt< bool > LowerToSMERoutines("aarch64-lower-to-sme-routines", cl::Hidden, cl::desc("Enable AArch64 SME memory operations " "to lower to librt functions"), cl::init(true))

static const int kSetTagLoopThreshold

Definition AArch64SelectionDAGInfo.cpp:281

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]

AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...

SDValue EmitMOPS(unsigned Opcode, SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Dst, SDValue SrcOrValue, SDValue Size, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const

Definition AArch64SelectionDAGInfo.cpp:112

SDValue EmitTargetCodeForMemmove(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const override

Emit target-specific code that performs a memmove.

Definition AArch64SelectionDAGInfo.cpp:249

SDValue EmitTargetCodeForMemcpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo) const override

Emit target-specific code that performs a memcpy.

Definition AArch64SelectionDAGInfo.cpp:210

void verifyTargetNode(const SelectionDAG &DAG, const SDNode *N) const override

Checks that the given target-specific node is valid. Aborts if it is not.

Definition AArch64SelectionDAGInfo.cpp:33

SDValue EmitTargetCodeForMemset(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVolatile, bool AlwaysInline, MachinePointerInfo DstPtrInfo) const override

Emit target-specific code that performs a memset.

Definition AArch64SelectionDAGInfo.cpp:229

SDValue EmitStreamingCompatibleMemLibCall(SelectionDAG &DAG, const SDLoc &DL, SDValue Chain, SDValue Op0, SDValue Op1, SDValue Size, RTLIB::Libcall LC) const

Definition AArch64SelectionDAGInfo.cpp:158

SDValue EmitTargetCodeForSetTag(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, MachinePointerInfo DstPtrInfo, bool ZeroData) const override

Definition AArch64SelectionDAGInfo.cpp:335

AArch64SelectionDAGInfo()

Definition AArch64SelectionDAGInfo.cpp:30

std::pair< SDValue, SDValue > EmitTargetCodeForMemchr(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Src, SDValue Char, SDValue Length, MachinePointerInfo SrcPtrInfo) const override

Emit target-specific code that performs a memchr, in cases where that is faster than a libcall.

Definition AArch64SelectionDAGInfo.cpp:268

const AArch64TargetLowering * getTargetLowering() const override

LLVM_ABI IntegerType * getIntPtrType(LLVMContext &C, unsigned AddressSpace=0) const

Returns an integer type with size at least as big as that of a pointer in the given address space.

const TargetSubtargetInfo & getSubtarget() const

getSubtarget - Return the subtarget for which this machine code is being compiled.

MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)

getMachineMemOperand - Allocate a new MachineMemOperand.

Ty * getInfo()

getInfo - Keep track of various per-function pieces of information for backends that would like to do...

A description of a memory reference used in the backend.

@ MOVolatile

The memory access is volatile.

@ MOLoad

The memory access reads data.

@ MOStore

The memory access writes data.

An SDNode that represents everything that will be needed to construct a MachineInstr.

Class to represent pointers.

static PointerType * getUnqual(Type *ElementType)

This constructs a pointer to an object of the specified type in the default address space (address sp...

Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...

Represents one node in the SelectionDAG.

Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.

EVT getValueType() const

Return the ValueType of the referenced return value.

unsigned getOpcode() const

SMEAttrs is a utility class to parse the SME ACLE attributes on functions.

SelectionDAGGenTargetInfo(const SDNodeInfo &GenNodeInfo)

void verifyTargetNode(const SelectionDAG &DAG, const SDNode *N) const override

Checks that the given target-specific node is valid. Aborts if it is not.

This is used to represent a portion of an LLVM function in a low-level Data Dependence DAG representa...

LLVM_ABI SDValue getMergeValues(ArrayRef< SDValue > Ops, const SDLoc &dl)

Create a MERGE_VALUES node from the given operands.

LLVM_ABI SDVTList getVTList(EVT VT)

Return an SDVTList that represents the list of values specified.

LLVM_ABI MachineSDNode * getMachineNode(unsigned Opcode, const SDLoc &dl, EVT VT)

These are used for target selectors to create a new node with specified return type(s),...

LLVM_ABI SDValue getRegister(Register Reg, EVT VT)

LLVM_ABI SDValue getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, SDVTList VTList, ArrayRef< SDValue > Ops, EVT MemVT, MachinePointerInfo PtrInfo, Align Alignment, MachineMemOperand::Flags Flags=MachineMemOperand::MOLoad|MachineMemOperand::MOStore, LocationSize Size=LocationSize::precise(0), const AAMDNodes &AAInfo=AAMDNodes())

Creates a MemIntrinsicNode that may produce a result and takes a list of operands.

LLVM_ABI void setNodeMemRefs(MachineSDNode *N, ArrayRef< MachineMemOperand * > NewMemRefs)

Mutate the specified machine node's memory references to the provided list.

const DataLayout & getDataLayout() const

SDValue getTargetFrameIndex(int FI, EVT VT)

LLVM_ABI SDValue getMemBasePlusOffset(SDValue Base, TypeSize Offset, const SDLoc &DL, const SDNodeFlags Flags=SDNodeFlags())

Returns sum of the base pointer and offset.

LLVM_ABI SDValue getExternalSymbol(const char *Sym, EVT VT)

LLVM_ABI SDValue getNode(unsigned Opcode, const SDLoc &DL, EVT VT, ArrayRef< SDUse > Ops)

Gets or creates the specified node.

SDValue getTargetConstant(uint64_t Val, const SDLoc &DL, EVT VT, bool isOpaque=false)

MachineFunction & getMachineFunction() const

LLVM_ABI SDValue getZExtOrTrunc(SDValue Op, const SDLoc &DL, EVT VT)

Convert Op, which must be of integer type, to the integer type VT, by either zero-extending or trunca...

LLVMContext * getContext() const

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

std::vector< ArgListEntry > ArgListTy

static constexpr TypeSize getFixed(ScalarTy ExactSize)

static LLVM_ABI IntegerType * getInt32Ty(LLVMContext &C)

@ C

The default llvm calling convention, compatible with C.

@ ANY_EXTEND

ANY_EXTEND - Used for integer types. The high bits are undefined.

@ TokenFactor

TokenFactor - This node takes multiple tokens as input and produces a single token result.

initializer< Ty > init(const Ty &Val)

This is an optimization pass for GlobalISel generic memory operations.

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

This struct is a compact representation of a valid (non-zero power of two) alignment.

ElementCount getVectorElementCount() const

TypeSize getSizeInBits() const

Return the size of the specified value type in bits.

bool isVector() const

Return true if this is a vector value type.

bool isInteger() const

Return true if this is an integer or a vector integer type.

This class contains a discriminated union of information about pointers in memory operands,...

This structure contains all information that is necessary for lowering calls.

CallLoweringInfo & setLibCallee(CallingConv::ID CC, Type *ResultType, SDValue Target, ArgListTy &&ArgsList)

CallLoweringInfo & setDebugLoc(const SDLoc &dl)

CallLoweringInfo & setChain(SDValue InChain)