RISCVInterleavedAccess.cpp Source File (original) (raw)

21#include "llvm/IR/IntrinsicsRISCV.h"

25using namespace llvm;

28 VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace,

33 return false;

37 Alignment))

38 return false;

43 if (!Subtarget.useRVVForFixedLengthVectors())

44 return false;

47 if (FVTy->getNumElements() < 2)

48 return false;

51 }

55 if (Fractional)

56 return true;

57 return Factor * LMUL <= 8;

58}

61 Intrinsic::riscv_seg2_load_mask, Intrinsic::riscv_seg3_load_mask,

62 Intrinsic::riscv_seg4_load_mask, Intrinsic::riscv_seg5_load_mask,

63 Intrinsic::riscv_seg6_load_mask, Intrinsic::riscv_seg7_load_mask,

64 Intrinsic::riscv_seg8_load_mask};

67 Intrinsic::riscv_sseg2_load_mask, Intrinsic::riscv_sseg3_load_mask,

68 Intrinsic::riscv_sseg4_load_mask, Intrinsic::riscv_sseg5_load_mask,

69 Intrinsic::riscv_sseg6_load_mask, Intrinsic::riscv_sseg7_load_mask,

70 Intrinsic::riscv_sseg8_load_mask};

73 Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,

74 Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,

75 Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,

76 Intrinsic::riscv_vlseg8_mask};

79 Intrinsic::riscv_seg2_store_mask, Intrinsic::riscv_seg3_store_mask,

80 Intrinsic::riscv_seg4_store_mask, Intrinsic::riscv_seg5_store_mask,

81 Intrinsic::riscv_seg6_store_mask, Intrinsic::riscv_seg7_store_mask,

82 Intrinsic::riscv_seg8_store_mask};

85 Intrinsic::riscv_sseg2_store_mask, Intrinsic::riscv_sseg3_store_mask,

86 Intrinsic::riscv_sseg4_store_mask, Intrinsic::riscv_sseg5_store_mask,

87 Intrinsic::riscv_sseg6_store_mask, Intrinsic::riscv_sseg7_store_mask,

88 Intrinsic::riscv_sseg8_store_mask};

91 Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,

92 Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,

93 Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,

94 Intrinsic::riscv_vsseg8_mask};

98 if (N == 1)

99 return true;

100

102

106 C && C % N == 0)

107 return true;

108

112 }

113

114 return false;

115}

116

117

118

122

127 assert(LI->isSimple());

128 Ptr = LI->getPointerOperand();

129 Alignment = LI->getAlign();

130 assert(!Mask && "Unexpected mask on a load");

131 Mask = Builder.getAllOnesMask(EC);

134 return true;

135 }

138 Ptr = SI->getPointerOperand();

139 Alignment = SI->getAlign();

140 assert(!Mask && "Unexpected mask on a store");

141 Mask = Builder.getAllOnesMask(EC);

144 return true;

145 }

146

148 switch (II->getIntrinsicID()) {

149 default:

151 case Intrinsic::vp_load:

152 case Intrinsic::vp_store: {

154 Ptr = VPLdSt->getMemoryPointerParam();

155 Alignment = VPLdSt->getPointerAlignment().value_or(

156 DL.getABITypeAlign(VTy->getElementType()));

157

158 assert(Mask && "vp.load and vp.store needs a mask!");

159

160 Value *WideEVL = VPLdSt->getVectorLengthParam();

161

162

163 if ( isMultipleOfN (WideEVL, I->getDataLayout(), Factor))

164 return false;

165

166 auto *FactorC = ConstantInt::get(WideEVL->getType(), Factor);

167 VL = Builder.CreateZExt(Builder.CreateExactUDiv(WideEVL, FactorC), XLenTy);

168 return true;

169 }

170 case Intrinsic::masked_load: {

171 Ptr = II->getOperand(0);

172 Alignment = II->getParamAlign(0).valueOrOne();

173

175 return false;

176

177 assert(Mask && "masked.load needs a mask!");

178

180 ? Builder.CreateElementCount(XLenTy, VTy->getElementCount())

182 return true;

183 }

184 case Intrinsic::masked_store: {

185 Ptr = II->getOperand(1);

186 Alignment = II->getParamAlign(1).valueOrOne();

187

188 assert(Mask && "masked.store needs a mask!");

189

191 ? Builder.CreateElementCount(XLenTy, VTy->getElementCount())

193 return true;

194 }

195 }

196}

197

198

199

200

201

202

203

204

205

206

207

208

209

215

216

217

218 unsigned MaskFactor = GapMask.popcount();

219 if (MaskFactor < 2 || !GapMask.isMask())

220 return false;

222

223 const DataLayout &DL = Load->getDataLayout();

225 auto *XLenTy = Builder.getIntNTy(Subtarget.getXLen());

226

229 if ( getMemOperands (MaskFactor, VTy, XLenTy, Load, Ptr, Mask, VL, Alignment))

230 return false;

231

235 return false;

236

238 if (MaskFactor < Factor) {

239

240 unsigned ScalarSizeInBytes = DL.getTypeStoreSize(VTy->getElementType());

241 Value *Stride = ConstantInt::get(XLenTy, Factor * ScalarSizeInBytes);

242 SegLoad = Builder.CreateIntrinsic(FixedVlssegIntrIds[MaskFactor - 2],

243 {VTy, PtrTy, XLenTy, XLenTy},

244 {Ptr, Stride, Mask, VL});

245 } else {

246

248 {VTy, PtrTy, XLenTy}, {Ptr, Mask, VL});

249 }

250

251 for (unsigned i = 0; i < Shuffles.size(); i++) {

252 unsigned FactorIdx = Indices[i];

253 if (FactorIdx >= MaskFactor) {

254

256 } else {

257 Value *SubVec = Builder.CreateExtractValue(SegLoad, FactorIdx);

258 Shuffles[i]->replaceAllUsesWith(SubVec);

259 }

260 }

261

262 return true;

263}

264

265

266

267

268

269

270

271

272

273

274

275

276

277

278

279

280

284 unsigned Factor,

285 const APInt &GapMask) const {

287

288

289

290 unsigned MaskFactor = GapMask.popcount();

291 if (MaskFactor < 2 || !GapMask.isMask())

292 return false;

293

295 const DataLayout &DL = Store->getDataLayout();

298

300 ShuffleVTy->getNumElements() / Factor);

301 auto *XLenTy = Builder.getIntNTy(Subtarget.getXLen());

302

305 if ( getMemOperands (MaskFactor, VTy, XLenTy, Store, Ptr, LaneMask, VL,

306 Alignment))

307 return false;

308

312 return false;

313

315 if (MaskFactor < Factor)

316

319 {VTy, PtrTy, XLenTy, XLenTy});

320 else

321

324 {VTy, PtrTy, XLenTy});

325

328

329 for (unsigned i = 0; i < MaskFactor; i++) {

330

331 for (unsigned j = 0; j < VTy->getNumElements(); j++)

332 NewShuffleMask.push_back(Mask[i + Factor * j]);

333

334 Value *Shuffle = Builder.CreateShuffleVector(

336 Ops.push_back(Shuffle);

337

338 NewShuffleMask.clear();

339 }

340 Ops.push_back(Ptr);

341 if (MaskFactor < Factor) {

342

343 unsigned ScalarSizeInBytes = DL.getTypeStoreSize(VTy->getElementType());

344 Ops.push_back(ConstantInt::get(XLenTy, Factor * ScalarSizeInBytes));

345 }

346 Ops.append({LaneMask, VL});

347 Builder.CreateCall(SegStoreFunc, Ops);

348

349 return true;

350}

351

355 if (Factor > 8)

356 return false;

357

359

361

362 const DataLayout &DL = Load->getDataLayout();

363 auto *XLenTy = Builder.getIntNTy(Subtarget.getXLen());

364

367 if ( getMemOperands (Factor, ResVTy, XLenTy, Load, Ptr, Mask, VL, Alignment))

368 return false;

369

373 return false;

374

378 {ResVTy, PtrTy, XLenTy}, {Ptr, Mask, VL});

379 } else {

380 unsigned SEW = DL.getTypeSizeInBits(ResVTy->getElementType());

383 Load->getContext(), "riscv.vector.tuple",

385 Factor);

388 {VecTupTy, PtrTy, Mask->getType(), VL->getType()});

389

390 Value *Operands[] = {

392 Ptr,

393 Mask,

394 VL,

395 ConstantInt::get(XLenTy,

397 ConstantInt::get(XLenTy, Log2_64(SEW))};

398

399 CallInst *Vlseg = Builder.CreateCall(VlsegNFunc, Operands);

400

403 for (unsigned i = 0; i < Factor; ++i) {

404 Value *VecExtract = Builder.CreateIntrinsic(

405 Intrinsic::riscv_tuple_extract, {ResVTy, VecTupTy},

406 {Vlseg, Builder.getInt32(i)});

407 Return = Builder.CreateInsertValue(Return, VecExtract, i);

408 }

409 }

410

412 return true;

413}

414

417 unsigned Factor = InterleaveValues.size();

418 if (Factor > 8)

419 return false;

420

422

424 const DataLayout &DL = Store->getDataLayout();

425 Type *XLenTy = Builder.getIntNTy(Subtarget.getXLen());

426

429 if ( getMemOperands (Factor, InVTy, XLenTy, Store, Ptr, Mask, VL, Alignment))

430 return false;

434 return false;

435

439 {InVTy, PtrTy, XLenTy});

441 Ops.append({Ptr, Mask, VL});

442 Builder.CreateCall(VssegNFunc, Ops);

443 return true;

444 }

445 unsigned SEW = DL.getTypeSizeInBits(InVTy->getElementType());

446 unsigned NumElts = InVTy->getElementCount().getKnownMinValue();

448 Store->getContext(), "riscv.vector.tuple",

450

452 for (unsigned i = 0; i < Factor; ++i)

453 StoredVal = Builder.CreateIntrinsic(

454 Intrinsic::riscv_tuple_insert, {VecTupTy, InVTy},

455 {StoredVal, InterleaveValues[i], Builder.getInt32(i)});

456

459 {VecTupTy, PtrTy, Mask->getType(), VL->getType()});

460

461 Value *Operands[] = {StoredVal, Ptr, Mask, VL,

462 ConstantInt::get(XLenTy, Log2_64(SEW))};

463 Builder.CreateCall(VssegNFunc, Operands);

464 return true;

465}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

Module.h This file contains the declarations for the Module class.

const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]

uint64_t IntrinsicInst * II

static const Intrinsic::ID FixedVlsegIntrIds[]

Definition RISCVInterleavedAccess.cpp:60

static bool isMultipleOfN(const Value *V, const DataLayout &DL, unsigned N)

Definition RISCVInterleavedAccess.cpp:96

static const Intrinsic::ID ScalableVlsegIntrIds[]

Definition RISCVInterleavedAccess.cpp:72

static const Intrinsic::ID ScalableVssegIntrIds[]

Definition RISCVInterleavedAccess.cpp:90

static const Intrinsic::ID FixedVlssegIntrIds[]

Definition RISCVInterleavedAccess.cpp:66

static bool getMemOperands(unsigned Factor, VectorType *VTy, Type *XLenTy, Instruction *I, Value *&Ptr, Value *&Mask, Value *&VL, Align &Alignment)

Do the common operand retrieval and validition required by the routines below.

Definition RISCVInterleavedAccess.cpp:119

static const Intrinsic::ID FixedVsssegIntrIds[]

Definition RISCVInterleavedAccess.cpp:84

static const Intrinsic::ID FixedVssegIntrIds[]

Definition RISCVInterleavedAccess.cpp:78

static SymbolRef::Type getType(const Symbol *Sym)

Class for arbitrary precision integers.

unsigned popcount() const

Count the number of bits set.

unsigned getBitWidth() const

Return the number of bits in the APInt.

bool isMask(unsigned numBits) const

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

This class represents a function call, abstracting a target machine's calling convention.

static LLVM_ABI Constant * getAllOnesValue(Type *Ty)

A parsed version of the target data layout string in and methods for querying it.

static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

A wrapper class for inspecting calls to intrinsic functions.

Intrinsic::ID getIntrinsicID() const

Return the intrinsic ID of this intrinsic.

static LLVM_ABI PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

bool lowerDeinterleaveIntrinsicToLoad(Instruction *Load, Value *Mask, IntrinsicInst *DI) const override

Lower a deinterleave intrinsic to a target specific load intrinsic.

Definition RISCVInterleavedAccess.cpp:352

bool lowerInterleavedStore(Instruction *Store, Value *Mask, ShuffleVectorInst *SVI, unsigned Factor, const APInt &GapMask) const override

Lower an interleaved store into a vssegN intrinsic.

Definition RISCVInterleavedAccess.cpp:281

MVT getContainerForFixedLengthVector(MVT VT) const

bool lowerInterleavedLoad(Instruction *Load, Value *Mask, ArrayRef< ShuffleVectorInst * > Shuffles, ArrayRef< unsigned > Indices, unsigned Factor, const APInt &GapMask) const override

Lower an interleaved load into a vlsegN intrinsic.

Definition RISCVInterleavedAccess.cpp:210

bool lowerInterleaveIntrinsicToStore(Instruction *Store, Value *Mask, ArrayRef< Value * > InterleaveValues) const override

Lower an interleave intrinsic to a target specific store intrinsic.

Definition RISCVInterleavedAccess.cpp:415

bool isLegalElementTypeForRVV(EVT ScalarTy) const

bool isLegalInterleavedAccessType(VectorType *VTy, unsigned Factor, Align Alignment, unsigned AddrSpace, const DataLayout &) const

Returns whether or not generating a interleaved load/store intrinsic for this type will be legal.

Definition RISCVInterleavedAccess.cpp:27

static RISCVVType::VLMUL getLMUL(MVT VT)

static LLVM_ABI ScalableVectorType * get(Type *ElementType, unsigned MinNumElts)

This instruction constructs a fixed permutation of two input vectors.

VectorType * getType() const

Overload to return most specific vector type.

static LLVM_ABI void getShuffleMask(const Constant *Mask, SmallVectorImpl< int > &Result)

Convert the input shuffle mask operand to a vector of integers.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

static LLVM_ABI StructType * get(LLVMContext &Context, ArrayRef< Type * > Elements, bool isPacked=false)

This static method is the primary way to create a literal StructType.

static LLVM_ABI TargetExtType * get(LLVMContext &Context, StringRef Name, ArrayRef< Type * > Types={}, ArrayRef< unsigned > Ints={})

Return a target extension type having the specified name and optional type and integer parameters.

EVT getValueType(const DataLayout &DL, Type *Ty, bool AllowUnknown=false) const

Return the EVT corresponding to this LLVM type.

bool isTypeLegal(EVT VT) const

Return true if the target has native support for the specified value type.

bool allowsMemoryAccessForAlignment(LLVMContext &Context, const DataLayout &DL, EVT VT, unsigned AddrSpace=0, Align Alignment=Align(1), MachineMemOperand::Flags Flags=MachineMemOperand::MONone, unsigned *Fast=nullptr) const

This function returns true if the memory access is aligned or if the target allows this specific unal...

The instances of the Type class are immutable: once they are created, they are never changed.

LLVM_ABI unsigned getPointerAddressSpace() const

Get the address space of this pointer or pointer vector type.

LLVMContext & getContext() const

Return the LLVMContext in which this type was uniqued.

static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)

Value * getOperand(unsigned i) const

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

LLVM_ABI void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

Base class of all SIMD vector types.

ElementCount getElementCount() const

Return an ElementCount instance to represent the (possibly scalable) number of elements in the vector...

Type * getElementType() const

constexpr ScalarTy getKnownMinValue() const

Returns the minimum value this quantity can represent.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ C

The default llvm calling convention, compatible with C.

LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})

Look up the Function declaration of the intrinsic id in the Module M.

bool match(Val *V, const Pattern &P)

class_match< ConstantInt > m_ConstantInt()

Match an arbitrary ConstantInt and ignore it.

OverflowingBinaryOp_match< LHS, RHS, Instruction::Mul, OverflowingBinaryOperator::NoUnsignedWrap > m_NUWMul(const LHS &L, const RHS &R)

class_match< Value > m_Value()

Match an arbitrary value and ignore it.

match_combine_or< LTy, RTy > m_CombineOr(const LTy &L, const RTy &R)

Combine two pattern matchers matching L || R.

LLVM_ABI std::pair< unsigned, bool > decodeVLMUL(VLMUL VLMul)

This is an optimization pass for GlobalISel generic memory operations.

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

unsigned Log2_64(uint64_t Value)

Return the floor log base 2 of the specified value, -1 if the value is zero.

unsigned Log2_32(uint32_t Value)

Return the floor log base 2 of the specified value, -1 if the value is zero.

constexpr bool isPowerOf2_32(uint32_t Value)

Return true if the argument is a power of two > 0.

LLVM_ABI void computeKnownBits(const Value *V, KnownBits &Known, const DataLayout &DL, AssumptionCache *AC=nullptr, const Instruction *CxtI=nullptr, const DominatorTree *DT=nullptr, bool UseInstrInfo=true, unsigned Depth=0)

Determine which bits of V are known to be either zero or one and return them in the KnownZero/KnownOn...

LLVM_ABI unsigned getDeinterleaveIntrinsicFactor(Intrinsic::ID ID)

Returns the corresponding factor of llvm.vector.deinterleaveN intrinsics.

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

LLVM_ABI VectorType * getDeinterleavedVectorType(IntrinsicInst *DI)

Given a deinterleaveN intrinsic, return the (narrow) vector type of each factor.

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

This struct is a compact representation of a valid (non-zero power of two) alignment.

MVT getSimpleVT() const

Return the SimpleValueType held in the specified simple EVT.

EVT getScalarType() const

If this is a vector type, return the element type, otherwise return this.

unsigned countMinTrailingZeros() const

Returns the minimum number of trailing zero bits.