X86ShuffleDecode.cpp Source File (original) (raw)

24namespace llvm {

27 bool SrcIsMem) {

29 ShuffleMask.push_back(0);

30 ShuffleMask.push_back(1);

31 ShuffleMask.push_back(2);

32 ShuffleMask.push_back(3);

35 unsigned ZMask = Imm & 15;

36 unsigned CountD = (Imm >> 4) & 3;

37 unsigned CountS = SrcIsMem ? 0 : (Imm >> 6) & 3;

40 unsigned InVal = 4 + CountS;

42 ShuffleMask[CountD] = InVal;

48}

52 assert((Idx + Len) <= NumElts && "Insertion out of range");

54 for (unsigned i = 0; i != NumElts; ++i)

55 ShuffleMask.push_back(i);

56 for (unsigned i = 0; i != Len; ++i)

57 ShuffleMask[Idx + i] = NumElts + i;

58}

62 for (unsigned i = NElts / 2; i != NElts; ++i)

63 ShuffleMask.push_back(NElts + i);

65 for (unsigned i = NElts / 2; i != NElts; ++i)

66 ShuffleMask.push_back(i);

67}

71 for (unsigned i = 0; i != NElts / 2; ++i)

72 ShuffleMask.push_back(i);

74 for (unsigned i = 0; i != NElts / 2; ++i)

75 ShuffleMask.push_back(NElts + i);

76}

79 for (int i = 0, e = NumElts / 2; i < e; ++i) {

80 ShuffleMask.push_back(2 * i);

81 ShuffleMask.push_back(2 * i);

82 }

83}

86 for (int i = 0, e = NumElts / 2; i < e; ++i) {

87 ShuffleMask.push_back(2 * i + 1);

88 ShuffleMask.push_back(2 * i + 1);

89 }

90}

93 const unsigned NumLaneElts = 2;

95 for (unsigned l = 0; l < NumElts; l += NumLaneElts)

96 for (unsigned i = 0; i < NumLaneElts; ++i)

97 ShuffleMask.push_back(l);

98}

102 const unsigned NumLaneElts = 16;

103

104 for (unsigned l = 0; l < NumElts; l += NumLaneElts)

105 for (unsigned i = 0; i < NumLaneElts; ++i) {

107 if (i >= Imm) M = i - Imm + l;

108 ShuffleMask.push_back(M);

109 }

110}

111

114 const unsigned NumLaneElts = 16;

115

116 for (unsigned l = 0; l < NumElts; l += NumLaneElts)

117 for (unsigned i = 0; i < NumLaneElts; ++i) {

118 unsigned Base = i + Imm;

119 int M = Base + l;

121 ShuffleMask.push_back(M);

122 }

123}

124

127 const unsigned NumLaneElts = 16;

128

129 for (unsigned l = 0; l != NumElts; l += NumLaneElts) {

130 for (unsigned i = 0; i != NumLaneElts; ++i) {

131 unsigned Base = i + Imm;

132

133 if (Base >= NumLaneElts) Base += NumElts - NumLaneElts;

134 ShuffleMask.push_back(Base + l);

135 }

136 }

137}

138

141

143 Imm = Imm & (NumElts - 1);

144 for (unsigned i = 0; i != NumElts; ++i)

145 ShuffleMask.push_back(i + Imm);

146}

147

148void DecodePSHUFMask(unsigned NumElts, unsigned ScalarBits, unsigned Imm,

150 unsigned Size = NumElts * ScalarBits;

151 unsigned NumLanes = Size / 128;

152 if (NumLanes == 0) NumLanes = 1;

153 unsigned NumLaneElts = NumElts / NumLanes;

154

155 uint32_t SplatImm = (Imm & 0xff) * 0x01010101;

156 for (unsigned l = 0; l != NumElts; l += NumLaneElts) {

157 for (unsigned i = 0; i != NumLaneElts; ++i) {

158 ShuffleMask.push_back(SplatImm % NumLaneElts + l);

159 SplatImm /= NumLaneElts;

160 }

161 }

162}

163

166 for (unsigned l = 0; l != NumElts; l += 8) {

167 unsigned NewImm = Imm;

168 for (unsigned i = 0, e = 4; i != e; ++i) {

169 ShuffleMask.push_back(l + i);

170 }

171 for (unsigned i = 4, e = 8; i != e; ++i) {

172 ShuffleMask.push_back(l + 4 + (NewImm & 3));

173 NewImm >>= 2;

174 }

175 }

176}

177

180 for (unsigned l = 0; l != NumElts; l += 8) {

181 unsigned NewImm = Imm;

182 for (unsigned i = 0, e = 4; i != e; ++i) {

183 ShuffleMask.push_back(l + (NewImm & 3));

184 NewImm >>= 2;

185 }

186 for (unsigned i = 4, e = 8; i != e; ++i) {

187 ShuffleMask.push_back(l + i);

188 }

189 }

190}

191

193 unsigned NumHalfElts = NumElts / 2;

194

195 for (unsigned l = 0; l != NumHalfElts; ++l)

196 ShuffleMask.push_back(l + NumHalfElts);

197 for (unsigned h = 0; h != NumHalfElts; ++h)

198 ShuffleMask.push_back(h);

199}

200

203 unsigned NumLaneElts = 128 / ScalarBits;

204

205 unsigned NewImm = Imm;

206 for (unsigned l = 0; l != NumElts; l += NumLaneElts) {

207

208 for (unsigned s = 0; s != NumElts * 2; s += NumElts) {

209 for (unsigned i = 0; i != NumLaneElts / 2; ++i) {

210 ShuffleMask.push_back(NewImm % NumLaneElts + s + l);

211 NewImm /= NumLaneElts;

212 }

213 }

214 if (NumLaneElts == 4) NewImm = Imm;

215 }

216}

217

220

221

222 unsigned NumLanes = (NumElts * ScalarBits) / 128;

223 if (NumLanes == 0) NumLanes = 1;

224 unsigned NumLaneElts = NumElts / NumLanes;

225

226 for (unsigned l = 0; l != NumElts; l += NumLaneElts) {

227 for (unsigned i = l + NumLaneElts / 2, e = l + NumLaneElts; i != e; ++i) {

228 ShuffleMask.push_back(i);

229 ShuffleMask.push_back(i + NumElts);

230 }

231 }

232}

233

236

237

238 unsigned NumLanes = (NumElts * ScalarBits) / 128;

239 if (NumLanes == 0 ) NumLanes = 1;

240 unsigned NumLaneElts = NumElts / NumLanes;

241

242 for (unsigned l = 0; l != NumElts; l += NumLaneElts) {

243 for (unsigned i = l, e = l + NumLaneElts / 2; i != e; ++i) {

244 ShuffleMask.push_back(i);

245 ShuffleMask.push_back(i + NumElts);

246 }

247 }

248}

249

252 ShuffleMask.append(NumElts, 0);

253}

254

257 unsigned Scale = DstNumElts / SrcNumElts;

258

259 for (unsigned i = 0; i != Scale; ++i)

260 for (unsigned j = 0; j != SrcNumElts; ++j)

261 ShuffleMask.push_back(j);

262}

263

265 unsigned Imm,

267 unsigned NumElementsInLane = 128 / ScalarSize;

268 unsigned NumLanes = NumElts / NumElementsInLane;

269

270 for (unsigned l = 0; l != NumElts; l += NumElementsInLane) {

271 unsigned Index = (Imm % NumLanes) * NumElementsInLane;

272 Imm /= NumLanes;

273

274 if (l >= (NumElts / 2))

275 Index += NumElts;

276 for (unsigned i = 0; i != NumElementsInLane; ++i)

277 ShuffleMask.push_back(Index + i);

278 }

279}

280

283 unsigned HalfSize = NumElts / 2;

284

285 for (unsigned l = 0; l != 2; ++l) {

286 unsigned HalfMask = Imm >> (l * 4);

287 unsigned HalfBegin = (HalfMask & 0x3) * HalfSize;

288 for (unsigned i = HalfBegin, e = HalfBegin + HalfSize; i != e; ++i)

289 ShuffleMask.push_back((HalfMask & 8) ? SM_SentinelZero : (int)i);

290 }

291}

292

295 for (int i = 0, e = RawMask.size(); i < e; ++i) {

297 if (UndefElts[i]) {

299 continue;

300 }

301

302

303 int Base = (i / 16) * 16;

304

305 if (M & (1 << 7))

307 else {

308

309 int Index = Base + (M & 0xf);

310 ShuffleMask.push_back(Index);

311 }

312 }

313}

314

317 for (unsigned i = 0; i < NumElts; ++i) {

318

319

320 unsigned Bit = i % 8;

321 ShuffleMask.push_back(((Imm >> Bit) & 1) ? NumElts + i : i);

322 }

323}

324

327 assert(RawMask.size() == 16 && "Illegal VPPERM shuffle mask size");

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342 for (int i = 0, e = RawMask.size(); i < e; ++i) {

343 if (UndefElts[i]) {

345 continue;

346 }

347

349 uint64_t PermuteOp = (M >> 5) & 0x7;

350 if (PermuteOp == 4) {

352 continue;

353 }

354 if (PermuteOp != 0) {

355 ShuffleMask.clear();

356 return;

357 }

358

360 ShuffleMask.push_back((int)Index);

361 }

362}

363

366 for (unsigned l = 0; l != NumElts; l += 4)

367 for (unsigned i = 0; i != 4; ++i)

368 ShuffleMask.push_back(l + ((Imm >> (2 * i)) & 3));

369}

370

372 unsigned NumDstElts, bool IsAnyExtend,

374 unsigned Scale = DstScalarBits / SrcScalarBits;

375 assert(SrcScalarBits < DstScalarBits &&

376 "Expected zero extension mask to increase scalar size");

377

379 for (unsigned i = 0; i != NumDstElts; i++) {

380 ShuffleMask.push_back(i);

381 ShuffleMask.append(Scale - 1, Sentinel);

382 }

383}

384

387 ShuffleMask.push_back(0);

389}

390

393

394

395 ShuffleMask.push_back(NumElts);

396 for (unsigned i = 1; i < NumElts; i++)

397 ShuffleMask.push_back(IsLoad ? static_cast<int>(SM_SentinelZero) : i);

398}

399

400void DecodeEXTRQIMask(unsigned NumElts, unsigned EltSize, int Len, int Idx,

402 unsigned HalfElts = NumElts / 2;

403

404

405 Len &= 0x3F;

406 Idx &= 0x3F;

407

408

409

410 if (0 != (Len % EltSize) || 0 != (Idx % EltSize))

411 return;

412

413

414 if (Len == 0)

415 Len = 64;

416

417

418 if ((Len + Idx) > 64) {

420 return;

421 }

422

423

424 Len /= EltSize;

425 Idx /= EltSize;

426

427

428

429 for (int i = 0; i != Len; ++i)

430 ShuffleMask.push_back(i + Idx);

431 for (int i = Len; i != (int)HalfElts; ++i)

433 for (int i = HalfElts; i != (int)NumElts; ++i)

435}

436

439 unsigned HalfElts = NumElts / 2;

440

441

442 Len &= 0x3F;

443 Idx &= 0x3F;

444

445

446

447 if (0 != (Len % EltSize) || 0 != (Idx % EltSize))

448 return;

449

450

451 if (Len == 0)

452 Len = 64;

453

454

455 if ((Len + Idx) > 64) {

457 return;

458 }

459

460

461 Len /= EltSize;

462 Idx /= EltSize;

463

464

465

466

467 for (int i = 0; i != Idx; ++i)

468 ShuffleMask.push_back(i);

469 for (int i = 0; i != Len; ++i)

470 ShuffleMask.push_back(i + NumElts);

471 for (int i = Idx + Len; i != (int)HalfElts; ++i)

472 ShuffleMask.push_back(i);

473 for (int i = HalfElts; i != (int)NumElts; ++i)

475}

476

480 unsigned VecSize = NumElts * ScalarBits;

481 unsigned NumLanes = VecSize / 128;

482 unsigned NumEltsPerLane = NumElts / NumLanes;

483 assert((VecSize == 128 || VecSize == 256 || VecSize == 512) &&

484 "Unexpected vector size");

485 assert((ScalarBits == 32 || ScalarBits == 64) && "Unexpected element size");

486

487 for (unsigned i = 0, e = RawMask.size(); i < e; ++i) {

488 if (UndefElts[i]) {

490 continue;

491 }

493 M = (ScalarBits == 64 ? ((M >> 1) & 0x1) : (M & 0x3));

494 unsigned LaneOffset = i & ~(NumEltsPerLane - 1);

495 ShuffleMask.push_back((int)(LaneOffset + M));

496 }

497}

498

502 unsigned VecSize = NumElts * ScalarBits;

503 unsigned NumLanes = VecSize / 128;

504 unsigned NumEltsPerLane = NumElts / NumLanes;

505 assert((VecSize == 128 || VecSize == 256) && "Unexpected vector size");

506 assert((ScalarBits == 32 || ScalarBits == 64) && "Unexpected element size");

507 assert((NumElts == RawMask.size()) && "Unexpected mask size");

508

509 for (unsigned i = 0, e = RawMask.size(); i < e; ++i) {

510 if (UndefElts[i]) {

512 continue;

513 }

514

515

516

517

518

519 uint64_t Selector = RawMask[i];

520 unsigned MatchBit = (Selector >> 3) & 0x1;

521

522

523

524

525

526

527

528 if ((M2Z & 0x2) != 0 && MatchBit != (M2Z & 0x1)) {

530 continue;

531 }

532

533 int Index = i & ~(NumEltsPerLane - 1);

534 if (ScalarBits == 64)

535 Index += (Selector >> 1) & 0x1;

536 else

537 Index += Selector & 0x3;

538

539 int Src = (Selector >> 2) & 0x1;

540 Index += Src * NumElts;

541 ShuffleMask.push_back(Index);

542 }

543}

544

548 for (int i = 0, e = RawMask.size(); i != e; ++i) {

549 if (UndefElts[i]) {

551 continue;

552 }

554 M &= EltMaskSize;

555 ShuffleMask.push_back((int)M);

556 }

557}

558

561 uint64_t EltMaskSize = (RawMask.size() * 2) - 1;

562 for (int i = 0, e = RawMask.size(); i != e; ++i) {

563 if (UndefElts[i]) {

565 continue;

566 }

568 M &= EltMaskSize;

569 ShuffleMask.push_back((int)M);

570 }

571}

572

573}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

This file implements a class to represent arbitrary precision integral constant values and operations...

This file defines the SmallVector class.

Class for arbitrary precision integers.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

This is an optimization pass for GlobalISel generic memory operations.

void DecodeZeroExtendMask(unsigned SrcScalarBits, unsigned DstScalarBits, unsigned NumDstElts, bool IsAnyExtend, SmallVectorImpl< int > &ShuffleMask)

Decode a zero extension instruction as a shuffle mask.

Definition X86ShuffleDecode.cpp:371

void DecodeMOVHLPSMask(unsigned NElts, SmallVectorImpl< int > &ShuffleMask)

Decode a MOVHLPS instruction as a v2f64/v4f32 shuffle mask.

Definition X86ShuffleDecode.cpp:61

void DecodeZeroMoveLowMask(unsigned NumElts, SmallVectorImpl< int > &ShuffleMask)

Decode a move lower and zero upper instruction as a shuffle mask.

Definition X86ShuffleDecode.cpp:385

void DecodeInsertElementMask(unsigned NumElts, unsigned Idx, unsigned Len, SmallVectorImpl< int > &ShuffleMask)

Definition X86ShuffleDecode.cpp:50

void DecodeVPERMILPMask(unsigned NumElts, unsigned ScalarBits, ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)

Decode a VPERMILPD/VPERMILPS variable mask from a raw array of constants.

Definition X86ShuffleDecode.cpp:477

void DecodePSHUFLWMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)

Decodes the shuffle masks for pshuflw.

Definition X86ShuffleDecode.cpp:178

void DecodeVPERMV3Mask(ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)

Decode a VPERMT2 W/D/Q/PS/PD mask from a raw array of constants.

Definition X86ShuffleDecode.cpp:559

void DecodeBLENDMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)

Decode a BLEND immediate mask into a shuffle mask.

Definition X86ShuffleDecode.cpp:315

void decodeVSHUF64x2FamilyMask(unsigned NumElts, unsigned ScalarSize, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)

Decode a shuffle packed values at 128-bit granularity (SHUFF32x4/SHUFF64x2/SHUFI32x4/SHUFI64x2) immed...

Definition X86ShuffleDecode.cpp:264

void DecodeVPERMMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)

Decodes the shuffle masks for VPERMQ/VPERMPD.

Definition X86ShuffleDecode.cpp:364

void DecodeEXTRQIMask(unsigned NumElts, unsigned EltSize, int Len, int Idx, SmallVectorImpl< int > &ShuffleMask)

Decode a SSE4A EXTRQ instruction as a shuffle mask.

Definition X86ShuffleDecode.cpp:400

void DecodePSRLDQMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)

Definition X86ShuffleDecode.cpp:112

void DecodeINSERTPSMask(unsigned Imm, SmallVectorImpl< int > &ShuffleMask, bool SrcIsMem)

Decode a 128-bit INSERTPS instruction as a v4f32 shuffle mask.

Definition X86ShuffleDecode.cpp:26

void DecodeVPERM2X128Mask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)

Definition X86ShuffleDecode.cpp:281

void DecodeVPERMIL2PMask(unsigned NumElts, unsigned ScalarBits, unsigned M2Z, ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)

Decode a VPERMIL2PD/VPERMIL2PS variable mask from a raw array of constants.

Definition X86ShuffleDecode.cpp:499

void DecodeMOVLHPSMask(unsigned NElts, SmallVectorImpl< int > &ShuffleMask)

Decode a MOVLHPS instruction as a v2f64/v4f32 shuffle mask.

Definition X86ShuffleDecode.cpp:70

constexpr bool isPowerOf2_32(uint32_t Value)

Return true if the argument is a power of two > 0.

void DecodePSWAPMask(unsigned NumElts, SmallVectorImpl< int > &ShuffleMask)

Decodes a PSWAPD 3DNow! instruction.

Definition X86ShuffleDecode.cpp:192

void DecodeINSERTQIMask(unsigned NumElts, unsigned EltSize, int Len, int Idx, SmallVectorImpl< int > &ShuffleMask)

Decode a SSE4A INSERTQ instruction as a shuffle mask.

Definition X86ShuffleDecode.cpp:437

void DecodeVPERMVMask(ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)

Decode a VPERM W/D/Q/PS/PD mask from a raw array of constants.

Definition X86ShuffleDecode.cpp:545

void DecodeVALIGNMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)

Definition X86ShuffleDecode.cpp:139

void DecodeScalarMoveMask(unsigned NumElts, bool IsLoad, SmallVectorImpl< int > &ShuffleMask)

Decode a scalar float move instruction as a shuffle mask.

Definition X86ShuffleDecode.cpp:391

void DecodeVPPERMMask(ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)

Decode a VPPERM mask from a raw array of constants such as from BUILD_VECTOR.

Definition X86ShuffleDecode.cpp:325

void DecodePALIGNRMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)

Definition X86ShuffleDecode.cpp:125

void DecodeMOVSLDUPMask(unsigned NumElts, SmallVectorImpl< int > &ShuffleMask)

Definition X86ShuffleDecode.cpp:78

void DecodeSubVectorBroadcast(unsigned DstNumElts, unsigned SrcNumElts, SmallVectorImpl< int > &ShuffleMask)

Decodes a broadcast of a subvector to a larger vector type.

Definition X86ShuffleDecode.cpp:255

void DecodeUNPCKLMask(unsigned NumElts, unsigned ScalarBits, SmallVectorImpl< int > &ShuffleMask)

Decodes the shuffle masks for unpcklps/unpcklpd and punpckl*.

Definition X86ShuffleDecode.cpp:234

void DecodePSLLDQMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)

Definition X86ShuffleDecode.cpp:100

void DecodeUNPCKHMask(unsigned NumElts, unsigned ScalarBits, SmallVectorImpl< int > &ShuffleMask)

Decodes the shuffle masks for unpckhps/unpckhpd and punpckh*.

Definition X86ShuffleDecode.cpp:218

void DecodePSHUFMask(unsigned NumElts, unsigned ScalarBits, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)

Decodes the shuffle masks for pshufd/pshufw/vpermilpd/vpermilps.

Definition X86ShuffleDecode.cpp:148

void DecodeMOVDDUPMask(unsigned NumElts, SmallVectorImpl< int > &ShuffleMask)

Definition X86ShuffleDecode.cpp:92

void DecodeVectorBroadcast(unsigned NumElts, SmallVectorImpl< int > &ShuffleMask)

Decodes a broadcast of the first element of a vector.

Definition X86ShuffleDecode.cpp:250

void DecodeSHUFPMask(unsigned NumElts, unsigned ScalarBits, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)

Decodes the shuffle masks for shufp*.

Definition X86ShuffleDecode.cpp:201

void DecodePSHUFHWMask(unsigned NumElts, unsigned Imm, SmallVectorImpl< int > &ShuffleMask)

Decodes the shuffle masks for pshufhw.

Definition X86ShuffleDecode.cpp:164

void DecodeMOVSHDUPMask(unsigned NumElts, SmallVectorImpl< int > &ShuffleMask)

Definition X86ShuffleDecode.cpp:85

void DecodePSHUFBMask(ArrayRef< uint64_t > RawMask, const APInt &UndefElts, SmallVectorImpl< int > &ShuffleMask)

Decode a PSHUFB mask from a raw array of constants such as from BUILD_VECTOR.

Definition X86ShuffleDecode.cpp:293