X86FastPreTileConfig.cpp Source File (original) (raw)

34using namespace llvm;

36#define DEBUG_TYPE "fastpretileconfig"

38 STATISTIC(NumStores, "Number of stores added");

39 STATISTIC(NumLoads, "Number of loads added");

41namespace {

52 int CfgSS = -1;

53 struct PHIInfo {

57 };

58 DenseMap<MachineInstr *, struct PHIInfo> VisitedPHIs;

61 IndexedMap<int, VirtReg2IndexFunctor> StackSlotForVirtReg;

65 BitVector MayLiveAcrossBlocks;

67 int getStackSpaceFor(Register VirtReg);

68 void InitializeTileConfigStackSpace();

69 bool mayLiveOut(Register VirtReg, MachineInstr *CfgMI);

72 MachineOperand *RowMO, MachineOperand *ColMO);

73 void canonicalizePHIs(MachineBasicBlock &MBB);

74 void convertPHI(MachineBasicBlock *MBB, MachineInstr &PHI);

75 void convertPHIs(MachineBasicBlock &MBB);

76 bool configBasicBlock(MachineBasicBlock &MBB);

78public:

79 X86FastPreTileConfig() : MachineFunctionPass(ID), StackSlotForVirtReg(-1) {}

82 StringRef getPassName() const override {

83 return "Fast Tile Register Preconfigure";

84 }

87 bool runOnMachineFunction(MachineFunction &MFunc) override;

89 static char ID;

90};

92}

94char X86FastPreTileConfig::ID = 0;

97 "Fast Tile Register Preconfigure", false, false)

99 "Fast Tile Register Preconfigure", false, false)

100

104 auto MBBEnd = MBB.end();

105 if (B == MBBEnd)

106 return true;

107

109 for (; &*I != A && &*I != B; ++I)

110 ;

111

112 return &*I == A;

113}

114

115

116

117int X86FastPreTileConfig::getStackSpaceFor(Register VirtReg) {

118

119 int SS = StackSlotForVirtReg[VirtReg];

120

121 if (SS != -1)

122 return SS;

123

124

125 const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);

126 unsigned Size = TRI->getSpillSize(RC);

127 Align Alignment = TRI->getSpillAlign(RC);

129

130

131 StackSlotForVirtReg[VirtReg] = FrameIdx;

132 return FrameIdx;

133}

134

135

136

137

138bool X86FastPreTileConfig::mayLiveOut(Register VirtReg, MachineInstr *CfgMI) {

140 return true;

141

142 for (const MachineInstr &UseInst : MRI->use_nodbg_instructions(VirtReg)) {

143 if (UseInst.getParent() != MBB) {

145 return true;

146 }

147

148

149

150

151 if (CfgMI) {

154 return true;

155 }

156 }

157 }

158

159 return false;

160}

161

162void X86FastPreTileConfig::InitializeTileConfigStackSpace() {

163 MachineBasicBlock &MBB = MF->front();

167 Register Zmm = MRI->createVirtualRegister(&X86::VR512RegClass);

171 } else if (ST->hasAVX2()) {

172 Register Ymm = MRI->createVirtualRegister(&X86::VR256RegClass);

177 32)

179 } else {

180 assert(ST->hasSSE2() && "AMX should assume SSE2 enabled");

181 unsigned StoreOpc = ST->hasAVX() ? X86::VMOVUPSmr : X86::MOVUPSmr;

182 Register Xmm = MRI->createVirtualRegister(&X86::VR128RegClass);

192 }

193

196}

197

198

199

201 Register VirtReg, bool Kill) {

203 int FI = getStackSpaceFor(VirtReg);

204 LLVM_DEBUG(dbgs() << " to stack slot #" << FI << '\n');

205

206 const TargetRegisterClass &RC = *MRI->getRegClass(VirtReg);

207

208

210 ++NumStores;

211

212

213}

214

215

217 Register OrigReg, MachineOperand *RowMO,

218 MachineOperand *ColMO) {

219 int FI = getStackSpaceFor(OrigReg);

220 const TargetRegisterClass &RC = *MRI->getRegClass(OrigReg);

222

223

224

225

226

227

228

229

232 else

233 TileReg = MRI->createVirtualRegister(&RC);

234

235

236

237 unsigned Opc = X86::PTILELOADDV;

238 Register StrideReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);

239

241 TII->get(X86::MOV64ri), StrideReg)

247 FI);

248 MachineOperand &MO = NewMI->getOperand(5);

249 MO.setReg(StrideReg);

253

256 } else {

257

259 if (MO.isReg() && MO.getReg() == OrigReg)

261 }

262 }

263

264 ++NumLoads;

267}

268

270 if (Reg.isVirtual() &&

271 (MRI->getRegClass(Reg)->getID() == X86::TILERegClassID)) {

272 return true;

273 }

274

275 if (Reg >= X86::TMM0 && Reg <= X86::TMM7)

276 return true;

277

278 return false;

279}

280

282

283 if (MI.isDebugInstr() || MI.getNumOperands() < 3 || .isPseudo())

284 return false;

286

288 return false;

289

291}

292

298 return ShapeT(RowMO, ColMO, MRI);

299 } else if (MI->isCopy()) {

300 TileReg = MI->getOperand(1).getReg();

302 }

303

304

305

306 assert(MI->isPHI() && "Unexpected PHI when get shape.");

308}

309

310

311

312

313

314

315

316

317

318

319

320

321

322

323

324void X86FastPreTileConfig::convertPHI(MachineBasicBlock *MBB,

325 MachineInstr &PHI) {

326

327

328

329

330

331 Register StackAddrReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);

333 TII->get(X86::PHI), StackAddrReg);

334 Register RowReg = MRI->createVirtualRegister(&X86::GR16RegClass);

336 TII->get(X86::PHI), RowReg);

337 Register ColReg = MRI->createVirtualRegister(&X86::GR16RegClass);

339 TII->get(X86::PHI), ColReg);

340

341 VisitedPHIs[&PHI] = {RowReg, ColReg, StackAddrReg};

342

343 for (unsigned I = 1, E = PHI.getNumOperands(); I != E; I += 2) {

344

345 Register InTileReg = PHI.getOperand(I).getReg();

346

347

348

350 MachineBasicBlock *InMBB = PHI.getOperand(I + 1).getMBB();

351

352 MachineInstr *TileDefMI = MRI->getVRegDef(InTileReg);

354 if (TileDefMI->isPHI()) {

356 if (auto It = VisitedPHIs.find(TileDefMI);

357 It != VisitedPHIs.end()) {

358

359

360

361

362

363

364

365

366

367 Register InRowReg = It->second.Row;

368 Register InColReg = It->second.Col;

369 Register InStackAddrReg = It->second.StackAddr;

372 AddrPHI.addReg(InStackAddrReg).addMBB(InMBB);

373 continue;

374 } else {

375

376 convertPHI(TileDefMI->getParent(), *TileDefMI);

377

378

379 MachineInstr *TileLoad = MRI->getVRegDef(InTileReg);

380 assert(TileLoad && TileLoad->getOpcode() == X86::PTILELOADDV);

386 AddrPHI.addReg(InStackAddrReg).addMBB(InMBB);

387 }

388 } else {

390

391

397

398

399

400 int FI = getStackSpaceFor(InTileReg);

402 MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);

404 TII->get(X86::LEA64r), InStackAddrReg)

406 0);

407 AddrPHI.addReg(InStackAddrReg).addMBB(InMBB);

408 }

409 }

410

412 Register StrideReg = MRI->createVirtualRegister(&X86::GR64_NOSPRegClass);

415 Register TileReg = PHI.getOperand(0).getReg();

420 StackAddrReg);

421 MachineOperand &MO = NewMI->getOperand(5);

422 MO.setReg(StrideReg);

424 PHI.eraseFromParent();

425 VisitedPHIs.erase(&PHI);

426}

427

431 return true;

432 return false;

433}

434

435void X86FastPreTileConfig::canonicalizePHIs(MachineBasicBlock &MBB) {

436 SmallVector<MachineInstr *, 8> PHIs;

437

438 for (MachineInstr &MI : MBB) {

439 if (.isPHI())

440 break;

442 continue;

444 }

445

446

447

448

449

450

451

452

453

454

455 while (!PHIs.empty()) {

457

458

459

460 MachineOperand *InMO = nullptr;

461 MachineInstr *DefMI = nullptr;

462 for (unsigned I = 1, E = PHI->getNumOperands(); I != E; I += 2) {

463 Register InTileReg = PHI->getOperand(I).getReg();

464 MachineBasicBlock *InMBB = PHI->getOperand(I + 1).getMBB();

465 DefMI = MRI->getVRegDef(InTileReg);

467 continue;

468

469 InMO = &PHI->getOperand(I);

470 break;

471 }

472

473 if (!InMO)

474 continue;

475

476

477

480 MachineBasicBlock *InMBB = PHI->getOperand(I + 1).getMBB();

481 if (InMBB != &MBB)

482 continue;

484 InMO->setReg(DefTileReg);

485 break;

486 }

487 }

488}

489

490void X86FastPreTileConfig::convertPHIs(MachineBasicBlock &MBB) {

491 SmallVector<MachineInstr *, 8> PHIs;

492 for (MachineInstr &MI : MBB) {

493 if (.isPHI())

494 break;

496 continue;

498 }

499 while (!PHIs.empty()) {

501 VisitedPHIs.clear();

502 convertPHI(&MBB, *MI);

503 }

504}

505

506

507

508bool X86FastPreTileConfig::configBasicBlock(MachineBasicBlock &MBB) {

510 bool Change = false;

511 MachineInstr *LastShapeMI = nullptr;

512 MachineInstr *LastTileCfg = nullptr;

513 bool HasUnconfigTile = false;

514

515 auto Config = [&](MachineInstr &Before) {

516 if (CfgSS == -1)

521 LastShapeMI = nullptr;

522 Change = true;

523 };

524 auto HasTileOperand = [](MachineRegisterInfo *MRI, MachineInstr &MI) {

525 for (const MachineOperand &MO : MI.operands()) {

527 continue;

530 return true;

531 }

532 return false;

533 };

535

536 if (MI.isPHI())

537 break;

538

539

540

541

542

543

544

545

546

547

548

549

550 if (HasTileOperand(MRI, MI))

551 HasUnconfigTile = true;

552

553

554 if (MI.isCall() && HasUnconfigTile) {

558 else {

559

560

561 auto UseIt = MI.getIterator();

562 while (UseIt != MBB.end()) {

563 if (HasTileOperand(MRI, *UseIt))

564 break;

565 ++UseIt;

566 }

567 I = UseIt;

568 }

569 Config(*I);

570 HasUnconfigTile = false;

571 continue;

572 }

574 continue;

575

576

577

578

579

580

581

582

583

584

585

586

587

588

589

590

591

592

593

594

595

596

597

598

599

600

601

602

603

604

605

606

607

608

609

612 MachineOperand *RowMO = &MI.getOperand(1);

613 MachineOperand *ColMO = &MI.getOperand(2);

614 MachineInstr *RowMI = MRI->getVRegDef(RowMO->getReg());

615 MachineInstr *ColMI = MRI->getVRegDef(ColMO->getReg());

616

617

619 if (!LastShapeMI)

620 LastShapeMI = RowMI;

622 LastShapeMI = RowMI;

623 }

625 if (!LastShapeMI)

626 LastShapeMI = ColMI;

628 LastShapeMI = ColMI;

629 }

630

631

632

633 Register TileReg = MI.getOperand(0).getReg();

634 if (mayLiveOut(TileReg, LastTileCfg))

635 spill(++MI.getIterator(), TileReg, false);

636 for (MachineInstr &UseMI : MRI->use_instructions(TileReg)) {

637 if (UseMI.getParent() == &MBB) {

638

640 continue;

641

642 reload(UseMI.getIterator(), TileReg, RowMO, ColMO);

643 } else {

644

645

646 if ( UseMI .isPHI())

647 reload(UseMI.getIterator(), TileReg, RowMO, ColMO);

648 }

649 }

650 }

651

652

653 if (HasUnconfigTile) {

654 MachineInstr *Before;

655 if (LastShapeMI == nullptr || LastShapeMI->isPHI())

657 else

658 Before = &*(++LastShapeMI->getIterator());

659

660 Config(*Before);

661 }

662

663 return Change;

664}

665

666bool X86FastPreTileConfig::runOnMachineFunction(MachineFunction &MFunc) {

667 X86FI = MFunc.getInfo();

668

669 if (X86FI->getAMXProgModel() != AMXProgModelEnum::ManagedRA)

670 return false;

671

672 MF = &MFunc;

678 CfgSS = -1;

679

680 unsigned NumVirtRegs = MRI->getNumVirtRegs();

681

682 StackSlotForVirtReg.resize(NumVirtRegs);

683 MayLiveAcrossBlocks.clear();

684

685

686

687 MayLiveAcrossBlocks.resize(NumVirtRegs * 3);

688 bool Change = false;

690

691

692 for (MachineBasicBlock &MBB : MFunc)

693 canonicalizePHIs(MBB);

694

695

696

697

698 ReversePostOrderTraversal<MachineFunction *> RPOT(MF);

699 for (MachineBasicBlock *MBB : RPOT) {

700 convertPHIs(*MBB);

701 Change |= configBasicBlock(*MBB);

702 }

703

704 if (Change)

705 InitializeTileConfigStackSpace();

706

707 StackSlotForVirtReg.clear();

708 return Change;

709}

710

712 return new X86FastPreTileConfig();

713}

unsigned const MachineRegisterInfo * MRI

MachineInstrBuilder & UseMI

MachineInstrBuilder MachineInstrBuilder & DefMI

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

const TargetInstrInfo & TII

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

Promote Memory to Register

#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)

#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)

This file builds on the ADT/GraphTraits.h file to build a generic graph post order iterator.

static bool dominates(InstrPosIndexes &PosIndexes, const MachineInstr &A, const MachineInstr &B)

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

static bool isTileRegDef(MachineRegisterInfo *MRI, MachineInstr &MI)

Definition X86FastPreTileConfig.cpp:428

static ShapeT getShape(MachineRegisterInfo *MRI, Register TileReg)

Definition X86FastPreTileConfig.cpp:293

static bool isTileDef(MachineRegisterInfo *MRI, MachineInstr &MI)

Definition X86FastPreTileConfig.cpp:281

static bool isTileRegister(MachineRegisterInfo *MRI, Register Reg)

Definition X86FastPreTileConfig.cpp:269

bool test(unsigned Idx) const

void resize(unsigned N, bool t=false)

resize - Grow or shrink the bitvector.

void clear()

clear - Removes all bits from the bitvector.

FunctionPass class - This class is used to implement most global optimizations.

const MCInstrDesc & get(unsigned Opcode) const

Return the machine instruction descriptor that corresponds to the specified instruction opcode.

MachineInstrBundleIterator< const MachineInstr > const_iterator

LLVM_ABI iterator getFirstNonPHI()

Returns a pointer to the first instruction in this block that is not a PHINode instruction.

MachineInstrBundleIterator< MachineInstr > iterator

The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.

LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)

Create a new statically sized stack object, returning a nonnegative identifier to represent it.

LLVM_ABI int CreateSpillStackObject(uint64_t Size, Align Alignment)

Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...

MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...

const TargetSubtargetInfo & getSubtarget() const

getSubtarget - Return the subtarget for which this machine code is being compiled.

MachineFrameInfo & getFrameInfo()

getFrameInfo - Return the frame info object for the current function.

MachineRegisterInfo & getRegInfo()

getRegInfo - Return information about the registers currently in use.

Ty * getInfo()

getInfo - Keep track of various per-function pieces of information for backends that would like to do...

const MachineBasicBlock & front() const

const MachineInstrBuilder & addImm(int64_t Val) const

Add a new immediate operand.

const MachineInstrBuilder & addFrameIndex(int Idx) const

const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const

Add a new virtual register operand.

const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const

Representation of each machine instruction.

unsigned getOpcode() const

Returns the opcode of this MachineInstr.

const MachineBasicBlock * getParent() const

unsigned getNumOperands() const

Retuns the total number of operands.

LLVM_ABI void eraseFromParent()

Unlink 'this' from the containing basic block and delete it.

const MachineOperand & getOperand(unsigned i) const

MachineOperand class - Representation of each machine instruction operand.

bool isReg() const

isReg - Tests if this is a MO_Register operand.

LLVM_ABI void setReg(Register Reg)

Change the register this operand corresponds to.

void setIsKill(bool Val=true)

getReg - Returns the register number.

MachineRegisterInfo - Keep track of information for virtual and physical registers,...

Wrapper class representing virtual and physical registers.

unsigned virtRegIndex() const

Convert a virtual register number to a 0-based index.

constexpr bool isVirtual() const

Return true if the specified register number is in the virtual register namespace.

MachineOperand * getRow() const

MachineOperand * getCol() const

void push_back(const T &Elt)

TargetInstrInfo - Interface to description of machine instruction set.

virtual void storeRegToStackSlot(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, Register SrcReg, bool isKill, int FrameIndex, const TargetRegisterClass *RC, Register VReg, MachineInstr::MIFlag Flags=MachineInstr::NoFlags) const

Store the specified register of the given register class to the specified stack frame index.

TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...

X86MachineFunctionInfo - This class is derived from MachineFunction and contains private X86 target-s...

AMXProgModelEnum getAMXProgModel() const

unsigned getTileConfigSize() const

Align getTileConfigAlignment() const

const X86InstrInfo * getInstrInfo() const override

const X86RegisterInfo * getRegisterInfo() const override

self_iterator getIterator()

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

constexpr char Align[]

Key for Kernel::Arg::Metadata::mAlign.

This is an optimization pass for GlobalISel generic memory operations.

MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)

Builder interface. Specify how to create the initial instruction itself.

static const MachineInstrBuilder & addFrameReference(const MachineInstrBuilder &MIB, int FI, int Offset=0, bool mem=true)

addFrameReference - This function is used to add a reference to the base of an abstract object on the...

FunctionPass * createX86FastPreTileConfigPass()

Return a pass that preconfig the tile registers before fast reg allocation.

Definition X86FastPreTileConfig.cpp:711

auto reverse(ContainerTy &&C)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

static const MachineInstrBuilder & addOffset(const MachineInstrBuilder &MIB, int Offset)

static const MachineInstrBuilder & addDirectMem(const MachineInstrBuilder &MIB, Register Reg)

addDirectMem - This function is used to add a direct memory reference to the current instruction – th...

LLVM_ABI Printable printReg(Register Reg, const TargetRegisterInfo *TRI=nullptr, unsigned SubIdx=0, const MachineRegisterInfo *MRI=nullptr)

Prints virtual and physical registers with or without a TRI instance.