HexagonVLIWPacketizer.cpp Source File (original) (raw)

50#include

51#include

52#include

54using namespace llvm;

56#define DEBUG_TYPE "packets"

60 cl::desc("Disable Hexagon packetizer pass"));

64 cl::desc("Allow slot1 store and slot0 load"));

68 cl::desc("Allow non-solo packetization of volatile memory references"));

72 cl::desc("Generate all instruction with TC"));

76 cl::desc("Disable vector double new-value-stores"));

80namespace llvm {

85}

87namespace {

90 public:

91 static char ID;

93 HexagonPacketizer(bool Min = false)

105 }

106

109

112 MachineFunctionProperties::Property::NoVRegs);

113 }

114

115 private:

118 const bool Minimal = false;

119 };

120

121}

122

123char HexagonPacketizer::ID = 0;

124

126 "Hexagon Packetizer", false, false)

133

138 Minimal(Minimal) {

141

142 addMutation(std::make_uniqueHexagonSubtarget::UsrOverflowMutation());

143 addMutation(std::make_uniqueHexagonSubtarget::HVXMemLatencyMutation());

144 addMutation(std::make_uniqueHexagonSubtarget::BankConflictMutation());

145}

146

147

151 for (auto &MO : FirstI.operands()) {

152 if (!MO.isReg() || !MO.isDef())

153 continue;

156 return true;

157 }

158 return false;

159}

160

161

167 else

168 InsertPt = std::next(BundleIt).getInstrIterator();

169

171

172

173 assert(MI.isBundledWithPred());

174 if (MI.isBundledWithSucc()) {

177 } else {

178

179

180

181 MI.unbundleFromPred();

182 }

183 B.splice(InsertPt, &B, MI.getIterator());

184

185

188 unsigned Size = 0;

189 for (++I; I != E && I->isBundledWithPred(); ++I)

191

192

193

194 if (Size > 1)

195 return BundleIt;

196

197

199 MachineInstr &SingleI = *BundleIt->getNextNode();

202 BundleIt->eraseFromParent();

203 return NextIt;

204}

205

206bool HexagonPacketizer::runOnMachineFunction(MachineFunction &MF) {

207

209 MachineFunctionProperties::Property::FailsVerification);

210

213 HRI = HST.getRegisterInfo();

214 auto &MLI = getAnalysis().getLI();

215 auto *AA = &getAnalysis().getAAResults();

216 auto *MBPI =

217 &getAnalysis().getMBPI();

218

220 HII->genAllInsnTimingClasses(MF);

221

222

223 bool MinOnly = Minimal || DisablePacketizer || !HST.usePackets() ||

226

227

228 assert(Packetizer.getResourceTracker() && "Empty DFA table!");

229

230

231

232

233

234

235

236

237

240 if (MI.isKill())

241 MB.erase(&MI);

242 }

243

244

245 if (HST.isTinyCoreWithDuplex())

246 HII->translateInstrsForDup(MF, true);

247

248

249 for (auto &MB : MF) {

250 auto Begin = MB.begin(), End = MB.end();

251 while (Begin != End) {

252

253

255 while (RB != End && HII->isSchedulingBoundary(*RB, &MB, MF))

256 ++RB;

257

258

260 while (RE != End && !HII->isSchedulingBoundary(*RE, &MB, MF))

261 ++RE;

262

263 if (RE != End)

264 ++RE;

265

266 if (RB != End)

267 Packetizer.PacketizeMIs(&MB, RB, RE);

268

269 Begin = RE;

270 }

271 }

272

273

274 if (HST.isTinyCoreWithDuplex())

275 HII->translateInstrsForDup(MF, false);

276

277 Packetizer.unpacketizeSoloInstrs(MF);

278 return true;

279}

280

281

282

286}

287

290}

291

292

293

297 if (Reserve && Avail)

300 return Avail;

301}

302

304 SDep::Kind DepType, unsigned DepReg) {

305

306 if (DepReg == HRI->getRARegister())

307 return true;

308

309 if (HII->isDeallocRet(MI))

311 return true;

312

313

314

315

316

317

320 if (MO.isReg() && MO.getReg() == DepReg && !MO.isImplicit())

321 return true;

322 }

323

324 return false;

325}

326

330}

331

333 return MI.getOpcode() == Hexagon::J2_jump;

334}

335

337 switch (MI.getOpcode()) {

338 case Hexagon::Y2_barrier:

339 return true;

340 }

341 return false;

342}

343

345 return MI.getDesc().isTerminator() || MI.getDesc().isCall();

346}

347

348

352 for (auto *CSR = TRI->getCalleeSavedRegs(&MF); CSR && *CSR; ++CSR)

353 if (MI.modifiesRegister(*CSR, TRI))

354 return true;

355 return false;

356}

357

358

359

362

363

364 if (NewRC == &Hexagon::PredRegsRegClass) {

365 if (HII->isHVXVec(MI) && MI.mayStore())

366 return false;

367 return HII->isPredicated(MI) && HII->getDotNewPredOp(MI, nullptr) > 0;

368 }

369

370 return HII->mayBeNewStore(MI);

371}

372

373

374

375

380 int CurOpcode = HII->getDotCurOp(MI);

381 MI.setDesc(HII->get(CurOpcode));

382 return true;

383}

384

388 LLVM_DEBUG(dbgs() << "Cleanup packet has "; BI->dump(););

389 if (HII->isDotCurInst(*BI)) {

390 MI = BI;

391 continue;

392 }

393 if (MI) {

394 for (auto &MO : BI->operands())

395 if (MO.isReg() && MO.getReg() == MI->getOperand(0).getReg())

396 return;

397 }

398 }

399 if ()

400 return;

401

402 MI->setDesc(HII->get(HII->getNonDotCurOp(*MI)));

404}

405

406

410 if (!HII->isHVXVec(MI))

411 return false;

412 if (!HII->isHVXVec(*MII))

413 return false;

414

415

416 if (HII->isDotCurInst(MI) && !HII->mayBeCurLoad(MI))

417 return false;

418

419 if (!HII->mayBeCurLoad(MI))

420 return false;

421

422

424 return false;

425

426

428 dbgs() << "in packet\n";);

431 dbgs() << "Checking CUR against ";

432 MJ.dump();

433 });

434 Register DestReg = MI.getOperand(0).getReg();

435 bool FoundMatch = false;

436 for (auto &MO : MJ.operands())

437 if (MO.isReg() && MO.getReg() == DestReg)

438 FoundMatch = true;

439 if (!FoundMatch)

440 return false;

441

442

443

447 return false;

448 }

449

451

452 return true;

453}

454

455

456

457

462 int NewOpcode;

463 if (RC == &Hexagon::PredRegsRegClass)

464 NewOpcode = HII->getDotNewPredOp(MI, MBPI);

465 else

466 NewOpcode = HII->getDotNewOp(MI);

467 MI.setDesc(HII->get(NewOpcode));

468 return true;

469}

470

472 int NewOpcode = HII->getDotOldOp(MI);

473 MI.setDesc(HII->get(NewOpcode));

474 return true;

475}

476

478 unsigned Opc = MI.getOpcode();

479 switch (Opc) {

480 case Hexagon::S2_storerd_io:

481 case Hexagon::S2_storeri_io:

482 case Hexagon::S2_storerh_io:

483 case Hexagon::S2_storerb_io:

484 break;

485 default:

487 }

491 if (HII->isValidOffset(Opc, NewOff, HRI)) {

492 Off.setImm(NewOff);

493 return true;

494 }

495 return false;

496}

497

499 unsigned Opc = MI.getOpcode();

500 switch (Opc) {

501 case Hexagon::S2_storerd_io:

502 case Hexagon::S2_storeri_io:

503 case Hexagon::S2_storerh_io:

504 case Hexagon::S2_storerb_io:

505 break;

506 default:

508 }

512}

513

514

515

520

521 unsigned BPI, OPI;

522 if (!HII->getBaseAndOffsetPosition(MI, BPI, OPI))

523 return false;

524 unsigned BPJ, OPJ;

525 if (!HII->getBaseAndOffsetPosition(MJ, BPJ, OPJ))

526 return false;

527 Register Reg = MI.getOperand(BPI).getReg();

529 return false;

530

531

532

533 for (const auto &PI : SUI->Preds)

535 (PI.getKind() != SDep::Data || PI.getReg() != Reg))

536 return false;

537 int Incr;

538 if (!HII->getIncrementValue(MJ, Incr))

539 return false;

540

541 int64_t Offset = MI.getOperand(OPI).getImm();

542 if (!HII->isValidOffset(MI.getOpcode(), Offset+Incr, HRI))

543 return false;

544

545 MI.getOperand(OPI).setImm(Offset + Incr);

546 ChangedOffset = Offset;

547 return true;

548}

549

550

551

553 unsigned BP, OP;

554 if (!HII->getBaseAndOffsetPosition(MI, BP, OP))

555 llvm_unreachable("Unable to find base and offset operands.");

556 MI.getOperand(OP).setImm(ChangedOffset);

557}

558

564

565

566

574}

575

579#ifndef NDEBUG

580

581

582

584 for (auto &MO : MI.operands())

585 if (MO.isReg() && MO.isDef())

586 DefRegsSet.insert(MO.getReg());

587

588 for (auto &MO : MI.operands())

589 if (MO.isReg() && MO.isUse() && DefRegsSet.count(MO.getReg()))

590 return MO;

591#else

592 if (MI.mayLoad()) {

594

595 assert(Op1.isReg() && "Post increment operand has be to a register.");

596 return Op1;

597 }

598 if (MI.getDesc().mayStore()) {

600

601 assert(Op0.isReg() && "Post increment operand has be to a register.");

602 return Op0;

603 }

604#endif

605

606 llvm_unreachable("mayLoad or mayStore not set for Post Increment operation");

607}

608

609

611

612 return MI.getOperand(MI.getNumOperands()-1);

613}

614

616 unsigned Opc = MI.getOpcode();

617 switch (Opc) {

618 case Hexagon::L4_loadrd_ap:

619 case Hexagon::L4_loadrb_ap:

620 case Hexagon::L4_loadrh_ap:

621 case Hexagon::L4_loadrub_ap:

622 case Hexagon::L4_loadruh_ap:

623 case Hexagon::L4_loadri_ap:

624 return true;

625 }

626 return false;

627}

628

631 return MI.getOperand(1);

632}

633

634

635

636

637

638

639

640

641

642

643

644

645

646

647

648

649

650

652 const MachineInstr &PacketMI, unsigned DepReg) {

653

654 if (!HII->mayBeNewStore(MI))

655 return false;

656

657

659 if (Val.isReg() && Val.getReg() != DepReg)

660 return false;

661

663

664

666

667 if (PacketRC == &Hexagon::DoubleRegsRegClass)

668 return false;

669

670

671

675 return false;

676 }

677

678

679

680 if (HII->isPostIncrement(MI) &&

682 return false;

683 }

684

685 if (HII->isPostIncrement(PacketMI) && PacketMI.mayLoad() &&

687

688

689

690

691

692 return false;

693 }

694

696 return false;

697

698

699

700 if (HII->isPredicated(PacketMI)) {

701 if (!HII->isPredicated(MI))

702 return false;

703

704

705

706 unsigned predRegNumSrc = 0;

707 unsigned predRegNumDst = 0;

709

710

711 for (auto &MO : PacketMI.operands()) {

712 if (!MO.isReg())

713 continue;

714 predRegNumSrc = MO.getReg();

715 predRegClass = HRI->getMinimalPhysRegClass(predRegNumSrc);

716 if (predRegClass == &Hexagon::PredRegsRegClass)

717 break;

718 }

719 assert((predRegClass == &Hexagon::PredRegsRegClass) &&

720 "predicate register not found in a predicated PacketMI instruction");

721

722

723 for (auto &MO : MI.operands()) {

724 if (!MO.isReg())

725 continue;

726 predRegNumDst = MO.getReg();

727 predRegClass = HRI->getMinimalPhysRegClass(predRegNumDst);

728 if (predRegClass == &Hexagon::PredRegsRegClass)

729 break;

730 }

731 assert((predRegClass == &Hexagon::PredRegsRegClass) &&

732 "predicate register not found in a predicated MI instruction");

733

734

735

736

737

738

739

740

741

742 if (predRegNumDst != predRegNumSrc ||

743 HII->isDotNewInst(PacketMI) != HII->isDotNewInst(MI) ||

745 return false;

746 }

747

748

749

750

751

752

753

754

755

756 unsigned StartCheck = 0;

757

761

762

763

764

765 if (&TempMI != &PacketMI && !StartCheck)

766 continue;

767

768 StartCheck = 1;

769 if (&TempMI == &PacketMI)

770 continue;

771

772 for (auto &MO : MI.operands())

774 return false;

775 }

776

777

778

779

780

781

782

783 if (!HII->isPostIncrement(MI)) {

784 for (unsigned opNum = 0; opNum < MI.getNumOperands()-1; opNum++) {

787 return false;

788 }

789 }

790

791

792

793

794

795 for (auto &MO : PacketMI.operands()) {

796 if (MO.isRegMask() && MO.clobbersPhysReg(DepReg))

797 return false;

798 if (!MO.isReg() || !MO.isDef() || !MO.isImplicit())

799 continue;

801 if (R == DepReg || HRI->isSuperRegister(DepReg, R))

802 return false;

803 }

804

805

806

807

808

809

810 for (auto &MO : MI.operands()) {

811 if (MO.isReg() && MO.isUse() && MO.isImplicit() && MO.getReg() == DepReg)

812 return false;

813 }

814

815

816 return true;

817}

818

819

821 const SUnit *PacketSU, unsigned DepReg,

823 if (!HII->mayBeNewStore(MI))

824 return false;

825

826

829 return true;

830

831

832

833 return false;

834}

835

837 unsigned DepReg) {

838 for (auto &MO : I.operands()) {

839 if (CheckDef && MO.isRegMask() && MO.clobbersPhysReg(DepReg))

840 return true;

841 if (!MO.isReg() || MO.getReg() != DepReg || !MO.isImplicit())

842 continue;

843 if (CheckDef == MO.isDef())

844 return true;

845 }

846 return false;

847}

848

849

853

854 if (HII->isDotNewInst(MI) && !HII->mayBeNewStore(MI))

855 return false;

856

858 return false;

859

861

862

864 return false;

865

866

867

869 return false;

870

871

872

875 return false;

876

880 return false;

881

882

883 if (RC == &Hexagon::PredRegsRegClass)

884 return HII->predCanBeUsedAsDotNew(PI, DepReg);

885

886 if (RC != &Hexagon::PredRegsRegClass && !HII->mayBeNewStore(MI))

887 return false;

888

889

890

891 int NewOpcode = (RC != &Hexagon::PredRegsRegClass) ? HII->getDotNewOp(MI) :

892 HII->getDotNewPredOp(MI, MBPI);

897 if (!ResourcesAvailable)

898 return false;

899

900

902 return false;

903

904 return true;

905}

906

907

908

909

910

911

912

913

914

915

916

917

918

919

921 unsigned DepReg) {

923

925

926 if (!HII->isPredicated(*I))

927 continue;

928

929

931

932

933

934

935 if (PacketSU->isSucc(PacketSUDep)) {

936 for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {

937 auto &Dep = PacketSU->Succs[i];

938 if (Dep.getSUnit() == PacketSUDep && Dep.getKind() == SDep::Anti &&

939 Dep.getReg() == DepReg)

940 return true;

941 }

942 }

943 }

944

945 return false;

946}

947

948

951

952

954

955 for (auto &Op : MI.operands()) {

956 if (Op.isReg() && Op.getReg() && Op.isUse() &&

957 Hexagon::PredRegsRegClass.contains(Op.getReg()))

958 return Op.getReg();

959 }

960

962 return 0;

963}

964

965

966

969

970

973 return false;

974

975

977

978

979

980

981

982

983

984

985

986

987

988

989

990

991

992

993

995

997

998

999 if (PacketSU->isSucc(SU)) {

1000 for (unsigned i = 0; i < PacketSU->Succs.size(); ++i) {

1001 auto Dep = PacketSU->Succs[i];

1002

1003

1004

1005

1006 if (Dep.getSUnit() == SU && Dep.getKind() == SDep::Data &&

1007 Hexagon::PredRegsRegClass.contains(Dep.getReg())) {

1008

1009

1010

1011

1012

1014 return false;

1015 }

1016 }

1017 }

1018 }

1019

1020

1021

1022

1023

1026 return PReg1 == PReg2 &&

1027 Hexagon::PredRegsRegClass.contains(PReg1) &&

1028 Hexagon::PredRegsRegClass.contains(PReg2) &&

1030 HII->isDotNewInst(MI1) == HII->isDotNewInst(MI2);

1031}

1032

1033

1036 PromotedToDotNew = false;

1037 GlueToNewValueJump = false;

1038 GlueAllocframeStore = false;

1039 FoundSequentialDependence = false;

1041}

1042

1043

1046 if (MI.isDebugInstr())

1047 return true;

1048

1049 if (MI.isCFIInstruction())

1050 return false;

1051

1052

1053 if (MI.isInlineAsm())

1054 return false;

1055

1056 if (MI.isImplicitDef())

1057 return false;

1058

1059

1060

1064}

1065

1067

1068 if (MI.isBundle())

1069 return true;

1070

1071 if (MI.isEHLabel() || MI.isCFIInstruction())

1072 return true;

1073

1074

1075

1076

1077

1078

1080 return true;

1081

1083 return true;

1084

1085 if (HII->isSolo(MI))

1086 return true;

1087

1088 if (MI.getOpcode() == Hexagon::PATCHABLE_FUNCTION_ENTER ||

1089 MI.getOpcode() == Hexagon::PATCHABLE_FUNCTION_EXIT ||

1090 MI.getOpcode() == Hexagon::PATCHABLE_TAIL_CALL)

1091 return true;

1092

1093 if (MI.getOpcode() == Hexagon::A2_nop)

1094 return true;

1095

1096 return false;

1097}

1098

1099

1100

1101

1102

1103

1104

1105

1111 return true;

1112

1113

1114

1116 return true;

1117

1118

1119

1120

1121

1122 if (MI.isInlineAsm())

1125

1126

1128 return true;

1129

1130 switch (MI.getOpcode()) {

1131 case Hexagon::S2_storew_locked:

1132 case Hexagon::S4_stored_locked:

1133 case Hexagon::L2_loadw_locked:

1134 case Hexagon::L4_loadd_locked:

1135 case Hexagon::Y2_dccleana:

1136 case Hexagon::Y2_dccleaninva:

1137 case Hexagon::Y2_dcinva:

1138 case Hexagon::Y2_dczeroa:

1139 case Hexagon::Y4_l2fetch:

1140 case Hexagon::Y5_l2fetch: {

1141

1142

1143

1144 unsigned TJ = HII.getType(MJ);

1148 return true;

1149 break;

1150 }

1151 default:

1152 break;

1153 }

1154

1155

1156

1157 return false;

1158}

1159

1160

1164}

1165

1167 for (auto &B : MF) {

1170 if (MI.isBundle())

1171 BundleIt = MI.getIterator();

1172 if (.isInsideBundle())

1173 continue;

1174

1175

1176

1177

1178

1179

1180

1181 bool InsertBeforeBundle;

1182 if (MI.isInlineAsm())

1184 else if (MI.isDebugInstr())

1185 InsertBeforeBundle = true;

1186 else

1187 continue;

1188

1189 BundleIt = moveInstrOut(MI, BundleIt, InsertBeforeBundle);

1190 }

1191 }

1192}

1193

1194

1196 unsigned Opc = MI.getOpcode();

1197 switch (Opc) {

1198 case Hexagon::Y2_barrier:

1199 case Hexagon::Y2_dcfetchbo:

1200 case Hexagon::Y4_l2fetch:

1201 case Hexagon::Y5_l2fetch:

1202 return true;

1203 }

1204 return false;

1205}

1206

1209

1210

1211

1212 if (I.isCall() || J.isCall())

1213 return false;

1214 if (HII->isPredicated(I) || HII->isPredicated(J))

1215 return false;

1216

1217 BitVector DeadDefs(Hexagon::NUM_TARGET_REGS);

1218 for (auto &MO : I.operands()) {

1219 if (!MO.isReg() || !MO.isDef() || !MO.isDead())

1220 continue;

1221 DeadDefs[MO.getReg()] = true;

1222 }

1223

1224 for (auto &MO : J.operands()) {

1225 if (!MO.isReg() || !MO.isDef() || !MO.isDead())

1226 continue;

1228 if (R != Hexagon::USR_OVF && DeadDefs[R])

1229 return true;

1230 }

1231 return false;

1232}

1233

1236

1237

1238 if ((HII->isSaveCalleeSavedRegsCall(I) &&

1240 (HII->isSaveCalleeSavedRegsCall(J) &&

1242 return true;

1243

1244

1246 return true;

1247

1248

1249

1250

1251 auto isBadForLoopN = [this] (const MachineInstr &MI) -> bool {

1252 if (MI.isCall() || HII->isDeallocRet(MI) || HII->isNewValueJump(MI))

1253 return true;

1254 if (HII->isPredicated(MI) && HII->isPredicatedNew(MI) && HII->isJumpR(MI))

1255 return true;

1256 return false;

1257 };

1258

1259 if (HII->isLoopN(I) && isBadForLoopN(J))

1260 return true;

1261 if (HII->isLoopN(J) && isBadForLoopN(I))

1262 return true;

1263

1264

1265

1266 return HII->isDeallocRet(I) &&

1268}

1269

1272

1273

1274

1275

1276

1277

1278

1279

1280

1281

1282

1284 if (!OpJ.isRegMask())

1285 continue;

1286 assert((J.isCall() || HII->isTailCall(J)) && "Regmask on a non-call");

1288 if (OpI.isReg()) {

1289 if (OpJ.clobbersPhysReg(OpI.getReg()))

1290 return true;

1291 } else if (OpI.isRegMask()) {

1292

1293 return true;

1294 }

1295 }

1296 }

1297 return false;

1298}

1299

1303 bool StoreI = I.mayStore(), StoreJ = J.mayStore();

1304 if ((SysI && StoreJ) || (SysJ && StoreI))

1305 return true;

1306

1307 if (StoreI && StoreJ) {

1308 if (HII->isNewValueInst(J) || HII->isMemOp(J) || HII->isMemOp(I))

1309 return true;

1310 } else {

1311

1312

1313 bool MopStI = HII->isMemOp(I) || StoreI;

1314 bool MopStJ = HII->isMemOp(J) || StoreJ;

1315 if (MopStI && MopStJ)

1316 return true;

1317 }

1318

1319 return (StoreJ && HII->isDeallocRet(I)) || (StoreI && HII->isDeallocRet(J));

1320}

1321

1322

1323

1324

1329

1330

1332 IgnoreDepMIs.clear();

1333

1335

1336

1338

1340 return false;

1341

1344 return false;

1345

1346

1347

1348

1349

1352 return false;

1353

1354

1355

1356

1359 return false;

1360

1361

1363 ++NextMII;

1364 if (NextMII != I.getParent()->end() && HII->isNewValueJump(*NextMII)) {

1366

1367 bool secondRegMatch = false;

1370

1371 if (NOp1.isReg() && I.getOperand(0).getReg() == NOp1.getReg())

1372 secondRegMatch = true;

1373

1375

1376 if (PI->isCall()) {

1378 break;

1379 }

1380

1381

1382

1383

1384

1385

1386

1387

1388 if (PI->getOpcode() == Hexagon::S2_allocframe || PI->mayStore() ||

1389 HII->isLoopN(*PI)) {

1391 break;

1392 }

1393

1394 const MachineOperand &OpR = secondRegMatch ? NOp0 : NOp1;

1395 if (OpR.isReg() && PI->modifiesRegister(OpR.getReg(), HRI)) {

1397 break;

1398 }

1399 }

1400

1401 GlueToNewValueJump = true;

1403 return false;

1404 }

1405

1406

1407 if (!SUJ->isSucc(SUI))

1408 return true;

1409

1410 for (unsigned i = 0; i < SUJ->Succs.size(); ++i) {

1411 if (FoundSequentialDependence)

1412 break;

1413

1414 if (SUJ->Succs[i].getSUnit() != SUI)

1415 continue;

1416

1418

1419

1420

1421

1422

1423

1424

1425

1426

1427

1428

1429

1430

1431

1432

1433 unsigned DepReg = 0;

1436 DepReg = SUJ->Succs[i].getReg();

1437 RC = HRI->getMinimalPhysRegClass(DepReg);

1438 }

1439

1440 if (I.isCall() || HII->isJumpR(I) || I.isReturn() || HII->isTailCall(I)) {

1442 continue;

1444 continue;

1445 }

1446

1450 continue;

1451 }

1452

1453

1454 if (DepType == SDep::Data && HII->isDotCurInst(J)) {

1455 if (HII->isHVXVec(I))

1456 continue;

1457 }

1458

1459

1463 PromotedToDotNew = true;

1465 FoundSequentialDependence = true;

1466 continue;

1467 }

1468 }

1469 if (HII->isNewValueJump(I))

1470 continue;

1471 }

1472

1473

1474

1475 if (HII->isPredicated(I) && HII->isPredicated(J) &&

1477

1478

1479

1480

1481

1482

1483

1484

1485

1486

1487

1488

1489 auto Itr = find(IgnoreDepMIs, &J);

1490 if (Itr != IgnoreDepMIs.end()) {

1492 return false;

1493 }

1494 IgnoreDepMIs.push_back(&I);

1495 continue;

1496 }

1497

1498

1499

1502 continue;

1503

1504

1505

1506 if (I.isConditionalBranch() && DepType != SDep::Data &&

1508 continue;

1509

1511 FoundSequentialDependence = true;

1512 break;

1513 }

1514

1515

1516

1517

1518

1519

1520

1524 if (OrdRefs) {

1525 FoundSequentialDependence = true;

1526 break;

1527 }

1528 }

1529

1531 bool LoadI = I.mayLoad(), StoreI = I.mayStore();

1532 bool NVStoreJ = HII->isNewValueStore(J);

1533 bool NVStoreI = HII->isNewValueStore(I);

1534 bool IsVecJ = HII->isHVXVec(J);

1535 bool IsVecI = HII->isHVXVec(I);

1536

1537

1538

1539 if (LoadJ && LoadI && HII->isPureSlot0(J)) {

1540 FoundSequentialDependence = true;

1541 break;

1542 }

1543

1545 ((LoadJ && StoreI && !NVStoreI) ||

1546 (StoreJ && LoadI && !NVStoreJ)) &&

1547 (J.getOpcode() != Hexagon::S2_allocframe &&

1548 I.getOpcode() != Hexagon::S2_allocframe) &&

1549 (J.getOpcode() != Hexagon::L2_deallocframe &&

1550 I.getOpcode() != Hexagon::L2_deallocframe) &&

1551 (!HII->isMemOp(J) && !HII->isMemOp(I)) && (!IsVecJ && !IsVecI))

1553 else

1554 if (StoreJ && LoadI && alias(J, I)) {

1555 FoundSequentialDependence = true;

1556 break;

1557 }

1558

1559 if (!StoreJ)

1560 if (!LoadJ || (!LoadI && !StoreI)) {

1561

1562

1563 FoundSequentialDependence = true;

1564 break;

1565 }

1566

1567

1568

1569

1570 continue;

1571 }

1572

1573

1574

1575

1576

1578 unsigned Opc = I.getOpcode();

1579 switch (Opc) {

1580 case Hexagon::S2_storerd_io:

1581 case Hexagon::S2_storeri_io:

1582 case Hexagon::S2_storerh_io:

1583 case Hexagon::S2_storerb_io:

1585

1586

1587

1588

1590 if (GlueAllocframeStore)

1591 continue;

1592 }

1593 break;

1594 default:

1595 break;

1596 }

1597 }

1598

1599

1600

1601

1602

1603

1604

1606

1607

1608

1610 if (Op.isReg() && Op.isDef()) {

1613 continue;

1614 } else if (.isRegMask()) {

1615

1616 continue;

1617 }

1618 FoundSequentialDependence = true;

1619 break;

1620 }

1621 }

1622

1623

1624

1625

1626

1627

1628

1630 FoundSequentialDependence = true;

1631 break;

1632 }

1633 }

1634

1635 if (FoundSequentialDependence) {

1637 return false;

1638 }

1639

1640 return true;

1641}

1642

1647

1649

1651 return true;

1652

1653

1654

1655 if (PromotedToDotNew)

1657

1659

1660

1661

1662 if (GlueAllocframeStore) {

1664 GlueAllocframeStore = false;

1665 }

1666

1669

1670 if (GlueToNewValueJump) {

1671

1672

1673 GlueToNewValueJump = false;

1674 return false;

1675 }

1676

1677 if (!Coexist)

1678 return false;

1679

1681 FoundSequentialDependence = false;

1683 return true;

1684 }

1685

1686 return false;

1687}

1688

1689

1691 bool FoundLoad = false;

1692 bool FoundStore = false;

1693

1695 unsigned Opc = MJ->getOpcode();

1696 if (Opc == Hexagon::S2_allocframe || Opc == Hexagon::L2_deallocframe)

1697 continue;

1698 if (HII->isMemOp(*MJ))

1699 continue;

1700 if (MJ->mayLoad())

1701 FoundLoad = true;

1702 if (MJ->mayStore() && !HII->isNewValueStore(*MJ))

1703 FoundStore = true;

1704 }

1705 return FoundLoad && FoundStore;

1706}

1707

1708

1713

1715 PacketStalls = false;

1716 PacketStallCycles = 0;

1717 }

1719 PacketStallCycles = std::max(PacketStallCycles, calcStall(MI));

1720

1721 if (MI.isImplicitDef()) {

1722

1723

1725 return MII;

1726 }

1728

1729 bool ExtMI = HII->isExtended(MI) || HII->isConstExtended(MI);

1730 bool Good = true;

1731

1732 if (GlueToNewValueJump) {

1734

1735

1736

1737

1739 if (ExtMI)

1741

1742 bool ExtNvjMI = HII->isExtended(NvjMI) || HII->isConstExtended(NvjMI);

1743 if (Good) {

1746 else

1747 Good = false;

1748 }

1749 if (Good && ExtNvjMI)

1751

1752 if (!Good) {

1756 if (ExtMI) {

1759 }

1762 if (ExtNvjMI) {

1765 }

1766 }

1769 return MII;

1770 }

1771

1775 if (PromotedToDotNew)

1777 if (GlueAllocframeStore) {

1779 GlueAllocframeStore = false;

1780 }

1783 }

1784

1786 return MII;

1787}

1788

1791

1794 dbgs() << "Finalizing packet:\n";

1795 unsigned Idx = 0;

1796 for (MachineInstr *MI : CurrentPacketMIs) {

1797 unsigned R = ResourceTracker->getUsedResources(Idx++);

1798 dbgs() << " * [res:0x" << utohexstr(R) << "] " << *MI;

1799 }

1800 }

1801 });

1802

1803 bool memShufDisabled = getmemShufDisabled();

1804 if (memShufDisabled && !foundLSInPacket()) {

1805 setmemShufDisabled(false);

1806 LLVM_DEBUG(dbgs() << " Not added to NoShufPacket\n");

1807 }

1808 memShufDisabled = getmemShufDisabled();

1809

1810 OldPacketMIs.clear();

1813 for (auto &I : make_range(HII->expandVGatherPseudo(*MI), NextMI))

1814 OldPacketMIs.push_back(&I);

1815 }

1816 CurrentPacketMIs.clear();

1817

1818 if (OldPacketMIs.size() > 1) {

1822 auto BundleMII = std::prev(FirstMI);

1823 if (memShufDisabled)

1824 HII->setBundleNoShuf(BundleMII);

1825

1826 setmemShufDisabled(false);

1827 }

1828

1829 PacketHasDuplex = false;

1830 PacketHasSLOT0OnlyInsn = false;

1831 ResourceTracker->clearResources();

1833}

1834

1836 if (Minimal)

1837 return false;

1838

1840 return false;

1841

1842

1843

1844 auto &HST = MI.getParent()->getParent()->getSubtarget<HexagonSubtarget>();

1845

1846

1847

1848

1849

1850

1851

1852 if (HST.isTinyCoreWithDuplex() && CurrentPacketMIs.size() > 0 &&

1853 !PacketHasDuplex) {

1854

1856 PacketHasSLOT0OnlyInsn |= HII->isPureSlot0(*MJ);

1857

1858 int Opcode = HII->getDuplexOpcode(MI, false);

1859 if (Opcode >= 0) {

1860

1862 if (HII->isDuplexPair(MI, *MJ) && !PacketHasSLOT0OnlyInsn) {

1863 PacketHasDuplex = true;

1864 return true;

1865 }

1866 }

1867

1868

1870 MIRef.setDesc(HII->get(Opcode));

1872 }

1873 }

1874

1875 return true;

1876}

1877

1878

1880

1881

1882

1883

1884

1885

1886

1887 if (!OldPacketMIs.empty()) {

1888 auto *OldBB = OldPacketMIs.front()->getParent();

1889 auto *ThisBB = I.getParent();

1891 return 0;

1892 }

1893

1895 if (!SUI)

1896 return 0;

1897

1898

1899

1900

1901

1902

1903

1904

1905

1906

1907

1908

1909

1910

1911

1912

1913

1914

1915

1916

1917

1918

1921 for (auto &Pred : SUI->Preds)

1922 if (Pred.getSUnit() == SUJ)

1923 if ((Pred.getLatency() == 0 && Pred.isAssignedRegDep()) ||

1924 HII->isNewValueJump(I) || HII->isToBeScheduledASAP(*J, I))

1925 return 0;

1926 }

1927

1928

1929

1930 for (auto *J : OldPacketMIs) {

1932 for (auto &Pred : SUI->Preds)

1933 if (Pred.getSUnit() == SUJ && Pred.getLatency() > 1)

1934 return Pred.getLatency();

1935 }

1936

1937 return 0;

1938}

1939

1943 return false;

1944

1945 if (PacketStalls)

1946 return Latency > PacketStallCycles;

1947 return true;

1948}

1949

1950

1951

1952

1953

1955 return new HexagonPacketizer(Minimal);

1956}

This file implements the BitVector class.

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")

This file defines the DenseSet and SmallDenseSet classes.

cl::opt< bool > ScheduleInlineAsm("hexagon-sched-inline-asm", cl::Hidden, cl::init(false), cl::desc("Do not consider inline-asm a scheduling/" "packetization boundary."))

#define HEXAGON_LRFP_SIZE

static bool cannotCoexistAsymm(const MachineInstr &MI, const MachineInstr &MJ, const HexagonInstrInfo &HII)

static bool isDirectJump(const MachineInstr &MI)

static MachineBasicBlock::iterator moveInstrOut(MachineInstr &MI, MachineBasicBlock::iterator BundleIt, bool Before)

static bool isRegDependence(const SDep::Kind DepType)

static const MachineOperand & getStoreValueOperand(const MachineInstr &MI)

static cl::opt< bool > EnableGenAllInsnClass("enable-gen-insn", cl::Hidden, cl::desc("Generate all instruction with TC"))

static bool isControlFlow(const MachineInstr &MI)

hexagon Hexagon Packetizer

static cl::opt< bool > DisableVecDblNVStores("disable-vecdbl-nv-stores", cl::Hidden, cl::desc("Disable vector double new-value-stores"))

static PredicateKind getPredicateSense(const MachineInstr &MI, const HexagonInstrInfo *HII)

Returns true if an instruction is predicated on p0 and false if it's predicated on !...

static unsigned getPredicatedRegister(MachineInstr &MI, const HexagonInstrInfo *QII)

Gets the predicate register of a predicated instruction.

static cl::opt< bool > DisablePacketizer("disable-packetizer", cl::Hidden, cl::desc("Disable Hexagon packetizer pass"))

static cl::opt< bool > Slot1Store("slot1-store-slot0-load", cl::Hidden, cl::init(true), cl::desc("Allow slot1 store and slot0 load"))

static cl::opt< bool > PacketizeVolatiles("hexagon-packetize-volatiles", cl::Hidden, cl::init(true), cl::desc("Allow non-solo packetization of volatile memory references"))

cl::opt< bool > ScheduleInlineAsm

static bool hasWriteToReadDep(const MachineInstr &FirstI, const MachineInstr &SecondI, const TargetRegisterInfo *TRI)

static bool doesModifyCalleeSavedReg(const MachineInstr &MI, const TargetRegisterInfo *TRI)

Returns true if the instruction modifies a callee-saved register.

static bool isLoadAbsSet(const MachineInstr &MI)

static const MachineOperand & getAbsSetOperand(const MachineInstr &MI)

static const MachineOperand & getPostIncrementOperand(const MachineInstr &MI, const HexagonInstrInfo *HII)

static bool isImplicitDependency(const MachineInstr &I, bool CheckDef, unsigned DepReg)

static bool isSchedBarrier(const MachineInstr &MI)

static bool isSystemInstr(const MachineInstr &MI)

unsigned const TargetRegisterInfo * TRI

static unsigned getReg(const MCDisassembler *D, unsigned RC, unsigned RegNo)

uint64_t IntrinsicInst * II

#define INITIALIZE_PASS_DEPENDENCY(depName)

#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)

#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

A wrapper pass to provide the legacy pass manager access to a suitably prepared AAResults object.

Represent the analysis usage information of a pass.

AnalysisUsage & addRequired()

AnalysisUsage & addPreserved()

Add the specified Pass class to the set of analyses preserved by this pass.

void setPreservesCFG()

This function should be called by the pass, iff they do not:

const InstrItineraryData * getInstrItins() const

bool canReserveResources(const MCInstrDesc *MID)

void reserveResources(const MCInstrDesc *MID)

This class represents an Operation in the Expression.

Implements a dense probed hash-table based set.

Dependence - This class represents a dependence between two memory memory references in a function.

FunctionPass class - This class is used to implement most global optimizations.

bool isPredicated(const MachineInstr &MI) const override

Returns true if the instruction is already predicated.

bool isHVXMemWithAIndirect(const MachineInstr &I, const MachineInstr &J) const

bool isRestrictNoSlot1Store(const MachineInstr &MI) const

bool isPureSlot0(const MachineInstr &MI) const

bool isPostIncrement(const MachineInstr &MI) const override

Return true for post-incremented instructions.

uint64_t getType(const MachineInstr &MI) const

bool isPredicatedTrue(const MachineInstr &MI) const

bool isNewValueStore(const MachineInstr &MI) const

bool arePredicatesComplements(MachineInstr &MI1, MachineInstr &MI2)

bool updateOffset(SUnit *SUI, SUnit *SUJ)

Return true if we can update the offset in MI so that MI and MJ can be packetized together.

void endPacket(MachineBasicBlock *MBB, MachineBasicBlock::iterator MI) override

bool isCallDependent(const MachineInstr &MI, SDep::Kind DepType, unsigned DepReg)

bool promoteToDotCur(MachineInstr &MI, SDep::Kind DepType, MachineBasicBlock::iterator &MII, const TargetRegisterClass *RC)

bool promoteToDotNew(MachineInstr &MI, SDep::Kind DepType, MachineBasicBlock::iterator &MII, const TargetRegisterClass *RC)

bool isLegalToPacketizeTogether(SUnit *SUI, SUnit *SUJ) override

bool canPromoteToDotCur(const MachineInstr &MI, const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII, const TargetRegisterClass *RC)

void useCalleesSP(MachineInstr &MI)

bool demoteToDotOld(MachineInstr &MI)

bool cannotCoexist(const MachineInstr &MI, const MachineInstr &MJ)

const MachineLoopInfo * MLI

bool isSoloInstruction(const MachineInstr &MI) override

bool isLegalToPruneDependencies(SUnit *SUI, SUnit *SUJ) override

void initPacketizerState() override

bool hasControlDependence(const MachineInstr &I, const MachineInstr &J)

bool restrictingDepExistInPacket(MachineInstr &, unsigned)

bool producesStall(const MachineInstr &MI)

void undoChangedOffset(MachineInstr &MI)

Undo the changed offset.

bool hasDualStoreDependence(const MachineInstr &I, const MachineInstr &J)

unsigned int calcStall(const MachineInstr &MI)

bool canPromoteToDotNew(const MachineInstr &MI, const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII, const TargetRegisterClass *RC)

bool canPromoteToNewValue(const MachineInstr &MI, const SUnit *PacketSU, unsigned DepReg, MachineBasicBlock::iterator &MII)

bool ignorePseudoInstruction(const MachineInstr &MI, const MachineBasicBlock *MBB) override

void unpacketizeSoloInstrs(MachineFunction &MF)

const MachineBranchProbabilityInfo * MBPI

A handle to the branch probability pass.

bool shouldAddToPacket(const MachineInstr &MI) override

bool canReserveResourcesForConstExt()

bool useCallersSP(MachineInstr &MI)

bool canPromoteToNewValueStore(const MachineInstr &MI, const MachineInstr &PacketMI, unsigned DepReg)

bool tryAllocateResourcesForConstExt(bool Reserve)

void setmemShufDisabled(bool val)

void reserveResourcesForConstExt()

MachineBasicBlock::iterator addToPacket(MachineInstr &MI) override

bool hasDeadDependence(const MachineInstr &I, const MachineInstr &J)

bool isNewifiable(const MachineInstr &MI, const TargetRegisterClass *NewRC)

bool hasRegMaskDependence(const MachineInstr &I, const MachineInstr &J)

const HexagonInstrInfo * getInstrInfo() const override

bool hasV60OpsOnly() const

const InstrStage * beginStage(unsigned ItinClassIndx) const

Return the first stage of the itinerary.

LoopT * getLoopFor(const BlockT *BB) const

Return the inner most loop that BB lives in.

Describe properties that are true of each instruction in the target description file.

unsigned getSchedClass() const

Return the scheduling class for this instruction.

Instructions::iterator instr_iterator

Instructions::const_iterator const_instr_iterator

Analysis pass which computes a MachineDominatorTree.

uint64_t getStackSize() const

Return the number of bytes that must be allocated to hold all of the fixed size frame objects.

MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...

void getAnalysisUsage(AnalysisUsage &AU) const override

getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.

virtual bool runOnMachineFunction(MachineFunction &MF)=0

runOnMachineFunction - This method must be overloaded to perform the desired machine code transformat...

virtual MachineFunctionProperties getRequiredProperties() const

Properties which a MachineFunction may have at a given point in time.

MachineFunctionProperties & set(Property P)

const TargetSubtargetInfo & getSubtarget() const

getSubtarget - Return the subtarget for which this machine code is being compiled.

MachineInstr * CreateMachineInstr(const MCInstrDesc &MCID, DebugLoc DL, bool NoImplicit=false)

CreateMachineInstr - Allocate a new MachineInstr.

MachineFrameInfo & getFrameInfo()

getFrameInfo - Return the frame info object for the current function.

void deleteMachineInstr(MachineInstr *MI)

DeleteMachineInstr - Delete the given MachineInstr.

Function & getFunction()

Return the LLVM function that this machine code represents.

const MachineFunctionProperties & getProperties() const

Get the function properties.

instr_iterator getInstrIterator() const

Representation of each machine instruction.

unsigned getOpcode() const

Returns the opcode of this MachineInstr.

bool isTerminator(QueryType Type=AnyInBundle) const

Returns true if this instruction part of the terminator for a basic block.

bool isImplicitDef() const

bool readsRegister(Register Reg, const TargetRegisterInfo *TRI) const

Return true if the MachineInstr reads the specified register.

bool isBarrier(QueryType Type=AnyInBundle) const

Returns true if the specified instruction stops control flow from executing the instruction immediate...

bool isCall(QueryType Type=AnyInBundle) const

bool isBranch(QueryType Type=AnyInBundle) const

Returns true if this is a conditional, unconditional, or indirect branch.

bool modifiesRegister(Register Reg, const TargetRegisterInfo *TRI) const

Return true if the MachineInstr modifies (fully define or partially define) the specified register.

void unbundleFromPred()

Break bundle above this instruction.

bool mayLoad(QueryType Type=AnyInBundle) const

Return true if this instruction could possibly read memory.

const MCInstrDesc & getDesc() const

Returns the target instruction descriptor of this MachineInstr.

void setDesc(const MCInstrDesc &TID)

Replace the instruction descriptor (thus opcode) of the current instruction with a new one.

iterator_range< mop_iterator > operands()

bool hasOrderedMemoryRef() const

Return true if this instruction may have an ordered or volatile memory reference, or if the informati...

bool mayStore(QueryType Type=AnyInBundle) const

Return true if this instruction could possibly modify memory.

bool isBundledWithSucc() const

Return true if this instruction is part of a bundle, and it is not the last instruction in the bundle...

const MachineOperand & getOperand(unsigned i) const

MachineOperand class - Representation of each machine instruction operand.

bool isReg() const

isReg - Tests if this is a MO_Register operand.

getReg - Returns the register number.

PassRegistry - This class manages the registration and intitialization of the pass subsystem as appli...

virtual StringRef getPassName() const

getPassName - Return a nice clean name for a pass.

Wrapper class representing virtual and physical registers.

Kind

These are the different kinds of scheduling dependencies.

@ Output

A register output-dependence (aka WAW).

@ Order

Any other ordering dependency.

@ Anti

A register anti-dependence (aka WAR).

@ Data

Regular data dependence (aka true-dependence).

Scheduling unit. This is a node in the scheduling DAG.

bool isSucc(const SUnit *N) const

Tests if node N is a successor of this node.

SmallVector< SDep, 4 > Succs

All sunit successors.

SmallVector< SDep, 4 > Preds

All sunit predecessors.

MachineInstr * getInstr() const

Returns the representative MachineInstr for this SUnit.

StringRef - Represent a constant reference to a string, i.e.

TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...

virtual const TargetRegisterInfo * getRegisterInfo() const

getRegisterInfo - If register information is available, return it.

bool alias(const MachineInstr &MI1, const MachineInstr &MI2, bool UseTBAA=true) const

std::vector< MachineInstr * > CurrentPacketMIs

std::map< MachineInstr *, SUnit * > MIToSUnit

DFAPacketizer * ResourceTracker

std::pair< iterator, bool > insert(const ValueT &V)

size_type count(const_arg_type_t< ValueT > V) const

Return 1 if the specified key is in the set, 0 otherwise.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

initializer< Ty > init(const Ty &Val)

This is an optimization pass for GlobalISel generic memory operations.

void finalizeBundle(MachineBasicBlock &MBB, MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)

finalizeBundle - Finalize a machine instruction bundle which includes a sequence of instructions star...

auto find(R &&Range, const T &Val)

Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.

iterator_range< T > make_range(T x, T y)

Convenience function for iterating over sub-ranges.

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

void initializeHexagonPacketizerPass(PassRegistry &)

FunctionPass * createHexagonPacketizer(bool Minimal)

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

FuncUnits getUnits() const

Returns the choice of FUs.