RewriteStatepointsForGC.cpp Source File (original) (raw)

66#include

67#include

68#include

69#include

70#include

71#include

72#include

73#include

74#include

76#define DEBUG_TYPE "rewrite-statepoints-for-gc"

78using namespace llvm;

96#ifdef EXPENSIVE_CHECKS

98#else

100#endif

101

105

109

112

113

114

115

116

117

118

119

120

121

122

123

125

126

128

130

136

137 if (F.isDeclaration() || F.empty())

138 continue;

139

140

141

143 continue;

144

149 }

152

153

154

155

157

161 return PA;

162}

163

164namespace {

165

166struct GCPtrLivenessData {

167

169

170

171

173

174

175

177

178

179

181};

182

183

184

185

186

187

188

189

190

191

192

197using RematerializedValueMapTy =

199

200struct PartiallyConstructedSafepointRecord {

201

202 StatepointLiveSetTy LiveSet;

203

204

205

207

208

209

211

212

213

214

215 RematerializedValueMapTy RematerializedValues;

216};

217

218struct RematerizlizationCandidateRecord {

219

221

222 Value *RootOfChain;

223

225};

227

228}

229

231 std::optional DeoptBundle =

233

234 if (!DeoptBundle) {

236 "Found non-leaf call without deopt info!");

237 return {};

238 }

239

240 return DeoptBundle->Inputs;

241}

242

243

246

247

248

250 StatepointLiveSetTy &out, GCStrategy *GC);

251

253 assert(GC && "GC Strategy for isGCPointerType cannot be null");

254

256 return false;

257

258

259 return GC->isGCManagedPointer(T).value_or(true);

260}

261

262

263

264

265

267

269 return true;

270

271

274 return true;

275 return false;

276}

277

278#ifndef NDEBUG

279

280

283 return true;

290 [GC](Type *Ty) { return containsGCPtrType(Ty, GC); });

291 return false;

292}

293

294

295

296

300#endif

301

302

303

306 return V->hasName() ? (V->getName() + Suffix).str() : DefaultName.str();

307}

308

309

310

311

312

315 PartiallyConstructedSafepointRecord &Result, GCStrategy *GC) {

316 StatepointLiveSetTy LiveSet;

318

320 dbgs() << "Live Variables:\n";

321 for (Value *V : LiveSet)

322 dbgs() << " " << V->getName() << " " << *V << "\n";

323 }

325 dbgs() << "Safepoint For: " << Call->getCalledOperand()->getName() << "\n";

326 dbgs() << "Number live values: " << LiveSet.size() << "\n";

327 }

328 Result.LiveSet = LiveSet;

329}

330

331

332static bool isKnownBase(Value *V, const IsKnownBaseMapTy &KnownBases);

333

334

335

337 IsKnownBaseMapTy &KnownBases);

338

340 IsKnownBaseMapTy &KnownBases);

341

342

343

344

345

346

347

348

349

350

352 IsKnownBaseMapTy &KnownBases) {

353

354

355

356 auto Cached = Cache.find(I);

357 if (Cached != Cache.end())

358 return Cached->second;

359

361

362 Cache[I] = I;

363 setKnownBase(I, true, KnownBases);

364 return I;

365 }

366

368

369

371 Cache[I] = CAZ;

372 setKnownBase(CAZ, true, KnownBases);

373 return CAZ;

374 }

375

377 Cache[I] = I;

378 setKnownBase(I, true, KnownBases);

379 return I;

380 }

381

383

384

385

386 Cache[I] = I;

387 setKnownBase(I, false, KnownBases);

388 return I;

389 }

390

392

393

394

395

396

397 Cache[I] = I;

398 setKnownBase(I, false, KnownBases);

399 return I;

400 }

401

402

403

405 auto *BDV =

407 Cache[GEP] = BDV;

408 return BDV;

409 }

410

411

412

415 Cache[Freeze] = BDV;

416 return BDV;

417 }

418

419

420

423 Cache[BC] = BDV;

424 return BDV;

425 }

426

427

428

429

431 Cache[I] = I;

432 setKnownBase(I, true, KnownBases);

433 return I;

434 }

435

436

437

439 "unknown vector instruction - no base found for vector element");

440 Cache[I] = I;

441 setKnownBase(I, false, KnownBases);

442 return I;

443}

444

445

446

447

448

450 IsKnownBaseMapTy &KnownBases) {

451 assert(I->getType()->isPtrOrPtrVectorTy() &&

452 "Illegal to ask for the base pointer of a non-pointer type");

453 auto Cached = Cache.find(I);

454 if (Cached != Cache.end())

455 return Cached->second;

456

457 if (I->getType()->isVectorTy())

459

461

462

463 Cache[I] = I;

464 setKnownBase(I, true, KnownBases);

465 return I;

466 }

467

469

470

471

472

473

474

475

476

477

478

480 Cache[I] = CPN;

481 setKnownBase(CPN, true, KnownBases);

482 return CPN;

483 }

484

485

486

487

488

489

491 Cache[I] = I;

492 setKnownBase(I, true, KnownBases);

493 return I;

494 }

495

497 Value *Def = CI->stripPointerCasts();

498

499

502 "unsupported addrspacecast");

503

504

505

508 Cache[CI] = BDV;

509 return BDV;

510 }

511

513

514 Cache[I] = I;

515 setKnownBase(I, true, KnownBases);

516 return I;

517 }

518

520

521 auto *BDV =

523 Cache[GEP] = BDV;

524 return BDV;

525 }

526

529 Cache[Freeze] = BDV;

530 return BDV;

531 }

532

534 switch (II->getIntrinsicID()) {

535 default:

536

537 break;

538 case Intrinsic::experimental_gc_statepoint:

540 case Intrinsic::experimental_gc_relocate:

541

542

543

544 llvm_unreachable("repeat safepoint insertion is not supported");

545 case Intrinsic::gcroot:

546

547

548

550 "interaction with the gcroot mechanism is not supported");

551 case Intrinsic::experimental_gc_get_pointer_base:

553 Cache[II] = BDV;

554 return BDV;

555 }

556 }

557

558

559

561 Cache[I] = I;

562 setKnownBase(I, true, KnownBases);

563 return I;

564 }

565

566

567

569

571

572

573

574 Cache[I] = I;

575 setKnownBase(I, true, KnownBases);

576 return I;

577 }

578

581 "Only Xchg is allowed for pointer values");

582

583

584 Cache[I] = I;

585 setKnownBase(I, true, KnownBases);

586 return I;

587 }

588

589

590

591

593 Cache[I] = I;

594 setKnownBase(I, true, KnownBases);

595 return I;

596 }

597

598

599

601 "Base pointer for a struct is meaningless");

602

603

604

605 bool IsKnownBase =

607 setKnownBase(I, IsKnownBase, KnownBases);

608 Cache[I] = I;

609

610

611

612

613

615

616

617

618 return I;

619

620

621

622

623

625 "missing instruction case in findBaseDefiningValue");

626 return I;

627}

628

629

631 IsKnownBaseMapTy &KnownBases) {

632 if (!Cache.contains(I)) {

634 Cache[I] = BDV;

635 LLVM_DEBUG(dbgs() << "fBDV-cached: " << I->getName() << " -> "

636 << Cache[I]->getName() << ", is known base = "

637 << KnownBases[I] << "\n");

638 }

639 assert(Cache[I] != nullptr);

640 assert(KnownBases.contains(Cache[I]) &&

641 "Cached value must be present in known bases map");

642 return Cache[I];

643}

644

645

646

648 IsKnownBaseMapTy &KnownBases) {

650 auto Found = Cache.find(Def);

651 if (Found != Cache.end()) {

652

653 return Found->second;

654 }

655

656 return Def;

657}

658

659#ifndef NDEBUG

660

661

668#endif

669

671 auto It = KnownBases.find(V);

672 assert(It != KnownBases.end() && "Value not present in the map");

673 return It->second;

674}

675

677 IsKnownBaseMapTy &KnownBases) {

678#ifndef NDEBUG

679 auto It = KnownBases.find(V);

680 if (It != KnownBases.end())

681 assert(It->second == IsKnownBase && "Changing already present value");

682#endif

683 KnownBases[V] = IsKnownBase;

684}

685

686

691

692namespace {

693

694

695

696

697class BDVState {

698public:

699 enum StatusTy {

700

702

703

704

706

707 Conflict

708 };

709

710 BDVState() {

712 }

713

714 explicit BDVState(Value *OriginalValue)

715 : OriginalValue(OriginalValue) {}

716 explicit BDVState(Value *OriginalValue, StatusTy Status, Value *BaseValue = nullptr)

717 : OriginalValue(OriginalValue), Status(Status), BaseValue(BaseValue) {

719 }

720

721 StatusTy getStatus() const { return Status; }

722 Value *getOriginalValue() const { return OriginalValue; }

723 Value *getBaseValue() const { return BaseValue; }

724

725 bool isBase() const { return getStatus() == Base; }

726 bool isUnknown() const { return getStatus() == Unknown; }

727 bool isConflict() const { return getStatus() == Conflict; }

728

729

730

731

732 void meet(const BDVState &Other) {

733 auto markConflict = [&]() {

734 Status = BDVState::Conflict;

735 BaseValue = nullptr;

736 };

737

738 if (isConflict())

739 return;

740

741 if (isUnknown()) {

742 Status = Other.getStatus();

743 BaseValue = Other.getBaseValue();

744 return;

745 }

746

747 assert(isBase() && "Unknown state");

748

749 if (Other.isUnknown())

750 return;

751

752 if (Other.isConflict())

753 return markConflict();

754

755 assert(Other.isBase() && "Unknown state");

756

757 if (getBaseValue() != Other.getBaseValue())

758 return markConflict();

759

760 }

761

763 return OriginalValue == Other.OriginalValue && BaseValue == Other.BaseValue &&

764 Status == Other.Status;

765 }

766

767 bool operator!=(const BDVState &other) const { return !(*this == other); }

768

770 void dump() const {

772 dbgs() << '\n';

773 }

774

775 void print(raw_ostream &OS) const {

776 switch (getStatus()) {

778 OS << "U";

779 break;

781 OS << "B";

782 break;

783 case Conflict:

784 OS << "C";

785 break;

786 }

787 OS << " (base " << getBaseValue() << " - "

788 << (getBaseValue() ? getBaseValue()->getName() : "nullptr") << ")"

789 << " for " << OriginalValue->getName() << ":";

790 }

791

792private:

793 AssertingVH OriginalValue;

794 StatusTy Status = Unknown;

795 AssertingVH BaseValue = nullptr;

796};

797

798}

799

800#ifndef NDEBUG

802 State.print(OS);

803 return OS;

804}

805#endif

806

807

808

809

810

812 IsKnownBaseMapTy &KnownBases) {

814

816 return Def;

817

818

819

820

821

822

823

824

825

826

827

828

829

830

831

832

833

834

835

836

837

838

839

840#ifndef NDEBUG

841 auto isExpectedBDVType = [](Value *BDV) {

845 };

846#endif

847

848

849

850

851

852

854

855#ifndef NDEBUG

856 auto VerifyStates = [&]() {

857 for (auto &Entry : States) {

858 assert(Entry.first == Entry.second.getOriginalValue());

859 }

860 };

861#endif

862

863 auto visitBDVOperands = [](Value *BDV, std::function<void (Value*)> F) {

865 for (Value *InVal : PN->incoming_values())

866 F(InVal);

868 F(SI->getTrueValue());

869 F(SI->getFalseValue());

871 F(EE->getVectorOperand());

873 F(IE->getOperand(0));

874 F(IE->getOperand(1));

876

877

878 F(SV->getOperand(0));

879 if (!SV->isZeroEltSplat())

880 F(SV->getOperand(1));

881 } else {

883 }

884 };

885

886

887

888

889 {

892 States.insert({Def, BDVState(Def)});

893 while (!Worklist.empty()) {

896

897 auto visitIncomingValue = [&](Value *InVal) {

900

901

902

903

904

905 return;

906 assert(isExpectedBDVType(Base) && "the only non-base values "

907 "we see should be base defining values");

908 if (States.insert(std::make_pair(Base, BDVState(Base))).second)

910 };

911

912 visitBDVOperands(Current, visitIncomingValue);

913 }

914 }

915

916#ifndef NDEBUG

917 VerifyStates();

918 LLVM_DEBUG(dbgs() << "States after initialization:\n");

919 for (const auto &Pair : States) {

920 LLVM_DEBUG(dbgs() << " " << Pair.second << " for " << *Pair.first << "\n");

921 }

922#endif

923

924

925

926

927

928

930 do {

932 for (auto Pair : States) {

933 Value *BDV = Pair.first;

934 auto canPruneInput = [&](Value *V) {

935

936

937 if (V->stripPointerCasts() == BDV)

938 return true;

940 if (V->stripPointerCasts() != VBDV)

941 return false;

942

943

944 return States.count(VBDV) == 0;

945 };

946

947 bool CanPrune = true;

948 visitBDVOperands(BDV, [&](Value *Op) {

949 CanPrune = CanPrune && canPruneInput(Op);

950 });

951 if (CanPrune)

953 }

955 States.erase(V);

956

957 Cache[V] = V;

958 }

960

961

962 if (!States.count(Def))

963 return Def;

964

965

966

967 auto GetStateForBDV = [&](Value *BaseValue, Value *Input) {

968 auto I = States.find(BaseValue);

969 if (I != States.end())

970 return I->second;

972 return BDVState(BaseValue, BDVState::Base, BaseValue);

973 };

974

975

976

977

978

979

980

981

982

983

984

985

986

987

988

989

990

991

992

994

996 return true;

997

998

1000 return true;

1001

1002

1003

1004

1005

1006

1007

1009 return true;

1010 return false;

1011 };

1012

1013 bool Progress = true;

1014 while (Progress) {

1015#ifndef NDEBUG

1016 const size_t OldSize = States.size();

1017#endif

1018 Progress = false;

1019

1020

1021

1022

1023 for (auto Pair : States) {

1024 Value *BDV = Pair.first;

1025

1026

1027

1030 "why did it get added?");

1031

1032 BDVState NewState(BDV);

1033 visitBDVOperands(BDV, [&](Value *Op) {

1035 auto OpState = GetStateForBDV(BDV, Op);

1036 NewState.meet(OpState);

1037 });

1038

1039

1040

1041

1043 auto BV = NewState.getBaseValue();

1044 if (BV && MarkConflict(I, BV))

1045 NewState = BDVState(I, BDVState::Conflict);

1046

1047 BDVState OldState = Pair.second;

1048 if (OldState != NewState) {

1049 Progress = true;

1050 States[BDV] = NewState;

1051 }

1052 }

1053

1054 assert(OldSize == States.size() &&

1055 "fixed point shouldn't be adding any new nodes to state");

1056 }

1057

1058#ifndef NDEBUG

1059 VerifyStates();

1060 LLVM_DEBUG(dbgs() << "States after meet iteration:\n");

1061 for (const auto &Pair : States) {

1062 LLVM_DEBUG(dbgs() << " " << Pair.second << " for " << *Pair.first << "\n");

1063 }

1064

1065

1066

1067 for (auto Pair : States) {

1069 BDVState State = Pair.second;

1070 auto *BaseValue = State.getBaseValue();

1071

1072

1073

1076 "why did it get added?");

1077 assert(!State.isUnknown() && "Optimistic algorithm didn't complete!");

1078 }

1079#endif

1080

1081

1082

1083 for (auto Pair : States) {

1085 BDVState State = Pair.second;

1086

1087

1088

1091 "why did it get added?");

1092 assert(!State.isUnknown() && "Optimistic algorithm didn't complete!");

1093

1094

1095

1096

1098

1099 if (!State.isConflict())

1100 continue;

1101

1102 auto getMangledName = [](Instruction *I) -> std::string {

1111 } else {

1113 }

1114 };

1115

1118 BaseInst->setName(getMangledName(I));

1119

1121 States[I] = BDVState(I, BDVState::Conflict, BaseInst);

1122 setKnownBase(BaseInst, true, KnownBases);

1123 }

1124

1125#ifndef NDEBUG

1126 VerifyStates();

1127#endif

1128

1129

1130

1131

1132

1133

1134

1135

1136

1140 if (auto It = States.find(BDV); It == States.end()) {

1143 } else {

1144

1145 Base = It->second.getBaseValue();

1146 }

1148

1149 if (Base->getType() != Input->getType() && InsertPt)

1151 InsertPt->getIterator());

1152 return Base;

1153 };

1154

1155

1156

1157

1158 for (auto Pair : States) {

1160 BDVState State = Pair.second;

1161

1162

1163

1164

1167 "why did it get added?");

1168 assert(!State.isUnknown() && "Optimistic algorithm didn't complete!");

1169 if (!State.isConflict())

1170 continue;

1171

1175

1176

1177

1178

1179

1181 for (unsigned i = 0; i < NumPHIValues; i++) {

1184 auto [It, Inserted] = BlockToValue.try_emplace(InBB);

1185 if (Inserted)

1186 It->second = getBaseForInput(InVal, InBB->getTerminator());

1187 else {

1188#ifndef NDEBUG

1189 Value *OldBase = It->second;

1190 Value *Base = getBaseForInput(InVal, nullptr);

1191

1192

1193

1194 auto StripBitCasts = [](Value *V) -> Value * {

1196 V = BC->getOperand(0);

1197 return V;

1198 };

1199

1200

1201

1202

1203

1204

1205 assert(StripBitCasts(Base) == StripBitCasts(OldBase) &&

1206 "findBaseOrBDV should be pure!");

1207#endif

1208 }

1210 BasePHI->setIncomingValue(i, Base);

1211 }

1215

1216

1217

1218 BaseSI->setTrueValue(getBaseForInput(SI->getTrueValue(), BaseSI));

1219 BaseSI->setFalseValue(getBaseForInput(SI->getFalseValue(), BaseSI));

1220 } else if (auto *BaseEE =

1223

1224

1225 BaseEE->setOperand(0, getBaseForInput(InVal, BaseEE));

1228 auto UpdateOperand = [&](int OperandIdx) {

1229 Value *InVal = BdvIE->getOperand(OperandIdx);

1230 Value *Base = getBaseForInput(InVal, BaseIE);

1231 BaseIE->setOperand(OperandIdx, Base);

1232 };

1233 UpdateOperand(0);

1234 UpdateOperand(1);

1235 } else {

1238 auto UpdateOperand = [&](int OperandIdx) {

1239 Value *InVal = BdvSV->getOperand(OperandIdx);

1240 Value *Base = getBaseForInput(InVal, BaseSV);

1241 BaseSV->setOperand(OperandIdx, Base);

1242 };

1243 UpdateOperand(0);

1244 if (!BdvSV->isZeroEltSplat())

1245 UpdateOperand(1);

1246 else {

1247

1248 Value *InVal = BdvSV->getOperand(1);

1250 }

1251 }

1252 }

1253

1254#ifndef NDEBUG

1255 VerifyStates();

1256#endif

1257

1258

1259 [[maybe_unused]] auto &DL =

1261

1262

1263

1264 for (auto Pair : States) {

1265 auto *BDV = Pair.first;

1266 Value *Base = Pair.second.getBaseValue();

1268

1269

1271 DL.getTypeAllocSize(Base->getType()) &&

1272 "Derived and base values should have same size");

1273

1274

1275

1278 "why did it get added?");

1279

1281 dbgs() << "Updating base value cache"

1282 << " for: " << BDV->getName() << " from: "

1283 << (Cache.count(BDV) ? Cache[BDV]->getName().str() : "none")

1284 << " to: " << Base->getName() << "\n");

1285

1286 Cache[BDV] = Base;

1287 }

1288 assert(Cache.count(Def));

1289 return Cache[Def];

1290}

1291

1292

1293

1294

1295

1296

1297

1298

1299

1300

1301

1302

1303

1304

1305

1306

1308 PointerToBaseTy &PointerToBase, DominatorTree *DT,

1309 DefiningValueMapTy &DVCache,

1310 IsKnownBaseMapTy &KnownBases) {

1311 for (Value *ptr : live) {

1313 assert(base && "failed to find base pointer");

1314 PointerToBase[ptr] = base;

1318 "The base we found better dominate the derived pointer");

1319 }

1320}

1321

1322

1323

1326 PartiallyConstructedSafepointRecord &result,

1327 PointerToBaseTy &PointerToBase,

1328 IsKnownBaseMapTy &KnownBases) {

1329 StatepointLiveSetTy PotentiallyDerivedPointers = result.LiveSet;

1330

1331

1332

1333

1334

1336 for (Value *V : Opt->Inputs) {

1337 if (!PotentiallyDerivedPointers.count(V))

1338 continue;

1339 PotentiallyDerivedPointers.remove(V);

1340 PointerToBase[V] = V;

1341 }

1342 findBasePointers(PotentiallyDerivedPointers, PointerToBase, &DT, DVCache,

1343 KnownBases);

1344}

1345

1346

1347

1350 PartiallyConstructedSafepointRecord &result,

1351 PointerToBaseTy &PointerToBase,

1353

1357 PointerToBaseTy &PointerToBase, GCStrategy *GC) {

1358

1359

1360 GCPtrLivenessData RevisedLivenessData;

1362 for (size_t i = 0; i < records.size(); i++) {

1363 struct PartiallyConstructedSafepointRecord &info = records[i];

1365 GC);

1366 }

1367}

1368

1369

1370

1371

1374 Value *RootOfChain,

1375 Value *AlternateLiveBase) {

1378

1380

1381

1382

1383

1385

1386 Instruction *ClonedValue = Instr->clone();

1388 ClonedValue->setName(Instr->getName() + ".remat");

1389

1390

1391

1392 if (LastClonedValue) {

1395#ifndef NDEBUG

1396 for (auto *OpValue : ClonedValue->operand_values()) {

1397

1398

1400 "incorrect use in rematerialization chain");

1401

1402

1403 assert(OpValue != RootOfChain && OpValue != AlternateLiveBase);

1404 }

1405#endif

1406 } else {

1407

1408

1409

1410

1411

1412 if (RootOfChain != AlternateLiveBase)

1414 }

1415

1416 LastClonedValue = ClonedValue;

1417 LastValue = Instr;

1418 }

1419 assert(LastClonedValue);

1420 return LastClonedValue;

1421}

1422

1423

1424

1425

1426

1427

1428

1435

1436

1437

1440 "All PHI nodes should have been removed!");

1441

1442

1443

1444 return Ret;

1445}

1446

1447

1448

1449

1450

1451

1453 {Attribute::Memory, Attribute::NoSync, Attribute::NoFree};

1454

1455

1456

1458 AttributeList StatepointAL) {

1459 AttributeList OrigAL = Call->getAttributes();

1460 if (OrigAL.isEmpty())

1461 return StatepointAL;

1462

1463

1465 AttrBuilder FnAttrs(Ctx, OrigAL.getFnAttrs());

1467 FnAttrs.removeAttribute(Attr);

1468

1469 for (Attribute A : OrigAL.getFnAttrs()) {

1471 FnAttrs.removeAttribute(A);

1472 }

1473

1474 StatepointAL = StatepointAL.addFnAttributes(Ctx, FnAttrs);

1475

1476

1477

1478

1479 if (IsMemIntrinsic)

1480 return StatepointAL;

1481

1482

1483

1484

1486 StatepointAL = StatepointAL.addParamAttributes(

1488 AttrBuilder(Ctx, OrigAL.getParamAttrs(I)));

1489

1490

1491 return StatepointAL;

1492}

1493

1494

1495

1496

1497

1498

1499

1500

1501

1507 return;

1508

1510 auto ValIt = llvm::find(LiveVec, Val);

1511 assert(ValIt != LiveVec.end() && "Val not found in LiveVec!");

1512 size_t Index = std::distance(LiveVec.begin(), ValIt);

1513 assert(Index < LiveVec.size() && "Bug in std::find?");

1514 return Index;

1515 };

1517

1518

1519

1520

1521

1522

1523

1524

1525 auto getGCRelocateDecl = [&](Type *Ty) {

1527 auto AS = Ty->getScalarType()->getPointerAddressSpace();

1533 M, Intrinsic::experimental_gc_relocate, {NewTy});

1534 };

1535

1536

1537

1538

1540

1541 for (unsigned i = 0; i < LiveVariables.size(); i++) {

1542

1543 Value *BaseIdx = Builder.getInt32(FindIndex(LiveVariables, BasePtrs[i]));

1544 Value *LiveIdx = Builder.getInt32(i);

1545

1547 auto [It, Inserted] = TypeToDeclMap.try_emplace(Ty);

1548 if (Inserted)

1549 It->second = getGCRelocateDecl(Ty);

1550 Function *GCRelocateDecl = It->second;

1551

1552

1554 GCRelocateDecl, {StatepointToken, BaseIdx, LiveIdx},

1556

1557

1559 }

1560}

1561

1562namespace {

1563

1564

1565

1566class DeferredReplacement {

1567 AssertingVH Old;

1568 AssertingVH New;

1569 bool IsDeoptimize = false;

1570

1571 DeferredReplacement() = default;

1572

1573public:

1574 static DeferredReplacement createRAUW(Instruction *Old, Instruction *New) {

1575 assert(Old != New && Old && New &&

1576 "Cannot RAUW equal values or to / from null!");

1577

1578 DeferredReplacement D;

1579 D.Old = Old;

1580 D.New = New;

1581 return D;

1582 }

1583

1584 static DeferredReplacement createDelete(Instruction *ToErase) {

1585 DeferredReplacement D;

1586 D.Old = ToErase;

1587 return D;

1588 }

1589

1590 static DeferredReplacement createDeoptimizeReplacement(Instruction *Old) {

1591#ifndef NDEBUG

1593 assert(F && F->getIntrinsicID() == Intrinsic::experimental_deoptimize &&

1594 "Only way to construct a deoptimize deferred replacement");

1595#endif

1596 DeferredReplacement D;

1597 D.Old = Old;

1598 D.IsDeoptimize = true;

1599 return D;

1600 }

1601

1602

1603 void doReplacement() {

1606

1607 assert(OldI != NewI && "Disallowed at construction?!");

1608 assert((!IsDeoptimize || !New) &&

1609 "Deoptimize intrinsics are not replaced!");

1610

1611 Old = nullptr;

1612 New = nullptr;

1613

1614 if (NewI)

1616

1617 if (IsDeoptimize) {

1618

1619

1621 new UnreachableInst(RI->getContext(), RI->getIterator());

1622 RI->eraseFromParent();

1623 }

1624

1626 }

1627};

1628

1629}

1630

1632 const char *DeoptLowering = "deopt-lowering";

1633 if (Call->hasFnAttr(DeoptLowering)) {

1634

1635

1636 const AttributeList &CSAS = Call->getAttributes();

1637 if (CSAS.hasFnAttr(DeoptLowering))

1638 return CSAS.getFnAttr(DeoptLowering).getValueAsString();

1640 assert(F && F->hasFnAttribute(DeoptLowering));

1641 return F->getFnAttribute(DeoptLowering).getValueAsString();

1642 }

1643 return "live-through";

1644}

1645

1646static void

1650 PartiallyConstructedSafepointRecord &Result,

1651 std::vector &Replacements,

1652 const PointerToBaseTy &PointerToBase,

1655

1656

1657

1658

1659

1661

1666

1668 std::optional<ArrayRef> DeoptArgs;

1670 DeoptArgs = Bundle->Inputs;

1671 std::optional<ArrayRef> TransitionArgs;

1673 TransitionArgs = Bundle->Inputs;

1674

1676 }

1677

1678

1679

1680

1681 bool IsDeoptimize = false;

1682 bool IsMemIntrinsic = false;

1683

1690

1691

1693 if (DeoptLowering == "live-in")

1695 else {

1696 assert(DeoptLowering == "live-through" && "Unsupported value!");

1697 }

1698

1701 auto IID = F->getIntrinsicID();

1702 if (IID == Intrinsic::experimental_deoptimize) {

1703

1704

1705

1706

1708 for (Value *Arg : CallArgs)

1709 DomainTy.push_back(Arg->getType());

1711 false);

1712

1713

1714

1715

1716

1717 CallTarget = F->getParent()

1718 ->getOrInsertFunction("__llvm_deoptimize", FTy);

1719

1720 IsDeoptimize = true;

1721 } else if (IID == Intrinsic::memcpy_element_unordered_atomic ||

1722 IID == Intrinsic::memmove_element_unordered_atomic) {

1723 IsMemIntrinsic = true;

1724

1725

1726

1727

1728

1729

1730

1731

1732

1733

1734

1735

1736

1737

1738

1739

1740

1741 auto &Context = Call->getContext();

1742 auto &DL = Call->getDataLayout();

1743 auto GetBaseAndOffset = [&](Value *Derived) {

1745

1746

1747

1748

1752 else {

1753 assert(PointerToBase.count(Derived));

1754 Base = PointerToBase.find(Derived)->second;

1755 }

1756 unsigned AddressSpace = Derived->getType()->getPointerAddressSpace();

1757 unsigned IntPtrSize = DL.getPointerSizeInBits(AddressSpace);

1758 Value *Base_int = Builder.CreatePtrToInt(

1760 Value *Derived_int = Builder.CreatePtrToInt(

1762 return std::make_pair(Base, Builder.CreateSub(Derived_int, Base_int));

1763 };

1764

1765 auto *Dest = CallArgs[0];

1766 Value *DestBase, *DestOffset;

1767 std::tie(DestBase, DestOffset) = GetBaseAndOffset(Dest);

1768

1769 auto *Source = CallArgs[1];

1770 Value *SourceBase, *SourceOffset;

1771 std::tie(SourceBase, SourceOffset) = GetBaseAndOffset(Source);

1772

1773 auto *LengthInBytes = CallArgs[2];

1775

1776 CallArgs.clear();

1780 CallArgs.push_back(SourceOffset);

1781 CallArgs.push_back(LengthInBytes);

1782

1784 for (Value *Arg : CallArgs)

1785 DomainTy.push_back(Arg->getType());

1787 false);

1788

1790 uint64_t ElementSize = ElementSizeCI->getZExtValue();

1791 if (IID == Intrinsic::memcpy_element_unordered_atomic) {

1792 switch (ElementSize) {

1793 case 1:

1794 return "__llvm_memcpy_element_unordered_atomic_safepoint_1";

1795 case 2:

1796 return "__llvm_memcpy_element_unordered_atomic_safepoint_2";

1797 case 4:

1798 return "__llvm_memcpy_element_unordered_atomic_safepoint_4";

1799 case 8:

1800 return "__llvm_memcpy_element_unordered_atomic_safepoint_8";

1801 case 16:

1802 return "__llvm_memcpy_element_unordered_atomic_safepoint_16";

1803 default:

1805 }

1806 }

1807 assert(IID == Intrinsic::memmove_element_unordered_atomic);

1808 switch (ElementSize) {

1809 case 1:

1810 return "__llvm_memmove_element_unordered_atomic_safepoint_1";

1811 case 2:

1812 return "__llvm_memmove_element_unordered_atomic_safepoint_2";

1813 case 4:

1814 return "__llvm_memmove_element_unordered_atomic_safepoint_4";

1815 case 8:

1816 return "__llvm_memmove_element_unordered_atomic_safepoint_8";

1817 case 16:

1818 return "__llvm_memmove_element_unordered_atomic_safepoint_16";

1819 default:

1821 }

1822 };

1823

1824 CallTarget =

1825 F->getParent()

1826 ->getOrInsertFunction(GetFunctionName(IID, ElementSizeCI), FTy);

1827 }

1828 }

1829

1830

1833 CallInst *SPCall = Builder.CreateGCStatepointCall(

1834 StatepointID, NumPatchBytes, CallTarget, Flags, CallArgs,

1835 TransitionArgs, DeoptArgs, GCLive, "safepoint_token");

1836

1839

1840

1841

1844

1846

1847

1848

1849 assert(CI->getNextNode() && "Not a terminator, must have next!");

1850 Builder.SetInsertPoint(CI->getNextNode());

1851 Builder.SetCurrentDebugLocation(CI->getNextNode()->getDebugLoc());

1852 } else {

1854

1855

1856

1857

1858 InvokeInst *SPInvoke = Builder.CreateGCStatepointInvoke(

1859 StatepointID, NumPatchBytes, CallTarget, II->getNormalDest(),

1860 II->getUnwindDest(), Flags, CallArgs, TransitionArgs, DeoptArgs,

1861 GCLive, "statepoint_token");

1862

1864

1865

1866

1869

1871

1872

1873 BasicBlock *UnwindBlock = II->getUnwindDest();

1876 "can't safely insert in this block!");

1877

1878 Builder.SetInsertPoint(UnwindBlock, UnwindBlock->getFirstInsertionPt());

1879 Builder.SetCurrentDebugLocation(II->getDebugLoc());

1880

1881

1883 Result.UnwindToken = ExceptionalToken;

1884

1886

1887

1888 BasicBlock *NormalDest = II->getNormalDest();

1891 "can't safely insert in this block!");

1892

1894

1895

1896

1897 }

1898 assert(Token && "Should be set in one of the above branches!");

1899

1900 if (IsDeoptimize) {

1901

1902

1903

1904 Replacements.push_back(

1905 DeferredReplacement::createDeoptimizeReplacement(Call));

1906 } else {

1907 Token->setName("statepoint_token");

1908 if ( Call ->getType()->isVoidTy() && Call ->use_empty()) {

1910 CallInst *GCResult = Builder.CreateGCResult(Token, Call->getType(), Name);

1912 AttributeList::get(GCResult->getContext(), AttributeList::ReturnIndex,

1913 Call->getAttributes().getRetAttrs()));

1914

1915

1916

1917

1918

1919

1920

1921 Replacements.emplace_back(

1922 DeferredReplacement::createRAUW(Call, GCResult));

1923 } else {

1924 Replacements.emplace_back(DeferredReplacement::createDelete(Call));

1925 }

1926 }

1927

1928 Result.StatepointToken = Token;

1929

1930

1932}

1933

1934

1935

1936

1937

1938

1939static void

1941 PartiallyConstructedSafepointRecord &Result,

1942 std::vector &Replacements,

1943 const PointerToBaseTy &PointerToBase, GCStrategy *GC) {

1944 const auto &LiveSet = Result.LiveSet;

1945

1946

1948 LiveVec.reserve(LiveSet.size());

1949 BaseVec.reserve(LiveSet.size());

1950 for (Value *L : LiveSet) {

1952 assert(PointerToBase.count(L));

1953 Value *Base = PointerToBase.find(L)->second;

1955 }

1957

1958

1960 PointerToBase, GC);

1961}

1962

1963

1964

1965

1966

1967

1968

1969static void

1973 for (User *U : GCRelocs) {

1975 if (!Relocate)

1976 continue;

1977

1980 Value *Alloca = AllocaMap[OriginalValue];

1981

1982

1984 "Should always have one since it's not a terminator");

1986

1987#ifndef NDEBUG

1988 VisitedLiveValues.insert(OriginalValue);

1989#endif

1990 }

1991}

1992

1993

1994

1996 const RematerializedValueMapTy &RematerializedValues,

1999 for (auto RematerializedValuePair: RematerializedValues) {

2000 Instruction *RematerializedValue = RematerializedValuePair.first;

2001 Value *OriginalValue = RematerializedValuePair.second;

2002

2003 assert(AllocaMap.count(OriginalValue) &&

2004 "Can not find alloca for rematerialized value");

2005 Value *Alloca = AllocaMap[OriginalValue];

2006

2007 new StoreInst(RematerializedValue, Alloca,

2008 std::next(RematerializedValue->getIterator()));

2009

2010#ifndef NDEBUG

2011 VisitedLiveValues.insert(OriginalValue);

2012#endif

2013 }

2014}

2015

2016

2020#ifndef NDEBUG

2021

2022

2023 int InitialAllocaNum = 0;

2026 InitialAllocaNum++;

2027#endif

2028

2029

2032

2033 std::size_t NumRematerializedValues = 0;

2035

2036

2037

2039 auto emitAllocaFor = [&](Value *LiveValue) {

2041 new AllocaInst(LiveValue->getType(), DL.getAllocaAddrSpace(), "",

2042 F.getEntryBlock().getFirstNonPHIIt());

2043 AllocaMap[LiveValue] = Alloca;

2044 PromotableAllocas.push_back(Alloca);

2045 };

2046

2047

2048 for (Value *V : Live)

2049 emitAllocaFor(V);

2050

2051

2052 for (const auto &Info : Records)

2053 for (auto RematerializedValuePair : Info.RematerializedValues) {

2054 Value *OriginalValue = RematerializedValuePair.second;

2055 if (AllocaMap.contains(OriginalValue))

2056 continue;

2057

2058 emitAllocaFor(OriginalValue);

2059 ++NumRematerializedValues;

2060 }

2061

2062

2063

2064

2065

2066

2067

2068

2069

2070

2071 for (const auto &Info : Records) {

2072 Value *Statepoint = Info.StatepointToken;

2073

2074

2076

2077

2079

2080

2081

2084 VisitedLiveValues);

2085 }

2086

2087

2089 VisitedLiveValues);

2090

2092

2093

2094

2095

2096

2098 for (auto Pair : AllocaMap) {

2099 Value *Def = Pair.first;

2101

2102

2103 if (VisitedLiveValues.count(Def)) {

2104 continue;

2105 }

2107 }

2108

2110 for (auto *AI : ToClobber) {

2111 auto AT = AI->getAllocatedType();

2113 if (AT->isVectorTy())

2115 else

2118 }

2119 };

2120

2121

2122

2124 InsertClobbersAt(II->getNormalDest()->getFirstInsertionPt());

2125 InsertClobbersAt(II->getUnwindDest()->getFirstInsertionPt());

2126 } else {

2127 InsertClobbersAt(

2129 }

2130 }

2131 }

2132

2133

2134 for (auto Pair : AllocaMap) {

2135 Value *Def = Pair.first;

2137

2138

2139

2140

2142

2143 Uses.reserve(Def->getNumUses());

2144 for (User *U : Def->users()) {

2146

2147

2148

2149

2150

2152 }

2153 }

2154

2158

2162 for (unsigned i = 0; i < Phi->getNumIncomingValues(); i++) {

2163 if (Def == Phi->getIncomingValue(i)) {

2166 Phi->getIncomingBlock(i)->getTerminator()->getIterator());

2167 Phi->setIncomingValue(i, Load);

2168 }

2169 }

2170 } else {

2172 Use->getIterator());

2173 Use->replaceUsesOfWith(Def, Load);

2174 }

2175 }

2176

2177

2178

2179

2181 DL.getABITypeAlign(Def->getType()));

2184

2185

2186 BasicBlock *NormalDest = Invoke->getNormalDest();

2188 } else {

2189 assert(!Inst->isTerminator() &&

2190 "The only terminator that can produce a value is "

2191 "InvokeInst which is handled above.");

2192 Store->insertAfter(Inst->getIterator());

2193 }

2194 } else {

2197 }

2198 }

2199

2200 assert(PromotableAllocas.size() == Live.size() + NumRematerializedValues &&

2201 "we must have the same allocas with lives");

2202 (void) NumRematerializedValues;

2203 if (!PromotableAllocas.empty()) {

2204

2206 }

2207

2208#ifndef NDEBUG

2209 for (auto &I : F.getEntryBlock())

2211 InitialAllocaNum--;

2212 assert(InitialAllocaNum == 0 && "We must not introduce any extra allocas");

2213#endif

2214}

2215

2216

2217

2220 if (Values.empty())

2221

2222 return;

2223

2225

2229

2232 return;

2233 }

2234

2235

2238 Func, Values, "", II->getNormalDest()->getFirstInsertionPt()));

2240 Func, Values, "", II->getUnwindDest()->getFirstInsertionPt()));

2241}

2242

2247 GCPtrLivenessData OriginalLivenessData;

2249 for (size_t i = 0; i < records.size(); i++) {

2250 struct PartiallyConstructedSafepointRecord &info = records[i];

2252 }

2253}

2254

2255

2256

2257

2258

2259

2260

2263 Value *CurrentValue) {

2267 GEP->getPointerOperand());

2268 }

2269

2271 if (!CI->isNoopCast(CI->getDataLayout()))

2272 return CI;

2273

2276 CI->getOperand(0));

2277 }

2278

2279

2280

2281 return CurrentValue;

2282}

2283

2284

2285

2290

2293 assert(CI->isNoopCast(CI->getDataLayout()) &&

2294 "non noop cast is found during rematerialization");

2295

2296 Type *SrcTy = CI->getOperand(0)->getType();

2297 Cost += TTI.getCastInstrCost(CI->getOpcode(), CI->getType(), SrcTy,

2300

2302

2303 Cost += TTI.getAddressComputationCost(

2304 GEP->getType(), nullptr, nullptr,

2306

2307

2308

2309

2310 if ( GEP ->hasAllConstantIndices())

2311 Cost += 2;

2312

2313 } else {

2314 llvm_unreachable("unsupported instruction type during rematerialization");

2315 }

2316 }

2317

2318 return Cost;

2319}

2320

2325 return false;

2326

2327

2329 for (unsigned i = 0; i < PhiNum; i++)

2332

2333

2334

2335 for (unsigned i = 0; i < PhiNum; i++) {

2336 auto CIVI =

2338 if (CIVI == CurrentIncomingValues.end())

2339 return false;

2340 BasicBlock *CurrentIncomingBB = CIVI->second;

2341 if (CurrentIncomingBB != AlternateRootPhi.getIncomingBlock(i))

2342 return false;

2343 }

2344 return true;

2345}

2346

2347

2348

2349static void

2351 RematCandTy &RematerizationCandidates,

2353 const unsigned int ChainLengthThreshold = 10;

2354

2355 for (auto P2B : PointerToBase) {

2356 auto *Derived = P2B.first;

2357 auto *Base = P2B.second;

2358

2359 if (Derived == Base)

2360 continue;

2361

2362

2364 Value *RootOfChain =

2366

2367

2368 if ( ChainToBase.size() == 0 ||

2369 ChainToBase.size() > ChainLengthThreshold)

2370 continue;

2371

2372

2373

2374 if (Value *BaseVal = PointerToBase[Derived]; RootOfChain != BaseVal) {

2377 if (!OrigRootPhi || !AlternateRootPhi)

2378 continue;

2379

2380

2381

2382

2383

2384

2385

2386

2387

2389 continue;

2390 }

2391

2393

2394

2395

2396

2397

2398

2399 RematerizlizationCandidateRecord Record;

2400 Record.ChainToBase = ChainToBase;

2401 Record.RootOfChain = RootOfChain;

2403 RematerizationCandidates.insert({ Derived, Record });

2404 }

2405}

2406

2407

2408

2409

2410

2412 RematCandTy &RematerizationCandidates,

2414 PointerToBaseTy &PointerToBase) {

2416 return;

2417

2419

2420 LLVM_DEBUG(dbgs() << "Rematerialize derived pointers at uses, "

2421 << "Num statepoints: " << Records.size() << '\n');

2422

2423 for (auto &It : RematerizationCandidates) {

2425 auto &Record = It.second;

2426

2428 continue;

2429

2431 continue;

2432

2435 if (U->getParent() == Cand->getParent())

2436 continue;

2437

2438

2440 [](const auto *U) { return isa(U); }))

2441 continue;

2442

2443 LLVM_DEBUG(dbgs() << "Trying cand " << *Cand << " ... ");

2444

2445

2446

2447

2448

2449

2450

2452 Records, [Cand](const auto &R) { return R.LiveSet.contains(Cand); });

2453 unsigned NumUses = Cand->getNumUses();

2454

2455 LLVM_DEBUG(dbgs() << "Num uses: " << NumUses << " Num live statepoints: "

2456 << NumLiveStatepoints << " ");

2457

2458 if (NumLiveStatepoints < NumUses) {

2460 continue;

2461 }

2462

2463

2464

2465

2466 if (NumLiveStatepoints == NumUses && Record.Cost > 0) {

2468 continue;

2469 }

2470

2472

2473

2474

2475

2476

2477

2478 if (Record.ChainToBase.size() > 1) {

2479 Record.ChainToBase.clear();

2481 }

2482

2483

2484

2485

2486

2487

2488

2489

2490

2491

2496 Record.RootOfChain, PointerToBase[Cand]);

2498 PointerToBase[RematChain] = PointerToBase[Cand];

2499 }

2500 LiveValuesToBeDeleted.push_back(Cand);

2501 }

2502

2503 LLVM_DEBUG(dbgs() << "Rematerialized " << LiveValuesToBeDeleted.size()

2504 << " derived pointers\n");

2505 for (auto *Cand : LiveValuesToBeDeleted) {

2506 assert(Cand->use_empty() && "Unexpected user remain");

2507 RematerizationCandidates.erase(Cand);

2508 for (auto &R : Records) {

2509 assert(!R.LiveSet.contains(Cand) ||

2510 R.LiveSet.contains(PointerToBase[Cand]));

2511 R.LiveSet.remove(Cand);

2512 }

2513 }

2514

2515

2516

2517 if (!LiveValuesToBeDeleted.empty()) {

2518 for (auto &P : RematerizationCandidates) {

2519 auto &R = P.second;

2520 if (R.ChainToBase.size() > 1) {

2521 R.ChainToBase.clear();

2523 }

2524 }

2525 }

2526}

2527

2528

2529

2530

2531

2533 PartiallyConstructedSafepointRecord &Info,

2534 PointerToBaseTy &PointerToBase,

2535 RematCandTy &RematerizationCandidates,

2537

2538

2540

2541 for (Value *LiveValue : Info.LiveSet) {

2542 auto It = RematerizationCandidates.find(LiveValue);

2543 if (It == RematerizationCandidates.end())

2544 continue;

2545

2546 RematerizlizationCandidateRecord &Record = It->second;

2547

2549

2550

2552 Cost *= 2;

2553

2554

2556 continue;

2557

2558

2559 LiveValuesToBeDeleted.push_back(LiveValue);

2560

2561

2562

2563

2564

2567 assert(InsertBefore);

2570 Record.RootOfChain, PointerToBase[LiveValue]);

2571 Info.RematerializedValues[RematerializedValue] = LiveValue;

2572 } else {

2574

2576 Invoke->getNormalDest()->getFirstInsertionPt();

2578 Invoke->getUnwindDest()->getFirstInsertionPt();

2579

2580 Instruction *NormalRematerializedValue =

2582 Record.RootOfChain, PointerToBase[LiveValue]);

2583 Instruction *UnwindRematerializedValue =

2585 Record.RootOfChain, PointerToBase[LiveValue]);

2586

2587 Info.RematerializedValues[NormalRematerializedValue] = LiveValue;

2588 Info.RematerializedValues[UnwindRematerializedValue] = LiveValue;

2589 }

2590 }

2591

2592

2593 for (auto *LiveValue: LiveValuesToBeDeleted) {

2594 Info.LiveSet.remove(LiveValue);

2595 }

2596}

2597

2600 DefiningValueMapTy &DVCache,

2601 IsKnownBaseMapTy &KnownBases) {

2602 auto &Context = F.getContext();

2603 auto &DL = F.getDataLayout();

2605

2606 for (auto *Callsite : Intrinsics)

2607 switch (Callsite->getIntrinsicID()) {

2608 case Intrinsic::experimental_gc_get_pointer_base: {

2611 findBasePointer(Callsite->getOperand(0), DVCache, KnownBases);

2612 assert(!DVCache.count(Callsite));

2613 Callsite->replaceAllUsesWith(Base);

2614 if ( Base ->hasName())

2615 Base->takeName(Callsite);

2616 Callsite->eraseFromParent();

2617 break;

2618 }

2619 case Intrinsic::experimental_gc_get_pointer_offset: {

2621 Value *Derived = Callsite->getOperand(0);

2623 assert(!DVCache.count(Callsite));

2625 unsigned IntPtrSize = DL.getPointerSizeInBits(AddressSpace);

2627 Value *BaseInt =

2630 Value *DerivedInt =

2631 Builder.CreatePtrToInt(Derived, Type::getIntNTy(Context, IntPtrSize),

2633 Value *Offset = Builder.CreateSub(DerivedInt, BaseInt);

2634 Callsite->replaceAllUsesWith(Offset);

2635 Offset->takeName(Callsite);

2636 Callsite->eraseFromParent();

2637 break;

2638 }

2639 default:

2641 }

2642

2644}

2645

2649 DefiningValueMapTy &DVCache,

2650 IsKnownBaseMapTy &KnownBases) {

2652

2653#ifndef NDEBUG

2654

2655 std::set<CallBase *> Uniqued;

2656 Uniqued.insert(ToUpdate.begin(), ToUpdate.end());

2657 assert(Uniqued.size() == ToUpdate.size() && "no duplicates please!");

2658

2661#endif

2662

2663

2664

2665

2666

2669 if ()

2670 continue;

2673 }

2674

2675

2676

2678

2679

2680

2681

2682

2685

2688 "support for FCA unimplemented");

2691 }

2692

2694 }

2695

2697

2698

2699

2701

2702

2703 PointerToBaseTy PointerToBase;

2704

2705

2706 for (size_t i = 0; i < Records.size(); i++) {

2707 PartiallyConstructedSafepointRecord &info = Records[i];

2708 findBasePointers(DT, DVCache, ToUpdate[i], info, PointerToBase, KnownBases);

2709 }

2711 errs() << "Base Pairs (w/o Relocation):\n";

2712 for (auto &Pair : PointerToBase) {

2713 errs() << " derived ";

2714 Pair.first->printAsOperand(errs(), false);

2715 errs() << " base ";

2716 Pair.second->printAsOperand(errs(), false);

2717 errs() << "\n";

2718 ;

2719 }

2720 }

2721

2722

2723

2724

2725

2726

2727

2728

2729

2730

2731

2732

2733

2734

2736 for (size_t i = 0; i < Records.size(); i++) {

2737 PartiallyConstructedSafepointRecord &Info = Records[i];

2738

2740 for (auto *Derived : Info.LiveSet) {

2741 assert(PointerToBase.count(Derived) && "Missed base for derived pointer");

2742 Bases.push_back(PointerToBase[Derived]);

2743 }

2744

2746 }

2747

2748

2749

2750

2752

2754 errs() << "Base Pairs: (w/Relocation)\n";

2755 for (auto Pair : PointerToBase) {

2756 errs() << " derived ";

2757 Pair.first->printAsOperand(errs(), false);

2758 errs() << " base ";

2759 Pair.second->printAsOperand(errs(), false);

2760 errs() << "\n";

2761 }

2762 }

2763

2764

2765

2766

2767

2768

2769

2770

2771

2772 for (auto &Info : Records) {

2773 Info.LiveSet.remove_if([&](Value *LiveV) {

2774 assert(PointerToBase.count(LiveV) && "Missed base for derived pointer");

2776 });

2777 }

2778

2779 for (CallInst *CI : Holders)

2780 CI->eraseFromParent();

2781

2782 Holders.clear();

2783

2784

2785 RematCandTy RematerizationCandidates;

2787

2788

2789

2790

2791

2793 PointerToBase);

2794 for (size_t i = 0; i < Records.size(); i++)

2796 RematerizationCandidates, TTI);

2797

2798

2799

2800

2801 std::vector Replacements;

2802

2803

2804

2805

2806

2807

2808

2809 for (size_t i = 0; i < Records.size(); i++)

2811 PointerToBase, GC.get());

2812

2813 ToUpdate.clear();

2814

2815 for (auto &PR : Replacements)

2816 PR.doReplacement();

2817

2818 Replacements.clear();

2819

2820 for (auto &Info : Records) {

2821

2822

2823

2824

2825

2826

2827

2828

2829 Info.LiveSet.clear();

2830 }

2831 PointerToBase.clear();

2832

2833

2834

2835

2837 for (const PartiallyConstructedSafepointRecord &Info : Records) {

2838

2839

2840

2841

2842

2844#ifndef NDEBUG

2845

2846

2847

2848

2850 "statepoint must be reachable or liveness is meaningless");

2851 for (Value *V : Info.StatepointToken->gc_live()) {

2853

2854 continue;

2857 "unreachable values should never be live");

2859 "basic SSA liveness expectation violated by liveness analysis");

2860 }

2861#endif

2862 }

2863

2864#ifndef NDEBUG

2865

2866 for (auto *Ptr : Live)

2868 "must be a gc pointer type");

2869#endif

2870

2872 return !Records.empty();

2873}

2874

2875

2876

2877

2880 R.addAttribute(Attribute::Dereferenceable);

2881 R.addAttribute(Attribute::DereferenceableOrNull);

2882 R.addAttribute(Attribute::ReadNone);

2883 R.addAttribute(Attribute::ReadOnly);

2884 R.addAttribute(Attribute::WriteOnly);

2885 R.addAttribute(Attribute::NoAlias);

2886 R.addAttribute(Attribute::NoFree);

2887 return R;

2888}

2889

2892

2893

2894

2895

2896

2897

2900 return;

2901 }

2902

2906 F.removeParamAttrs(A.getArgNo(), R);

2907

2909 F.removeRetAttrs(R);

2910

2912 F.removeFnAttr(Attr);

2913}

2914

2915

2916

2917

2920 return;

2921

2922

2923

2924

2925

2926

2927

2928

2929

2930

2931

2932

2933

2934 unsigned ValidMetadataAfterRS4GC[] = {LLVMContext::MD_tbaa,

2935 LLVMContext::MD_range,

2936 LLVMContext::MD_alias_scope,

2937 LLVMContext::MD_nontemporal,

2938 LLVMContext::MD_nonnull,

2939 LLVMContext::MD_align,

2940 LLVMContext::MD_type};

2941

2942

2943 I.dropUnknownNonDebugMetadata(ValidMetadataAfterRS4GC);

2944}

2945

2947 if (F.empty())

2948 return;

2949

2952

2953

2954

2956

2958

2959

2960

2961

2962

2963

2965 if (II->getIntrinsicID() == Intrinsic::invariant_start) {

2966 InvariantStartInstructions.push_back(II);

2967 continue;

2968 }

2969

2970 if (MDNode *Tag = I.getMetadata(LLVMContext::MD_tbaa)) {

2971 MDNode *MutableTBAA = Builder.createMutableTBAAAccessTag(Tag);

2972 I.setMetadata(LLVMContext::MD_tbaa, MutableTBAA);

2973 }

2974

2976

2979 for (int i = 0, e = Call->arg_size(); i != e; i++)

2981 Call->removeParamAttrs(i, R);

2983 Call->removeRetAttrs(R);

2984 }

2985 }

2986

2987

2988 for (auto *II : InvariantStartInstructions) {

2990 II->eraseFromParent();

2991 }

2992}

2993

2994

2995

2997 if (.hasGC())

2998 return nullptr;

2999

3001}

3002

3003

3004

3006 if (.hasGC())

3007 return false;

3008

3009 std::unique_ptr Strategy = findGCStrategy(F);

3010

3011 assert(Strategy && "GC strategy is required by function, but was not found");

3012

3013 return Strategy->useRS4GC();

3014}

3015

3017#ifndef NDEBUG

3019#endif

3020

3023

3026}

3027

3031 assert(.isDeclaration() && .empty() &&

3032 "need function body to rewrite statepoints in");

3034

3035 auto NeedsRewrite = [&TLI](Instruction &I) {

3038 return false;

3040 return false;

3041

3042

3043

3044

3045

3046

3047

3048

3052 "Don't expect any other calls here!");

3053 return false;

3054 }

3055 return true;

3056 }

3057 return false;

3058 };

3059

3060

3061

3062

3063 DomTreeUpdater DTU(DT, DomTreeUpdater::UpdateStrategy::Lazy);

3065

3067

3068

3069

3070

3074

3075 if (NeedsRewrite(I)) {

3076

3077

3078

3079

3081 "no unreachable blocks expected");

3083 }

3085 if (CI->getIntrinsicID() == Intrinsic::experimental_gc_get_pointer_base ||

3086 CI->getIntrinsicID() == Intrinsic::experimental_gc_get_pointer_offset)

3088 }

3089

3090

3091 if (ParsePointNeeded.empty() && Intrinsics.empty())

3092 return MadeChange;

3093

3094

3095

3096

3097

3099 if (BB.getUniquePredecessor())

3101

3102

3103

3104

3105

3106

3107

3108

3109

3110

3111

3112

3113

3114

3117 if (BI->isConditional())

3119

3120 return nullptr;

3121 };

3124 if (auto *Cond = getConditionInst(TI))

3125

3126

3128 MadeChange = true;

3130 }

3131 }

3132

3133

3134

3135

3136

3137

3140 continue;

3141

3142 unsigned VF = 0;

3143 for (unsigned i = 0; i < I.getNumOperands(); i++)

3148 }

3149

3150

3151

3152 if (.getOperand(0)->getType()->isVectorTy() && VF != 0) {

3154 auto *Splat = B.CreateVectorSplat(VF, I.getOperand(0));

3155 I.setOperand(0, Splat);

3156 MadeChange = true;

3157 }

3158 }

3159

3160

3161

3162

3163

3164 DefiningValueMapTy DVCache;

3165

3166

3167

3168 IsKnownBaseMapTy KnownBases;

3169

3170 if (!Intrinsics.empty())

3171

3172

3174

3175 if (!ParsePointNeeded.empty())

3176 MadeChange |=

3178

3179 return MadeChange;

3180}

3181

3182

3183

3184

3185

3186

3187

3188

3189

3193 for (auto &I : make_range(Begin, End)) {

3194

3196

3197

3198

3200 continue;

3201

3202

3203 for (Value *V : I.operands()) {

3205 "support for FCA unimplemented");

3207

3208

3209

3210

3211

3212

3213

3214

3215

3216

3218 }

3219 }

3220 }

3221}

3222

3226 for (auto &I : *Succ) {

3228 if (!PN)

3229 break;

3230

3233 "support for FCA unimplemented");

3236 }

3237 }

3238}

3239

3245 return KillSet;

3246}

3247

3248#ifndef NDEBUG

3249

3250

3252 Instruction *TI, bool TermOkay = false) {

3253 for (Value *V : Live) {

3255

3256

3257

3258 if (TermOkay && TI == I)

3259 continue;

3261 "basic SSA liveness expectation violated by liveness analysis");

3262 }

3263 }

3264}

3265

3266

3267

3268

3275#endif

3276

3280

3281

3284 auto &LiveSet = Data.LiveSet[&BB];

3285 LiveSet.clear();

3287

3288#ifndef NDEBUG

3289 for (Value *Kill : Data.KillSet[&BB])

3290 assert( Data .LiveSet[&BB].count(Kill) && "live set contains kill");

3291#endif

3292

3295 auto &In = Data.LiveIn[&BB] = Data.LiveSet[&BB];

3296 In.set_union(Out);

3297 In.set_subtract(Data.KillSet[&BB]);

3298 if (!In.empty())

3300 }

3301

3302

3303 while (!Worklist.empty()) {

3305

3306

3307

3309 const auto OldLiveOutSize = LiveOut.size();

3313 }

3314

3315 if (OldLiveOutSize == LiveOut.size()) {

3316

3317

3318

3319 continue;

3320 }

3321

3322

3326

3329

3330 if (LiveIn.size() != LiveTmp.size()) {

3331 LiveIn = std::move(LiveTmp);

3333 }

3334 }

3335

3336#ifndef NDEBUG

3337

3338

3341#endif

3342}

3343

3345 StatepointLiveSetTy &Out, GCStrategy *GC) {

3347

3348

3351

3352

3353

3354

3355

3357 GC);

3358 LiveOut.remove(Inst);

3359 Out.insert_range(LiveOut);

3360}

3361

3364 PartiallyConstructedSafepointRecord &Info,

3365 PointerToBaseTy &PointerToBase,

3367 StatepointLiveSetTy Updated;

3369

3370

3371

3372 for (auto *V : Updated)

3373 PointerToBase.insert({ V, V });

3374

3375 Info.LiveSet = Updated;

3376}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

ReachingDefInfo InstSet & ToRemove

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

static void print(raw_ostream &Out, object::Archive::Kind Kind, T Val)

Expand Atomic instructions

This file contains the simple types necessary to represent the attributes associated with functions a...

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

Analysis containing CSE Info

#define LLVM_DUMP_METHOD

Mark debug helper function definitions like dump() that should not be stripped from debug builds.

This file contains the declarations for the subclasses of Constant, which represent the different fla...

This file defines the DenseMap class.

This file defines the DenseSet and SmallDenseSet classes.

Module.h This file contains the declarations for the Module class.

This file implements a map that provides insertion order iteration.

uint64_t IntrinsicInst * II

FunctionAnalysisManager FAM

const SmallVectorImpl< MachineOperand > & Cond

Remove Loads Into Fake Uses

static void makeStatepointExplicitImpl(CallBase *Call, const SmallVectorImpl< Value * > &BasePtrs, const SmallVectorImpl< Value * > &LiveVariables, PartiallyConstructedSafepointRecord &Result, std::vector< DeferredReplacement > &Replacements, const PointerToBaseTy &PointerToBase, GCStrategy *GC)

Definition RewriteStatepointsForGC.cpp:1647

static void rematerializeLiveValues(CallBase *Call, PartiallyConstructedSafepointRecord &Info, PointerToBaseTy &PointerToBase, RematCandTy &RematerizationCandidates, TargetTransformInfo &TTI)

Definition RewriteStatepointsForGC.cpp:2532

static void findRematerializationCandidates(PointerToBaseTy PointerToBase, RematCandTy &RematerizationCandidates, TargetTransformInfo &TTI)

Definition RewriteStatepointsForGC.cpp:2350

static std::unique_ptr< GCStrategy > findGCStrategy(Function &F)

Looks up the GC strategy for a given function, returning null if the function doesn't have a GC tag.

Definition RewriteStatepointsForGC.cpp:2996

static void stripNonValidDataFromBody(Function &F)

Definition RewriteStatepointsForGC.cpp:2946

static bool isKnownBase(Value *V, const IsKnownBaseMapTy &KnownBases)

Returns true if V is a known base.

Definition RewriteStatepointsForGC.cpp:670

static Value * findBasePointer(Value *I, DefiningValueMapTy &Cache, IsKnownBaseMapTy &KnownBases)

For a given value or instruction, figure out what base ptr its derived from.

Definition RewriteStatepointsForGC.cpp:811

static cl::opt< bool, true > ClobberNonLiveOverride("rs4gc-clobber-non-live", cl::location(ClobberNonLive), cl::Hidden)

static void insertRelocationStores(iterator_range< Value::user_iterator > GCRelocs, DenseMap< Value *, AllocaInst * > &AllocaMap, DenseSet< Value * > &VisitedLiveValues)

Definition RewriteStatepointsForGC.cpp:1970

static BasicBlock * normalizeForInvokeSafepoint(BasicBlock *BB, BasicBlock *InvokeParent, DominatorTree &DT)

Definition RewriteStatepointsForGC.cpp:1430

static void analyzeParsePointLiveness(DominatorTree &DT, GCPtrLivenessData &OriginalLivenessData, CallBase *Call, PartiallyConstructedSafepointRecord &Result, GCStrategy *GC)

Definition RewriteStatepointsForGC.cpp:313

static void computeLiveOutSeed(BasicBlock *BB, SetVector< Value * > &LiveTmp, GCStrategy *GC)

Definition RewriteStatepointsForGC.cpp:3223

static void relocationViaAlloca(Function &F, DominatorTree &DT, ArrayRef< Value * > Live, ArrayRef< PartiallyConstructedSafepointRecord > Records)

Do all the relocation update via allocas and mem2reg.

Definition RewriteStatepointsForGC.cpp:2017

static bool AreEquivalentPhiNodes(PHINode &OrigRootPhi, PHINode &AlternateRootPhi)

Definition RewriteStatepointsForGC.cpp:2321

static cl::opt< unsigned > RematerializationThreshold("spp-rematerialization-threshold", cl::Hidden, cl::init(6))

static Value * findBaseOrBDV(Value *I, DefiningValueMapTy &Cache, IsKnownBaseMapTy &KnownBases)

Return a base pointer for this value if known.

Definition RewriteStatepointsForGC.cpp:647

static Value * findBaseDefiningValueCached(Value *I, DefiningValueMapTy &Cache, IsKnownBaseMapTy &KnownBases)

Returns the base defining value for this value.

Definition RewriteStatepointsForGC.cpp:630

static void insertUseHolderAfter(CallBase *Call, const ArrayRef< Value * > Values, SmallVectorImpl< CallInst * > &Holders)

Insert holders so that each Value is obviously live through the entire lifetime of the call.

Definition RewriteStatepointsForGC.cpp:2218

static AttributeList legalizeCallAttributes(CallBase *Call, bool IsMemIntrinsic, AttributeList StatepointAL)

Definition RewriteStatepointsForGC.cpp:1457

static void insertRematerializationStores(const RematerializedValueMapTy &RematerializedValues, DenseMap< Value *, AllocaInst * > &AllocaMap, DenseSet< Value * > &VisitedLiveValues)

Definition RewriteStatepointsForGC.cpp:1995

static bool insertParsePoints(Function &F, DominatorTree &DT, TargetTransformInfo &TTI, SmallVectorImpl< CallBase * > &ToUpdate, DefiningValueMapTy &DVCache, IsKnownBaseMapTy &KnownBases)

Definition RewriteStatepointsForGC.cpp:2646

static void findBasePointers(const StatepointLiveSetTy &live, PointerToBaseTy &PointerToBase, DominatorTree *DT, DefiningValueMapTy &DVCache, IsKnownBaseMapTy &KnownBases)

Definition RewriteStatepointsForGC.cpp:1307

static bool shouldRewriteStatepointsIn(Function &F)

Returns true if this function should be rewritten by this pass.

Definition RewriteStatepointsForGC.cpp:3005

static cl::opt< bool > RematDerivedAtUses("rs4gc-remat-derived-at-uses", cl::Hidden, cl::init(true))

static ArrayRef< Use > GetDeoptBundleOperands(const CallBase *Call)

Definition RewriteStatepointsForGC.cpp:230

static Instruction * rematerializeChain(ArrayRef< Instruction * > ChainToBase, BasicBlock::iterator InsertBefore, Value *RootOfChain, Value *AlternateLiveBase)

Definition RewriteStatepointsForGC.cpp:1372

static void stripNonValidAttributesFromPrototype(Function &F)

Definition RewriteStatepointsForGC.cpp:2890

static void findLiveSetAtInst(Instruction *inst, GCPtrLivenessData &Data, StatepointLiveSetTy &out, GCStrategy *GC)

Given results from the dataflow liveness computation, find the set of live Values at a particular ins...

Definition RewriteStatepointsForGC.cpp:3344

static void computeLiveInValues(DominatorTree &DT, Function &F, GCPtrLivenessData &Data, GCStrategy *GC)

Compute the live-in set for every basic block in the function.

Definition RewriteStatepointsForGC.cpp:3277

static void stripInvalidMetadataFromInstruction(Instruction &I)

Certain metadata on instructions are invalid after running RS4GC.

Definition RewriteStatepointsForGC.cpp:2918

static constexpr Attribute::AttrKind FnAttrsToStrip[]

Definition RewriteStatepointsForGC.cpp:1452

static bool areBothVectorOrScalar(Value *First, Value *Second)

Definition RewriteStatepointsForGC.cpp:687

static void rematerializeLiveValuesAtUses(RematCandTy &RematerizationCandidates, MutableArrayRef< PartiallyConstructedSafepointRecord > Records, PointerToBaseTy &PointerToBase)

Definition RewriteStatepointsForGC.cpp:2411

static bool isHandledGCPointerType(Type *T, GCStrategy *GC)

Definition RewriteStatepointsForGC.cpp:266

static Value * findRematerializableChainToBasePointer(SmallVectorImpl< Instruction * > &ChainToBase, Value *CurrentValue)

Definition RewriteStatepointsForGC.cpp:2261

static cl::opt< bool > PrintLiveSetSize("spp-print-liveset-size", cl::Hidden, cl::init(false))

static Value * findBaseDefiningValueOfVector(Value *I, DefiningValueMapTy &Cache, IsKnownBaseMapTy &KnownBases)

Return a base defining value for the 'Index' element of the given vector instruction 'I'.

Definition RewriteStatepointsForGC.cpp:351

static void stripNonValidData(Module &M)

The IR fed into RewriteStatepointsForGC may have had attributes and metadata implying dereferenceabil...

Definition RewriteStatepointsForGC.cpp:3016

static InstructionCost chainToBasePointerCost(SmallVectorImpl< Instruction * > &Chain, TargetTransformInfo &TTI)

Definition RewriteStatepointsForGC.cpp:2287

static bool isUnhandledGCPointerType(Type *Ty, GCStrategy *GC)

Definition RewriteStatepointsForGC.cpp:297

static SetVector< Value * > computeKillSet(BasicBlock *BB, GCStrategy *GC)

Definition RewriteStatepointsForGC.cpp:3240

static bool ClobberNonLive

Definition RewriteStatepointsForGC.cpp:99

static cl::opt< bool > PrintBasePointers("spp-print-base-pointers", cl::Hidden, cl::init(false))

static bool isOriginalBaseResult(Value *V)

This value is a base pointer that is not generated by RS4GC, i.e.

Definition RewriteStatepointsForGC.cpp:662

static cl::opt< bool > PrintLiveSet("spp-print-liveset", cl::Hidden, cl::init(false))

static void setKnownBase(Value *V, bool IsKnownBase, IsKnownBaseMapTy &KnownBases)

Caches the IsKnownBase flag for a value and asserts that it wasn't present in the cache before.

Definition RewriteStatepointsForGC.cpp:676

static cl::opt< bool > AllowStatepointWithNoDeoptInfo("rs4gc-allow-statepoint-with-no-deopt-info", cl::Hidden, cl::init(true))

static void makeStatepointExplicit(DominatorTree &DT, CallBase *Call, PartiallyConstructedSafepointRecord &Result, std::vector< DeferredReplacement > &Replacements, const PointerToBaseTy &PointerToBase, GCStrategy *GC)

Definition RewriteStatepointsForGC.cpp:1940

static std::string suffixed_name_or(Value *V, StringRef Suffix, StringRef DefaultName)

Definition RewriteStatepointsForGC.cpp:304

static void CreateGCRelocates(ArrayRef< Value * > LiveVariables, ArrayRef< Value * > BasePtrs, Instruction *StatepointToken, IRBuilder<> &Builder, GCStrategy *GC)

Helper function to place all gc relocates necessary for the given statepoint.

Definition RewriteStatepointsForGC.cpp:1502

static void checkBasicSSA(DominatorTree &DT, SetVector< Value * > &Live, Instruction *TI, bool TermOkay=false)

Check that the items in 'Live' dominate 'TI'.

Definition RewriteStatepointsForGC.cpp:3251

static StringRef getDeoptLowering(CallBase *Call)

Definition RewriteStatepointsForGC.cpp:1631

static void findLiveReferences(Function &F, DominatorTree &DT, ArrayRef< CallBase * > toUpdate, MutableArrayRef< struct PartiallyConstructedSafepointRecord > records, GCStrategy *GC)

Definition RewriteStatepointsForGC.cpp:2243

static AttributeMask getParamAndReturnAttributesToRemove()

Definition RewriteStatepointsForGC.cpp:2878

static bool inlineGetBaseAndOffset(Function &F, SmallVectorImpl< CallInst * > &Intrinsics, DefiningValueMapTy &DVCache, IsKnownBaseMapTy &KnownBases)

Definition RewriteStatepointsForGC.cpp:2598

static Value * findBaseDefiningValue(Value *I, DefiningValueMapTy &Cache, IsKnownBaseMapTy &KnownBases)

Helper function for findBasePointer - Will return a value which either a) defines the base pointer fo...

Definition RewriteStatepointsForGC.cpp:449

static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData, CallBase *Call, PartiallyConstructedSafepointRecord &result, PointerToBaseTy &PointerToBase, GCStrategy *GC)

Given an updated version of the dataflow liveness results, update the liveset and base pointer maps f...

Definition RewriteStatepointsForGC.cpp:3362

static unsigned getNumElements(Type *Ty)

verify safepoint Safepoint IR static false bool isGCPointerType(Type *T)

static bool containsGCPtrType(Type *Ty)

Provides some synthesis utilities to produce sequences of values.

This file implements a set that has insertion order iteration characteristics.

This file defines the SmallVector class.

This pass exposes codegen information to IR-level passes.

The Input class is used to parse a yaml document into in-memory structs and vectors.

an instruction to allocate memory on the stack

Type * getAllocatedType() const

Return the type that is being allocated by the instruction.

PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)

Get the result of an analysis pass for a given IR unit.

This class represents an incoming formal argument to a Function.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

bool empty() const

empty - Check if the array is empty.

Value handle that asserts if the Value is deleted.

This class stores enough information to efficiently remove some attributes from an existing AttrBuild...

Functions, function parameters, and return types can have attributes to indicate how they should be t...

AttrKind

This enumeration lists the attributes that can be associated with parameters, function results,...

LLVM Basic Block Representation.

iterator begin()

Instruction iterator methods.

LLVM_ABI const LandingPadInst * getLandingPadInst() const

Return the landingpad instruction associated with the landing pad.

LLVM_ABI const_iterator getFirstInsertionPt() const

Returns an iterator to the first instruction in this block that is suitable for inserting a non-PHI i...

reverse_iterator rbegin()

LLVM_ABI InstListType::const_iterator getFirstNonPHIIt() const

Returns an iterator to the first instruction in this block that is not a PHINode instruction.

InstListType::reverse_iterator reverse_iterator

LLVM_ABI const BasicBlock * getUniquePredecessor() const

Return the predecessor of this block if it has a unique predecessor block.

InstListType::iterator iterator

Instruction iterators...

const Instruction * getTerminator() const LLVM_READONLY

Returns the terminator instruction if the block is well formed or null if the block is not well forme...

This class represents a no-op cast from one type to another.

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

void setCallingConv(CallingConv::ID CC)

void setAttributes(AttributeList A)

Set the attributes for this call.

AttributeList getAttributes() const

Return the attributes for this call.

This class represents a function call, abstracting a target machine's calling convention.

void setTailCallKind(TailCallKind TCK)

static CallInst * Create(FunctionType *Ty, Value *F, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

This is the base class for all instructions that perform data casts.

static LLVM_ABI ConstantAggregateZero * get(Type *Ty)

This is the shared class of boolean and integer constants.

static LLVM_ABI ConstantPointerNull * get(PointerType *T)

Static factory methods - Return objects of the specified value.

This is an important base class in LLVM.

A parsed version of the target data layout string in and methods for querying it.

iterator find(const_arg_type_t< KeyT > Val)

std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)

size_type count(const_arg_type_t< KeyT > Val) const

Return 1 if the specified key is in the map, 0 otherwise.

bool contains(const_arg_type_t< KeyT > Val) const

Return true if the specified key is in the map, false otherwise.

Implements a dense probed hash-table based set.

Analysis pass which computes a DominatorTree.

Concrete subclass of DominatorTreeBase that is used to compute a normal dominator tree.

LLVM_ABI bool isReachableFromEntry(const Use &U) const

Provide an overload for a Use.

LLVM_ABI bool dominates(const BasicBlock *BB, const Use &U) const

Return true if the (end of the) basic block BB dominates the use U.

static LLVM_ABI FixedVectorType * get(Type *ElementType, unsigned NumElts)

A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...

static LLVM_ABI FunctionType * get(Type *Result, ArrayRef< Type * > Params, bool isVarArg)

This static method is the primary way of constructing a FunctionType.

Represents calls to the gc.relocate intrinsic.

LLVM_ABI Value * getDerivedPtr() const

Represents a gc.statepoint intrinsic call.

GCStrategy describes a garbage collector algorithm's code generation requirements,...

DomTreeT & getDomTree()

Flush DomTree updates and return DomTree.

an instruction for type-safe pointer arithmetic to access elements of arrays and structs

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

LLVM_ABI const Module * getModule() const

Return the module owning the function this instruction belongs to or nullptr it the function does not...

LLVM_ABI void insertBefore(InstListType::iterator InsertPos)

Insert an unlinked instruction into a basic block immediately before the specified position.

LLVM_ABI InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

LLVM_ABI void setMetadata(unsigned KindID, MDNode *Node)

Set the metadata of the specified kind to the specified node.

A wrapper class for inspecting calls to intrinsic functions.

This is an important class for using LLVM in a threaded context.

An instruction for reading from memory.

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

This class implements a map that also provides access to all stored values in a deterministic order.

size_type count(const KeyT &Key) const

iterator find(const KeyT &Key)

std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)

A Module instance is used to store all the information related to an LLVM module.

MutableArrayRef - Represent a mutable reference to an array (0 or more elements consecutively in memo...

Value * getIncomingValueForBlock(const BasicBlock *BB) const

BasicBlock * getIncomingBlock(unsigned i) const

Return incoming basic block number i.

Value * getIncomingValue(unsigned i) const

Return incoming value number x.

unsigned getNumIncomingValues() const

Return the number of incoming edges.

static LLVM_ABI PointerType * get(Type *ElementType, unsigned AddressSpace)

This constructs a pointer to an object of the specified type in a numbered address space.

static LLVM_ABI PoisonValue * get(Type *T)

Static factory methods - Return an 'poison' object of the specified type.

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

PreservedAnalyses & preserve()

Mark an analysis as preserved.

This class represents the LLVM 'select' instruction.

A vector that has set insertion semantics.

ArrayRef< value_type > getArrayRef() const

bool remove(const value_type &X)

Remove an item from the set vector.

size_type size() const

Determine the number of elements in the SetVector.

void insert_range(Range &&R)

bool set_union(const STy &S)

Compute This := This u S, return whether 'This' changed.

bool empty() const

Determine if the SetVector is empty or not.

void set_subtract(const STy &S)

Compute This := This - B TODO: We should be able to use set_subtract from SetOperations....

bool insert(const value_type &X)

Insert a new element into the SetVector.

value_type pop_back_val()

A SetVector that performs no allocations if smaller than a certain size.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

reference emplace_back(ArgTypes &&... Args)

void reserve(size_type N)

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

An instruction for storing to memory.

StringRef - Represent a constant reference to a string, i.e.

std::string str() const

str - Get the contents as an std::string.

Class to represent struct types.

Analysis pass providing the TargetTransformInfo.

Analysis pass providing the TargetLibraryInfo.

Provides information about what library functions are available for the current target.

This pass provides access to the codegen interfaces that are needed for IR-level transformations.

static LLVM_ABI CastContextHint getCastContextHint(const Instruction *I)

Calculates a CastContextHint from I.

@ TCK_SizeAndLatency

The weighted sum of size and latency.

The instances of the Type class are immutable: once they are created, they are never changed.

LLVM_ABI unsigned getPointerAddressSpace() const

Get the address space of this pointer or pointer vector type.

static LLVM_ABI Type * getVoidTy(LLVMContext &C)

static LLVM_ABI IntegerType * getIntNTy(LLVMContext &C, unsigned N)

A Use represents the edge between a Value definition and its users.

LLVM_ABI bool replaceUsesOfWith(Value *From, Value *To)

Replace uses of one Value with another.

iterator_range< value_op_iterator > operand_values()

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

user_iterator user_begin()

LLVM_ABI void setName(const Twine &Name)

Change the name of the value.

bool hasOneUse() const

Return true if there is exactly one use of this value.

LLVM_ABI void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

iterator_range< user_iterator > users()

LLVM_ABI User * getUniqueUndroppableUser()

Return true if there is exactly one unique user of this value that cannot be dropped (that user can h...

LLVM_ABI LLVMContext & getContext() const

All values hold a context through their type.

LLVM_ABI unsigned getNumUses() const

This method computes the number of uses of this Value.

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

std::pair< iterator, bool > insert(const ValueT &V)

size_type count(const_arg_type_t< ValueT > V) const

Return 1 if the specified key is in the set, 0 otherwise.

const ParentTy * getParent() const

self_iterator getIterator()

NodeTy * getNextNode()

Get the next node, or nullptr for the list tail.

A range adaptor for a pair of iterators.

This class implements an extremely fast bulk output stream that can only output to a stream.

This provides a very simple, boring adaptor for a begin and end iterator into a range type.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ Cold

Attempts to make code in the caller as efficient as possible under the assumption that the call is no...

LLVM_ABI Function * getOrInsertDeclaration(Module *M, ID id, ArrayRef< Type * > Tys={})

Look up the Function declaration of the intrinsic id in the Module M.

LLVM_ABI AttributeList getAttributes(LLVMContext &C, ID id, FunctionType *FT)

Return the attributes for an intrinsic.

initializer< Ty > init(const Ty &Val)

LocationClass< Ty > location(Ty &L)

friend class Instruction

Iterator for Instructions in a `BasicBlock.

This is an optimization pass for GlobalISel generic memory operations.

void dump(const SparseBitVector< ElementSize > &LHS, raw_ostream &out)

FunctionAddr VTableAddr Value

auto find(R &&Range, const T &Val)

Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.

LLVM_ABI void PromoteMemToReg(ArrayRef< AllocaInst * > Allocas, DominatorTree &DT, AssumptionCache *AC=nullptr)

Promote the specified list of alloca instructions into scalar registers, inserting PHI nodes as appro...

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

auto successors(const MachineBasicBlock *BB)

bool operator!=(uint64_t V1, const APInt &V2)

iterator_range< T > make_range(T x, T y)

Convenience function for iterating over sub-ranges.

InnerAnalysisManagerProxy< FunctionAnalysisManager, Module > FunctionAnalysisManagerModuleProxy

Provide the FunctionAnalysisManager to Module proxy.

auto unique(Range &&R, Predicate P)

bool operator==(const AddressRangeValuePair &LHS, const AddressRangeValuePair &RHS)

bool any_of(R &&range, UnaryPredicate P)

Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.

auto reverse(ContainerTy &&C)

void sort(IteratorTy Start, IteratorTy End)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeList AS)

Parse out statepoint directives from the function attributes present in AS.

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

LLVM_ABI raw_fd_ostream & errs()

This returns a reference to a raw_ostream for standard error.

LLVM_ABI BasicBlock * SplitBlockPredecessors(BasicBlock *BB, ArrayRef< BasicBlock * > Preds, const char *Suffix, DominatorTree *DT, LoopInfo *LI=nullptr, MemorySSAUpdater *MSSAU=nullptr, bool PreserveLCSSA=false)

This method introduces at least one new basic block into the function and moves some of the predecess...

@ First

Helpers to iterate all locations in the MemoryEffectsBase class.

FunctionAddr VTableAddr uintptr_t uintptr_t Data

DWARFExpression::Operation Op

LLVM_ABI bool FoldSingleEntryPHINodes(BasicBlock *BB, MemoryDependenceResults *MemDep=nullptr)

We know that BB has one predecessor.

raw_ostream & operator<<(raw_ostream &OS, const APFixedPoint &FX)

@ DeoptLiveIn

Mark the deopt arguments associated with the statepoint as only being "live-in".

@ GCTransition

Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware.

auto count_if(R &&Range, UnaryPredicate P)

Wrapper function around std::count_if to count the number of times an element satisfying a given pred...

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

LLVM_ABI std::unique_ptr< GCStrategy > getGCStrategy(const StringRef Name)

Lookup the GCStrategy object associated with the given gc name.

auto predecessors(const MachineBasicBlock *BB)

bool is_contained(R &&Range, const E &Element)

Returns true if Element is found in Range.

auto seq(T Begin, T End)

Iterate over an integral type from Begin up to - but not including - End.

bool isStatepointDirectiveAttr(Attribute Attr)

Return true if the Attr is an attribute that is a statepoint directive.

LLVM_ABI bool removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU=nullptr, MemorySSAUpdater *MSSAU=nullptr)

Remove all blocks that can not be reached from the function's entry.

LLVM_ABI bool callsGCLeafFunction(const CallBase *Call, const TargetLibraryInfo &TLI)

Return true if this call calls a gc leaf function.

AnalysisManager< Module > ModuleAnalysisManager

Convenience typedef for the Module analysis manager.

bool runOnFunction(Function &F, DominatorTree &, TargetTransformInfo &, const TargetLibraryInfo &)

Definition RewriteStatepointsForGC.cpp:3028

PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)

Definition RewriteStatepointsForGC.cpp:131

Call sites that get wrapped by a gc.statepoint (currently only in RewriteStatepointsForGC and potenti...

std::optional< uint32_t > NumPatchBytes

std::optional< uint64_t > StatepointID

static const uint64_t DefaultStatepointID