StatepointLowering.cpp Source File (original) (raw)

47#include

48#include

49#include

50#include

51#include

52#include

54using namespace llvm;

56#define DEBUG_TYPE "statepoint-lowering"

59 "Number of stack slots allocated for statepoints");

60 STATISTIC(NumOfStatepoints, "Number of statepoint nodes encountered");

62 "Maximum number of stack slots required for a singe statepoint");

66 cl::desc("Allow using registers for non pointer deopt args"));

69 "use-registers-for-gc-values-in-landing-pad", cl::Hidden, cl::init(false),

70 cl::desc("Allow using registers for gc pointer in landing pad"));

74 cl::desc("Max number of VRegs allowed to pass GC pointer meta args in"));

80 SDLoc L = Builder.getCurSDLoc();

81 Ops.push_back(Builder.DAG.getTargetConstant(StackMaps::ConstantOp, L,

82 MVT::i64));

83 Ops.push_back(Builder.DAG.getTargetConstant(Value, L, MVT::i64));

84}

88 assert(PendingGCRelocateCalls.empty() &&

89 "Trying to visit statepoint before finished processing previous one");

90 Locations.clear();

91 NextSlotToAllocate = 0;

95 AllocatedStackSlots.clear();

96 AllocatedStackSlots.resize(Builder.FuncInfo.StatepointStackSlots.size());

97}

100 Locations.clear();

101 AllocatedStackSlots.clear();

102 assert(PendingGCRelocateCalls.empty() &&

103 "cleared before statepoint sequence completed");

104}

105

109 NumSlotsAllocatedForStatepoints++;

110 MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo();

111

112 unsigned SpillSize = ValueType.getStoreSize();

113 assert((SpillSize * 8) ==

114 (-8u & (7 + ValueType.getSizeInBits())) &&

115 "Size not in bytes?");

116

117

118

119

120

121 const size_t NumSlots = AllocatedStackSlots.size();

122 assert(NextSlotToAllocate <= NumSlots && "Broken invariant");

123

124 assert(AllocatedStackSlots.size() ==

125 Builder.FuncInfo.StatepointStackSlots.size() &&

126 "Broken invariant");

127

128 for (; NextSlotToAllocate < NumSlots; NextSlotToAllocate++) {

129 if (!AllocatedStackSlots.test(NextSlotToAllocate)) {

130 const int FI = Builder.FuncInfo.StatepointStackSlots[NextSlotToAllocate];

132 AllocatedStackSlots.set(NextSlotToAllocate);

133

134 return Builder.DAG.getFrameIndex(FI, ValueType);

135 }

136 }

137 }

138

139

140

141 SDValue SpillSlot = Builder.DAG.CreateStackTemporary(ValueType);

144

145 Builder.FuncInfo.StatepointStackSlots.push_back(FI);

146 AllocatedStackSlots.resize(AllocatedStackSlots.size()+1, true);

147 assert(AllocatedStackSlots.size() ==

148 Builder.FuncInfo.StatepointStackSlots.size() &&

149 "Broken invariant");

150

151 StatepointMaxSlotsRequired.updateMax(

152 Builder.FuncInfo.StatepointStackSlots.size());

153

154 return SpillSlot;

155}

156

157

158

159

162 int LookUpDepth) {

163

164 if (LookUpDepth <= 0)

165 return std::nullopt;

166

167

169 const Value *Statepoint = Relocate->getStatepoint();

171 "GetStatepoint must return one of two types");

173 return std::nullopt;

174

175 const auto &RelocationMap = Builder.FuncInfo.StatepointRelocationMaps

177

178 auto It = RelocationMap.find(Relocate);

179 if (It == RelocationMap.end())

180 return std::nullopt;

181

182 auto &Record = It->second;

184 return std::nullopt;

185

186 return Record.payload.FI;

187 }

188

189

192

193

194

195

197 std::optional MergedResult;

198

199 for (const auto &IncomingValue : Phi->incoming_values()) {

200 std::optional SpillSlot =

202 if (!SpillSlot)

203 return std::nullopt;

204

205 if (MergedResult && *MergedResult != *SpillSlot)

206 return std::nullopt;

207

208 MergedResult = SpillSlot;

209 }

210 return MergedResult;

211 }

212

213

214

215

216

217

218

219

220

221

222

223

224

225

226

227

228

229

230

231

232

233

234

235

236

237

238

239

240

241 return std::nullopt;

242}

243

244

245

246

248

249

251 return true;

252

253

254

255

256

257 if (Incoming.getValueType().getSizeInBits() > 64)

258 return false;

259

261}

262

263

264

265

266

267

271

272

273

275 return;

276

277 SDValue OldLocation = Builder.StatepointLowering.getLocation(Incoming);

278 if (OldLocation.getNode())

279

280 return;

281

282 const int LookUpDepth = 6;

283 std::optional Index =

285 if (!Index)

286 return;

287

288 const auto &StatepointSlots = Builder.FuncInfo.StatepointStackSlots;

289

290 auto SlotIt = find(StatepointSlots, *Index);

291 assert(SlotIt != StatepointSlots.end() &&

292 "Value spilled to the unknown stack slot");

293

294

295 const int Offset = std::distance(StatepointSlots.begin(), SlotIt);

296 if (Builder.StatepointLowering.isStackSlotAllocated(Offset)) {

297

298

299

300

301

302

303 return;

304 }

305

306 Builder.StatepointLowering.reserveStackSlot(Offset);

307

308

309

311 Builder.DAG.getTargetFrameIndex(*Index, Builder.getFrameIndexTy());

312 Builder.StatepointLowering.setLocation(Incoming, Loc);

313}

314

315

316

317

321 SDValue ReturnValue, CallEndVal;

322 std::tie(ReturnValue, CallEndVal) =

323 Builder.lowerInvokable(SI.CLI, SI.EHPadBB);

325

326

327

328

329

330

331

332

333

334

335

336

337

338

339

340

341

342 if (CallEnd->getOpcode() == ISD::EH_LABEL)

344

345 bool HasDef = .CLI.RetTy->isVoidTy();

346 if (HasDef) {

347 if (CallEnd->getOpcode() == ISD::LOAD)

349 else

352 }

353

354 assert(CallEnd->getOpcode() == ISD::CALLSEQ_END && "expected!");

355 return std::make_pair(ReturnValue, CallEnd->getOperand(0).getNode());

356}

357

365 MFI.getObjectSize(FI.getIndex()),

366 MFI.getObjectAlign(FI.getIndex()));

367}

368

369

370

371

372

373

374

375static std::tuple<SDValue, SDValue, MachineMemOperand*>

380

381

382 if ( Loc .getNode()) {

383 Loc = Builder.StatepointLowering.allocateStackSlot(Incoming.getValueType(),

384 Builder);

386

387 Loc = Builder.DAG.getTargetFrameIndex(Index, Builder.getFrameIndexTy());

388

389

390

391

392

393

394 MachineFrameInfo &MFI = Builder.DAG.getMachineFunction().getFrameInfo();

396 (-8 & (7 +

397 (int64_t)Incoming.getValueSizeInBits())) &&

398 "Bad spill: stack slot does not match!");

399

400

401

402

403 auto &MF = Builder.DAG.getMachineFunction();

405 auto *StoreMMO = MF.getMachineMemOperand(

408 Chain = Builder.DAG.getStore(Chain, Builder.getCurSDLoc(), Incoming, Loc,

409 StoreMMO);

410

412

413 Builder.StatepointLowering.setLocation(Incoming, Loc);

414 }

415

417 return std::make_tuple(Loc, Chain, MMO);

418}

419

420

421

422

423static void

428

431

432

433

434 assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&

435 "Incoming value is a frame index!");

436 Ops.push_back(Builder.DAG.getTargetFrameIndex(FI->getIndex(),

437 Builder.getFrameIndexTy()));

438

439 auto &MF = Builder.DAG.getMachineFunction();

442 return;

443 }

444

445 assert(Incoming.getValueType().getSizeInBits() <= 64);

446

448

449

450

451

453 return;

454 }

455

456

457

458

459

462 return;

465 C->getValueAPF().bitcastToAPInt().getZExtValue());

466 return;

467 }

468

470 }

471

472

473

474 if (!RequireSpillSlot) {

475

476

477

478

479

480

481

483 } else {

484

485

486

487

488

489 SDValue Chain = Builder.getRoot();

491 Ops.push_back(std::get<0>(Res));

492 if (auto *MMO = std::get<2>(Res))

494 Chain = std::get<1>(Res);

495 Builder.DAG.setRoot(Chain);

496 }

497

498}

499

500

501

503 auto *Ty = V->getType();

504 if (!Ty->isPtrOrPtrVectorTy())

505 return false;

506 if (auto *GFI = Builder.GFI)

507 if (auto IsManaged = GFI->getStrategy().isGCManagedPointer(Ty))

508 return *IsManaged;

509 return true;

510}

511

512

513

514

515

516

517

518

519static void

526

527

528

529

530

531

532

533

534

535

536

537

538

539 const bool LiveInDeopt =

541

542

544

545

546

549 if (const auto *StInvoke =

552 for (const auto *Relocate : SI.GCRelocates)

553 if (Relocate->getOperand(0) == LPI) {

554 LPadPointers.insert(Builder.getValue(Relocate->getBasePtr()));

555 LPadPointers.insert(Builder.getValue(Relocate->getDerivedPtr()));

556 }

557 }

558

559 LLVM_DEBUG(dbgs() << "Deciding how to lower GC Pointers:\n");

560

561

563

565

566 unsigned CurNumVRegs = 0;

567

568 auto canPassGCPtrOnVReg = [&](SDValue SD) {

569 if (SD.getValueType().isVector())

570 return false;

571 if (LPadPointers.count(SD))

572 return false;

574 };

575

576 auto processGCPtr = [&](const Value *V) {

578 if (!LoweredGCPtrs.insert(PtrSD))

579 return;

580 GCPtrIndexMap[PtrSD] = LoweredGCPtrs.size() - 1;

581

582 assert(!LowerAsVReg.count(PtrSD) && "must not have been seen");

583 if (LowerAsVReg.size() == MaxVRegPtrs)

584 return;

586 "IR and SD types disagree");

587 if (!canPassGCPtrOnVReg(PtrSD)) {

589 return;

590 }

592 LowerAsVReg[PtrSD] = CurNumVRegs++;

593 };

594

595

596 for (const Value *V : SI.Ptrs)

597 processGCPtr(V);

598 for (const Value *V : SI.Bases)

599 processGCPtr(V);

600

601 LLVM_DEBUG(dbgs() << LowerAsVReg.size() << " pointers will go in vregs\n");

602

603 auto requireSpillSlot = [&](const Value *V) {

604 if (!Builder.DAG.getTargetLoweringInfo().isTypeLegal(

605 Builder.getValue(V).getValueType()))

606 return true;

608 return !LowerAsVReg.count(Builder.getValue(V));

610 };

611

612

613

614

615

616

617 for (const Value *V : SI.DeoptState) {

618 if (requireSpillSlot(V))

620 }

621

622 for (const Value *V : SI.Ptrs) {

624 if (!LowerAsVReg.count(SDV))

626 }

627

628 for (const Value *V : SI.Bases) {

630 if (!LowerAsVReg.count(SDV))

632 }

633

634

635

636

637 const int NumVMSArgs = SI.DeoptState.size();

639

640

641

643 for (const Value *V : SI.DeoptState) {

645

646

648 int FI = Builder.FuncInfo.getArgumentFrameIndex(Arg);

649 if (FI != INT_MAX)

650 Incoming = Builder.DAG.getFrameIndex(FI, Builder.getFrameIndexTy());

651 }

653 Incoming = Builder.getValue(V);

655 << " requireSpillSlot = " << requireSpillSlot(V) << "\n");

657 Builder);

658 }

659

660

662 for (SDValue SDV : LoweredGCPtrs)

664 Builder);

665

666

668

669

670

671

672

673

675 for (Value *V : SI.GCLives) {

678

679 assert(Incoming.getValueType() == Builder.getFrameIndexTy() &&

680 "Incoming value is a frame index!");

681 Allocas.push_back(Builder.DAG.getTargetFrameIndex(

682 FI->getIndex(), Builder.getFrameIndexTy()));

683

684 auto &MF = Builder.DAG.getMachineFunction();

687 }

688 }

690 Ops.append(Allocas.begin(), Allocas.end());

691

692

694 SDLoc L = Builder.getCurSDLoc();

695 for (unsigned i = 0; i < SI.Ptrs.size(); ++i) {

697 assert(GCPtrIndexMap.count(Base) && "base not found in index map");

698 Ops.push_back(

699 Builder.DAG.getTargetConstant(GCPtrIndexMap[Base], L, MVT::i64));

701 assert(GCPtrIndexMap.count(Derived) && "derived not found in index map");

702 Ops.push_back(

703 Builder.DAG.getTargetConstant(GCPtrIndexMap[Derived], L, MVT::i64));

704 }

705}

706

709

710

711

712

713 NumOfStatepoints++;

714

716 assert(SI.Bases.size() == SI.Ptrs.size() && "Pointer without base!");

718 "No gc specified, so cannot relocate pointers!");

719

721 << "Lowering statepoint " << *SI.StatepointInstr << "\n");

722#ifndef NDEBUG

723 for (const auto *Reloc : SI.GCRelocates)

724 if (Reloc->getParent() == SI.StatepointInstr->getParent())

726#endif

727

728

729

730

732

735

738 SI, *this);

739

740

741

743

744

748

749

750

751

752

754

756 bool CallHasIncomingGlue = CallNode->getGluedNode();

757 if (CallHasIncomingGlue) {

758

760 }

761

762

763

764

765

766

767

768

769 const bool IsGCTransition =

772 if (IsGCTransition) {

774

775

777

778

779 for (const Value *V : SI.GCTransitionArgs) {

781 if (V->getType()->isPointerTy())

783 }

784

785

786 if (CallHasIncomingGlue)

788

789 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);

790

791 SDValue GCTransitionStart =

792 DAG.getNode(ISD::GC_TRANSITION_START, getCurSDLoc(), NodeTys, TSOps);

793

794 Chain = GCTransitionStart.getValue(0);

795 Glue = GCTransitionStart.getValue(1);

796 }

797

798

799

800

802

803

805 Ops.push_back(

806 DAG.getTargetConstant(SI.NumPatchBytes, getCurSDLoc(), MVT::i32));

807

808

809

810 unsigned NumCallRegArgs =

811 CallNode->getNumOperands() - (CallHasIncomingGlue ? 4 : 3);

812 Ops.push_back(DAG.getTargetConstant(NumCallRegArgs, getCurSDLoc(), MVT::i32));

813

814

816 Ops.push_back(CallTarget);

817

818

819

821 if (CallHasIncomingGlue)

822 RegMaskIt = CallNode->op_end() - 2;

823 else

824 RegMaskIt = CallNode->op_end() - 1;

825 Ops.insert(Ops.end(), CallNode->op_begin() + 2, RegMaskIt);

826

827

829

830

833 "Unknown flag used");

835

836

838

839

840 Ops.push_back(*RegMaskIt);

841

842

843 Ops.push_back(Chain);

844

845

847 Ops.push_back(Glue);

848

849

850

852 for (auto SD : LoweredGCArgs) {

853 if (!LowerAsVReg.count(SD))

854 continue;

855 NodeTys.push_back(SD.getValueType());

856 }

857 LLVM_DEBUG(dbgs() << "Statepoint has " << NodeTys.size() << " results\n");

858 assert(NodeTys.size() == LowerAsVReg.size() && "Inconsistent GC Ptr lowering");

861

862 unsigned NumResults = NodeTys.size();

864 DAG.getMachineNode(TargetOpcode::STATEPOINT, getCurSDLoc(), NodeTys, Ops);

865 DAG.setNodeMemRefs(StatepointMCNode, MemRefs);

866

867

868

869

871 for (const auto *Relocate : SI.GCRelocates) {

872 Value *Derived = Relocate->getDerivedPtr();

874 auto It = LowerAsVReg.find(SD);

875 if (It == LowerAsVReg.end())

876 continue;

877

878 SDValue Relocated = SDValue(StatepointMCNode, It->second);

879

880

881

882 if (SI.StatepointInstr->getParent() == Relocate->getParent()) {

884 if (Res)

885 assert(Res == Relocated);

886 else

888 continue;

889 }

890

891

892 auto [VRegIt, Inserted] = VirtRegs.try_emplace(SD);

893 if (!Inserted)

894 continue;

895

896 auto *RetTy = Relocate->getType();

899 DAG.getDataLayout(), Reg, RetTy, std::nullopt);

902 PendingExports.push_back(Chain);

903

904 VRegIt->second = Reg;

905 }

906

907

908

909 const Instruction *StatepointInstr = SI.StatepointInstr;

910 auto &RelocationMap = FuncInfo.StatepointRelocationMaps[StatepointInstr];

915

916 bool IsLocal = (Relocate->getParent() == StatepointInstr->getParent());

917

919 if (LowerAsVReg.count(SDV)) {

920 if (IsLocal) {

921

923 } else {

925 auto It = VirtRegs.find(SDV);

927 Record.payload.Reg = It->second;

928 }

929 } else if (Loc.getNode()) {

932 } else {

934

935

936

937

940 }

941 RelocationMap[Relocate] = Record;

942 }

943

944

945

946 SDNode *SinkNode = StatepointMCNode;

947

948

949

950

951

952 if (IsGCTransition) {

954

955

957

958

959 for (const Value *V : SI.GCTransitionArgs) {

961 if (V->getType()->isPointerTy())

963 }

964

965

967

968 SDVTList NodeTys = DAG.getVTList(MVT::Other, MVT::Glue);

969

970 SDValue GCTransitionStart =

971 DAG.getNode(ISD::GC_TRANSITION_END, getCurSDLoc(), NodeTys, TEOps);

972

973 SinkNode = GCTransitionStart.getNode();

974 }

975

976

977

978

979 unsigned NumSinkValues = SinkNode->getNumValues();

980 SDValue StatepointValues[2] = {SDValue(SinkNode, NumSinkValues - 2),

981 SDValue(SinkNode, NumSinkValues - 1)};

982 DAG.ReplaceAllUsesWith(CallNode, StatepointValues);

983

984 DAG.DeleteNode(CallNode);

985

986

987

989

990

991

992

993

994

995

996 return ReturnVal;

997}

998

999

1000

1001

1002static std::pair<const GCResultInst*, const GCResultInst*>

1004 std::pair<const GCResultInst *, const GCResultInst*> Res(nullptr, nullptr);

1005 for (const auto *U : S.users()) {

1007 if (!GRI)

1008 continue;

1009 if (GRI->getParent() == S.getParent())

1010 Res.first = GRI;

1011 else

1012 Res.second = GRI;

1013 }

1014 return Res;

1015}

1016

1017void

1019 const BasicBlock *EHPadBB ) {

1021 "anyregcc is not supported on statepoints!");

1022

1023#ifndef NDEBUG

1024

1025 assert(GFI->getStrategy().useStatepoints() &&

1026 "GCStrategy does not expect to encounter statepoints");

1027#endif

1028

1031

1032 if (I.getNumPatchBytes() > 0) {

1033

1034

1035

1036

1037

1038 ActualCallee = DAG.getUNDEF(Callee.getValueType());

1039 } else {

1040 ActualCallee = Callee;

1041 }

1042

1045 if (GCResultLocality.first)

1046 retAttrs = GCResultLocality.first->getAttributes().getRetAttrs();

1047

1050 I.getNumCallArgs(), ActualCallee,

1051 I.getActualReturnType(), retAttrs,

1052 false);

1053

1054

1055

1056

1057

1058

1059

1060

1061

1062

1063

1064

1065

1066

1068 for (const GCRelocateInst *Relocate : I.getGCRelocates()) {

1069 SI.GCRelocates.push_back(Relocate);

1070

1071 SDValue DerivedSD = getValue(Relocate->getDerivedPtr());

1072 if (Seen.insert(DerivedSD).second) {

1073 SI.Bases.push_back(Relocate->getBasePtr());

1074 SI.Ptrs.push_back(Relocate->getDerivedPtr());

1075 }

1076 }

1077

1078

1079

1080

1081

1082

1083

1084 for (Value *V : I.deopt_operands()) {

1086 continue;

1088 SI.Bases.push_back(V);

1089 SI.Ptrs.push_back(V);

1090 }

1091 }

1092

1094 SI.StatepointInstr = &I;

1095 SI.ID = I.getID();

1096

1099 I.gc_transition_args_end());

1100

1101 SI.StatepointFlags = I.getFlags();

1102 SI.NumPatchBytes = I.getNumPatchBytes();

1103 SI.EHPadBB = EHPadBB;

1104

1106

1107

1108 if (!GCResultLocality.first && !GCResultLocality.second) {

1109

1110

1112 return;

1113 }

1114

1115 if (GCResultLocality.first) {

1116

1117

1118

1120 }

1121

1122 if (!GCResultLocality.second)

1123 return;

1124

1125

1126

1127

1128

1129

1130

1131

1132 Type *RetTy = GCResultLocality.second->getType();

1135 DAG.getDataLayout(), Reg, RetTy,

1136 I.getCallingConv());

1138

1140 PendingExports.push_back(Chain);

1142}

1143

1146 bool VarArgDisallowed, bool ForceVoidReturnTy) {

1148 unsigned ArgBeginIndex = Call->arg_begin() - Call->op_begin();

1150 SI.CLI, Call, ArgBeginIndex, Call->arg_size(), Callee,

1152 Call->getAttributes().getRetAttrs(), false);

1153 if (!VarArgDisallowed)

1154 SI.CLI.IsVarArg = Call->getFunctionType()->isVarArg();

1155

1157

1159

1161 SI.ID = SD.StatepointID.value_or(DefaultID);

1162 SI.NumPatchBytes = SD.NumPatchBytes.value_or(0);

1163

1164 SI.DeoptState =

1165 ArrayRef(DeoptBundle.Inputs.begin(), DeoptBundle.Inputs.end());

1167 SI.EHPadBB = EHPadBB;

1168

1169

1170

1171 LLVM_DEBUG(dbgs() << "Lowering call with deopt bundle " << *Call << "\n");

1175 }

1176}

1177

1181 false,

1182 false);

1183}

1184

1185void SelectionDAGBuilder::visitGCResult(const GCResultInst &CI) {

1186

1187

1190 "GetStatepoint must return one of two types");

1192 return;

1193

1196 return;

1197 }

1198

1199

1200

1201

1202

1203

1206

1207 assert(CopyFromReg.getNode());

1209}

1210

1211void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) {

1212 const Value *Statepoint = Relocate.getStatepoint();

1213#ifndef NDEBUG

1214

1215

1216

1217

1219 "GetStatepoint must return one of two types");

1221 return;

1222

1225#endif

1226

1227 const Value *DerivedPtr = Relocate.getDerivedPtr();

1228 auto &RelocationMap =

1230 auto SlotIt = RelocationMap.find(&Relocate);

1231 assert(SlotIt != RelocationMap.end() && "Relocating not lowered gc value");

1232 const RecordType &Record = SlotIt->second;

1233

1234

1238 "Nonlocal gc.relocate mapped via SDValue");

1242 return;

1243 }

1246 RegsForValue RFV(*DAG.getContext(), DAG.getTargetLoweringInfo(),

1247 DAG.getDataLayout(), InReg, Relocate.getType(),

1248 std::nullopt);

1249

1250

1251

1254 Chain, nullptr, nullptr);

1255 setValue(&Relocate, Relocation);

1256 return;

1257 }

1258

1262

1263

1264

1265

1266

1267

1268

1269

1270 const SDValue Chain = DAG.getRoot();

1271

1272 auto &MF = DAG.getMachineFunction();

1273 auto &MFI = MF.getFrameInfo();

1276 MFI.getObjectSize(Index),

1277 MFI.getObjectAlign(Index));

1278

1279 auto LoadVT = DAG.getTargetLoweringInfo().getValueType(DAG.getDataLayout(),

1281

1283 DAG.getLoad(LoadVT, getCurSDLoc(), Chain, SpillSlot, LoadMMO);

1285

1287 setValue(&Relocate, SpillLoad);

1288 return;

1289 }

1290

1293

1295

1296

1297 setValue(&Relocate, DAG.getConstant(0xFEFEFEFE, SDLoc(SD), MVT::i64));

1298 return;

1299 }

1300

1301

1302

1304}

1305

1307 const auto &TLI = DAG.getTargetLoweringInfo();

1308 SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE),

1309 TLI.getPointerTy(DAG.getDataLayout()));

1310

1311

1312

1313

1315 true,

1316 true);

1317}

1318

1320

1321

1322

1323 if (DAG.getTarget().Options.TrapUnreachable)

1324 DAG.setRoot(

1325 DAG.getNode(ISD::TRAP, getCurSDLoc(), MVT::Other, DAG.getRoot()));

1326}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static const Function * getParent(const Value *V)

const AbstractManglingParser< Derived, Alloc >::OperatorInfo AbstractManglingParser< Derived, Alloc >::Ops[]

Promote Memory to Register

This file implements a set that has insertion order iteration characteristics.

This file implements the SmallBitVector class.

This file defines the SmallSet class.

This file defines the SmallVector class.

static cl::opt< bool > UseRegistersForGCPointersInLandingPad("use-registers-for-gc-values-in-landing-pad", cl::Hidden, cl::init(false), cl::desc("Allow using registers for gc pointer in landing pad"))

static void lowerIncomingStatepointValue(SDValue Incoming, bool RequireSpillSlot, SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand * > &MemRefs, SelectionDAGBuilder &Builder)

Lower a single value incoming to a statepoint node.

Definition StatepointLowering.cpp:424

static std::optional< int > findPreviousSpillSlot(const Value *Val, SelectionDAGBuilder &Builder, int LookUpDepth)

Utility function for reservePreviousStackSlotForValue.

Definition StatepointLowering.cpp:160

static void pushStackMapConstant(SmallVectorImpl< SDValue > &Ops, SelectionDAGBuilder &Builder, uint64_t Value)

Definition StatepointLowering.cpp:78

static bool isGCValue(const Value *V, SelectionDAGBuilder &Builder)

Return true if value V represents the GC value.

Definition StatepointLowering.cpp:502

static bool willLowerDirectly(SDValue Incoming)

Return true if-and-only-if the given SDValue can be lowered as either a constant argument or a stack ...

Definition StatepointLowering.cpp:247

static void lowerStatepointMetaArgs(SmallVectorImpl< SDValue > &Ops, SmallVectorImpl< MachineMemOperand * > &MemRefs, SmallVectorImpl< SDValue > &GCPtrs, DenseMap< SDValue, int > &LowerAsVReg, SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)

Lower deopt state and gc pointer arguments of the statepoint.

Definition StatepointLowering.cpp:520

static std::pair< const GCResultInst *, const GCResultInst * > getGCResultLocality(const GCStatepointInst &S)

Return two gc.results if present.

Definition StatepointLowering.cpp:1003

static cl::opt< bool > UseRegistersForDeoptValues("use-registers-for-deopt-values", cl::Hidden, cl::init(false), cl::desc("Allow using registers for non pointer deopt args"))

static cl::opt< unsigned > MaxRegistersForGCPointers("max-registers-for-gc-values", cl::Hidden, cl::init(0), cl::desc("Max number of VRegs allowed to pass GC pointer meta args in"))

static void reservePreviousStackSlotForValue(const Value *IncomingValue, SelectionDAGBuilder &Builder)

Try to find existing copies of the incoming values in stack slots used for statepoint spilling.

Definition StatepointLowering.cpp:268

FunctionLoweringInfo::StatepointRelocationRecord RecordType

Definition StatepointLowering.cpp:76

static MachineMemOperand * getMachineMemOperand(MachineFunction &MF, FrameIndexSDNode &FI)

Definition StatepointLowering.cpp:358

static std::pair< SDValue, SDNode * > lowerCallFromStatepointLoweringInfo(SelectionDAGBuilder::StatepointLoweringInfo &SI, SelectionDAGBuilder &Builder)

Extract call from statepoint, lower it and return pointer to the call node.

Definition StatepointLowering.cpp:318

static std::tuple< SDValue, SDValue, MachineMemOperand * > spillIncomingStatepointValue(SDValue Incoming, SDValue Chain, SelectionDAGBuilder &Builder)

Spill a value incoming to the statepoint.

Definition StatepointLowering.cpp:376

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

This file describes how to lower LLVM code to machine code.

This class represents an incoming formal argument to a Function.

This class holds the attributes for a particular argument, parameter, function, or return value.

LLVM Basic Block Representation.

This class represents a no-op cast from one type to another.

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

This class represents a function call, abstracting a target machine's calling convention.

iterator find(const_arg_type_t< KeyT > Val)

std::pair< iterator, bool > try_emplace(KeyT &&Key, Ts &&...Args)

size_type count(const_arg_type_t< KeyT > Val) const

Return 1 if the specified key is in the map, 0 otherwise.

LLVM_ABI const Value * getStatepoint() const

The statepoint with which this gc.relocate is associated.

Represents calls to the gc.relocate intrinsic.

LLVM_ABI Value * getDerivedPtr() const

Represents calls to the gc.result intrinsic.

Represents a gc.statepoint intrinsic call.

The landingpad instruction holds all of the information necessary to generate correct exception handl...

The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.

void markAsStatepointSpillSlotObjectIndex(int ObjectIdx)

Align getObjectAlign(int ObjectIdx) const

Return the alignment of the specified stack object.

int64_t getObjectSize(int ObjectIdx) const

Return the size of the specified object.

MachineMemOperand * getMachineMemOperand(MachinePointerInfo PtrInfo, MachineMemOperand::Flags f, LLT MemTy, Align base_alignment, const AAMDNodes &AAInfo=AAMDNodes(), const MDNode *Ranges=nullptr, SyncScope::ID SSID=SyncScope::System, AtomicOrdering Ordering=AtomicOrdering::NotAtomic, AtomicOrdering FailureOrdering=AtomicOrdering::NotAtomic)

getMachineMemOperand - Allocate a new MachineMemOperand.

MachineFrameInfo & getFrameInfo()

getFrameInfo - Return the frame info object for the current function.

A description of a memory reference used in the backend.

@ MOVolatile

The memory access is volatile.

@ MOLoad

The memory access reads data.

@ MOStore

The memory access writes data.

An SDNode that represents everything that will be needed to construct a MachineInstr.

Wrapper class representing virtual and physical registers.

Wrapper class for IR location info (IR ordering and DebugLoc) to be passed into SDNode creation funct...

Represents one node in the SelectionDAG.

unsigned getOpcode() const

Return the SelectionDAG opcode value for this node.

unsigned getNumValues() const

Return the number of values defined/returned by this operator.

unsigned getNumOperands() const

Return the number of values used by this operation.

const SDValue & getOperand(unsigned Num) const

SDNode * getGluedNode() const

If this node has a glue operand, return the node to which the glue operand points.

op_iterator op_end() const

op_iterator op_begin() const

Unlike LLVM values, Selection DAG nodes may return multiple values as the result of a computation.

SDNode * getNode() const

get the SDNode which holds the desired result

SDValue getValue(unsigned R) const

EVT getValueType() const

Return the ValueType of the referenced return value.

SelectionDAGBuilder - This is the common target-independent lowering implementation that is parameter...

SDValue getValue(const Value *V)

getValue - Return an SDValue for the given Value.

MVT getFrameIndexTy()

Returns the type of FrameIndex and TargetFrameIndex nodes.

void LowerStatepoint(const GCStatepointInst &I, const BasicBlock *EHPadBB=nullptr)

Definition StatepointLowering.cpp:1018

SDValue lowerRangeToAssertZExt(SelectionDAG &DAG, const Instruction &I, SDValue Op)

void LowerDeoptimizeCall(const CallInst *CI)

Definition StatepointLowering.cpp:1306

void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB)

Definition StatepointLowering.cpp:1178

void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB, bool VarArgDisallowed, bool ForceVoidReturnTy)

Definition StatepointLowering.cpp:1144

StatepointLoweringState StatepointLowering

State used while lowering a statepoint sequence (gc_statepoint, gc_relocate, and gc_result).

void populateCallLoweringInfo(TargetLowering::CallLoweringInfo &CLI, const CallBase *Call, unsigned ArgIdx, unsigned NumArgs, SDValue Callee, Type *ReturnTy, AttributeSet RetAttrs, bool IsPatchPoint)

Populate a CallLowerinInfo (into CLI) based on the properties of the call being lowered.

SmallVector< SDValue, 8 > PendingLoads

Loads are not emitted to the program immediately.

GCFunctionInfo * GFI

Garbage collection metadata for the function.

SDValue getRoot()

Similar to getMemoryRoot, but also flushes PendingConstrainedFP(Strict) items.

void ExportFromCurrentBlock(const Value *V)

ExportFromCurrentBlock - If this condition isn't known to be exported from the current basic block,...

SDLoc getCurSDLoc() const

SDValue getCopyFromRegs(const Value *V, Type *Ty)

If there was virtual register allocated for the value V emit CopyFromReg of the specified type Ty.

void LowerDeoptimizingReturn()

Definition StatepointLowering.cpp:1319

FunctionLoweringInfo & FuncInfo

Information about the function as a whole.

void setValue(const Value *V, SDValue NewN)

SDValue getControlRoot()

Similar to getRoot, but instead of flushing all the PendingLoad items, flush all the PendingExports (...

SDValue LowerAsSTATEPOINT(StatepointLoweringInfo &SI)

Lower SLI into a STATEPOINT instruction.

Definition StatepointLowering.cpp:707

size_type size() const

Determine the number of elements in the SetVector.

Vector takeVector()

Clear the SetVector and return the underlying vector.

bool insert(const value_type &X)

Insert a new element into the SetVector.

A SetVector that performs no allocations if smaller than a certain size.

SmallSet - This maintains a set of unique values, optimizing for the case when the set is small (less...

size_type count(const T &V) const

count - Return 1 if the element is in the set, 0 otherwise.

std::pair< const_iterator, bool > insert(const T &V)

insert - Insert an element into the set if it isn't already there.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

void clear()

Clear the memory usage of this object.

Definition StatepointLowering.cpp:99

SDValue allocateStackSlot(EVT ValueType, SelectionDAGBuilder &Builder)

Get a stack slot we can use to store an value of type ValueType.

Definition StatepointLowering.cpp:107

void startNewStatepoint(SelectionDAGBuilder &Builder)

Reset all state tracking for a newly encountered safepoint.

Definition StatepointLowering.cpp:86

The instances of the Type class are immutable: once they are created, they are never changed.

static LLVM_ABI Type * getVoidTy(LLVMContext &C)

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

iterator_range< user_iterator > users()

const ParentTy * getParent() const

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ AnyReg

OBSOLETED - Used for stack based JavaScript calls.

@ C

The default llvm calling convention, compatible with C.

@ CopyFromReg

CopyFromReg - This node indicates that the input value is a virtual or physical register that is defi...

initializer< Ty > init(const Ty &Val)

This is an optimization pass for GlobalISel generic memory operations.

auto find(R &&Range, const T &Val)

Provide wrappers to std::find which take ranges instead of having to pass begin/end explicitly.

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

bool isIntOrFPConstant(SDValue V)

Return true if V is either a integer or FP constant.

void append_range(Container &C, Range &&R)

Wrapper function to append range R to container C.

auto dyn_cast_or_null(const Y &Val)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

StatepointDirectives parseStatepointDirectivesFromAttrs(AttributeList AS)

Parse out statepoint directives from the function attributes present in AS.

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

ArrayRef(const T &OneElt) -> ArrayRef< T >

@ MaskAll

A bitmask that includes all valid flags.

@ DeoptLiveIn

Mark the deopt arguments associated with the statepoint as only being "live-in".

@ GCTransition

Indicates that this statepoint is a transition from GC-aware code to code that is not GC-aware.

decltype(auto) cast(const From &Val)

cast - Return the argument parameter cast to the specified type.

PointerUnion< const Value *, const PseudoSourceValue * > ValueType

TypeSize getSizeInBits() const

Return the size of the specified value type in bits.

bool isVector() const

Return true if this is a vector value type.

Helper object to track which of three possible relocation mechanisms are used for a particular value ...

union llvm::FunctionLoweringInfo::StatepointRelocationRecord::payload_t payload

enum llvm::FunctionLoweringInfo::StatepointRelocationRecord::RelocType type

Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...

static LLVM_ABI MachinePointerInfo getFixedStack(MachineFunction &MF, int FI, int64_t Offset=0)

Return a MachinePointerInfo record that refers to the specified FrameIndex.

This struct represents the registers (physical or virtual) that a particular set of values is assigne...

void getCopyToRegs(SDValue Val, SelectionDAG &DAG, const SDLoc &dl, SDValue &Chain, SDValue *Glue, const Value *V=nullptr, ISD::NodeType PreferredExtendType=ISD::ANY_EXTEND) const

Emit a series of CopyToReg nodes that copies the specified value into the registers specified by this...

This represents a list of ValueType's that has been intern'd by a SelectionDAG.

Describes a gc.statepoint or a gc.statepoint like thing for the purposes of lowering into a STATEPOIN...

static const uint64_t DeoptBundleStatepointID