FunctionImport.cpp Source File (original) (raw)

47#include

48#include

49#include

50#include <system_error>

51#include

52#include

54using namespace llvm;

56#define DEBUG_TYPE "function-import"

59 "Number of functions thin link decided to import");

61 "Number of hot functions thin link decided to import");

62 STATISTIC(NumImportedCriticalFunctionsThinLink,

63 "Number of critical functions thin link decided to import");

65 "Number of global variables thin link decided to import");

66 STATISTIC(NumImportedFunctions, "Number of functions imported in backend");

68 "Number of global variables imported in backend");

69 STATISTIC(NumImportedModules, "Number of modules imported from");

70 STATISTIC(NumDeadSymbols, "Number of dead stripped symbols in index");

71 STATISTIC(NumLiveSymbols, "Number of live symbols in index");

76 cl::desc("Only import functions with less than N instructions"));

80 cl::desc("Only import first N functions if N>=0 (default -1)"));

84 cl::desc("Import functions with noinline attribute"));

89 cl::desc("As we import functions, multiply the "

90 "`import-instr-limit` threshold by this factor "

91 "before processing newly imported functions"));

96 cl::desc("As we import functions called from hot callsite, multiply the "

97 "`import-instr-limit` threshold by this factor "

98 "before processing newly imported functions"));

102 cl::desc("Multiply the `import-instr-limit` threshold for hot callsites"));

103

108 "Multiply the `import-instr-limit` threshold for critical callsites"));

109

110

113 cl::desc("Multiply the `import-instr-limit` threshold for cold callsites"));

114

116 cl::desc("Print imported functions"));

117

120 cl::desc("Print information for functions rejected for importing"));

121

123 cl::desc("Compute dead symbols"));

124

127 cl::desc("Enable import metadata like 'thinlto_src_module' and "

128 "'thinlto_src_file'"));

129

130

131

134 cl::desc("The summary file to use for function importing."));

135

136

137

140 cl::desc("Import all external functions in index."));

141

142

143

144

145

146

147

150 cl::desc("If true, import function declaration as fallback if the function "

151 "definition is not imported."));

152

153

154

155

156

157

158

159

160

162 "thinlto-workload-def",

163 cl::desc("Pass a workload definition. This is a file containing a JSON "

164 "dictionary. The keys are root functions, the values are lists of "

165 "functions to import in the module defining the root. It is "

166 "assumed -funique-internal-linkage-names was used, to ensure "

167 "local linkage functions have unique names. For example: \n"

168 "{\n"

169 " \"rootFunction_1\": [\"function_to_import_1\", "

170 "\"function_to_import_2\"], \n"

171 " \"rootFunction_2\": [\"function_to_import_3\", "

172 "\"function_to_import_4\"] \n"

173 "}"),

175

177

178namespace llvm {

180}

181

182

183static std::unique_ptr loadFile(const std::string &FileName,

186 LLVM_DEBUG(dbgs() << "Loading '" << FileName << "'\n");

187

188

189 std::unique_ptr Result =

191 true);

192 if (!Result) {

193 Err.print("function-import", errs());

195 }

196

197 return Result;

198}

199

201 size_t NumDefs,

203

204 if (NumDefs == 1)

205 return false;

206

207

208

209

210

211

213 RefSummary->modulePath() != ImporterModule;

214}

215

216

217

218

219

222 ArrayRef<std::unique_ptr> CalleeSummaryList,

225 CalleeSummaryList,

226 [&Index, CalleeSummaryList,

227 CallerModulePath](const std::unique_ptr &SummaryPtr)

230 auto *GVSummary = SummaryPtr.get();

231 if (!Index.isGlobalValueLive(GVSummary))

232 return {FunctionImporter::ImportFailureReason::NotLive, GVSummary};

233

235 return {FunctionImporter::ImportFailureReason::InterposableLinkage,

236 GVSummary};

237

238 auto *Summary = dyn_cast(GVSummary->getBaseObject());

239

240

241

242

243

244

245 if (!Summary)

246 return {FunctionImporter::ImportFailureReason::GlobalVar, GVSummary};

247

248

249

250

251

252

253

254

255

256

257

258

259

261 CallerModulePath))

262 return {

263 FunctionImporter::ImportFailureReason::LocalLinkageNotInModule,

264 GVSummary};

265

266

267

268 if (Summary->notEligibleToImport())

269 return {FunctionImporter::ImportFailureReason::NotEligible,

270 GVSummary};

271

272 return {FunctionImporter::ImportFailureReason::None, GVSummary};

273 });

274}

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

293 ArrayRef<std::unique_ptr> CalleeSummaryList,

294 unsigned Threshold, StringRef CallerModulePath,

297

298 TooLargeOrNoInlineSummary = nullptr;

299 auto QualifiedCandidates =

301 for (auto QualifiedValue : QualifiedCandidates) {

302 Reason = QualifiedValue.first;

303

304 if (Reason != FunctionImporter::ImportFailureReason::None)

305 continue;

306 auto *Summary =

307 cast(QualifiedValue.second->getBaseObject());

308

309

310

311 if ((Summary->instCount() > Threshold) && !Summary->fflags().AlwaysInline &&

313 TooLargeOrNoInlineSummary = Summary;

314 Reason = FunctionImporter::ImportFailureReason::TooLarge;

315 continue;

316 }

317

318

320 TooLargeOrNoInlineSummary = Summary;

321 Reason = FunctionImporter::ImportFailureReason::NoInline;

322 continue;

323 }

324

325 return Summary;

326 }

327 return nullptr;

328}

329

330namespace {

331

332using EdgeInfo = std::tuple<const FunctionSummary *, unsigned >;

333

334}

335

339 auto [Def, Decl] = IDs.createImportIDs(FromModule, GUID);

340 if (!Imports.insert(Def).second)

341

343

344

345

348}

349

352 auto [Def, Decl] = IDs.createImportIDs(FromModule, GUID);

353

354

355 if (!Imports.contains(Def))

356 Imports.insert(Decl);

357}

358

362 for (const auto &[SrcMod, GUID, ImportType] : *this)

363 ModuleSet.insert(SrcMod);

366 return Modules;

367}

368

369std::optionalGlobalValueSummary::ImportKind

372 if (auto IDPair = IDs.getImportIDs(FromModule, GUID)) {

373 auto [Def, Decl] = *IDPair;

374 if (Imports.contains(Def))

376 if (Imports.contains(Decl))

378 }

379 return std::nullopt;

380}

381

382

383

388 IsPrevailing;

391

392 bool shouldImportGlobal(const ValueInfo &VI) {

393 const auto &GVS = DefinedGVSummaries.find(VI.getGUID());

394 if (GVS == DefinedGVSummaries.end())

395 return true;

396

397

398

399

400

401

402

403

404

405

406 if (VI.getSummaryList().size() > 1 &&

408 !IsPrevailing(VI.getGUID(), GVS->second))

409 return true;

410

411 return false;

412 }

413

414 void

417 for (const auto &VI : Summary.refs()) {

418 if (!shouldImportGlobal(VI)) {

420 dbgs() << "Ref ignored! Target already in destination module.\n");

421 continue;

422 }

423

425

426 for (const auto &RefSummary : VI.getSummaryList()) {

427 const auto *GVS = dyn_cast(RefSummary.get());

428

429

430

431

432

433 bool CanImportDecl = false;

434 if (!GVS ||

436 Summary.modulePath()) ||

437 Index .canImportGlobalVar(GVS, true,

438 CanImportDecl)) {

441 VI.getGUID());

442

443 continue;

444 }

445

446

447

448 if (ImportList.addDefinition(RefSummary->modulePath(), VI.getGUID()) !=

449 FunctionImporter::ImportMapTy::AddDefinitionStatus::Inserted)

450 break;

451

452

453

454 NumImportedGlobalVarsThinLink++;

455

456

457

458 if (ExportLists)

459 (*ExportLists)[RefSummary->modulePath()].insert(VI);

460

461

462

463 if ( Index .isWriteOnly(GVS))

465 break;

466 }

467 }

468 }

469

470public:

474 IsPrevailing,

477 : Index(Index), DefinedGVSummaries(DefinedGVSummaries),

478 IsPrevailing(IsPrevailing), ImportList(ImportList),

479 ExportLists(ExportLists) {}

480

483 onImportingSummaryImpl(Summary, Worklist);

484 while (!Worklist.empty())

485 onImportingSummaryImpl(*Worklist.pop_back_val(), Worklist);

486 }

487};

488

490

491

493protected:

498

501 IsPrevailing,

504 : IsPrevailing(IsPrevailing), Index(Index), ExportLists(ExportLists) {}

505

506public:

508

509

510

511

512 virtual void

513 computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,

516

517 static std::unique_ptr

519 IsPrevailing,

522 nullptr);

523};

524

525

526

527

529

530

531

533

534 void

535 computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries,

538 auto SetIter = Workloads.find(ModName);

539 if (SetIter == Workloads.end()) {

541 << " does not contain the root of any context.\n");

543 ModName, ImportList);

544 }

546 << " contains the root(s) of context(s).\n");

547

549 ExportLists);

550 auto &ValueInfos = SetIter->second;

553 auto It = DefinedGVSummaries.find(VI.getGUID());

554 if (It != DefinedGVSummaries.end() &&

555 IsPrevailing(VI.getGUID(), It->second)) {

557 dbgs() << "[Workload] " << VI.name()

558 << " has the prevailing variant already in the module "

559 << ModName << ". No need to import\n");

560 continue;

561 }

562 auto Candidates =

564

568 Candidates,

569 [&](const auto &Candidate) {

570 LLVM_DEBUG(dbgs() << "[Workflow] Candidate for " << VI.name()

571 << " from " << Candidate.second->modulePath()

572 << " ImportFailureReason: "

574 return Candidate.first ==

575 FunctionImporter::ImportFailureReason::None;

576 }),

577 [](const auto &Candidate) { return Candidate.second; });

578 if (PotentialCandidates.empty()) {

579 LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()

580 << " because can't find eligible Callee. Guid is: "

581 << Function::getGUID(VI.name()) << "\n");

582 continue;

583 }

584

585

586

587

588

589

590

591

592

593

594

595

596

597

599 PotentialCandidates, [&](const auto *Candidate) {

600 return IsPrevailing(VI.getGUID(), Candidate);

601 });

602 if (PrevailingCandidates.empty()) {

603 GVS = *PotentialCandidates.begin();

608 << "[Workload] Found multiple non-prevailing candidates for "

609 << VI.name()

610 << ". This is unexpected. Are module paths passed to the "

611 "compiler unique for the modules passed to the linker?");

612

613

614

615

616

618 } else {

620 GVS = *PrevailingCandidates.begin();

621 }

622

623 auto ExportingModule = GVS->modulePath();

624

625

626

627 if (ExportingModule == ModName) {

628 LLVM_DEBUG(dbgs() << "[Workload] Not importing " << VI.name()

629 << " because its defining module is the same as the "

630 "current module\n");

631 continue;

632 }

633 LLVM_DEBUG(dbgs() << "[Workload][Including]" << VI.name() << " from "

634 << ExportingModule << " : "

635 << Function::getGUID(VI.name()) << "\n");

636 ImportList.addDefinition(ExportingModule, VI.getGUID());

638 if (ExportLists)

639 (*ExportLists)[ExportingModule].insert(VI);

640 }

642 }

643

644 void loadFromJson() {

645

646

649 for (auto &I : Index) {

651 if (!NameToValueInfo.insert(std::make_pair(VI.name(), VI)).second)

653 }

654 auto DbgReportIfAmbiguous = [&](StringRef Name) {

656 dbgs() << "[Workload] Function name " << Name

657 << " present in the workload definition is ambiguous. Consider "

658 "compiling with -funique-internal-linkage-names.";

659 });

660 };

661 std::error_code EC;

663 if (std::error_code EC = BufferOrErr.getError()) {

665 return;

666 }

667 auto Buffer = std::move(BufferOrErr.get());

668 std::map<std::string, std::vectorstd::string> WorkloadDefs;

670

671

672

673

674

675

676 auto Parsed = json::parse(Buffer->getBuffer());

677 if (!Parsed)

679 if ( json::fromJSON (*Parsed, WorkloadDefs, NullRoot))

681 for (const auto &Workload : WorkloadDefs) {

682 const auto &Root = Workload.first;

683 DbgReportIfAmbiguous(Root);

684 LLVM_DEBUG(dbgs() << "[Workload] Root: " << Root << "\n");

685 const auto &AllCallees = Workload.second;

686 auto RootIt = NameToValueInfo.find(Root);

687 if (RootIt == NameToValueInfo.end()) {

689 << " not found in this linkage unit.\n");

690 continue;

691 }

692 auto RootVI = RootIt->second;

693 if (RootVI.getSummaryList().size() != 1) {

695 << " should have exactly one summary, but has "

696 << RootVI.getSummaryList().size() << ". Skipping.\n");

697 continue;

698 }

700 RootVI.getSummaryList().front()->modulePath();

701 LLVM_DEBUG(dbgs() << "[Workload] Root defining module for " << Root

702 << " is : " << RootDefiningModule << "\n");

703 auto &Set = Workloads[RootDefiningModule];

704 for (const auto &Callee : AllCallees) {

705 LLVM_DEBUG(dbgs() << "[Workload] " << Callee << "\n");

706 DbgReportIfAmbiguous(Callee);

707 auto ElemIt = NameToValueInfo.find(Callee);

708 if (ElemIt == NameToValueInfo.end()) {

709 LLVM_DEBUG(dbgs() << "[Workload] " << Callee << " not found\n");

710 continue;

711 }

712 Set.insert(ElemIt->second);

713 }

714 }

715 }

716

717 void loadFromCtxProf() {

718 std::error_code EC;

720 if (std::error_code EC = BufferOrErr.getError()) {

722 return;

723 }

724 auto Buffer = std::move(BufferOrErr.get());

725

728 if (!Ctx) {

730 return;

731 }

732 const auto &CtxMap = *Ctx;

734 for (const auto &[RootGuid, Root] : CtxMap) {

735

736

737 ContainedGUIDs.clear();

738

739 auto RootVI = Index.getValueInfo(RootGuid);

740 if (!RootVI) {

742 << " not found in this linkage unit.\n");

743 continue;

744 }

745 if (RootVI.getSummaryList().size() != 1) {

747 << " should have exactly one summary, but has "

748 << RootVI.getSummaryList().size() << ". Skipping.\n");

749 continue;

750 }

752 RootVI.getSummaryList().front()->modulePath();

753 LLVM_DEBUG(dbgs() << "[Workload] Root defining module for " << RootGuid

754 << " is : " << RootDefiningModule << "\n");

755 auto &Set = Workloads[RootDefiningModule];

756 Root.getContainedGuids(ContainedGUIDs);

757 for (auto Guid : ContainedGUIDs)

758 if (auto VI = Index.getValueInfo(Guid))

759 Set.insert(VI);

760 }

761 }

762

763public:

766 IsPrevailing,

772 "Pass only one of: -thinlto-pgo-ctx-prof or -thinlto-workload-def");

773 return;

774 }

776 loadFromCtxProf();

777 else

778 loadFromJson();

780 for (const auto &[Root, Set] : Workloads) {

781 dbgs() << "[Workload] Root: " << Root << " we have " << Set.size()

782 << " distinct callees.\n";

783 for (const auto &VI : Set) {

784 dbgs() << "[Workload] Root: " << Root

785 << " Would include: " << VI.getGUID() << "\n";

786 }

787 }

788 });

789 }

790};

791

794 IsPrevailing,

798 LLVM_DEBUG(dbgs() << "[Workload] Using the regular imports manager.\n");

799 return std::unique_ptr(

801 }

802 LLVM_DEBUG(dbgs() << "[Workload] Using the contextual imports manager.\n");

803 return std::make_unique(IsPrevailing, Index,

804 ExportLists);

805}

806

807static const char *

809 switch (Reason) {

811 return "None";

813 return "GlobalVar";

815 return "NotLive";

817 return "TooLarge";

819 return "InterposableLinkage";

821 return "LocalLinkageNotInModule";

823 return "NotEligible";

825 return "NoInline";

826 }

828}

829

830

831

832

835 const unsigned Threshold, const GVSummaryMapTy &DefinedGVSummaries,

837 isPrevailing,

843 static int ImportCount = 0;

844 for (const auto &Edge : Summary.calls()) {

846 LLVM_DEBUG(dbgs() << " edge -> " << VI << " Threshold:" << Threshold

847 << "\n");

848

851 << " reached.\n");

852 continue;

853 }

854

855 if (DefinedGVSummaries.count(VI.getGUID())) {

856

857

858

859 LLVM_DEBUG(dbgs() << "ignored! Target already in destination module.\n");

860 continue;

861 }

862

870 return 1.0;

871 };

872

873 const auto NewThreshold =

874 Threshold * GetBonusMultiplier(Edge.second.getHotness());

875

876 auto IT = ImportThresholds.insert(std::make_pair(

877 VI.getGUID(), std::make_tuple(NewThreshold, nullptr, nullptr)));

878 bool PreviouslyVisited = .second;

879 auto &ProcessedThreshold = std::get<0>(IT.first->second);

880 auto &CalleeSummary = std::get<1>(IT.first->second);

881 auto &FailureInfo = std::get<2>(IT.first->second);

882

883 bool IsHotCallsite =

885 bool IsCriticalCallsite =

887

889 if (CalleeSummary) {

890 assert(PreviouslyVisited);

891

892

893

894

895 if (NewThreshold <= ProcessedThreshold) {

897 dbgs() << "ignored! Target was already imported with Threshold "

898 << ProcessedThreshold << "\n");

899 continue;

900 }

901

902 ProcessedThreshold = NewThreshold;

903 ResolvedCalleeSummary = cast(CalleeSummary);

904 } else {

905

906

907 if (PreviouslyVisited && NewThreshold <= ProcessedThreshold) {

909 dbgs() << "ignored! Target was already rejected with Threshold "

910 << ProcessedThreshold << "\n");

912 assert(FailureInfo &&

913 "Expected FailureInfo for previously rejected candidate");

914 FailureInfo->Attempts++;

915 }

916 continue;

917 }

918

920

921

923 CalleeSummary =

924 selectCallee(Index, VI.getSummaryList(), NewThreshold,

925 Summary.modulePath(), SummaryForDeclImport, Reason);

926 if (!CalleeSummary) {

927

928

931

932

933

935 }

936

937

938

939 if (PreviouslyVisited) {

940 ProcessedThreshold = NewThreshold;

942 assert(FailureInfo &&

943 "Expected FailureInfo for previously rejected candidate");

944 FailureInfo->Reason = Reason;

945 FailureInfo->Attempts++;

946 FailureInfo->MaxHotness =

947 std::max(FailureInfo->MaxHotness, Edge.second.getHotness());

948 }

950 assert(!FailureInfo &&

951 "Expected no FailureInfo for newly rejected candidate");

952 FailureInfo = std::make_uniqueFunctionImporter::ImportFailureInfo(

953 VI, Edge.second.getHotness(), Reason, 1);

954 }

956 std::string Msg = std::string("Failed to import function ") +

957 VI.name().str() + " due to " +

959 auto Error = make_error(

962 "Error importing module: ");

963 break;

964 } else {

966 << "ignored! No qualifying callee with summary found.\n");

967 continue;

968 }

969 }

970

971

972 CalleeSummary = CalleeSummary->getBaseObject();

973 ResolvedCalleeSummary = cast(CalleeSummary);

974

976 (ResolvedCalleeSummary->instCount() <= NewThreshold)) &&

977 "selectCallee() didn't honor the threshold");

978

979 auto ExportModulePath = ResolvedCalleeSummary->modulePath();

980

981

982

983 if (ImportList.addDefinition(ExportModulePath, VI.getGUID()) !=

985 NumImportedFunctionsThinLink++;

986 if (IsHotCallsite)

987 NumImportedHotFunctionsThinLink++;

988 if (IsCriticalCallsite)

989 NumImportedCriticalFunctionsThinLink++;

990 }

991

992

993

994

995 if (ExportLists)

996 (*ExportLists)[ExportModulePath].insert(VI);

997 }

998

999 auto GetAdjustedThreshold = [](unsigned Threshold, bool IsHotCallsite) {

1000

1001

1002

1003 if (IsHotCallsite)

1006 };

1007

1008 const auto AdjThreshold = GetAdjustedThreshold(Threshold, IsHotCallsite);

1009

1010 ImportCount++;

1011

1012

1013 Worklist.emplace_back(ResolvedCalleeSummary, AdjThreshold);

1014 }

1015}

1016

1020

1021

1023 GlobalsImporter GVI(Index, DefinedGVSummaries, IsPrevailing, ImportList,

1024 ExportLists);

1026

1027

1028

1029 for (const auto &GVSummary : DefinedGVSummaries) {

1030#ifndef NDEBUG

1031

1032

1033 auto VI = Index.getValueInfo(GVSummary.first);

1034#endif

1035 if (!Index.isGlobalValueLive(GVSummary.second)) {

1036 LLVM_DEBUG(dbgs() << "Ignores Dead GUID: " << VI << "\n");

1037 continue;

1038 }

1039 auto *FuncSummary =

1040 dyn_cast(GVSummary.second->getBaseObject());

1041 if (!FuncSummary)

1042

1043 continue;

1044 LLVM_DEBUG(dbgs() << "Initialize import for " << VI << "\n");

1046 DefinedGVSummaries, IsPrevailing, Worklist, GVI,

1047 ImportList, ExportLists, ImportThresholds);

1048 }

1049

1050

1051 while (!Worklist.empty()) {

1053 auto *Summary = std::get<0>(GVInfo);

1054 auto Threshold = std::get<1>(GVInfo);

1055

1056 if (auto *FS = dyn_cast(Summary))

1058 IsPrevailing, Worklist, GVI, ImportList,

1059 ExportLists, ImportThresholds);

1060 }

1061

1062

1063

1065 dbgs() << "Missed imports into module " << ModName << "\n";

1066 for (auto &I : ImportThresholds) {

1067 auto &ProcessedThreshold = std::get<0>(I.second);

1068 auto &CalleeSummary = std::get<1>(I.second);

1069 auto &FailureInfo = std::get<2>(I.second);

1070 if (CalleeSummary)

1071 continue;

1072 assert(FailureInfo);

1074 if (!FailureInfo->VI.getSummaryList().empty())

1075 FS = dyn_cast(

1076 FailureInfo->VI.getSummaryList()[0]->getBaseObject());

1077 dbgs() << FailureInfo->VI

1078 << ": Reason = " << getFailureName(FailureInfo->Reason)

1079 << ", Threshold = " << ProcessedThreshold

1080 << ", Size = " << (FS ? (int)FS->instCount() : -1)

1081 << ", MaxHotness = " << getHotnessName(FailureInfo->MaxHotness)

1082 << ", Attempts = " << FailureInfo->Attempts << "\n";

1083 }

1084 }

1085}

1086

1087#ifndef NDEBUG

1089 auto SL = VI.getSummaryList();

1090 return SL.empty()

1093}

1094

1097 if (const auto &VI = Index.getValueInfo(G))

1099 return false;

1100}

1101

1102

1103static unsigned

1106 unsigned NumGVS = 0;

1107 for (auto &VI : ExportSet)

1109 ++NumGVS;

1110 return NumGVS;

1111}

1112

1114 unsigned NumGVS = 0;

1115 unsigned DefinedFS = 0;

1116 unsigned Count = 0;

1117};

1118

1119

1124

1125 for (const auto &[FromModule, GUID, Type] : ImportList) {

1127 ++Entry.Count;

1129 ++Entry.NumGVS;

1131 ++Entry.DefinedFS;

1132 }

1133 return Histogram;

1134}

1135#endif

1136

1137#ifndef NDEBUG

1143

1144 for (const auto &ImportPerModule : ImportLists)

1145 for (const auto &[FromModule, GUID, ImportType] : ImportPerModule.second)

1146 FlattenedImports.insert(GUID);

1147

1148

1149

1150

1151

1152

1153

1154 auto IsReadOrWriteOnlyVarNeedingImporting = [&](StringRef ModulePath,

1156 auto *GVS = dyn_cast_or_null(

1157 Index.findSummaryInModule(VI, ModulePath));

1158 return GVS && (Index.isReadOnly(GVS) || Index.isWriteOnly(GVS)) &&

1162 };

1163

1164 for (auto &ExportPerModule : ExportLists)

1165 for (auto &VI : ExportPerModule.second)

1166 if (!FlattenedImports.count(VI.getGUID()) &&

1167 IsReadOrWriteOnlyVarNeedingImporting(ExportPerModule.first, VI))

1168 return false;

1169

1170 return true;

1171}

1172#endif

1173

1174

1179 isPrevailing,

1183

1184 for (const auto &DefinedGVSummaries : ModuleToDefinedGVSummaries) {

1185 auto &ImportList = ImportLists[DefinedGVSummaries.first];

1186 LLVM_DEBUG(dbgs() << "Computing import for Module '"

1187 << DefinedGVSummaries.first << "'\n");

1188 MIS->computeImportForModule(DefinedGVSummaries.second,

1189 DefinedGVSummaries.first, ImportList);

1190 }

1191

1192

1193

1194

1195

1196

1197 for (auto &ELI : ExportLists) {

1198

1199

1201 const auto &DefinedGVSummaries =

1202 ModuleToDefinedGVSummaries.lookup(ELI.first);

1203 for (auto &EI : ELI.second) {

1204

1205

1206

1207

1208

1209

1210 auto DS = DefinedGVSummaries.find(EI.getGUID());

1211

1212

1213 assert(DS != DefinedGVSummaries.end());

1214 auto *S = DS->getSecond();

1215 S = S->getBaseObject();

1216 if (auto *GVS = dyn_cast(S)) {

1217

1218

1219

1220

1221 if (!Index.isWriteOnly(GVS))

1222 for (const auto &VI : GVS->refs())

1223 NewExports.insert(VI);

1224 } else {

1225 auto *FS = cast(S);

1226 for (const auto &Edge : FS->calls())

1227 NewExports.insert(Edge.first);

1228 for (const auto &Ref : FS->refs())

1230 }

1231 }

1232

1233

1234

1235

1236 for (auto EI = NewExports.begin(); EI != NewExports.end();) {

1237 if (!DefinedGVSummaries.count(EI->getGUID()))

1238 NewExports.erase(EI++);

1239 else

1240 ++EI;

1241 }

1242 ELI.second.insert(NewExports.begin(), NewExports.end());

1243 }

1244

1246#ifndef NDEBUG

1247 LLVM_DEBUG(dbgs() << "Import/Export lists for " << ImportLists.size()

1248 << " modules:\n");

1249 for (const auto &ModuleImports : ImportLists) {

1250 auto ModName = ModuleImports.first;

1251 auto &Exports = ExportLists[ModName];

1255 LLVM_DEBUG(dbgs() << "* Module " << ModName << " exports "

1256 << Exports.size() - NumGVS << " functions and " << NumGVS

1257 << " vars. Imports from " << Histogram.size()

1258 << " modules.\n");

1259 for (const auto &[SrcModName, Stats] : Histogram) {

1261 << " function definitions and "

1263 << " function declarations imported from " << SrcModName

1264 << "\n");

1266 << " global vars imported from " << SrcModName << "\n");

1267 }

1268 }

1269#endif

1270}

1271

1272#ifndef NDEBUG

1278 LLVM_DEBUG(dbgs() << "* Module " << ModulePath << " imports from "

1279 << Histogram.size() << " modules.\n");

1280 for (const auto &[SrcModName, Stats] : Histogram) {

1282 << " function definitions and "

1284 << " function declarations imported from " << SrcModName

1285 << "\n");

1287 << SrcModName << "\n");

1288 }

1289}

1290#endif

1291

1292

1293

1294

1295

1296

1297

1298

1299

1303 isPrevailing,

1306

1307

1309 Index.collectDefinedFunctionsForModule(ModulePath, FunctionSummaryMap);

1310

1311

1312 LLVM_DEBUG(dbgs() << "Computing import for Module '" << ModulePath << "'\n");

1314 MIS->computeImportForModule(FunctionSummaryMap, ModulePath, ImportList);

1315

1316#ifndef NDEBUG

1318#endif

1319}

1320

1321

1322

1323

1324

1325

1329 for (const auto &GlobalList : Index) {

1330

1331 if (GlobalList.second.SummaryList.empty())

1332 continue;

1333

1334 auto GUID = GlobalList.first;

1335 assert(GlobalList.second.SummaryList.size() == 1 &&

1336 "Expected individual combined index to have one summary per GUID");

1337 auto &Summary = GlobalList.second.SummaryList[0];

1338

1339

1340 if (Summary->modulePath() == ModulePath)

1341 continue;

1342

1343 ImportList.addGUID(Summary->modulePath(), GUID, Summary->importType());

1344 }

1345#ifndef NDEBUG

1347#endif

1348}

1349

1350

1351

1352

1353

1356 for (auto &EI : FS->mutableCalls()) {

1357 if (!EI.first.getSummaryList().empty())

1358 continue;

1359 auto GUID = Index.getGUIDFromOriginalID(EI.first.getGUID());

1360 if (GUID == 0)

1361 continue;

1362

1363 auto VI = Index.getValueInfo(GUID);

1365 VI.getSummaryList(),

1366 [&](const std::unique_ptr &SummaryPtr) {

1367

1368

1369

1370

1371

1372

1373

1374

1375

1376 return SummaryPtr->getSummaryKind() ==

1377 GlobalValueSummary::GlobalVarKind;

1378 }))

1379 continue;

1380 EI.first = VI;

1381 }

1382}

1383

1385 for (const auto &Entry : Index) {

1386 for (const auto &S : Entry.second.SummaryList) {

1387 if (auto *FS = dyn_cast(S.get()))

1389 }

1390 }

1391}

1392

1397 assert(!Index.withGlobalValueDeadStripping());

1399

1400 GUIDPreservedSymbols.empty()) {

1401

1403 return;

1404 }

1405 unsigned LiveSymbols = 0;

1407 Worklist.reserve(GUIDPreservedSymbols.size() * 2);

1408 for (auto GUID : GUIDPreservedSymbols) {

1409 ValueInfo VI = Index.getValueInfo(GUID);

1410 if (!VI)

1411 continue;

1412 for (const auto &S : VI.getSummaryList())

1413 S->setLive(true);

1414 }

1415

1416

1417 for (const auto &Entry : Index) {

1418 auto VI = Index.getValueInfo(Entry);

1419 for (const auto &S : Entry.second.SummaryList) {

1420 if (auto *FS = dyn_cast(S.get()))

1422 if (S->isLive()) {

1425 ++LiveSymbols;

1426 break;

1427 }

1428 }

1429 }

1430

1431

1433

1434

1435

1436

1437

1438

1439

1440

1441

1443 [](const std::unique_ptrllvm::GlobalValueSummary &S) {

1444 return S->isLive();

1445 }))

1446 return;

1447

1448

1449

1450

1451

1452

1454 bool KeepAliveLinkage = false;

1455 bool Interposable = false;

1456 for (const auto &S : VI.getSummaryList()) {

1460 KeepAliveLinkage = true;

1462 Interposable = true;

1463 }

1464

1465 if (!IsAliasee) {

1466 if (!KeepAliveLinkage)

1467 return;

1468

1469 if (Interposable)

1471 "Interposable and available_externally/linkonce_odr/weak_odr "

1472 "symbol");

1473 }

1474 }

1475

1476 for (const auto &S : VI.getSummaryList())

1477 S->setLive(true);

1478 ++LiveSymbols;

1480 };

1481

1482 while (!Worklist.empty()) {

1484 for (const auto &Summary : VI.getSummaryList()) {

1485 if (auto *AS = dyn_cast(Summary.get())) {

1486

1487

1488

1489 visit(AS->getAliaseeVI(), true);

1490 continue;

1491 }

1492 for (auto Ref : Summary->refs())

1494 if (auto *FS = dyn_cast(Summary.get()))

1495 for (auto Call : FS->calls())

1496 visit(Call.first, false);

1497 }

1498 }

1499 Index.setWithGlobalValueDeadStripping();

1500

1501 unsigned DeadSymbols = Index.size() - LiveSymbols;

1502 LLVM_DEBUG(dbgs() << LiveSymbols << " symbols Live, and " << DeadSymbols

1503 << " symbols Dead \n");

1504 NumDeadSymbols += DeadSymbols;

1505 NumLiveSymbols += LiveSymbols;

1506}

1507

1508

1513 bool ImportEnabled) {

1515 isPrevailing);

1516 if (ImportEnabled)

1517 Index.propagateAttributes(GUIDPreservedSymbols);

1518}

1519

1520

1521

1528

1529 ModuleToSummariesForIndex[std::string(ModulePath)] =

1530 ModuleToDefinedGVSummaries.lookup(ModulePath);

1531

1532

1533

1534

1535

1536

1539 auto It = Map.find(Key);

1540 if (It == Map.end())

1541 std::tie(It, std::ignore) =

1542 Map.try_emplace(std::string(Key), GVSummaryMapTy());

1543 return It->second;

1544 };

1545

1546

1547 for (const auto &[FromModule, GUID, ImportType] : ImportList) {

1548 auto &SummariesForIndex =

1549 LookupOrCreate(ModuleToSummariesForIndex, FromModule);

1550

1551 const auto &DefinedGVSummaries = ModuleToDefinedGVSummaries.at(FromModule);

1552 const auto &DS = DefinedGVSummaries.find(GUID);

1553 assert(DS != DefinedGVSummaries.end() &&

1554 "Expected a defined summary for imported global value");

1556 DecSummaries.insert(DS->second);

1557

1558 SummariesForIndex[GUID] = DS->second;

1559 }

1560}

1561

1562

1566 std::error_code EC;

1568 if (EC)

1571 for (const auto &ILI : ModuleToSummariesForIndex)

1572

1573

1574

1575 if (ILI.first != ModulePath)

1576 ImportsOS << ILI.first << "\n";

1578}

1579

1582 << "\n");

1583 if (Function *F = dyn_cast(&GV)) {

1584 F->deleteBody();

1585 F->clearMetadata();

1586 F->setComdat(nullptr);

1587 } else if (GlobalVariable *V = dyn_cast(&GV)) {

1588 V->setInitializer(nullptr);

1590 V->clearMetadata();

1591 V->setComdat(nullptr);

1592 } else {

1595 NewGV =

1599 else

1600 NewGV =

1603 nullptr, "",

1608 return false;

1609 }

1612 return true;

1613}

1614

1619 auto FinalizeInModule = [&](GlobalValue &GV, bool Propagate = false) {

1620

1621 const auto &GS = DefinedGlobals.find(GV.getGUID());

1622 if (GS == DefinedGlobals.end())

1623 return;

1624

1625 if (Propagate)

1626 if (FunctionSummary *FS = dyn_cast(GS->second)) {

1627 if (Function *F = dyn_cast(&GV)) {

1628

1629 if (FS->fflags().ReadNone && ->doesNotAccessMemory())

1630 F->setDoesNotAccessMemory();

1631

1632 if (FS->fflags().ReadOnly && ->onlyReadsMemory())

1633 F->setOnlyReadsMemory();

1634

1635 if (FS->fflags().NoRecurse && ->doesNotRecurse())

1636 F->setDoesNotRecurse();

1637

1638 if (FS->fflags().NoUnwind && ->doesNotThrow())

1639 F->setDoesNotThrow();

1640 }

1641 }

1642

1643 auto NewLinkage = GS->second->linkage();

1645

1646

1647

1649

1651 return;

1652

1653

1654

1655

1656

1659

1661 return;

1662

1663

1664

1665

1666

1667

1671

1672

1673

1675 } else {

1676

1677

1678

1679

1680

1682 GS->second->canAutoHide()) {

1685 }

1686

1688 << "` from " << GV.getLinkage() << " to " << NewLinkage

1689 << "\n");

1691 }

1692

1693

1694

1695 auto *GO = dyn_cast_or_null(&GV);

1696 if (GO && GO->isDeclarationForLinker() && GO->hasComdat()) {

1697 if (GO->getComdat()->getName() == GO->getName())

1698 NonPrevailingComdats.insert(GO->getComdat());

1699 GO->setComdat(nullptr);

1700 }

1701 };

1702

1703

1704 for (auto &GV : TheModule)

1706 for (auto &GV : TheModule.globals())

1707 FinalizeInModule(GV);

1708 for (auto &GV : TheModule.aliases())

1709 FinalizeInModule(GV);

1710

1711

1712

1713

1714 if (NonPrevailingComdats.empty())

1715 return;

1717 if (auto *C = GO.getComdat(); C && NonPrevailingComdats.count(C)) {

1718 GO.setComdat(nullptr);

1720 }

1721 }

1722 bool Changed;

1723 do {

1724 Changed = false;

1725

1726

1727

1728

1729 for (auto &GA : TheModule.aliases()) {

1730 if (GA.hasAvailableExternallyLinkage())

1731 continue;

1733 assert(Obj && "aliasee without an base object is unimplemented");

1736 Changed = true;

1737 }

1738 }

1739 } while (Changed);

1740}

1741

1742

1745

1746

1747 auto MustPreserveGV = [&](const GlobalValue &GV) -> bool {

1748

1749

1750

1751 if (isa(&GV) ||

1752 (isa(&GV) &&

1753 isa(cast(&GV)->getAliaseeObject())))

1754 return true;

1755

1756

1757 auto GS = DefinedGlobals.find(GV.getGUID());

1758 if (GS == DefinedGlobals.end()) {

1759

1760

1761

1762

1763

1770 if (GS == DefinedGlobals.end()) {

1771

1772

1773

1774

1775

1776

1778 assert(GS != DefinedGlobals.end());

1779 }

1780 }

1782 };

1783

1784

1785

1787}

1788

1789

1792

1795

1796

1801 return NewFn;

1802}

1803

1804

1805

1807 for (auto &GV : M.globals())

1808

1809

1810 if (!GV.isDeclaration() && GV.hasAttribute("thinlto-internalize")) {

1813 }

1814}

1815

1816

1817

1822 unsigned ImportedCount = 0, ImportedGVCount = 0;

1823

1824 IRMover Mover(DestModule);

1825

1826

1828

1830 if (!SrcModuleOrErr)

1831 return SrcModuleOrErr.takeError();

1832 std::unique_ptr SrcModule = std::move(*SrcModuleOrErr);

1833 assert(&DestModule.getContext() == &SrcModule->getContext() &&

1834 "Context mismatch");

1835

1836

1837

1838 if (Error Err = SrcModule->materializeMetadata())

1839 return std::move(Err);

1840

1841

1843 for (Function &F : *SrcModule) {

1844 if (.hasName())

1845 continue;

1846 auto GUID = F.getGUID();

1847 auto MaybeImportType = ImportList.getImportType(ModName, GUID);

1849

1850 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")

1851 << " importing function"

1852 << (ImportDefinition

1853 ? " definition "

1854 : (MaybeImportType ? " declaration " : " "))

1855 << GUID << " " << F.getName() << " from "

1856 << SrcModule->getSourceFileName() << "\n");

1857 if (ImportDefinition) {

1858 if (Error Err = F.materialize())

1859 return std::move(Err);

1860

1861

1863

1864

1865 F.setMetadata(

1866 "thinlto_src_module",

1868 {MDString::get(DestModule.getContext(),

1869 SrcModule->getModuleIdentifier())}));

1870 F.setMetadata(

1871 "thinlto_src_file",

1873 {MDString::get(DestModule.getContext(),

1874 SrcModule->getSourceFileName())}));

1875 }

1876 GlobalsToImport.insert(&F);

1877 }

1878 }

1880 if (!GV.hasName())

1881 continue;

1882 auto GUID = GV.getGUID();

1883 auto MaybeImportType = ImportList.getImportType(ModName, GUID);

1885

1886 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")

1887 << " importing global"

1888 << (ImportDefinition

1889 ? " definition "

1890 : (MaybeImportType ? " declaration " : " "))

1891 << GUID << " " << GV.getName() << " from "

1892 << SrcModule->getSourceFileName() << "\n");

1893 if (ImportDefinition) {

1894 if (Error Err = GV.materialize())

1895 return std::move(Err);

1896 ImportedGVCount += GlobalsToImport.insert(&GV);

1897 }

1898 }

1899 for (GlobalAlias &GA : SrcModule->aliases()) {

1900 if (!GA.hasName() || isa(GA.getAliaseeObject()))

1901 continue;

1902 auto GUID = GA.getGUID();

1903 auto MaybeImportType = ImportList.getImportType(ModName, GUID);

1905

1906 LLVM_DEBUG(dbgs() << (MaybeImportType ? "Is" : "Not")

1907 << " importing alias"

1908 << (ImportDefinition

1909 ? " definition "

1910 : (MaybeImportType ? " declaration " : " "))

1911 << GUID << " " << GA.getName() << " from "

1912 << SrcModule->getSourceFileName() << "\n");

1913 if (ImportDefinition) {

1914 if (Error Err = GA.materialize())

1915 return std::move(Err);

1916

1919 return std::move(Err);

1922 << GO->getName() << " from "

1923 << SrcModule->getSourceFileName() << "\n");

1925

1926

1927 Fn->setMetadata(

1928 "thinlto_src_module",

1930 {MDString::get(DestModule.getContext(),

1931 SrcModule->getModuleIdentifier())}));

1932 Fn->setMetadata(

1933 "thinlto_src_file",

1935 {MDString::get(DestModule.getContext(),

1936 SrcModule->getSourceFileName())}));

1937 }

1938 GlobalsToImport.insert(Fn);

1939 }

1940 }

1941

1942

1943

1945

1946

1947

1948

1949

1950 SrcModule->setPartialSampleProfileRatio(Index);

1951

1952

1954 &GlobalsToImport);

1955

1957 for (const auto *GV : GlobalsToImport)

1959 << " from " << SrcModule->getSourceFileName() << "\n";

1960 }

1961

1962 if (Error Err = Mover.move(std::move(SrcModule),

1964 true))

1966 Twine("Function Import: link error: ") +

1968

1969 ImportedCount += GlobalsToImport.size();

1970 NumImportedModules++;

1971 }

1972

1974

1975 NumImportedFunctions += (ImportedCount - ImportedGVCount);

1976 NumImportedGlobalVars += ImportedGVCount;

1977

1978

1979 LLVM_DEBUG(dbgs() << "Imported " << ImportedCount - ImportedGVCount

1980 << " functions for Module "

1983 << " global variables for Module "

1985 return ImportedCount;

1986}

1987

1990 isPrevailing) {

1992 report_fatal_error("error: -function-import requires -summary-file\n");

1995 if (!IndexPtrOrErr) {

1997 "Error loading file '" + SummaryFile + "': ");

1998 return false;

1999 }

2000 std::unique_ptr Index = std::move(*IndexPtrOrErr);

2001

2002

2005

2006

2007

2010 *Index, ImportList);

2011 else

2013 isPrevailing, *Index, ImportList);

2014

2015

2016

2017

2018

2019 for (auto &I : *Index) {

2020 for (auto &S : I.second.SummaryList) {

2023 }

2024 }

2025

2026

2027

2029 nullptr);

2030

2031

2032 auto ModuleLoader = [&M](StringRef Identifier) {

2033 return loadFile(std::string(Identifier), M.getContext());

2034 };

2036 false);

2038

2039

2040 if (!Result) {

2042 "Error importing module: ");

2043 return true;

2044 }

2045

2046 return true;

2047}

2048

2051

2052

2053

2054

2056 return true;

2057 };

2060

2062}

static cl::opt< ITMode > IT(cl::desc("IT block support"), cl::Hidden, cl::init(DefaultIT), cl::values(clEnumValN(DefaultIT, "arm-default-it", "Generate any type of IT block"), clEnumValN(RestrictedIT, "arm-restrict-it", "Disallow complex IT blocks")))

cl::opt< std::string > UseCtxProfile("use-ctx-profile", cl::init(""), cl::Hidden, cl::desc("Use the specified contextual profile file"))

static auto qualifyCalleeCandidates(const ModuleSummaryIndex &Index, ArrayRef< std::unique_ptr< GlobalValueSummary > > CalleeSummaryList, StringRef CallerModulePath)

Given a list of possible callee implementation for a call site, qualify the legality of importing eac...

static cl::opt< bool > EnableImportMetadata("enable-import-metadata", cl::init(false), cl::Hidden, cl::desc("Enable import metadata like 'thinlto_src_module' and " "'thinlto_src_file'"))

static cl::opt< float > ImportColdMultiplier("import-cold-multiplier", cl::init(0), cl::Hidden, cl::value_desc("N"), cl::desc("Multiply the `import-instr-limit` threshold for cold callsites"))

static DenseMap< StringRef, ImportStatistics > collectImportStatistics(const ModuleSummaryIndex &Index, const FunctionImporter::ImportMapTy &ImportList)

static cl::opt< float > ImportHotMultiplier("import-hot-multiplier", cl::init(10.0), cl::Hidden, cl::value_desc("x"), cl::desc("Multiply the `import-instr-limit` threshold for hot callsites"))

static unsigned numGlobalVarSummaries(const ModuleSummaryIndex &Index, FunctionImporter::ExportSetTy &ExportSet)

static bool checkVariableImport(const ModuleSummaryIndex &Index, FunctionImporter::ImportListsTy &ImportLists, DenseMap< StringRef, FunctionImporter::ExportSetTy > &ExportLists)

static bool doImportingForModuleForTest(Module &M, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing)

static cl::opt< float > ImportHotInstrFactor("import-hot-evolution-factor", cl::init(1.0), cl::Hidden, cl::value_desc("x"), cl::desc("As we import functions called from hot callsite, multiply the " "`import-instr-limit` threshold by this factor " "before processing newly imported functions"))

static void computeImportForFunction(const FunctionSummary &Summary, const ModuleSummaryIndex &Index, const unsigned Threshold, const GVSummaryMapTy &DefinedGVSummaries, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, SmallVectorImpl< EdgeInfo > &Worklist, GlobalsImporter &GVImporter, FunctionImporter::ImportMapTy &ImportList, DenseMap< StringRef, FunctionImporter::ExportSetTy > *ExportLists, FunctionImporter::ImportThresholdsTy &ImportThresholds)

Compute the list of functions to import for a given caller.

static cl::opt< float > ImportCriticalMultiplier("import-critical-multiplier", cl::init(100.0), cl::Hidden, cl::value_desc("x"), cl::desc("Multiply the `import-instr-limit` threshold for critical callsites"))

static cl::opt< int > ImportCutoff("import-cutoff", cl::init(-1), cl::Hidden, cl::value_desc("N"), cl::desc("Only import first N functions if N>=0 (default -1)"))

cl::opt< std::string > UseCtxProfile

static const char * getFailureName(FunctionImporter::ImportFailureReason Reason)

static cl::opt< float > ImportInstrFactor("import-instr-evolution-factor", cl::init(0.7), cl::Hidden, cl::value_desc("x"), cl::desc("As we import functions, multiply the " "`import-instr-limit` threshold by this factor " "before processing newly imported functions"))

static void internalizeGVsAfterImport(Module &M)

static cl::opt< bool > PrintImports("print-imports", cl::init(false), cl::Hidden, cl::desc("Print imported functions"))

void updateValueInfoForIndirectCalls(ModuleSummaryIndex &Index, FunctionSummary *FS)

static std::unique_ptr< Module > loadFile(const std::string &FileName, LLVMContext &Context)

static cl::opt< bool > ForceImportAll("force-import-all", cl::init(false), cl::Hidden, cl::desc("Import functions with noinline attribute"))

static bool shouldSkipLocalInAnotherModule(const GlobalValueSummary *RefSummary, size_t NumDefs, StringRef ImporterModule)

static bool isGlobalVarSummary(const ModuleSummaryIndex &Index, ValueInfo VI)

static cl::opt< bool > ImportDeclaration("import-declaration", cl::init(false), cl::Hidden, cl::desc("If true, import function declaration as fallback if the function " "definition is not imported."))

This is a test-only option.

static cl::opt< unsigned > ImportInstrLimit("import-instr-limit", cl::init(100), cl::Hidden, cl::value_desc("N"), cl::desc("Only import functions with less than N instructions"))

Limit on instruction count of imported functions.

static cl::opt< std::string > WorkloadDefinitions("thinlto-workload-def", cl::desc("Pass a workload definition. This is a file containing a JSON " "dictionary. The keys are root functions, the values are lists of " "functions to import in the module defining the root. It is " "assumed -funique-internal-linkage-names was used, to ensure " "local linkage functions have unique names. For example: \n" "{\n" " \"rootFunction_1\": [\"function_to_import_1\", " "\"function_to_import_2\"], \n" " \"rootFunction_2\": [\"function_to_import_3\", " "\"function_to_import_4\"] \n" "}"), cl::Hidden)

Pass a workload description file - an example of workload would be the functions executed to satisfy ...

static cl::opt< bool > ComputeDead("compute-dead", cl::init(true), cl::Hidden, cl::desc("Compute dead symbols"))

static cl::opt< std::string > SummaryFile("summary-file", cl::desc("The summary file to use for function importing."))

Summary file to use for function importing when using -function-import from the command line.

static cl::opt< bool > ImportAllIndex("import-all-index", cl::desc("Import all external functions in index."))

Used when testing importing from distributed indexes via opt.

static cl::opt< bool > PrintImportFailures("print-import-failures", cl::init(false), cl::Hidden, cl::desc("Print information for functions rejected for importing"))

static Function * replaceAliasWithAliasee(Module *SrcModule, GlobalAlias *GA)

Make alias a clone of its aliasee.

static void dumpImportListForModule(const ModuleSummaryIndex &Index, StringRef ModulePath, FunctionImporter::ImportMapTy &ImportList)

static void ComputeCrossModuleImportForModuleFromIndexForTest(StringRef ModulePath, const ModuleSummaryIndex &Index, FunctionImporter::ImportMapTy &ImportList)

Mark all external summaries in Index for import into the given module.

static const GlobalValueSummary * selectCallee(const ModuleSummaryIndex &Index, ArrayRef< std::unique_ptr< GlobalValueSummary > > CalleeSummaryList, unsigned Threshold, StringRef CallerModulePath, const GlobalValueSummary *&TooLargeOrNoInlineSummary, FunctionImporter::ImportFailureReason &Reason)

Given a list of possible callee implementation for a call site, select one that fits the Threshold fo...

static void ComputeCrossModuleImportForModuleForTest(StringRef ModulePath, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, const ModuleSummaryIndex &Index, FunctionImporter::ImportMapTy &ImportList)

Compute all the imports for the given module using the Index.

Module.h This file contains the declarations for the Module class.

This file supports working with JSON data.

block placement Basic Block Placement Stats

static cl::opt< std::string > OutputFilename("o", cl::desc("Output filename"), cl::value_desc("filename"), cl::init("-"))

static cl::opt< bool > PropagateAttrs("propagate-attrs", cl::init(true), cl::Hidden, cl::desc("Propagate attributes in index"))

ModuleSummaryIndex.h This file contains the declarations the classes that hold the module index and s...

Reader for contextual iFDO profile, which comes in bitstream format.

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

void visit(MachineFunction &MF, MachineBasicBlock &Start, std::function< void(MachineBasicBlock *)> op)

This file implements a set that has insertion order iteration characteristics.

This file defines the SmallVector class.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

Import globals referenced by a function or other globals that are being imported, if importing such g...

void onImportingSummary(const GlobalValueSummary &Summary)

GlobalsImporter(const ModuleSummaryIndex &Index, const GVSummaryMapTy &DefinedGVSummaries, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> IsPrevailing, FunctionImporter::ImportMapTy &ImportList, DenseMap< StringRef, FunctionImporter::ExportSetTy > *ExportLists)

Determine the list of imports and exports for each module.

DenseMap< StringRef, FunctionImporter::ExportSetTy > *const ExportLists

virtual ~ModuleImportsManager()=default

static std::unique_ptr< ModuleImportsManager > create(function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> IsPrevailing, const ModuleSummaryIndex &Index, DenseMap< StringRef, FunctionImporter::ExportSetTy > *ExportLists=nullptr)

function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> IsPrevailing

ModuleImportsManager(function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> IsPrevailing, const ModuleSummaryIndex &Index, DenseMap< StringRef, FunctionImporter::ExportSetTy > *ExportLists=nullptr)

const ModuleSummaryIndex & Index

virtual void computeImportForModule(const GVSummaryMapTy &DefinedGVSummaries, StringRef ModName, FunctionImporter::ImportMapTy &ImportList)

Given the list of globals defined in a module, compute the list of imports as well as the list of "ex...

A ModuleImportsManager that operates based on a workload definition (see -thinlto-workload-def).

WorkloadImportsManager(function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> IsPrevailing, const ModuleSummaryIndex &Index, DenseMap< StringRef, FunctionImporter::ExportSetTy > *ExportLists)

A container for analyses that lazily runs them and caches their results.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

ValueT lookup(const_arg_type_t< KeyT > Val) const

lookup - Return the entry for the specified key, or a default constructed value if no such entry exis...

iterator find(const_arg_type_t< KeyT > Val)

size_type count(const_arg_type_t< KeyT > Val) const

Return 1 if the specified key is in the map, 0 otherwise.

const ValueT & at(const_arg_type_t< KeyT > Val) const

at - Return the entry for the specified key, or abort if no such entry exists.

std::pair< iterator, bool > insert(const std::pair< KeyT, ValueT > &KV)

Implements a dense probed hash-table based set.

Lightweight error class with error context and mandatory checking.

static ErrorSuccess success()

Create a success value.

Tagged union holding either a T or a Error.

Error takeError()

Take ownership of the stored error.

PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)

std::pair< ImportIDTy, ImportIDTy > createImportIDs(StringRef FromModule, GlobalValue::GUID GUID)

The map maintains the list of imports.

AddDefinitionStatus addDefinition(StringRef FromModule, GlobalValue::GUID GUID)

void addGUID(StringRef FromModule, GlobalValue::GUID GUID, GlobalValueSummary::ImportKind ImportKind)

SmallVector< StringRef, 0 > getSourceModules() const

std::optional< GlobalValueSummary::ImportKind > getImportType(StringRef FromModule, GlobalValue::GUID GUID) const

void maybeAddDeclaration(StringRef FromModule, GlobalValue::GUID GUID)

The function importer is automatically importing function from other modules based on the provided su...

Expected< bool > importFunctions(Module &M, const ImportMapTy &ImportList)

Import functions in Module M based on the supplied import list.

ImportFailureReason

The different reasons selectCallee will chose not to import a candidate.

@ LocalLinkageNotInModule

Function summary information to aid decisions and implementation of importing.

unsigned instCount() const

Get the instruction count recorded for this function.

FFlags fflags() const

Get function summary flags.

static Function * Create(FunctionType *Ty, LinkageTypes Linkage, unsigned AddrSpace, const Twine &N="", Module *M=nullptr)

const GlobalObject * getAliaseeObject() const

Function and variable summary information to aid decisions and implementation of importing.

StringRef modulePath() const

Get the path to the module containing this function.

GlobalValue::LinkageTypes linkage() const

Return linkage type recorded for this global value.

VisibilityTypes getVisibility() const

bool isImplicitDSOLocal() const

static bool isLocalLinkage(LinkageTypes Linkage)

bool isDeclaration() const

Return true if the primary definition of this global value is outside of the current translation unit...

LinkageTypes getLinkage() const

uint64_t GUID

Declare a type to represent a global unique identifier for a global value.

ThreadLocalMode getThreadLocalMode() const

static bool isAvailableExternallyLinkage(LinkageTypes Linkage)

static GUID getGUID(StringRef GlobalName)

Return a 64-bit global unique ID constructed from global value name (i.e.

void setLinkage(LinkageTypes LT)

unsigned getAddressSpace() const

GUID getGUID() const

Return a 64-bit global unique ID constructed from global value name (i.e.

Module * getParent()

Get the module that this global value is contained inside of...

const GlobalObject * getAliaseeObject() const

void setDSOLocal(bool Local)

PointerType * getType() const

Global values are always pointers.

@ DefaultVisibility

The GV is visible.

@ HiddenVisibility

The GV is hidden.

void setVisibility(VisibilityTypes V)

static bool isInterposableLinkage(LinkageTypes Linkage)

Whether the definition of this global may be replaced by something non-equivalent at link time.

Error materialize()

Make sure this GlobalValue is fully read.

bool canBeOmittedFromSymbolTable() const

True if GV can be left out of the object symbol table.

bool hasAvailableExternallyLinkage() const

std::string getGlobalIdentifier() const

Return the modified name for this global value suitable to be used as the key for a global lookup (e....

@ InternalLinkage

Rename collisions when linking (static functions).

@ WeakODRLinkage

Same, but only replaced by something equivalent.

@ ExternalLinkage

Externally visible function.

@ AvailableExternallyLinkage

Available for inspection, not emission.

@ LinkOnceODRLinkage

Same, but only replaced by something equivalent.

Type * getValueType() const

Error move(std::unique_ptr< Module > Src, ArrayRef< GlobalValue * > ValuesToLink, LazyCallback AddLazyFor, bool IsPerformingImport)

Move in the provide values in ValuesToLink from Src.

This is an important class for using LLVM in a threaded context.

static MDTuple * get(LLVMContext &Context, ArrayRef< Metadata * > MDs)

static ErrorOr< std::unique_ptr< MemoryBuffer > > getFileOrSTDIN(const Twine &Filename, bool IsText=false, bool RequiresNullTerminator=true, std::optional< Align > Alignment=std::nullopt)

Open the specified file as a MemoryBuffer, or open stdin if the Filename is "-".

Class to hold module path string table and global value map, and encapsulate methods for operating on...

static StringRef getOriginalNameBeforePromote(StringRef Name)

Helper to obtain the unpromoted name for a global value (or the original name if not promoted).

A Module instance is used to store all the information related to an LLVM module.

LLVMContext & getContext() const

Get the global data context.

const std::string & getSourceFileName() const

Get the module's original source file name.

iterator_range< alias_iterator > aliases()

iterator_range< global_iterator > globals()

const std::string & getModuleIdentifier() const

Get the module identifier which is, essentially, the name of the module.

iterator_range< global_object_iterator > global_objects()

Expected< std::map< GlobalValue::GUID, PGOCtxProfContext > > loadContexts()

unsigned getAddressSpace() const

Return the address space of the Pointer type.

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

static PreservedAnalyses all()

Construct a special preserved set that preserves all passes.

Instances of this class encapsulate one diagnostic report, allowing printing to a raw_ostream as a ca...

A vector that has set insertion semantics.

ArrayRef< value_type > getArrayRef() const

size_type size() const

Determine the number of elements in the SetVector.

Vector takeVector()

Clear the SetVector and return the underlying vector.

void clear()

Completely clear the SetVector.

bool insert(const value_type &X)

Insert a new element into the SetVector.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

reference emplace_back(ArgTypes &&... Args)

void reserve(size_type N)

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringMap - This is an unconventional map that is specialized for handling keys that are "strings",...

iterator find(StringRef Key)

size_type count(StringRef Key) const

count - Return 1 if the element is in the map, 0 otherwise.

bool insert(MapEntryTy *KeyValue)

insert - Insert the specified key/value pair into the map.

StringRef - Represent a constant reference to a string, i.e.

constexpr size_t size() const

size - Get the string size.

char front() const

front - Get the first character in the string.

StringSet - A wrapper for StringMap that provides set-like functionality.

std::pair< typename Base::iterator, bool > insert(StringRef key)

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

The instances of the Type class are immutable: once they are created, they are never changed.

bool isFunctionTy() const

True if this is an instance of FunctionType.

void replaceAllUsesWith(Value *V)

Change all uses of this to point to a new Value.

StringRef getName() const

Return a constant reference to the value's name.

void takeName(Value *V)

Transfer the name from V to this value.

std::pair< iterator, bool > insert(const ValueT &V)

bool erase(const ValueT &V)

size_type count(const_arg_type_t< ValueT > V) const

Return 1 if the specified key is in the set, 0 otherwise.

An efficient, type-erasing, non-owning reference to a callable.

The root is the trivial Path to the root value.

A raw_ostream that writes to a file descriptor.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

@ C

The default llvm calling convention, compatible with C.

initializer< Ty > init(const Ty &Val)

llvm::Expected< Value > parse(llvm::StringRef JSON)

Parses the provided JSON source, or returns a ParseError.

bool fromJSON(const Value &E, std::string &Out, Path P)

@ OF_Text

The file should be opened in text mode on platforms like z/OS that make this distinction.

This is an optimization pass for GlobalISel generic memory operations.

void logAllUnhandledErrors(Error E, raw_ostream &OS, Twine ErrorBanner={})

Log all errors (if any) in E to OS.

Error createFileError(const Twine &F, Error E)

Concatenate a source file path and/or name with an Error.

bool internalizeModule(Module &TheModule, std::function< bool(const GlobalValue &)> MustPreserveGV)

Helper function to internalize functions and variables in a Module.

std::error_code make_error_code(BitcodeError E)

const char * getHotnessName(CalleeInfo::HotnessType HT)

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

bool convertToDeclaration(GlobalValue &GV)

Converts value GV to declaration, or replaces with a declaration if it is an alias.

void renameModuleForThinLTO(Module &M, const ModuleSummaryIndex &Index, bool ClearDSOLocalOnDeclarations, SetVector< GlobalValue * > *GlobalsToImport=nullptr)

Perform in-place global value handling on the given Module for exported local functions renamed and p...

Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)

Create formatted StringError object.

auto map_range(ContainerTy &&C, FuncTy F)

void ComputeCrossModuleImport(const ModuleSummaryIndex &Index, const DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, function_ref< bool(GlobalValue::GUID, const GlobalValueSummary *)> isPrevailing, FunctionImporter::ImportListsTy &ImportLists, DenseMap< StringRef, FunctionImporter::ExportSetTy > &ExportLists)

Compute all the imports and exports for every module in the Index.

bool any_of(R &&range, UnaryPredicate P)

Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.

void sort(IteratorTy Start, IteratorTy End)

void computeDeadSymbolsAndUpdateIndirectCalls(ModuleSummaryIndex &Index, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols, function_ref< PrevailingType(GlobalValue::GUID)> isPrevailing)

Compute all the symbols that are "dead": i.e these that can't be reached in the graph from any of the...

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

void report_fatal_error(Error Err, bool gen_crash_diag=true)

Report a serious error, calling any installed error handler.

bool hasSingleElement(ContainerTy &&C)

Returns true if the given container only contains a single element.

iterator_range< filter_iterator< detail::IterOfRange< RangeT >, PredicateT > > make_filter_range(RangeT &&Range, PredicateT Pred)

Convenience function that takes a range of elements and a predicate, and return a new filter_iterator...

void updateIndirectCalls(ModuleSummaryIndex &Index)

Update call edges for indirect calls to local functions added from SamplePGO when needed.

raw_fd_ostream & errs()

This returns a reference to a raw_ostream for standard error.

@ Ref

The access may reference the value stored in memory.

cl::opt< bool > EnableMemProfContextDisambiguation

Enable MemProf context disambiguation for thin link.

std::unique_ptr< Module > getLazyIRFileModule(StringRef Filename, SMDiagnostic &Err, LLVMContext &Context, bool ShouldLazyLoadMetadata=false)

If the given file holds a bitcode image, return a Module for it which does lazy deserialization of fu...

void thinLTOInternalizeModule(Module &TheModule, const GVSummaryMapTy &DefinedGlobals)

Internalize TheModule based on the information recorded in the summaries during global summary-based ...

std::unordered_set< GlobalValueSummary * > GVSummaryPtrSet

A set of global value summary pointers.

void gatherImportedSummariesForModule(StringRef ModulePath, const DenseMap< StringRef, GVSummaryMapTy > &ModuleToDefinedGVSummaries, const FunctionImporter::ImportMapTy &ImportList, ModuleToSummariesForIndexTy &ModuleToSummariesForIndex, GVSummaryPtrSet &DecSummaries)

Compute the set of summaries needed for a ThinLTO backend compilation of ModulePath.

std::map< std::string, GVSummaryMapTy, std::less<> > ModuleToSummariesForIndexTy

Map of a module name to the GUIDs and summaries we will import from that module.

PrevailingType

PrevailingType enum used as a return type of callback passed to computeDeadSymbolsAndUpdateIndirectCa...

const char * toString(DWARFSectionKind Kind)

Error errorCodeToError(std::error_code EC)

Helper for converting an std::error_code to a Error.

bool UpgradeDebugInfo(Module &M)

Check the debug info version number, if it is out-dated, drop the debug info.

Function * CloneFunction(Function *F, ValueToValueMapTy &VMap, ClonedCodeInfo *CodeInfo=nullptr)

Return a copy of the specified function and add it to that function's module.

void computeDeadSymbolsWithConstProp(ModuleSummaryIndex &Index, const DenseSet< GlobalValue::GUID > &GUIDPreservedSymbols, function_ref< PrevailingType(GlobalValue::GUID)> isPrevailing, bool ImportEnabled)

Compute dead symbols and run constant propagation in combined index after that.

Error EmitImportsFiles(StringRef ModulePath, StringRef OutputFilename, const ModuleToSummariesForIndexTy &ModuleToSummariesForIndex)

Emit into OutputFilename the files module ModulePath will import from.

Expected< std::unique_ptr< ModuleSummaryIndex > > getModuleSummaryIndexForFile(StringRef Path, bool IgnoreEmptyThinLTOIndexFile=false)

Parse the module summary index out of an IR file and return the module summary index object if found,...

DenseMap< GlobalValue::GUID, GlobalValueSummary * > GVSummaryMapTy

Map of global value GUID to its summary, used to identify values defined in a particular module,...

void thinLTOFinalizeInModule(Module &TheModule, const GVSummaryMapTy &DefinedGlobals, bool PropagateAttrs)

Based on the information recorded in the summaries during global summary-based analysis:

Struct that holds a reference to a particular GUID in a global value summary.