SampleProfileMatcher.cpp Source File (original) (raw)

21#include <unordered_set>

23using namespace llvm;

26#define DEBUG_TYPE "sample-profile-matcher"

28namespace llvm {

32 cl::desc("Consider a profile matches a function if the similarity of their "

33 "callee sequences is above the specified percentile."));

37 cl::desc("The minimum number of basic blocks required for a function to "

38 "run stale profile call graph matching."));

42 cl::desc("The minimum number of call anchors required for a function to "

43 "run stale profile call graph matching."));

48 "Load top-level profiles that the sample reader initially skipped for "

49 "the call-graph matching (only meaningful for extended binary "

50 "format)"));

58 "salvage-stale-profile-max-callsites", cl::Hidden, cl::init(UINT_MAX),

59 cl::desc("The maximum number of callsites in a function, above which stale "

60 "profile matching will be skipped."));

62}

64void SampleProfileMatcher::findIRAnchors(const Function &F,

69 auto FindTopLevelInlinedCallsite = [](const DILocation *DIL) {

70 assert((DIL && DIL->getInlinedAt()) && "No inlined callsite");

72 do {

73 PrevDIL = DIL;

74 DIL = DIL->getInlinedAt();

75 } while (DIL->getInlinedAt());

79 StringRef CalleeName = PrevDIL->getSubprogramLinkageName();

80 return std::make_pair(Callsite, FunctionId(CalleeName));

81 };

83 auto GetCanonicalCalleeName = [](const CallBase *CB) {

84 StringRef CalleeName = UnknownIndirectCallee;

85 if (Function *Callee = CB->getCalledFunction())

87 return CalleeName;

88 };

91 for (auto &BB : F) {

92 for (auto &I : BB) {

93 DILocation *DIL = I.getDebugLoc();

94 if (!DIL)

95 continue;

100 if (DIL->getInlinedAt()) {

101 IRAnchors.emplace(FindTopLevelInlinedCallsite(DIL));

102 } else {

103

104 StringRef CalleeName;

106

108 CalleeName = GetCanonicalCalleeName(CB);

109 }

110 LineLocation Loc = LineLocation(Probe->Id, 0);

111 IRAnchors.emplace(Loc, FunctionId(CalleeName));

112 }

113 }

114 } else {

115

116

117

119 continue;

120

121 if (DIL->getInlinedAt()) {

122 IRAnchors.emplace(FindTopLevelInlinedCallsite(DIL));

123 } else {

127 IRAnchors.emplace(Callsite, FunctionId(CalleeName));

128 }

129 }

130 }

131 }

132}

133

134void SampleProfileMatcher::findProfileAnchors(const FunctionSamples &FS,

135 AnchorMap &ProfileAnchors) const {

136 auto isInvalidLineOffset = [](uint32_t LineOffset) {

137 return LineOffset & 0x8000;

138 };

139

140 auto InsertAnchor = [](const LineLocation &Loc, const FunctionId &CalleeName,

142 auto Ret = ProfileAnchors.try_emplace(Loc, CalleeName);

143 if (!Ret.second) {

144

145

146 Ret.first->second = FunctionId(UnknownIndirectCallee);

147 }

148 };

149

150 for (const auto &I : FS.getBodySamples()) {

151 const LineLocation &Loc = I.first;

152 if (isInvalidLineOffset(Loc.LineOffset))

153 continue;

154 for (const auto &C : I.second.getCallTargets())

155 InsertAnchor(Loc, C.first, ProfileAnchors);

156 }

157

158 for (const auto &I : FS.getCallsiteSamples()) {

159 const LineLocation &Loc = I.first;

160 if (isInvalidLineOffset(Loc.LineOffset))

161 continue;

162 for (const auto &C : I.second)

163 InsertAnchor(Loc, C.first, ProfileAnchors);

164 }

165}

166

167bool SampleProfileMatcher::functionHasProfile(const FunctionId &IRFuncName,

168 Function *&FuncWithoutProfile) {

169 FuncWithoutProfile = nullptr;

170 auto R = FunctionsWithoutProfile.find(IRFuncName);

171 if (R != FunctionsWithoutProfile.end())

172 FuncWithoutProfile = R->second;

173 return !FuncWithoutProfile;

174}

175

176bool SampleProfileMatcher::isProfileUnused(const FunctionId &ProfileFuncName) {

177 return SymbolMap->find(ProfileFuncName) == SymbolMap->end();

178}

179

180bool SampleProfileMatcher::functionMatchesProfile(

182 bool FindMatchedProfileOnly) {

183 if (IRFuncName == ProfileFuncName)

184 return true;

186 return false;

187

188

189

191 if (functionHasProfile(IRFuncName, IRFunc) ||

192 !isProfileUnused(ProfileFuncName))

193 return false;

194

195 assert(FunctionId(IRFunc->getName()) != ProfileFuncName &&

196 "IR function should be different from profile function to match");

197 return functionMatchesProfile(*IRFunc, ProfileFuncName,

198 FindMatchedProfileOnly);

199}

200

202SampleProfileMatcher::longestCommonSequence(const AnchorList &AnchorList1,

204 bool MatchUnusedFunction) {

207 AnchorList1, AnchorList2,

208 [&](const FunctionId &A, const FunctionId &B) {

209 return functionMatchesProfile(

210 A, B,

211 !MatchUnusedFunction

212 );

213 },

214 [&](LineLocation A, LineLocation B) {

215 MatchedAnchors.try_emplace(A, B);

216 });

217 return MatchedAnchors;

218}

219

220void SampleProfileMatcher::matchNonCallsiteLocs(

223 auto InsertMatching = [&](const LineLocation &From, const LineLocation &To) {

224

225 if (From != To)

226 IRToProfileLocationMap.insert({From, To});

227 };

228

229

230 int32_t LocationDelta = 0;

232 for (const auto &IR : IRAnchors) {

233 const auto &Loc = IR.first;

234 bool IsMatchedAnchor = false;

235

236 auto R = MatchedAnchors.find(Loc);

237 if (R != MatchedAnchors.end()) {

238 const auto &Candidate = R->second;

239 InsertMatching(Loc, Candidate);

240 LLVM_DEBUG(dbgs() << "Callsite with callee:" << IR.second.stringRef()

241 << " is matched from " << Loc << " to " << Candidate

242 << "\n");

243 LocationDelta = Candidate.LineOffset - Loc.LineOffset;

244

245

246

247

248

249 for (size_t I = (LastMatchedNonAnchors.size() + 1) / 2;

250 I < LastMatchedNonAnchors.size(); I++) {

251 const auto &L = LastMatchedNonAnchors[I];

252 uint32_t CandidateLineOffset = L.LineOffset + LocationDelta;

253 LineLocation Candidate(CandidateLineOffset, L.Discriminator);

254 InsertMatching(L, Candidate);

255 LLVM_DEBUG(dbgs() << "Location is rematched backwards from " << L

256 << " to " << Candidate << "\n");

257 }

258

259 IsMatchedAnchor = true;

260 LastMatchedNonAnchors.clear();

261 }

262

263

264 if (!IsMatchedAnchor) {

265 uint32_t CandidateLineOffset = Loc.LineOffset + LocationDelta;

266 LineLocation Candidate(CandidateLineOffset, Loc.Discriminator);

267 InsertMatching(Loc, Candidate);

268 LLVM_DEBUG(dbgs() << "Location is matched from " << Loc << " to "

269 << Candidate << "\n");

271 }

272 }

273}

274

275

276

277void SampleProfileMatcher::getFilteredAnchorList(

280 for (const auto &I : IRAnchors) {

281 if (I.second.stringRef().empty())

282 continue;

283 FilteredIRAnchorsList.emplace_back(I);

284 }

285

286 for (const auto &I : ProfileAnchors)

287 FilteredProfileAnchorList.emplace_back(I);

288}

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306

307void SampleProfileMatcher::runStaleProfileMatching(

310 bool RunCFGMatching, bool RunCGMatching) {

311 if (!RunCFGMatching && !RunCGMatching)

312 return;

313 LLVM_DEBUG(dbgs() << "Run stale profile matching for " << F.getName()

314 << "\n");

315 assert(IRToProfileLocationMap.empty() &&

316 "Run stale profile matching only once per function");

317

318 AnchorList FilteredProfileAnchorList;

320 getFilteredAnchorList(IRAnchors, ProfileAnchors, FilteredIRAnchorsList,

321 FilteredProfileAnchorList);

322

323 if (FilteredIRAnchorsList.empty() || FilteredProfileAnchorList.empty())

324 return;

325

328 LLVM_DEBUG(dbgs() << "Skip stale profile matching for " << F.getName()

329 << " because the number of callsites in the IR is "

330 << FilteredIRAnchorsList.size()

331 << " and in the profile is "

332 << FilteredProfileAnchorList.size() << "\n");

333 return;

334 }

335

336

337

338

339

340

341

342

343

344

345

347 longestCommonSequence(FilteredIRAnchorsList, FilteredProfileAnchorList,

348 RunCGMatching );

349

350

351

352

353

354 if (RunCFGMatching)

355 matchNonCallsiteLocs(MatchedAnchors, IRAnchors, IRToProfileLocationMap);

356}

357

358void SampleProfileMatcher::runOnFunction(Function &F) {

359

360

361

362

363

364

365 const auto *FSForMatching = getFlattenedSamplesFor(F);

367

368 auto R = FuncToProfileNameMap.find(&F);

369 if (R != FuncToProfileNameMap.end()) {

370 FSForMatching = getFlattenedSamplesFor(R->second);

371

372

374 FSForMatching = Reader.getSamplesFor(R->second.stringRef());

375 }

376 }

377 if (!FSForMatching)

378 return;

379

380

381

382

384 findIRAnchors(F, IRAnchors);

385

386

388 findProfileAnchors(*FSForMatching, ProfileAnchors);

389

390

392 recordCallsiteMatchStates(F, IRAnchors, ProfileAnchors, nullptr);

393

395 return;

396

397

399 !ProbeManager->profileIsValid(F, *FSForMatching);

400 bool RunCFGMatching =

403

404

405

406

408 F.addFnAttr("profile-checksum-mismatch");

409

410

411

412 auto &IRToProfileLocationMap = getIRToProfileLocationMap(F);

413 runStaleProfileMatching(F, IRAnchors, ProfileAnchors, IRToProfileLocationMap,

414 RunCFGMatching, RunCGMatching);

415

417 recordCallsiteMatchStates(F, IRAnchors, ProfileAnchors,

418 &IRToProfileLocationMap);

419}

420

421void SampleProfileMatcher::recordCallsiteMatchStates(

424 const LocToLocMap *IRToProfileLocationMap) {

425 bool IsPostMatch = IRToProfileLocationMap != nullptr;

426 auto &CallsiteMatchStates =

428

429 auto MapIRLocToProfileLoc = [&](const LineLocation &IRLoc) {

430

431 if (!IRToProfileLocationMap)

432 return IRLoc;

433 const auto &ProfileLoc = IRToProfileLocationMap->find(IRLoc);

434 if (ProfileLoc != IRToProfileLocationMap->end())

435 return ProfileLoc->second;

436 else

437 return IRLoc;

438 };

439

440 for (const auto &I : IRAnchors) {

441

442

443 const auto &ProfileLoc = MapIRLocToProfileLoc(I.first);

444 const auto &IRCalleeId = I.second;

445 const auto &It = ProfileAnchors.find(ProfileLoc);

446 if (It == ProfileAnchors.end())

447 continue;

448 const auto &ProfCalleeId = It->second;

449 if (IRCalleeId == ProfCalleeId) {

450 auto It = CallsiteMatchStates.find(ProfileLoc);

451 if (It == CallsiteMatchStates.end())

452 CallsiteMatchStates.emplace(ProfileLoc, MatchState::InitialMatch);

453 else if (IsPostMatch) {

454 if (It->second == MatchState::InitialMatch)

455 It->second = MatchState::UnchangedMatch;

456 else if (It->second == MatchState::InitialMismatch)

457 It->second = MatchState::RecoveredMismatch;

458 }

459 }

460 }

461

462

463

464 for (const auto &I : ProfileAnchors) {

465 const auto &Loc = I.first;

466 assert(.second.stringRef().empty() && "Callees should not be empty");

467 auto It = CallsiteMatchStates.find(Loc);

468 if (It == CallsiteMatchStates.end())

469 CallsiteMatchStates.emplace(Loc, MatchState::InitialMismatch);

470 else if (IsPostMatch) {

471

472

473 if (It->second == MatchState::InitialMismatch)

474 It->second = MatchState::UnchangedMismatch;

475 else if (It->second == MatchState::InitialMatch)

476 It->second = MatchState::RemovedMatch;

477 }

478 }

479}

480

481void SampleProfileMatcher::countMismatchedFuncSamples(const FunctionSamples &FS,

482 bool IsTopLevel) {

483 const auto *FuncDesc = ProbeManager->getDesc(FS.getGUID());

484

485 if (!FuncDesc)

486 return;

487

488 if (ProbeManager->profileIsHashMismatched(*FuncDesc, FS)) {

489 if (IsTopLevel)

490 NumStaleProfileFunc++;

491

492

493

494

495 MismatchedFunctionSamples += FS.getTotalSamples();

496 return;

497 }

498

499

500

501

502

503

504 for (const auto &I : FS.getCallsiteSamples())

505 for (const auto &CS : I.second)

506 countMismatchedFuncSamples(CS.second, false);

507}

508

509void SampleProfileMatcher::countMismatchedCallsiteSamples(

511 auto It = FuncCallsiteMatchStates.find(FS.getFuncName());

512

513 if (It == FuncCallsiteMatchStates.end() || It->second.empty())

514 return;

515 const auto &CallsiteMatchStates = It->second;

516

517 auto findMatchState = [&](const LineLocation &Loc) {

518 auto It = CallsiteMatchStates.find(Loc);

519 if (It == CallsiteMatchStates.end())

520 return MatchState::Unknown;

521 return It->second;

522 };

523

524 auto AttributeMismatchedSamples = [&](const enum MatchState &State,

525 uint64_t Samples) {

526 if (isMismatchState(State))

527 MismatchedCallsiteSamples += Samples;

528 else if (State == MatchState::RecoveredMismatch)

529 RecoveredCallsiteSamples += Samples;

530 };

531

532

533

534 for (const auto &I : FS.getBodySamples())

535 AttributeMismatchedSamples(findMatchState(I.first), I.second.getSamples());

536

537

538 for (const auto &I : FS.getCallsiteSamples()) {

539 auto State = findMatchState(I.first);

540 uint64_t CallsiteSamples = 0;

541 for (const auto &CS : I.second)

542 CallsiteSamples += CS.second.getTotalSamples();

543 AttributeMismatchedSamples(State, CallsiteSamples);

544

545 if (isMismatchState(State))

546 continue;

547

548

549

550

551 for (const auto &CS : I.second)

552 countMismatchedCallsiteSamples(CS.second);

553 }

554}

555

556void SampleProfileMatcher::countMismatchCallsites(const FunctionSamples &FS) {

557 auto It = FuncCallsiteMatchStates.find(FS.getFuncName());

558

559 if (It == FuncCallsiteMatchStates.end() || It->second.empty())

560 return;

561 const auto &MatchStates = It->second;

562 [[maybe_unused]] bool OnInitialState =

563 isInitialState(MatchStates.begin()->second);

564 for (const auto &I : MatchStates) {

565 TotalProfiledCallsites++;

567 (OnInitialState ? isInitialState(I.second) : isFinalState(I.second)) &&

568 "Profile matching state is inconsistent");

569

570 if (isMismatchState(I.second))

571 NumMismatchedCallsites++;

572 else if (I.second == MatchState::RecoveredMismatch)

573 NumRecoveredCallsites++;

574 }

575}

576

577void SampleProfileMatcher::countCallGraphRecoveredSamples(

579 std::unordered_set &CallGraphRecoveredProfiles) {

580 if (CallGraphRecoveredProfiles.count(FS.getFunction())) {

581 NumCallGraphRecoveredFuncSamples += FS.getTotalSamples();

582 return;

583 }

584

585 for (const auto &CM : FS.getCallsiteSamples()) {

586 for (const auto &CS : CM.second) {

587 countCallGraphRecoveredSamples(CS.second, CallGraphRecoveredProfiles);

588 }

589 }

590}

591

592void SampleProfileMatcher::computeAndReportProfileStaleness() {

594 return;

595

596 std::unordered_set CallGraphRecoveredProfiles;

598 for (const auto &I : FuncToProfileNameMap) {

599 CallGraphRecoveredProfiles.insert(I.second);

601 continue;

602 NumCallGraphRecoveredProfiledFunc++;

603 }

604 }

605

606

607 for (const auto &F : M) {

609 continue;

610

611

613 continue;

614 const auto *FS = Reader.getSamplesFor(F);

615 if (!FS)

616 continue;

617 TotalProfiledFunc++;

618 TotalFunctionSamples += FS->getTotalSamples();

619

621 countCallGraphRecoveredSamples(*FS, CallGraphRecoveredProfiles);

622

623

625 countMismatchedFuncSamples(*FS, true);

626

627

628 countMismatchCallsites(*FS);

629 countMismatchedCallsiteSamples(*FS);

630 }

631

634 errs() << "(" << NumStaleProfileFunc << "/" << TotalProfiledFunc

635 << ") of functions' profile are invalid and ("

636 << MismatchedFunctionSamples << "/" << TotalFunctionSamples

637 << ") of samples are discarded due to function hash mismatch.\n";

638 }

640 errs() << "(" << NumCallGraphRecoveredProfiledFunc << "/"

641 << TotalProfiledFunc << ") of functions' profile are matched and ("

642 << NumCallGraphRecoveredFuncSamples << "/" << TotalFunctionSamples

643 << ") of samples are reused by call graph matching.\n";

644 }

645

646 errs() << "(" << (NumMismatchedCallsites + NumRecoveredCallsites) << "/"

647 << TotalProfiledCallsites

648 << ") of callsites' profile are invalid and ("

649 << (MismatchedCallsiteSamples + RecoveredCallsiteSamples) << "/"

650 << TotalFunctionSamples

651 << ") of samples are discarded due to callsite location mismatch.\n";

652 errs() << "(" << NumRecoveredCallsites << "/"

653 << (NumRecoveredCallsites + NumMismatchedCallsites)

654 << ") of callsites and (" << RecoveredCallsiteSamples << "/"

655 << (RecoveredCallsiteSamples + MismatchedCallsiteSamples)

656 << ") of samples are recovered by stale profile matching.\n";

657 }

658

660 LLVMContext &Ctx = M.getContext();

661 MDBuilder MDB(Ctx);

662

665 ProfStatsVec.emplace_back("NumStaleProfileFunc", NumStaleProfileFunc);

666 ProfStatsVec.emplace_back("TotalProfiledFunc", TotalProfiledFunc);

667 ProfStatsVec.emplace_back("MismatchedFunctionSamples",

668 MismatchedFunctionSamples);

669 ProfStatsVec.emplace_back("TotalFunctionSamples", TotalFunctionSamples);

670 }

671

673 ProfStatsVec.emplace_back("NumCallGraphRecoveredProfiledFunc",

674 NumCallGraphRecoveredProfiledFunc);

675 ProfStatsVec.emplace_back("NumCallGraphRecoveredFuncSamples",

676 NumCallGraphRecoveredFuncSamples);

677 }

678

679 ProfStatsVec.emplace_back("NumMismatchedCallsites", NumMismatchedCallsites);

680 ProfStatsVec.emplace_back("NumRecoveredCallsites", NumRecoveredCallsites);

681 ProfStatsVec.emplace_back("TotalProfiledCallsites", TotalProfiledCallsites);

682 ProfStatsVec.emplace_back("MismatchedCallsiteSamples",

683 MismatchedCallsiteSamples);

684 ProfStatsVec.emplace_back("RecoveredCallsiteSamples",

685 RecoveredCallsiteSamples);

686

687 auto *MD = MDB.createLLVMStats(ProfStatsVec);

688 auto *NMD = M.getOrInsertNamedMetadata("llvm.stats");

689 NMD->addOperand(MD);

690 }

691}

692

693void SampleProfileMatcher::findFunctionsWithoutProfile() {

694

696 return;

697 StringSet<> NamesInProfile;

698 if (auto NameTable = Reader.getNameTable()) {

699 for (auto Name : *NameTable)

700 NamesInProfile.insert(Name.stringRef());

701 }

702

703 for (auto &F : M) {

704

705

706 if (F.isDeclaration())

707 continue;

708

710 const auto *FS = getFlattenedSamplesFor(F);

711 if (FS)

712 continue;

713

714

715

716

717 if (NamesInProfile.count(CanonFName))

718 continue;

719

720

721

722 if (PSL && PSL->contains(CanonFName))

723 continue;

724

726 << " is not in profile or profile symbol list.\n");

727 FunctionsWithoutProfile[FunctionId(CanonFName)] = &F;

728 }

729}

730

731bool SampleProfileMatcher::functionMatchesProfileHelper(

733

734

735 float Similarity = 0.0;

736

737

738

739 ItaniumPartialDemangler Demangler;

740

741

742 auto GetBaseName = [&](StringRef FName) {

743 auto FunctionName = FName.str();

744 if (Demangler.partialDemangle(FunctionName.c_str()))

745 return std::string();

746 size_t BaseNameSize = 0;

747

748

749

750 char *BaseNamePtr = Demangler.getFunctionBaseName(nullptr, &BaseNameSize);

751 std::string Result = (BaseNamePtr && BaseNameSize)

752 ? std::string(BaseNamePtr, BaseNameSize)

753 : std::string();

754 free(BaseNamePtr);

756 };

757 auto IRBaseName = GetBaseName(IRFunc.getName());

758 auto ProfBaseName = GetBaseName(ProfFunc.stringRef());

759 if (!IRBaseName.empty() && IRBaseName == ProfBaseName) {

761 << ProfFunc << "(Profile) share the same base name: "

762 << IRBaseName << ".\n");

763 return true;

764 }

765

766 const auto *FSForMatching = getFlattenedSamplesFor(ProfFunc);

767

768

769

770

771

773 DenseSet TopLevelFunc({ProfFunc.stringRef()});

774 if (std::error_code EC = Reader.read(TopLevelFunc))

775 return false;

776 FSForMatching = Reader.getSamplesFor(ProfFunc.stringRef());

778 if (FSForMatching)

779 dbgs() << "Read top-level function " << ProfFunc

780 << " for call-graph matching\n";

781 });

782 }

783 if (!FSForMatching)

784 return false;

785

786

787

790 return false;

791

792

793

795 const auto *FuncDesc = ProbeManager->getDesc(IRFunc);

796 if (FuncDesc &&

797 !ProbeManager->profileIsHashMismatched(*FuncDesc, *FSForMatching)) {

799 << "(IR) and " << ProfFunc << "(Profile) match.\n");

800

801 return true;

802 }

803 }

804

806 findIRAnchors(IRFunc, IRAnchors);

808 findProfileAnchors(*FSForMatching, ProfileAnchors);

809

811 AnchorList FilteredProfileAnchorList;

812 getFilteredAnchorList(IRAnchors, ProfileAnchors, FilteredIRAnchorsList,

813 FilteredProfileAnchorList);

814

815

818 return false;

819

820

821

822

823

824

826 longestCommonSequence(FilteredIRAnchorsList, FilteredProfileAnchorList,

827 false );

828

829 Similarity = static_cast<float>(MatchedAnchors.size()) /

830 FilteredProfileAnchorList.size();

831

833 << "(IR) and " << ProfFunc << "(profile) is "

834 << format("%.2f", Similarity) << "\n");

835 assert((Similarity >= 0 && Similarity <= 1.0) &&

836 "Similarity value should be in [0, 1]");

838}

839

840

841

842bool SampleProfileMatcher::functionMatchesProfile(Function &IRFunc,

844 bool FindMatchedProfileOnly) {

845 auto R = FuncProfileMatchCache.find({&IRFunc, ProfFunc});

846 if (R != FuncProfileMatchCache.end())

847 return R->second;

848

849 if (FindMatchedProfileOnly)

850 return false;

851

852 bool Matched = functionMatchesProfileHelper(IRFunc, ProfFunc);

853 FuncProfileMatchCache[{&IRFunc, ProfFunc}] = Matched;

854 if (Matched) {

855 FuncToProfileNameMap[&IRFunc] = ProfFunc;

857 << " matches profile:" << ProfFunc << "\n");

858 }

859

860 return Matched;

861}

862

863void SampleProfileMatcher::UpdateWithSalvagedProfiles() {

864 DenseSet ProfileSalvagedFuncs;

865

866 for (auto &I : FuncToProfileNameMap) {

867 assert(I.first && "New function is null");

868 FunctionId FuncName(I.first->getName());

869 ProfileSalvagedFuncs.insert(I.second.stringRef());

870 FuncNameToProfNameMap->emplace(FuncName, I.second);

871

872

873

874 SymbolMap->erase(FuncName);

875 SymbolMap->emplace(I.second, I.first);

876 }

877

878

879

880

881

882 Reader.read(ProfileSalvagedFuncs);

883 Reader.setFuncNameToProfNameMap(*FuncNameToProfNameMap);

884}

885

890 findFunctionsWithoutProfile();

891

892

893

894 std::vector<Function *> TopDownFunctionList;

895 TopDownFunctionList.reserve(M.size());

897 for (auto *F : TopDownFunctionList) {

899 continue;

900 runOnFunction(*F);

901 }

902

904 UpdateWithSalvagedProfiles();

905

907 distributeIRToProfileLocationMap();

908

909 computeAndReportProfileStaleness();

910}

911

912void SampleProfileMatcher::distributeIRToProfileLocationMap(

914 const auto ProfileMappings = FuncMappings.find(FS.getFuncName());

915 if (ProfileMappings != FuncMappings.end()) {

916 FS.setIRToProfileLocationMap(&(ProfileMappings->second));

917 }

918

919 for (auto &Callees :

921 for (auto &FS : Callees.second) {

922 distributeIRToProfileLocationMap(FS.second);

923 }

924 }

925}

926

927

928

929void SampleProfileMatcher::distributeIRToProfileLocationMap() {

930 for (auto &I : Reader.getProfiles()) {

931 distributeIRToProfileLocationMap(I.second);

932 }

933}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static GCRegistry::Add< ErlangGC > A("erlang", "erlang-compatible garbage collector")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

itanium_demangle::ManglingParser< DefaultAllocator > Demangler

Legalize the Machine IR a function s Machine IR

This file provides the interface for SampleProfileMatcher.

static bool isAvailableExternallyLinkage(LinkageTypes Linkage)

void runOnModule()

Definition SampleProfileMatcher.cpp:886

reference emplace_back(ArgTypes &&... Args)

iterator find(StringRef Key)

size_type count(StringRef Key) const

count - Return 1 if the element is in the map, 0 otherwise.

std::pair< typename Base::iterator, bool > insert(StringRef key)

LLVM_ABI StringRef getName() const

Return a constant reference to the value's name.

std::pair< iterator, bool > insert(const ValueT &V)

This class represents a function that is read from a sample profile.

StringRef stringRef() const

Convert to StringRef.

Representation of the samples collected for a function.

static LLVM_ABI bool ProfileIsCS

static LLVM_ABI bool ProfileIsProbeBased

static StringRef getCanonicalFnName(const Function &F)

Return the canonical name for a function, taking into account suffix elision policy attributes.

static LLVM_ABI bool ProfileIsFS

If this profile uses flow sensitive discriminators.

static LLVM_ABI LineLocation getCallSiteIdentifier(const DILocation *DIL, bool ProfileIsFS=false)

Returns a unique call site identifier for a given debug location of a call instruction.

static LLVM_ABI bool UseMD5

Whether the profile uses MD5 to represent string.

static void flattenProfile(SampleProfileMap &ProfileMap, bool ProfileIsCS=false)

@ C

The default llvm calling convention, compatible with C.

initializer< Ty > init(const Ty &Val)

std::map< LineLocation, FunctionSamplesMap > CallsiteSampleMap

std::unordered_map< LineLocation, LineLocation, LineLocationHash > LocToLocMap

This is an optimization pass for GlobalISel generic memory operations.

cl::opt< bool > ReportProfileStaleness("report-profile-staleness", cl::Hidden, cl::init(false), cl::desc("Compute and report stale profile statistical metrics."))

Definition SampleProfileMatcher.cpp:55

cl::opt< bool > PersistProfileStaleness("persist-profile-staleness", cl::Hidden, cl::init(false), cl::desc("Compute stale profile statistical metrics and write it into the " "native object file(.llvm_stats section)."))

Definition SampleProfileMatcher.cpp:54

std::map< LineLocation, FunctionId > AnchorMap

decltype(auto) dyn_cast(const From &Val)

dyn_cast - Return the argument parameter cast to the specified type.

static cl::opt< bool > LoadFuncProfileforCGMatching("load-func-profile-for-cg-matching", cl::Hidden, cl::init(true), cl::desc("Load top-level profiles that the sample reader initially skipped for " "the call-graph matching (only meaningful for extended binary " "format)"))

static void buildTopDownFuncOrder(LazyCallGraph &CG, std::vector< Function * > &FunctionOrderList)

@ ThinLTOPreLink

ThinLTO prelink (summary) phase.

static cl::opt< unsigned > MinCallCountForCGMatching("min-call-count-for-cg-matching", cl::Hidden, cl::init(3), cl::desc("The minimum number of call anchors required for a function to " "run stale profile call graph matching."))

LLVM_ABI std::optional< PseudoProbe > extractProbe(const Instruction &Inst)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

static cl::opt< unsigned > MinFuncCountForCGMatching("min-func-count-for-cg-matching", cl::Hidden, cl::init(5), cl::desc("The minimum number of basic blocks required for a function to " "run stale profile call graph matching."))

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

bool isa(const From &Val)

isa - Return true if the parameter to the template is an instance of one of the template type argu...

format_object< Ts... > format(const char *Fmt, const Ts &... Vals)

These are helper functions used to produce formatted output.

LLVM_ABI raw_fd_ostream & errs()

This returns a reference to a raw_ostream for standard error.

cl::opt< bool > SalvageStaleProfile("salvage-stale-profile", cl::Hidden, cl::init(false), cl::desc("Salvage stale profile by fuzzy matching and use the remapped " "location for sample profile query."))

Definition SampleProfileMatcher.cpp:52

void longestCommonSequence(AnchorList AnchorList1, AnchorList AnchorList2, llvm::function_ref< bool(const Function &, const Function &)> FunctionMatchesProfile, llvm::function_ref< void(Loc, Loc)> InsertMatching)

std::vector< std::pair< LineLocation, FunctionId > > AnchorList

static bool skipProfileForFunction(const Function &F)

cl::opt< bool > SalvageUnusedProfile("salvage-unused-profile", cl::Hidden, cl::init(false), cl::desc("Salvage unused profile by matching with new " "functions on call graph."))

Definition SampleProfileMatcher.cpp:53

static cl::opt< unsigned > SalvageStaleProfileMaxCallsites("salvage-stale-profile-max-callsites", cl::Hidden, cl::init(UINT_MAX), cl::desc("The maximum number of callsites in a function, above which stale " "profile matching will be skipped."))

static cl::opt< unsigned > FuncProfileSimilarityThreshold("func-profile-similarity-threshold", cl::Hidden, cl::init(80), cl::desc("Consider a profile matches a function if the similarity of their " "callee sequences is above the specified percentile."))