SampleProfWriter.cpp Source File (original) (raw)

31#include

32#include

33#include

34#include

35#include <system_error>

36#include

37#include

39#define DEBUG_TYPE "llvm-profdata"

41using namespace llvm;

42using namespace sampleprof;

44namespace llvm {

45namespace support {

47namespace {

50struct SeekableWriter {

56 template

58 std::string StringBuf;

60 Writer(SStream, Endian).write(Val);

61 OS.pwrite(StringBuf.data(), StringBuf.size(), Offset);

62 }

63};

65}

66}

67}

68}

74}

79 size_t NumToRemove = ProfileMap.size() - NewSize;

80 if (NumToRemove < 1)

81 NumToRemove = 1;

83 assert(NumToRemove <= SortedFunctions.size());

85 llvm::drop_begin(SortedFunctions, SortedFunctions.size() - NumToRemove))

87 SortedFunctions.resize(SortedFunctions.size() - NumToRemove);

88}

93 if (OutputSizeLimit == 0)

94 return write(ProfileMap);

96 size_t OriginalFunctionCount = ProfileMap.size();

98 std::unique_ptr<raw_ostream> OriginalOutputStream;

100

101 size_t IterationCount = 0;

102 size_t TotalSize;

103

105 do {

106 StringBuffer.clear();

108 if (std::error_code EC = write(ProfileMap))

109 return EC;

110

111 TotalSize = StringBuffer.size();

112

113

114

115#ifdef _WIN32

118#endif

119 if (TotalSize <= OutputSizeLimit)

120 break;

121

122 Strategy->Erase(TotalSize);

123 IterationCount++;

124 } while (ProfileMap.size() != 0);

125

126 if (ProfileMap.size() == 0)

128

131 LLVM_DEBUG(dbgs() << "Profile originally has " << OriginalFunctionCount

132 << " functions, reduced to " << ProfileMap.size() << " in "

133 << IterationCount << " iterations\n");

134

135 (void)OriginalFunctionCount;

136 (void)IterationCount;

138}

139

140std::error_code

142 std::vector V;

144 for (const auto &I : V) {

145 if (std::error_code EC = writeSample(*I.second))

146 return EC;

147 }

149}

150

152 if (std::error_code EC = writeHeader(ProfileMap))

153 return EC;

154

156 return EC;

157

159}

160

161

162

163

170 assert(Entry.Type == Type && "Unexpected section type");

171

174 return SectionStart;

175}

176

177std::error_code SampleProfileWriterExtBinaryBase::compressAndOutput() {

180 std::string &UncompressedStrings =

182 if (UncompressedStrings.size() == 0)

187 CompressedStrings,

191 OS << toStringRef(CompressedStrings);

192 UncompressedStrings.clear();

194}

195

196

197

198

203 assert(Entry.Type == Type && "Unexpected section type");

206 if (std::error_code EC = compressAndOutput())

207 return EC;

208 }

209 SecHdrTable.push_back({Type, Entry.Flags, SectionStart - FileStart,

210 OutputStream->tell() - SectionStart, LayoutIdx});

212}

213

214std::error_code

216

217

219 CSNameTable.clear();

220 SecHdrTable.clear();

221

222 if (std::error_code EC = writeHeader(ProfileMap))

223 return EC;

224

225 std::string LocalBuf;

226 LocalBufStream = std::make_unique<raw_string_ostream>(LocalBuf);

227 if (std::error_code EC = writeSections(ProfileMap))

228 return EC;

229

230 if (std::error_code EC = writeSecHdrTable())

231 return EC;

232

234}

235

240 else

242}

243

244std::error_code

246 const auto &Ret = CSNameTable.find(Context);

247 if (Ret == CSNameTable.end())

251}

252

253std::error_code

260}

261

264

265

267

268

271 return EC;

274 };

275

277

278

279

280 std::map<SampleContext, uint64_t> OrderedFuncOffsetTable(

281 FuncOffsetTable.begin(), FuncOffsetTable.end());

282 for (const auto &Entry : OrderedFuncOffsetTable) {

283 if (std::error_code EC = WriteItem(Entry.first, Entry.second))

284 return EC;

285 }

287 } else {

288 for (const auto &Entry : FuncOffsetTable) {

289 if (std::error_code EC = WriteItem(Entry.first, Entry.second))

290 return EC;

291 }

292 }

293

294 FuncOffsetTable.clear();

296}

297

302 return EC;

303

308 }

309

311

314 NumCallsites += J.second.size();

317 for (const auto &FS : J.second) {

322 return EC;

323 }

324 }

325 }

326

328}

329

335 for (const auto &Entry : Profiles) {

337 return EC;

338 }

340}

341

343 if (!UseMD5)

345

347 std::set V;

349

350

351

352

355 for (auto N : V)

356 Writer.write(N.getHashCode());

358}

359

362 for (const auto &I : ProfileMap) {

365 }

366

367

368

369

370

371 if (!UseMD5) {

375 break;

376 }

377 }

378 }

379

381 return EC;

383}

384

386

387 std::set OrderedContexts;

388 for (const auto &I : CSNameTable)

389 OrderedContexts.insert(I.first);

390 assert(OrderedContexts.size() == CSNameTable.size() &&

391 "Unmatched ordered and unordered contexts");

393 for (auto &Context : OrderedContexts)

394 CSNameTable[Context] = I++;

395

399 for (auto Context : OrderedContexts) {

400 auto Frames = Context.getContextFrames();

402 for (auto &Callsite : Frames) {

403 if (std::error_code EC = writeNameIdx(Callsite.Func))

404 return EC;

407 }

408 }

409

411}

412

413std::error_code

415 if (ProfSymList && ProfSymList->size() > 0)

417 return EC;

418

420}

421

424

438

440 switch (Type) {

444 return EC;

445 break;

448 return EC;

449 break;

452 return EC;

453 break;

457 return EC;

458 break;

461 return EC;

462 break;

465 return EC;

466 break;

469 return EC;

470 break;

471 default:

473 return EC;

474 break;

475 }

476 if (std::error_code EC = addNewSection(Type, LayoutIdx, SectionStart))

477 return EC;

479}

480

481std::error_code SampleProfileWriterExtBinary::writeDefaultLayout(

483

484

485

486

488 return EC;

490 return EC;

492 return EC;

494 return EC;

496 return EC;

498 return EC;

500 return EC;

502}

503

507 for (const auto &I : ProfileMap) {

508 if (I.second.getCallsiteSamples().size())

509 ContextProfileMap.insert({I.first, I.second});

510 else

511 NoContextProfileMap.insert({I.first, I.second});

512 }

513}

514

515std::error_code SampleProfileWriterExtBinary::writeCtxSplitLayout(

519

521 return EC;

523 return EC;

525 return EC;

527 return EC;

528

529

532 return EC;

533

534

537 return EC;

539 return EC;

541 return EC;

542

544}

545

546std::error_code SampleProfileWriterExtBinary::writeSections(

548 std::error_code EC;

550 EC = writeDefaultLayout(ProfileMap);

552 EC = writeCtxSplitLayout(ProfileMap);

553 else

555 return EC;

556}

557

558

559

560

561

562

563

564

565

570 else

572

573 if (Indent == 0)

575 OS << "\n";

577

579 for (const auto &I : SortedSamples.get()) {

585 else

587

589

591 OS << " " << J.first << ":" << J.second;

592 OS << "\n";

594 }

595

598 Indent += 1;

599 for (const auto &I : SortedCallsiteSamples.get())

600 for (const auto &FS : I->second) {

606 else

608 if (std::error_code EC = writeSample(CalleeSamples))

609 return EC;

610 }

611 Indent -= 1;

612

617 }

618

623 }

624

625 if (Indent == 0 && MarkFlatProfiles && S.getCallsiteSamples().size() == 0)

626 OS << " !Flat\n";

627

629}

630

631std::error_code

633 assert(!Context.hasContext() && "cs profile is not supported");

635}

636

639 const auto &Ret = NTable.find(FName);

640 if (Ret == NTable.end())

644}

645

648 NTable.insert(std::make_pair(FName, 0));

649}

650

653}

654

656

661 }

662

663

665 for (const auto &FS : J.second) {

669 }

670}

671

677 CSNameTable.insert(std::make_pair(Context, 0));

678 } else {

680 }

681}

682

685

687 V.insert(I.first);

688 int i = 0;

691}

692

695 std::set V;

697

698

700 for (auto N : V) {

703 }

705}

706

707std::error_code

710

714}

715

716std::error_code

718

719

721

723

726 return EC;

727

728

729 for (const auto &I : ProfileMap) {

732 }

733

736}

737

741}

742

745}

746

747void SampleProfileWriterExtBinaryBase::allocSecHdrTable() {

749

753 Writer.write(static_cast<uint64_t>(-1));

754 Writer.write(static_cast<uint64_t>(-1));

755 Writer.write(static_cast<uint64_t>(-1));

756 Writer.write(static_cast<uint64_t>(-1));

757 }

758}

759

760std::error_code SampleProfileWriterExtBinaryBase::writeSecHdrTable() {

762 "SecHdrTable entries doesn't match SectionHdrLayout");

764 for (uint32_t TableIdx = 0; TableIdx < SecHdrTable.size(); TableIdx++) {

765 IndexMap[SecHdrTable[TableIdx].LayoutIndex] = TableIdx;

766 }

767

768

769

770

771

772

773

774

775

776

777 support::endian::SeekableWriter Writer(

781 LayoutIdx++) {

782 assert(IndexMap[LayoutIdx] < SecHdrTable.size() &&

783 "Incorrect LayoutIdx in SecHdrTable");

784 auto Entry = SecHdrTable[IndexMap[LayoutIdx]];

786 SecHdrTableOffset + 4 * LayoutIdx * sizeof(uint64_t));

787 Writer.pwrite(static_cast<uint64_t>(Entry.Flags),

788 SecHdrTableOffset + (4 * LayoutIdx + 1) * sizeof(uint64_t));

789 Writer.pwrite(static_cast<uint64_t>(Entry.Offset),

790 SecHdrTableOffset + (4 * LayoutIdx + 2) * sizeof(uint64_t));

792 SecHdrTableOffset + (4 * LayoutIdx + 3) * sizeof(uint64_t));

793 }

794

796}

797

798std::error_code SampleProfileWriterExtBinaryBase::writeHeader(

801 FileStart = OS.tell();

803

804 allocSecHdrTable();

806}

807

817 for (auto Entry : Entries) {

821 }

823}

827 return EC;

828

830

831

842 uint64_t CalleeSamples = J.second;

843 if (std::error_code EC = writeNameIdx(Callee))

844 return EC;

846 }

847 }

848

849

852 NumCallsites += J.second.size();

855 for (const auto &FS : J.second) {

860 if (std::error_code EC = writeBody(CalleeSamples))

861 return EC;

862 }

863

865}

866

867

868

869

870std::error_code

874}

875

876

877

878

879

880

881

882

885 std::error_code EC;

886 std::unique_ptr<raw_ostream> OS;

889 else

891 if (EC)

892 return EC;

893

895}

896

897

898

899

900

901

902

903

907 std::error_code EC;

908 std::unique_ptr Writer;

909

910

914

923 else

925

926 if (EC)

927 return EC;

928

929 Writer->Format = Format;

930 return std::move(Writer);

931}

932

936}

static GCRegistry::Add< StatepointGC > D("statepoint-example", "an example strategy for statepoint")

Provides ErrorOr smart pointer.

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

static void splitProfileMapToTwo(const SampleProfileMap &ProfileMap, SampleProfileMap &ContextProfileMap, SampleProfileMap &NoContextProfileMap)

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

Represents either an error or a value T.

This class implements a map that also provides access to all stored values in a deterministic order.

static const ArrayRef< uint32_t > DefaultCutoffs

A vector of useful cutoff values for detailed summary.

std::unique_ptr< ProfileSummary > computeSummaryForProfiles(const sampleprof::SampleProfileMap &Profiles)

pointer data()

Return a pointer to the vector's buffer, even if empty().

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

The instances of the Type class are immutable: once they are created, they are never changed.

A raw_ostream that writes to a file descriptor.

uint64_t tell() const

tell - Return the current offset with the file.

raw_ostream & indent(unsigned NumSpaces)

indent - Insert 'NumSpaces' spaces.

An abstract base class for streams implementations that also support a pwrite operation.

void pwrite(const char *Ptr, size_t Size, uint64_t Offset)

A raw_ostream that writes to an std::string.

A raw_ostream that writes to an SmallVector or SmallString.

DefaultFunctionPruningStrategy(SampleProfileMap &ProfileMap, size_t OutputSizeLimit)

void Erase(size_t CurrentOutputSize) override

In this default implementation, functions with fewest samples are dropped first.

This class represents a function that is read from a sample profile.

When writing a profile with size limit, user may want to use a different strategy to reduce function ...

virtual void Erase(size_t CurrentOutputSize)=0

SampleProfileWriter::writeWithSizeLimit() calls this after every write iteration if the output size s...

SampleProfileMap & ProfileMap

Representation of the samples collected for a function.

static bool ProfileIsPreInlined

static constexpr const char * UniqSuffix

uint64_t getHeadSamples() const

For top-level functions, return the total number of branch samples that have the function as the bran...

FunctionId getFunction() const

Return the function name.

uint64_t getFunctionHash() const

static bool ProfileIsProbeBased

static bool ProfileIsFS

If this profile uses flow sensitive discriminators.

SampleContext & getContext() const

uint64_t getTotalSamples() const

Return the total number of samples collected inside the function.

const CallsiteSampleMap & getCallsiteSamples() const

Return all the callsite samples collected in the body of the function.

const BodySampleMap & getBodySamples() const

Return all the samples collected in the body of the function.

std::error_code write(raw_ostream &OS)

SampleContextFrames getContextFrames() const

FunctionId getFunction() const

uint32_t getAllAttributes()

std::string toString() const

This class provides operator overloads to the map container using MD5 as the key type,...

size_t erase(const SampleContext &Ctx)

void stablizeNameTable(MapVector< FunctionId, uint32_t > &NameTable, std::set< FunctionId > &V)

virtual void addContext(const SampleContext &Context)

virtual std::error_code writeNameTable()

virtual std::error_code writeMagicIdent(SampleProfileFormat Format)

std::error_code writeSummary()

MapVector< FunctionId, uint32_t > NameTable

void addNames(const FunctionSamples &S)

virtual std::error_code writeContextIdx(const SampleContext &Context)

std::error_code writeSample(const FunctionSamples &S) override

Write samples of a top-level function to a binary file.

std::error_code writeHeader(const SampleProfileMap &ProfileMap) override

Write a file header for the profile file.

virtual MapVector< FunctionId, uint32_t > & getNameTable()

std::error_code writeBody(const FunctionSamples &S)

std::error_code writeNameIdx(FunctionId FName)

void addName(FunctionId FName)

std::error_code writeNameTableSection(const SampleProfileMap &ProfileMap)

SmallVector< SecHdrTableEntry, 8 > SectionHdrLayout

std::error_code writeFuncMetadata(const SampleProfileMap &Profiles)

void setToCompressSection(SecType Type)

virtual std::error_code writeCustomSection(SecType Type)=0

virtual std::error_code writeOneSection(SecType Type, uint32_t LayoutIdx, const SampleProfileMap &ProfileMap)

std::error_code writeFuncOffsetTable()

std::error_code writeCSNameTableSection()

std::error_code writeCSNameIdx(const SampleContext &Context)

virtual std::error_code writeSections(const SampleProfileMap &ProfileMap)=0

std::error_code writeNameTable() override

void addSectionFlag(SecType Type, SecFlagType Flag)

std::error_code writeProfileSymbolListSection()

uint64_t markSectionStart(SecType Type, uint32_t LayoutIdx)

Return the current position and prepare to use it as the start position of a section given the sectio...

void addContext(const SampleContext &Context) override

std::error_code addNewSection(SecType Sec, uint32_t LayoutIdx, uint64_t SectionStart)

Add a new section into section header table given the section type Type, its position LayoutIdx in Se...

void setToCompressAllSections() override

uint64_t SecLBRProfileStart

std::error_code write(const SampleProfileMap &ProfileMap) override

Write all the sample profiles in the given map of samples.

std::error_code writeContextIdx(const SampleContext &Context) override

std::error_code writeSample(const FunctionSamples &S) override

Write samples of a top-level function to a binary file.

Sample-based profile writer (text format).

std::error_code writeSample(const FunctionSamples &S) override

Write samples to a text file.

std::unique_ptr< ProfileSummary > Summary

Profile summary.

virtual std::error_code writeSample(const FunctionSamples &S)=0

Write sample profiles in S.

SampleProfileFormat Format

Profile format.

std::error_code writeWithSizeLimitInternal(SampleProfileMap &ProfileMap, size_t OutputSizeLimit, FunctionPruningStrategy *Strategy)

void computeSummary(const SampleProfileMap &ProfileMap)

Compute summary for this profile.

virtual std::error_code writeFuncProfiles(const SampleProfileMap &ProfileMap)

std::unique_ptr< raw_ostream > OutputStream

Output stream where to emit the profile to.

size_t LineCount

For writeWithSizeLimit in text mode, each newline takes 1 additional byte on Windows when actually wr...

static ErrorOr< std::unique_ptr< SampleProfileWriter > > create(StringRef Filename, SampleProfileFormat Format)

Profile writer factory.

virtual std::error_code writeHeader(const SampleProfileMap &ProfileMap)=0

Write a file header for the profile file.

virtual std::error_code write(const SampleProfileMap &ProfileMap)

Write all the sample profiles in the given map of samples.

Representation of a single sample record.

const CallTargetMap & getCallTargets() const

uint64_t getSamples() const

const SortedCallTargetSet getSortedCallTargets() const

Sort a LocationT->SampleT map by LocationT.

const SamplesWithLocList & get() const

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

void compress(ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &CompressedBuffer, int Level=DefaultCompression)

constexpr int BestSizeCompression

void sortFuncProfiles(const SampleProfileMap &ProfileMap, std::vector< NameFunctionSamples > &SortedProfiles)

static uint64_t SPMagic(SampleProfileFormat Format=SPF_Binary)

std::pair< hash_code, const FunctionSamples * > NameFunctionSamples

static void addSecFlag(SecHdrTableEntry &Entry, SecFlagType Flag)

static bool hasSecFlag(const SecHdrTableEntry &Entry, SecFlagType Flag)

@ SecFlagIsPreInlined

SecFlagIsPreInlined means this profile contains ShouldBeInlined contexts thus this is CS preinliner c...

@ SecFlagFSDiscriminator

SecFlagFSDiscriminator means this profile uses flow-sensitive discriminators.

@ SecFlagFullContext

SecFlagContext means this is context-sensitive flat profile for CSSPGO.

static uint64_t SPVersion()

@ OF_TextWithCRLF

The file should be opened in text mode and use a carriage linefeed '\r '.

This is an optimization pass for GlobalISel generic memory operations.

auto drop_begin(T &&RangeOrContainer, size_t N=1)

Return a range covering RangeOrContainer with the first N elements excluded.

@ unsupported_writing_format

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

unsigned encodeULEB128(uint64_t Value, raw_ostream &OS, unsigned PadTo=0)

Utility function to encode a ULEB128 value to an output stream.

Represents the relative location of an instruction.

Adapter to write values to a stream in a particular byte order.

void write(ArrayRef< value_type > Val)

static uint64_t round(uint64_t Acc, uint64_t Input)