OffloadBundle.cpp Source File (original) (raw)

25using namespace llvm;

29 "Timer group for offload bundler");

38 size_t NextbundleStart = 0;

40 std::unique_ptr Buffer;

45 Buffer =

47 false);

51 Magic = "CCOB";

53 NextbundleStart = (*Buffer).getBuffer().find(Magic, Magic.size());

55 NextbundleStart = (*Buffer).getBuffer().size();

59 (*Buffer).getBuffer().take_front(NextbundleStart), FileName,

60 false);

61 if (std::error_code EC = CodeOrErr.getError())

66 if (!DecompressedBufferOrErr)

71 **DecompressedBufferOrErr, Offset, FileName, true);

72 if (!FatBundleOrErr)

73 return FatBundleOrErr.takeError();

76 Bundles.emplace_back(std::move(**FatBundleOrErr));

83 *Buffer, SectionOffset + Offset, FileName);

84 if (!FatBundleOrErr)

85 return FatBundleOrErr.takeError();

88 Bundles.emplace_back(std::move(**FatBundleOrErr));

90 Magic = "__CLANG_OFFLOAD_BUNDLE__";

91 NextbundleStart = (*Buffer).getBuffer().find(Magic, Magic.size());

92 }

95 Offset += NextbundleStart;

96 }

99}

100

104

106

107

111

112

113 if (auto EC = Reader.readInteger(NumOfEntries))

115

116 NumberOfEntries = NumOfEntries;

117

118

119 for (uint64_t I = 0; I < NumOfEntries; I++) {

124

126 return Err;

127

129 return Err;

130

132 return Err;

133

135 return Err;

136

137 auto Entry = std::make_unique(

138 EntryOffset + SectionOffset, EntrySize, EntryIDSize, EntryID);

139

140 Entries.push_back(*Entry);

141 }

142

144}

145

148 StringRef FileName, bool Decompress) {

151

152

157

158 std::unique_ptr TheBundle(

160

161

163 TheBundle->readEntries(Buf.getBuffer(), Decompress ? 0 : SectionOffset);

164 if (Err)

165 return Err;

166

167 return std::move(TheBundle);

168}

169

171

173

174 if (Entry.Size == 0)

175 continue;

176

177

178

180 "-size" + itostr(Entry.Size) + ".co";

183 return Err;

184 }

185

187}

188

191 assert((Obj.isELF() || Obj.isCOFF()) && "Invalid file type");

192

193

194 for (SectionRef Sec : Obj.sections()) {

196 if (!Buffer)

198

199

203

205 if (Obj.isELF()) {

207 } else if (Obj.isCOFF())

209 "COFF object files not supported");

210

213 Obj.getFileName(), Bundles))

214 return Err;

215 }

216 }

218}

219

224

225 if (!BufferOrErr)

227

230 return Err;

231

232 std::unique_ptr Buf = std::move(*BufferOrErr);

233 std::copy(InputBuffOrErr->getBufferStart() + Offset,

234 InputBuffOrErr->getBufferStart() + Offset + Size,

235 Buf->getBufferStart());

236 if (Error E = Buf->commit())

237 return E;

238

240}

241

246 if (!BufferOrErr)

248

249 std::unique_ptr Buf = std::move(*BufferOrErr);

252

253 return Buf->commit();

254}

255

256

257

259

262 if (!UriOrErr)

264

266 std::string OutputFile = Uri.FileName.str();

267 OutputFile +=

269

270

272 if (!ObjOrErr)

273 return ObjOrErr.takeError();

274

275 auto Obj = ObjOrErr->getBinary();

278 return Err;

279

281}

282

283

285 std::string Num = std::to_string(Value);

290 }

291 return Num;

292}

293

294Expected<std::unique_ptr>

300 Timer HashTimer("Hash Calculation Timer", "Hash calculation time",

302 if (VerboseStream)

304 MD5 Hash;

307 Hash.final(Result);

308 uint64_t TruncatedHash = Result.low();

309 if (VerboseStream)

311

314 reinterpret_cast<const uint8_t *>(Input.getBuffer().data()),

315 Input.getBuffer().size());

316 Timer CompressTimer("Compression Timer", "Compression time",

318 if (VerboseStream)

321 if (VerboseStream)

323

325

326

327 uint64_t UncompressedSize64 = Input.getBuffer().size();

329

330

332

333 if (UncompressedSize64 > std::numeric_limits<uint32_t>::max())

334 return createStringError("uncompressed size (%llu) exceeds version 2 "

335 "unsigned 32-bit integer limit",

336 UncompressedSize64);

337 TotalFileSize64 = MagicNumber.size() + sizeof(uint32_t) + sizeof(Version) +

338 sizeof(CompressionMethod) + sizeof(uint32_t) +

339 sizeof(TruncatedHash) + CompressedBuffer.size();

340 if (TotalFileSize64 > std::numeric_limits<uint32_t>::max())

341 return createStringError("total file size (%llu) exceeds version 2 "

342 "unsigned 32-bit integer limit",

343 TotalFileSize64);

344

345 } else {

346 TotalFileSize64 = MagicNumber.size() + sizeof(uint64_t) + sizeof(Version) +

347 sizeof(CompressionMethod) + sizeof(uint64_t) +

348 sizeof(TruncatedHash) + CompressedBuffer.size();

349 }

350

353 OS << MagicNumber;

355 OS.write(reinterpret_cast<const char *>(&CompressionMethod),

356 sizeof(CompressionMethod));

357

358

360 uint32_t TotalFileSize32 = static_cast<uint32_t>(TotalFileSize64);

361 uint32_t UncompressedSize32 = static_cast<uint32_t>(UncompressedSize64);

362 OS.write(reinterpret_cast<const char *>(&TotalFileSize32),

363 sizeof(TotalFileSize32));

364 OS.write(reinterpret_cast<const char *>(&UncompressedSize32),

365 sizeof(UncompressedSize32));

366 } else {

367 OS.write(reinterpret_cast<const char *>(&TotalFileSize64),

368 sizeof(TotalFileSize64));

369 OS.write(reinterpret_cast<const char *>(&UncompressedSize64),

370 sizeof(UncompressedSize64));

371 }

372

373 OS.write(reinterpret_cast<const char *>(&TruncatedHash),

374 sizeof(TruncatedHash));

375 OS.write(reinterpret_cast<const char *>(CompressedBuffer.data()),

376 CompressedBuffer.size());

377

378 if (VerboseStream) {

380 double CompressionRate =

381 static_cast<double>(UncompressedSize64) / CompressedBuffer.size();

383 double CompressionSpeedMBs =

384 (UncompressedSize64 / (1024.0 * 1024.0)) / CompressionTimeSeconds;

385 *VerboseStream << "Compressed bundle format version: " << Version << "\n"

386 << "Total file size (including headers): "

388 << "Compression method used: " << MethodUsed << "\n"

389 << "Compression level: " << P.level << "\n"

390 << "Binary size before compression: "

392 << "Binary size after compression: "

394 << "Compression rate: " << format("%.2lf", CompressionRate)

395 << "\n"

396 << "Compression ratio: "

397 << format("%.2lf%%", 100.0 / CompressionRate) << "\n"

398 << "Compression speed: "

399 << format("%.2lf MB/s", CompressionSpeedMBs) << "\n"

400 << "Truncated MD5 hash: " << format_hex(TruncatedHash, 16)

401 << "\n";

402 }

403

406}

407

408

409

444

445

447 switch (Version) {

448 case 1:

450 case 2:

452 case 3:

454 default:

456 }

457}

458

459ExpectedCompressedOffloadBundle::CompressedBundleHeader

463

465 std::memcpy(&Header, Blob.data(), std::min(Blob.size(), sizeof(Header)));

466

468 Normalized.Version = Header.Common.Version;

469

471

472 if (Blob.size() < RequiredSize)

473 return createStringError("compressed bundle header size too small");

474

475 switch (Normalized.Version) {

476 case 1:

478 Normalized.Hash = Header.V1.Hash;

479 break;

480 case 2:

481 Normalized.FileSize = Header.V2.FileSize;

483 Normalized.Hash = Header.V2.Hash;

484 break;

485 case 3:

486 Normalized.FileSize = Header.V3.FileSize;

488 Normalized.Hash = Header.V3.Hash;

489 break;

490 default:

492 }

493

494

495 switch (Header.Common.Method) {

500 break;

501 default:

503 }

504

505 return Normalized;

506}

507

512

513

516

518 if (VerboseStream)

519 *VerboseStream << "Uncompressed bundle\n";

521 }

522

525 if (!HeaderOrErr)

527

529 unsigned ThisVersion = Normalized.Version;

531

533

534 size_t TotalFileSize = Normalized.FileSize.value_or(0);

536 auto StoredHash = Normalized.Hash;

537

538 Timer DecompressTimer("Decompression Timer", "Decompression time",

540 if (VerboseStream)

542

545 Blob.substr(HeaderSize, TotalFileSize - HeaderSize);

546

549 DecompressedData, UncompressedSize))

550 return createStringError("could not decompress embedded file contents: " +

551 toString(std::move(DecompressionError)));

552

553 if (VerboseStream) {

555

556 double DecompressionTimeSeconds =

558

559

560 Timer HashRecalcTimer("Hash Recalculation Timer", "Hash recalculation time",

563 MD5 Hash;

566 Hash.final(Result);

567 uint64_t RecalculatedHash = Result.low();

569 bool HashMatch = (StoredHash == RecalculatedHash);

570

571 double CompressionRate =

572 static_cast<double>(UncompressedSize) / CompressedData.size();

573 double DecompressionSpeedMBs =

574 (UncompressedSize / (1024.0 * 1024.0)) / DecompressionTimeSeconds;

575

576 *VerboseStream << "Compressed bundle format version: " << ThisVersion

577 << "\n";

578 if (ThisVersion >= 2)

579 *VerboseStream << "Total file size (from header): "

581 *VerboseStream

582 << "Decompression method: "

584 << "\n"

585 << "Size before decompression: "

587 << "Size after decompression: " << formatWithCommas(UncompressedSize)

588 << " bytes\n"

589 << "Compression rate: " << format("%.2lf", CompressionRate) << "\n"

590 << "Compression ratio: " << format("%.2lf%%", 100.0 / CompressionRate)

591 << "\n"

592 << "Decompression speed: "

593 << format("%.2lf MB/s", DecompressionSpeedMBs) << "\n"

594 << "Stored hash: " << format_hex(StoredHash, 16) << "\n"

595 << "Recalculated hash: " << format_hex(RecalculatedHash, 16) << "\n"

596 << "Hashes match: " << (HashMatch ? "Yes" : "No") << "\n";

597 }

598

600}

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

#define LLVM_PACKED_START

Module.h This file contains the declarations for the Module class.

static LLVM_PACKED_END size_t getHeaderSize(uint16_t Version)

Definition OffloadBundle.cpp:446

Error extractOffloadBundle(MemoryBufferRef Contents, uint64_t SectionOffset, StringRef FileName, SmallVectorImpl< OffloadBundleFatBin > &Bundles)

Definition OffloadBundle.cpp:33

static std::string formatWithCommas(unsigned long long Value)

Definition OffloadBundle.cpp:284

static TimerGroup OffloadBundlerTimerGroup("Offload Bundler Timer Group", "Timer group for offload bundler")

The Input class is used to parse a yaml document into in-memory structs and vectors.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

Provides read only access to a subclass of BinaryStream.

Error readInteger(T &Dest)

Read an integer of the specified endianness into Dest and update the stream's offset.

LLVM_ABI Error readFixedString(StringRef &Dest, uint32_t Length)

Read a Length byte string into Dest.

Represents either an error or a value T.

std::error_code getError() const

Lightweight error class with error context and mandatory checking.

static ErrorSuccess success()

Create a success value.

Tagged union holding either a T or a Error.

Error takeError()

Take ownership of the stored error.

static LLVM_ABI Expected< std::unique_ptr< FileOutputBuffer > > create(StringRef FilePath, size_t Size, unsigned Flags=0)

Factory method to create an OutputBuffer object which manages a read/write buffer of the specified si...

LLVM_ABI void update(ArrayRef< uint8_t > Data)

Updates the hash for the byte stream provided.

LLVM_ABI void final(MD5Result &Result)

Finishes off the hash and puts the result in result.

size_t getBufferSize() const

const char * getBufferStart() const

StringRef getBuffer() const

This interface provides simple read-only access to a block of memory, and provides simple methods for...

static std::unique_ptr< MemoryBuffer > getMemBuffer(StringRef InputData, StringRef BufferName="", bool RequiresNullTerminator=true)

Open the specified memory range as a MemoryBuffer.

static std::unique_ptr< MemoryBuffer > getMemBufferCopy(StringRef InputData, const Twine &BufferName="")

Open the specified memory range as a MemoryBuffer, copying the contents and taking ownership of it.

This class consists of common code factored out of the SmallVector class to reduce code duplication b...

reference emplace_back(ArgTypes &&... Args)

pointer data()

Return a pointer to the vector's buffer, even if empty().

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

static constexpr size_t npos

std::string str() const

str - Get the contents as an std::string.

constexpr StringRef substr(size_t Start, size_t N=npos) const

Return a reference to the substring from [Start, Start + N).

StringRef drop_front(size_t N=1) const

Return a StringRef equal to 'this' but with the first N elements dropped.

constexpr size_t size() const

size - Get the string size.

constexpr const char * data() const

data - Get a pointer to the start of the string (which may not be null terminated).

double getWallTime() const

The TimerGroup class is used to group together related timers into a single report that is printed wh...

This class is used to track the amount of time spent between invocations of its startTimer()/stopTime...

LLVM_ABI void stopTimer()

Stop the timer.

LLVM_ABI void startTimer()

Start the timer running.

TimeRecord getTotalTime() const

Return the duration for which this timer has been running.

LLVM Value Representation.

static llvm::Expected< std::unique_ptr< llvm::MemoryBuffer > > decompress(const llvm::MemoryBuffer &Input, raw_ostream *VerboseStream=nullptr)

Definition OffloadBundle.cpp:509

static llvm::Expected< std::unique_ptr< llvm::MemoryBuffer > > compress(llvm::compression::Params P, const llvm::MemoryBuffer &Input, uint16_t Version, raw_ostream *VerboseStream=nullptr)

Definition OffloadBundle.cpp:295

uint64_t getOffset() const

This class is the base class for all object file types.

static Expected< OwningBinary< ObjectFile > > createObjectFile(StringRef ObjectPath)

LLVM_ABI Error readEntries(StringRef Section, uint64_t SectionOffset)

Definition OffloadBundle.cpp:101

OffloadBundleFatBin(MemoryBufferRef Source, StringRef File, bool Decompress=false)

LLVM_ABI Error extractBundle(const ObjectFile &Source)

Definition OffloadBundle.cpp:170

StringRef getFileName() const

static LLVM_ABI Expected< std::unique_ptr< OffloadBundleFatBin > > create(MemoryBufferRef, uint64_t SectionOffset, StringRef FileName, bool Decompress=false)

Definition OffloadBundle.cpp:147

This is a value type class that represents a single section in the list of sections in the object fil...

This class implements an extremely fast bulk output stream that can only output to a stream.

raw_ostream & write(unsigned char C)

A raw_ostream that writes to an SmallVector or SmallString.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

LLVM_ABI bool isAvailable()

LLVM_ABI Error decompress(DebugCompressionType T, ArrayRef< uint8_t > Input, uint8_t *Output, size_t UncompressedSize)

LLVM_ABI void compress(Params P, ArrayRef< uint8_t > Input, SmallVectorImpl< uint8_t > &Output)

LLVM_ABI Error extractCodeObject(const ObjectFile &Source, int64_t Offset, int64_t Size, StringRef OutputFileName)

Extract code object memory from the given Source object file at Offset and of Size,...

Definition OffloadBundle.cpp:220

LLVM_ABI Error extractOffloadBundleByURI(StringRef URIstr)

Extracts an Offload Bundle Entry given by URI.

Definition OffloadBundle.cpp:258

LLVM_ABI Error extractOffloadBundleFatBinary(const ObjectFile &Obj, SmallVectorImpl< OffloadBundleFatBin > &Bundles)

Extracts fat binary in binary clang-offload-bundler format from object Obj and return it in Bundles.

Definition OffloadBundle.cpp:189

This is an optimization pass for GlobalISel generic memory operations.

LLVM_ABI file_magic identify_magic(StringRef magic)

Identify the type of a binary file based on how magical it is.

Error createFileError(const Twine &F, Error E)

Concatenate a source file path and/or name with an Error.

ArrayRef< CharT > arrayRefFromStringRef(StringRef Input)

Construct a string ref from an array ref of unsigned chars.

Error createStringError(std::error_code EC, char const *Fmt, const Ts &... Vals)

Create formatted StringError object.

FunctionAddr VTableAddr uintptr_t uintptr_t Version

FormattedNumber format_hex(uint64_t N, unsigned Width, bool Upper=false)

format_hex - Output N as a fixed width hexadecimal.

format_object< Ts... > format(const char *Fmt, const Ts &... Vals)

These are helper functions used to produce formatted output.

std::string toString(const APInt &I, unsigned Radix, bool Signed, bool formatAsCLiteral=false, bool UpperCase=true, bool InsertSeparators=false)

LLVM_ABI Error errorCodeToError(std::error_code EC)

Helper for converting an std::error_code to a Error.

StringRef toStringRef(bool B)

Construct a string ref from a boolean.

std::string itostr(int64_t X)

@ offload_bundle

Clang offload bundle file.

@ offload_bundle_compressed

Compressed clang offload bundle file.

Bundle entry in binary clang-offload-bundler format.

static Expected< std::unique_ptr< OffloadBundleURI > > createOffloadBundleURI(StringRef Str, UriTypeT Type)