FileManager.cpp Source File (original) (raw)

21#include "llvm/ADT/SmallString.h"

22#include "llvm/ADT/Statistic.h"

23#include "llvm/Config/llvm-config.h"

24#include "llvm/Support/FileSystem.h"

25#include "llvm/Support/IOSandbox.h"

26#include "llvm/Support/MemoryBuffer.h"

27#include "llvm/Support/Path.h"

28#include "llvm/Support/raw_ostream.h"

29#include

30#include

31#include

32#include

33#include

34#include

35#include

37using namespace clang;

39#define DEBUG_TYPE "file-search"

47 : FS(std::move(FS)), FileSystemOpts(FSO), SeenDirEntries(64),

48 SeenFileEntries(64), NextFileUID(0) {

51 if (!this->FS)

52 this->FS = llvm::vfs::getRealFileSystem();

53}

58 assert(statCache && "No stat cache provided?");

59 StatCache = std::move(statCache);

60}

68 bool CacheFailure) {

69 if (Filename.empty())

70 return llvm::errorCodeToError(

73 if (llvm::sys::path::is_separator(Filename[Filename.size() - 1]))

74 return llvm::errorCodeToError(make_error_code(std::errc::is_a_directory));

76 StringRef DirName = llvm::sys::path::parent_path(Filename);

78 if (DirName.empty())

79 DirName = ".";

81 return FileMgr.getDirectoryRef(DirName, CacheFailure);

82}

84DirectoryEntry *&FileManager::getRealDirEntry(const llvm::vfs::Status &Status) {

85 assert(Status.isDirectory() && "The directory should exist!");

90 DirectoryEntry *&UDE = UniqueRealDirs[Status.getUniqueID()];

92 if (!UDE) {

95 UDE = new (DirsAlloc.Allocate()) DirectoryEntry();

96 }

97 return UDE;

98}

100

101

102void FileManager::addAncestorsAsVirtualDirs(StringRef Path) {

103 StringRef DirName = llvm::sys::path::parent_path(Path);

104 if (DirName.empty())

105 DirName = ".";

106

107 auto &NamedDirEnt = *SeenDirEntries.insert(

108 {DirName, std::errc::no_such_file_or_directory}).first;

109

110

111

112

113

114 if (NamedDirEnt.second)

115 return;

116

117

118 llvm::vfs::Status Status;

119 auto statError =

120 getStatValue(DirName, Status, false, nullptr );

121 if (statError) {

122

123

124 auto *UDE = new (DirsAlloc.Allocate()) DirectoryEntry();

125 NamedDirEnt.second = *UDE;

126 VirtualDirectoryEntries.push_back(UDE);

127 } else {

128

129 DirectoryEntry *&UDE = getRealDirEntry(Status);

130 NamedDirEnt.second = *UDE;

131 }

132

133

134 addAncestorsAsVirtualDirs(DirName);

135}

136

139

140

141

142 if (DirName.size() > 1 &&

143 DirName != llvm::sys::path::root_path(DirName) &&

144 llvm::sys::path::is_separator(DirName.back()))

145 DirName = DirName.drop_back();

146 std::optionalstd::string DirNameStr;

147 if (is_style_windows(llvm::sys::path::Style::native)) {

148

149

150 if (DirName.size() > 1 && DirName.back() == ':' &&

151 DirName.equals_insensitive(llvm::sys::path::root_name(DirName))) {

152 DirNameStr = DirName.str() + '.';

153 DirName = *DirNameStr;

154 }

155 }

156

157 ++NumDirLookups;

158

159

160

161 auto SeenDirInsertResult =

162 SeenDirEntries.insert({DirName, std::errc::no_such_file_or_directory});

163 if (!SeenDirInsertResult.second) {

164 if (SeenDirInsertResult.first->second)

166 return llvm::errorCodeToError(SeenDirInsertResult.first->second.getError());

167 }

168

169

170 ++NumDirCacheMisses;

171 auto &NamedDirEnt = *SeenDirInsertResult.first;

172 assert(!NamedDirEnt.second && "should be newly-created");

173

174

175

176 StringRef InterndDirName = NamedDirEnt.first();

177

178

179 llvm::vfs::Status Status;

180 auto statError = getStatValue(InterndDirName, Status, false,

181 nullptr );

182 if (statError) {

183

184 if (CacheFailure)

185 NamedDirEnt.second = statError;

186 else

187 SeenDirEntries.erase(DirName);

188 return llvm::errorCodeToError(statError);

189 }

190

191

193 NamedDirEnt.second = *UDE;

194

196}

197

199 bool openFile,

200 bool CacheFailure,

201 bool IsText) {

202 ++NumFileLookups;

203

204

205 auto SeenFileInsertResult =

206 SeenFileEntries.insert({Filename, std::errc::no_such_file_or_directory});

207 if (!SeenFileInsertResult.second) {

208 if (!SeenFileInsertResult.first->second)

209 return llvm::errorCodeToError(

210 SeenFileInsertResult.first->second.getError());

211 return FileEntryRef(*SeenFileInsertResult.first);

212 }

213

214

215 ++NumFileCacheMisses;

216 auto *NamedFileEnt = &*SeenFileInsertResult.first;

217 assert(!NamedFileEnt->second && "should be newly-created");

218

219

220

221 StringRef InterndFileName = NamedFileEnt->first();

222

223

224

225

226

227

229 if (!DirInfoOrErr) {

230 std::error_code Err = errorToErrorCode(DirInfoOrErr.takeError());

231 if (CacheFailure)

232 NamedFileEnt->second = Err;

233 else

234 SeenFileEntries.erase(Filename);

235

236 return llvm::errorCodeToError(Err);

237 }

239

240

241

242

243

244 std::unique_ptrllvm::vfs::File F;

245 llvm::vfs::Status Status;

246 auto statError = getStatValue(InterndFileName, Status, true,

247 openFile ? &F : nullptr, IsText);

248 if (statError) {

249

250 if (CacheFailure)

251 NamedFileEnt->second = statError;

252 else

253 SeenFileEntries.erase(Filename);

254

255 return llvm::errorCodeToError(statError);

256 }

257

258 assert((openFile || !F) && "undesired open file");

259

260

261

262 FileEntry *&UFE = UniqueRealFiles[Status.getUniqueID()];

263 bool ReusingEntry = UFE != nullptr;

264 if (!UFE)

265 UFE = new (FilesAlloc.Allocate()) FileEntry();

266

267 if (!Status.ExposesExternalVFSPath || Status.getName() == Filename) {

268

270 } else {

271

272

273

274

275

276

277

278

279

280

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

300

301

302

303

304

305

306 auto &Redirection =

307 *SeenFileEntries

309 .first;

311 "filename redirected to a non-canonical filename?");

313 "filename from getStatValue() refers to wrong file");

314

315

316

318 }

319

321 if (ReusingEntry) {

322 return ReturnedRef;

323 }

324

325

326 UFE->Size = Status.getSize();

327 UFE->ModTime = llvm::sys::toTimeT(Status.getLastModificationTime());

329 UFE->UID = NextFileUID++;

330 UFE->UniqueID = Status.getUniqueID();

331 UFE->IsNamedPipe = Status.getType() == llvm::sys::fs::file_type::fifo_file;

332 UFE->IsDeviceFile =

333 Status.getType() == llvm::sys::fs::file_type::character_file;

334 UFE->File = std::move(F);

335

336 if (UFE->File) {

337 if (auto PathName = UFE->File->getName())

338 fillRealPathName(UFE, *PathName);

339 } else if (!openFile) {

340

341 fillRealPathName(UFE, InterndFileName);

342 }

343 return ReturnedRef;

344}

345

347

348 if (STDIN)

349 return *STDIN;

350

351 auto ContentOrError = [] {

352 auto BypassSandbox = llvm::sys::sandbox::scopedDisable();

353 return llvm::MemoryBuffer::getSTDIN();

354 }();

355

356 if (!ContentOrError)

357 return llvm::errorCodeToError(ContentOrError.getError());

358

359 auto Content = std::move(*ContentOrError);

361 Content->getBufferSize(), 0);

363 FE.Content = std::move(Content);

364 FE.IsNamedPipe = true;

365 return *STDIN;

366}

367

369 FS->visit([Active](llvm::vfs::FileSystem &FileSys) {

370 if (auto *RFS = dyn_castllvm::vfs::RedirectingFileSystem(&FileSys))

371 RFS->setUsageTrackingActive(Active);

372 });

373}

374

376 time_t ModificationTime) {

377 ++NumFileLookups;

378

379

380 auto &NamedFileEnt = *SeenFileEntries.insert(

381 {Filename, std::errc::no_such_file_or_directory}).first;

382 if (NamedFileEnt.second) {

387 }

388

389

390 ++NumFileCacheMisses;

391 addAncestorsAsVirtualDirs(Filename);

393

394

395

396

397

398

399

401 *this, Filename.empty() ? "." : Filename, true));

402 assert(DirInfo &&

403 "The directory of a virtual file should already be in the cache.");

404

405

406 llvm::vfs::Status Status;

407 const char *InterndFileName = NamedFileEnt.first().data();

408 if (!getStatValue(InterndFileName, Status, true, nullptr)) {

409 Status = llvm::vfs::Status(

410 Status.getName(), Status.getUniqueID(),

411 llvm::sys::toTimePoint(ModificationTime),

412 Status.getUser(), Status.getGroup(), Size,

413 Status.getType(), Status.getPermissions());

414

415 auto &RealFE = UniqueRealFiles[Status.getUniqueID()];

416 if (RealFE) {

417

418

419

420 if (RealFE->File)

421 RealFE->closeFile();

422

423

424

425

428 }

429

430 RealFE = new (FilesAlloc.Allocate()) FileEntry();

431 RealFE->UniqueID = Status.getUniqueID();

432 RealFE->IsNamedPipe =

433 Status.getType() == llvm::sys::fs::file_type::fifo_file;

434 fillRealPathName(RealFE, Status.getName());

435

436 UFE = RealFE;

437 } else {

438

439 UFE = new (FilesAlloc.Allocate()) FileEntry();

440 VirtualFileEntries.push_back(UFE);

441 }

442

444 UFE->Size = Size;

445 UFE->ModTime = ModificationTime;

446 UFE->Dir = &DirInfo->getDirEntry();

447 UFE->UID = NextFileUID++;

448 UFE->File.reset();

450}

451

453

454 llvm::vfs::Status Status;

455 if (getStatValue(VF.getName(), Status, true, nullptr))

456 return std::nullopt;

457

458 if (!SeenBypassFileEntries)

459 SeenBypassFileEntries = std::make_unique<

460 llvm::StringMap<llvm::ErrorOrFileEntryRef::MapValue>>();

461

462

463 auto Insertion = SeenBypassFileEntries->insert(

464 {VF.getName(), std::errc::no_such_file_or_directory});

465 if (!Insertion.second)

467

468

470 BypassFileEntries.push_back(BFE);

472 BFE->Size = Status.getSize();

474 BFE->ModTime = llvm::sys::toTimeT(Status.getLastModificationTime());

475 BFE->UID = NextFileUID++;

476

477

479}

480

483 StringRef pathRef(Path.data(), Path.size());

484

485 if (FileSystemOpts.WorkingDir.empty()

486 || llvm::sys::path::is_absolute(pathRef))

487 return false;

488

490 llvm::sys::path::append(NewPath, pathRef);

491 Path = NewPath;

492 return true;

493}

494

497

498 if (!llvm::sys::path::is_absolute(StringRef(Path.data(), Path.size()))) {

499 FS->makeAbsolute(Path);

500 Changed = true;

501 }

502

503 return Changed;

504}

505

506void FileManager::fillRealPathName(FileEntry *UFE, llvm::StringRef FileName) {

508

509

510

511

513 llvm::sys::path::remove_dots(AbsPath, true);

514 UFE->RealPathName = std::string(AbsPath);

515}

516

517llvm::ErrorOr<std::unique_ptrllvm::MemoryBuffer>

519 bool RequiresNullTerminator,

520 std::optional<int64_t> MaybeLimit, bool IsText) {

522

523 if (Entry->Content)

524 return llvm::MemoryBuffer::getMemBuffer(Entry->Content->getMemBufferRef());

525

526 uint64_t FileSize = Entry->getSize();

527

528 if (MaybeLimit)

529 FileSize = *MaybeLimit;

530

531

532

534 FileSize = -1;

535

536 StringRef Filename = FE.getName();

537

538 if (Entry->File) {

539 auto Result = Entry->File->getBuffer(Filename, FileSize,

540 RequiresNullTerminator, isVolatile);

543 }

544

545

546 return getBufferForFileImpl(Filename, FileSize, isVolatile,

547 RequiresNullTerminator, IsText);

548}

549

550llvm::ErrorOr<std::unique_ptrllvm::MemoryBuffer>

551FileManager::getBufferForFileImpl(StringRef Filename, int64_t FileSize,

552 bool isVolatile, bool RequiresNullTerminator,

553 bool IsText) const {

554 if (FileSystemOpts.WorkingDir.empty())

555 return FS->getBufferForFile(Filename, FileSize, RequiresNullTerminator,

556 isVolatile, IsText);

557

560 return FS->getBufferForFile(FilePath, FileSize, RequiresNullTerminator,

561 isVolatile, IsText);

562}

563

564

565

566

567

568

569std::error_code FileManager::getStatValue(StringRef Path,

570 llvm::vfs::Status &Status,

571 bool isFile,

572 std::unique_ptrllvm::vfs::File *F,

573 bool IsText) {

574

575

576 if (FileSystemOpts.WorkingDir.empty())

578 *FS, IsText);

579

582

584 StatCache.get(), *FS, IsText);

585}

586

587std::error_code

589 llvm::vfs::Status &Result) {

592

593 llvm::ErrorOrllvm::vfs::Status S = FS->status(FilePath.c_str());

594 if (!S)

595 return S.getError();

597 return std::error_code();

598}

599

602 UIDToFiles.clear();

603 UIDToFiles.resize(NextFileUID);

604

605 for (const auto &Entry : SeenFileEntries) {

606

607 if (!Entry.getValue() || $isa<FileEntry \*>$ (Entry.getValue()->V))

608 continue;

610

611

613 if (!ExistingFE || FE.getName() < ExistingFE->getName())

614 ExistingFE = FE;

615 }

616}

617

621

625

627 llvm::DenseMap<const void *, llvm::StringRef>::iterator Known =

628 CanonicalNames.find(Entry);

629 if (Known != CanonicalNames.end())

630 return Known->second;

631

632

633

634 StringRef CanonicalName(Name);

635

638 if (!FS->getRealPath(Name, RealPathBuf)) {

639 if (is_style_windows(llvm::sys::path::Style::native)) {

640

641

642 AbsPathBuf = Name;

643 if (!FS->makeAbsolute(AbsPathBuf)) {

644 if (llvm::sys::path::root_name(RealPathBuf) ==

645 llvm::sys::path::root_name(AbsPathBuf)) {

646 CanonicalName = RealPathBuf.str().copy(CanonicalNameStorage);

647 } else {

648

649

650

651 llvm::sys::path::remove_dots(AbsPathBuf, true);

652 CanonicalName = AbsPathBuf.str().copy(CanonicalNameStorage);

653 }

654 }

655 } else {

656 CanonicalName = RealPathBuf.str().copy(CanonicalNameStorage);

657 }

658 }

659

660 CanonicalNames.insert({Entry, CanonicalName});

661 return CanonicalName;

662}

663

665 assert(&Other != this && "Collecting stats into the same FileManager");

666 NumDirLookups += Other.NumDirLookups;

667 NumFileLookups += Other.NumFileLookups;

668 NumDirCacheMisses += Other.NumDirCacheMisses;

669 NumFileCacheMisses += Other.NumFileCacheMisses;

670}

671

673 llvm::errs() << "\n*** File Manager Stats:\n";

674 llvm::errs() << UniqueRealFiles.size() << " real files found, "

675 << UniqueRealDirs.size() << " real dirs found.\n";

676 llvm::errs() << VirtualFileEntries.size() << " virtual files found, "

677 << VirtualDirectoryEntries.size() << " virtual dirs found.\n";

678 llvm::errs() << NumDirLookups << " dir lookups, "

679 << NumDirCacheMisses << " dir cache misses.\n";

680 llvm::errs() << NumFileLookups << " file lookups, "

681 << NumFileCacheMisses << " file cache misses.\n";

682

684 if (auto *T = dyn_cast_or_nullllvm::vfs::TracingFileSystem(&VFS))

685 llvm::errs() << "\n*** Virtual File System Stats:\n"

686 << T->NumStatusCalls << " status() calls\n"

687 << T->NumOpenFileForReadCalls << " openFileForRead() calls\n"

688 << T->NumDirBeginCalls << " dir_begin() calls\n"

689 << T->NumGetRealPathCalls << " getRealPath() calls\n"

690 << T->NumExistsCalls << " exists() calls\n"

691 << T->NumIsLocalCalls << " isLocal() calls\n";

692 });

693

694

695}

static llvm::Expected< DirectoryEntryRef > getDirectoryFromFile(FileManager &FileMgr, StringRef Filename, bool CacheFailure)

Retrieve the directory that the given file name resides in.

Definition FileManager.cpp:67

Defines the clang::FileManager interface and associated types.

Defines the FileSystemStatCache interface.

A reference to a DirectoryEntry that includes the name of the directory as it was accessed by the Fil...

StringRef getName() const

const DirectoryEntry & getDirEntry() const

Cached information about one directory (either on disk or in the virtual file system).

A reference to a FileEntry that includes the name of the file as it was accessed by the FileManager's...

const FileEntry & getFileEntry() const

StringRef getName() const

The name of this FileEntry.

DirectoryEntryRef getDir() const

Cached information about one file (either on disk or in the virtual file system).

bool isNamedPipe() const

Check whether the file is a named pipe (and thus can't be opened by the native FileManager methods).

Implements support for file system lookup, file system caching, and directory search management.

void AddStats(const FileManager &Other)

Import statistics from a child FileManager and add them to this current FileManager.

Definition FileManager.cpp:664

void trackVFSUsage(bool Active)

Enable or disable tracking of VFS usage.

Definition FileManager.cpp:368

void clearStatCache()

Removes the FileSystemStatCache object from the manager.

Definition FileManager.cpp:62

llvm::vfs::FileSystem & getVirtualFileSystem() const

llvm::ErrorOr< std::unique_ptr< llvm::MemoryBuffer > > getBufferForFile(FileEntryRef Entry, bool isVolatile=false, bool RequiresNullTerminator=true, std::optional< int64_t > MaybeLimit=std::nullopt, bool IsText=true)

Open the specified file as a MemoryBuffer, returning a new MemoryBuffer if successful,...

Definition FileManager.cpp:518

std::error_code getNoncachedStatValue(StringRef Path, llvm::vfs::Status &Result)

Get the 'stat' information for the given Path.

Definition FileManager.cpp:588

FileManager(const FileSystemOptions &FileSystemOpts, IntrusiveRefCntPtr< llvm::vfs::FileSystem > FS=nullptr)

Construct a file manager, optionally with a custom VFS.

Definition FileManager.cpp:45

llvm::Expected< FileEntryRef > getSTDIN()

Get the FileEntryRef for stdin, returning an error if stdin cannot be read.

Definition FileManager.cpp:346

StringRef getCanonicalName(DirectoryEntryRef Dir)

Retrieve the canonical name for a given directory.

Definition FileManager.cpp:618

void GetUniqueIDMapping(SmallVectorImpl< OptionalFileEntryRef > &UIDToFiles) const

Produce an array mapping from the unique IDs assigned to each file to the corresponding FileEntryRef.

Definition FileManager.cpp:600

bool makeAbsolutePath(SmallVectorImpl< char > &Path) const

Makes Path absolute taking into account FileSystemOptions and the working directory option.

Definition FileManager.cpp:495

llvm::Expected< DirectoryEntryRef > getDirectoryRef(StringRef DirName, bool CacheFailure=true)

Lookup, cache, and verify the specified directory (real or virtual).

Definition FileManager.cpp:138

void setStatCache(std::unique_ptr< FileSystemStatCache > statCache)

Installs the provided FileSystemStatCache object within the FileManager.

Definition FileManager.cpp:57

FileEntryRef getVirtualFileRef(StringRef Filename, off_t Size, time_t ModificationTime)

Retrieve a file entry for a "virtual" file that acts as if there were a file with the given name on d...

Definition FileManager.cpp:375

bool FixupRelativePath(SmallVectorImpl< char > &Path) const

If path is not absolute and FileSystemOptions set the working directory, the path is modified to be r...

void PrintStats() const

Definition FileManager.cpp:672

OptionalFileEntryRef getBypassFile(FileEntryRef VFE)

Retrieve a FileEntry that bypasses VFE, which is expected to be a virtual file entry,...

Definition FileManager.cpp:452

static bool fixupRelativePath(const FileSystemOptions &FileSystemOpts, SmallVectorImpl< char > &Path)

Definition FileManager.cpp:481

llvm::Expected< FileEntryRef > getFileRef(StringRef Filename, bool OpenFile=false, bool CacheFailure=true, bool IsText=true)

Lookup, cache, and verify the specified file (real or virtual).

Definition FileManager.cpp:198

Keeps track of options that affect how file operations are performed.

std::string WorkingDir

If set, paths are resolved as if the working directory was set to the value of WorkingDir.

static std::error_code get(StringRef Path, llvm::vfs::Status &Status, bool isFile, std::unique_ptr< llvm::vfs::File > *F, FileSystemStatCache *Cache, llvm::vfs::FileSystem &FS, bool IsText=true)

Get the 'stat' information for the specified path, using the cache to accelerate it if possible.

The JSON file list parser is used to communicate input to InstallAPI.

bool isa(CodeGen::Address addr)

CustomizableOptional< FileEntryRef > OptionalFileEntryRef

std::error_code make_error_code(BuildPreambleError Error)

@ Result

The result type of a method or function.

const FunctionProtoType * T

U cast(CodeGen::Address addr)

@ Other

Other implicit parameter.

Type stored in the StringMap.