CachePruning.cpp Source File (original) (raw)

23#define DEBUG_TYPE "cache-pruning"

25#include

26#include <system_error>

28using namespace llvm;

30namespace {

31struct FileInfo {

34 std::string Path;

39 return std::tie(Time, Other.Size, Path) <

40 std::tie(Other.Time, Size, Other.Path);

41 }

42};

43}

48 std::error_code EC;

50}

54 return make_error("Duration must not be empty",

60 return make_error("'" + NumStr + "' not an integer",

64 case 's':

65 return std::chrono::seconds(Num);

66 case 'm':

67 return std::chrono::minutes(Num);

68 case 'h':

69 return std::chrono::hours(Num);

70 default:

71 return make_error("'" + Duration +

72 "' must end with one of 's', 'm' or 'h'",

74 }

75}

80 std::pair<StringRef, StringRef> P = {"", PolicyStr};

81 while (.second.empty()) {

82 P = P.second.split(':');

85 std::tie(Key, Value) = P.first.split('=');

86 if (Key == "prune_interval") {

88 if (!DurationOrErr)

89 return DurationOrErr.takeError();

90 Policy.Interval = *DurationOrErr;

91 } else if (Key == "prune_after") {

93 if (!DurationOrErr)

94 return DurationOrErr.takeError();

96 } else if (Key == "cache_size") {

97 if (Value.back() != '%')

98 return make_error("'" + Value + "' must be a percentage",

103 return make_error("'" + SizeStr + "' not an integer",

105 if (Size > 100)

106 return make_error("'" + SizeStr +

107 "' must be between 0 and 100",

110 } else if (Key == "cache_size_bytes") {

112 switch (tolower(Value.back())) {

113 case 'k':

114 Mult = 1024;

116 break;

117 case 'm':

118 Mult = 1024 * 1024;

120 break;

121 case 'g':

122 Mult = 1024 * 1024 * 1024;

124 break;

125 }

128 return make_error("'" + Value + "' not an integer",

131 } else if (Key == "cache_size_files") {

133 return make_error("'" + Value + "' not an integer",

135 } else {

136 return make_error("Unknown key: '" + Key + "'",

138 }

139 }

140

141 return Policy;

142}

143

144

146 const std::vector<std::unique_ptr> &Files) {

147 using namespace std::chrono;

148

149 if (Path.empty())

150 return false;

151

152 bool isPathDir;

154 return false;

155

156 if (!isPathDir)

157 return false;

158

161

162 if (Policy.Expiration == seconds(0) &&

165 LLVM_DEBUG(dbgs() << "No pruning settings set, exit early\n");

166

167 return false;

168 }

169

170

174 const auto CurrentTime = system_clock::now();

175 if (auto EC = sys::fs::status(TimestampFile, FileStatus)) {

176 if (EC == errc::no_such_file_or_directory) {

177

179 } else {

180

181 return false;

182 }

183 } else {

185 return false;

186 if (Policy.Interval != seconds(0)) {

187

188

190 auto TimeStampAge = CurrentTime - TimeStampModTime;

191 if (TimeStampAge <= *Policy.Interval) {

193 << duration_cast(TimeStampAge).count()

194 << "s old), do not prune.\n");

195 return false;

196 }

197 }

198

199

200

202 }

203

204

205

206 std::set FileInfos;

208

209

210 std::error_code EC;

213

215 File != FileEnd && !EC; File.increment(EC)) {

216

217

218

219

222 continue;

223

224

225

227 if (!StatusOrErr) {

228 LLVM_DEBUG(dbgs() << "Ignore " << File->path() << " (can't stat)\n");

229 continue;

230 }

231

232

233 const auto FileAccessTime = StatusOrErr->getLastAccessedTime();

234 auto FileAge = CurrentTime - FileAccessTime;

236 LLVM_DEBUG(dbgs() << "Remove " << File->path() << " ("

237 << duration_cast(FileAge).count()

238 << "s old)\n");

240 continue;

241 }

242

243

244 TotalSize += StatusOrErr->getSize();

245 FileInfos.insert({FileAccessTime, StatusOrErr->getSize(), File->path()});

246 }

247

248 auto FileInfo = FileInfos.begin();

249 size_t NumFiles = FileInfos.size();

250

251 auto RemoveCacheFile = [&]() {

252

254

255 TotalSize -= FileInfo->Size;

256 NumFiles--;

257 LLVM_DEBUG(dbgs() << " - Remove " << FileInfo->Path << " (size "

258 << FileInfo->Size << "), new occupancy is " << TotalSize

259 << "%\n");

260 ++FileInfo;

261 };

262

263

264

265

266 const size_t ActualNums = Files.size();

269 << "ThinLTO cache pruning happens since the number of created files ("

270 << ActualNums << ") exceeds the maximum number of files ("

272 << "); consider adjusting --thinlto-cache-policy\n";

273

274

277 RemoveCacheFile();

278

279

282 if (!ErrOrSpaceInfo) {

284 }

286 auto AvailableSpace = TotalSize + SpaceInfo.free;

287

292 auto TotalSizeTarget = std::min<uint64_t>(

295

296 LLVM_DEBUG(dbgs() << "Occupancy: " << ((100 * TotalSize) / AvailableSpace)

297 << "% target is: "

300

301 size_t ActualSizes = 0;

302 for (const auto &File : Files)

303 if (File)

304 ActualSizes += File->getBufferSize();

305

306 if (ActualSizes > TotalSizeTarget)

308 << "ThinLTO cache pruning happens since the total size of the cache "

309 "files consumed by the current link job ("

310 << ActualSizes << " bytes) exceeds maximum cache size ("

311 << TotalSizeTarget

312 << " bytes); consider adjusting --thinlto-cache-policy\n";

313

314

315 while (TotalSize > TotalSizeTarget && FileInfo != FileInfos.end())

316 RemoveCacheFile();

317 }

318 return true;

319}

static Expected< std::chrono::seconds > parseDuration(StringRef Duration)

static void writeTimestampFile(StringRef TimestampFile)

Write a new timestamp file with the given path.

std::optional< std::vector< StOtherPiece > > Other

Represents either an error or a value T.

Tagged union holding either a T or a Error.

SmallString - A SmallString is just a SmallVector with methods and accessors that make it work better...

StringRef - Represent a constant reference to a string, i.e.

bool getAsInteger(unsigned Radix, T &Result) const

Parse the current string as an integer of the specified radix.

std::string str() const

str - Get the contents as an std::string.

bool starts_with(StringRef Prefix) const

Check if this string starts with the given Prefix.

LLVM Value Representation.

static raw_ostream & warning()

Convenience method for printing "warning: " to stderr.

A raw_ostream that writes to a file descriptor.

TimePoint getLastModificationTime() const

The file modification time as reported from the underlying file system.

directory_iterator - Iterates through the entries in path.

Represents the result of a call to sys::fs::status().

std::error_code status(const Twine &path, file_status &result, bool follow=true)

Get file status as if by POSIX stat().

std::error_code remove(const Twine &path, bool IgnoreNonExisting=true)

Remove path.

ErrorOr< space_info > disk_space(const Twine &Path)

Get disk space usage information.

bool is_directory(const basic_file_status &status)

Does status represent a directory?

StringRef filename(StringRef path LLVM_LIFETIME_BOUND, Style style=Style::native)

Get filename.

void append(SmallVectorImpl< char > &path, const Twine &a, const Twine &b="", const Twine &c="", const Twine &d="")

Append to path.

std::chrono::time_point< std::chrono::system_clock, D > TimePoint

A time point on the system clock.

This is an optimization pass for GlobalISel generic memory operations.

bool operator<(int64_t V1, const APSInt &V2)

std::error_code inconvertibleErrorCode()

The value returned by this function can be returned from convertToErrorCode for Error values where no...

Expected< CachePruningPolicy > parseCachePruningPolicy(StringRef PolicyStr)

Parse the given string as a cache pruning policy.

raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

void report_fatal_error(Error Err, bool gen_crash_diag=true)

Report a serious error, calling any installed error handler.

bool pruneCache(StringRef Path, CachePruningPolicy Policy, const std::vector< std::unique_ptr< MemoryBuffer > > &Files={})

Peform pruning using the supplied policy, returns true if pruning occurred, i.e.

auto count(R &&Range, const E &Element)

Wrapper function around std::count to count the number of times an element Element occurs in the give...

Policy for the pruneCache() function.

uint64_t MaxSizeFiles

The maximum number of files in the cache directory.

std::optional< std::chrono::seconds > Interval

The pruning interval.

std::chrono::seconds Expiration

The expiration for a file.

uint64_t MaxSizeBytes

The maximum size for the cache directory in bytes.

unsigned MaxSizePercentageOfAvailableSpace

The maximum size for the cache directory, in terms of percentage of the available space on the disk.

space_info - Self explanatory.