NumericalStabilitySanitizer.cpp Source File (original) (raw)

43#include

45using namespace llvm;

47#define DEBUG_TYPE "nsan"

50 "Number of instrumented floating-point loads");

53 "Number of instrumented floating-point calls");

55 "Number of instrumented floating-point returns");

57 "Number of instrumented floating-point stores");

59 "Number of instrumented non floating-point stores");

61 NumInstrumentedNonFTMemcpyStores,

62 "Number of instrumented non floating-point stores with memcpy semantics");

63 STATISTIC(NumInstrumentedFCmp, "Number of instrumented fcmps");

69 "nsan-shadow-type-mapping", cl::init("dqq"),

70 cl::desc("One shadow type id for each of `float`, `double`, `long double`. "

71 "`d`,`l`,`q`,`e` mean double, x86_fp80, fp128 (quad) and "

72 "ppc_fp128 (extended double) respectively. The default is to "

73 "shadow `float` as `double`, and `double` and `x86_fp80` as "

74 "`fp128`"),

79 cl::desc("Instrument floating-point comparisons"),

83 "check-functions-filter",

84 cl::desc("Only emit checks for arguments of functions "

85 "whose names match the given regular expression"),

89 "nsan-truncate-fcmp-eq", cl::init(true),

91 "This flag controls the behaviour of fcmp equality comparisons."

92 "For equality comparisons such as `x == 0.0f`, we can perform the "

93 "shadow check in the shadow (`x_shadow == 0.0) == (x == 0.0f)`) or app "

94 " domain (`(trunc(x_shadow) == 0.0f) == (x == 0.0f)`). This helps "

95 "catch the case when `x_shadow` is accurate enough (and therefore "

96 "close enough to zero) so that `trunc(x_shadow)` is zero even though "

97 "both `x` and `x_shadow` are not"),

100

101

102

103

104

105

106

107

108

110 cl::desc("Check floating-point load"),

112

114 cl::desc("Check floating-point stores"),

116

118 cl::desc("Check floating-point return values"),

120

121

122

123

124

125

126

128 "nsan-propagate-non-ft-const-stores-as-ft",

130 "Propagate non floating-point const stores as floating point values."

131 "For debugging purposes only"),

133

136

137

142

143namespace {

144

145

146

147class ShadowTypeConfig {

148public:

149 static std::unique_ptr fromNsanTypeId(char TypeId);

150

151

153

154

155 virtual char getNsanTypeId() const = 0;

156

157 virtual ~ShadowTypeConfig() = default;

158};

159

160template

161class ShadowTypeConfigImpl : public ShadowTypeConfig {

162public:

163 char getNsanTypeId() const override { return NsanTypeId; }

164 static constexpr const char kNsanTypeId = NsanTypeId;

165};

166

167

168class F64ShadowConfig : public ShadowTypeConfigImpl<'d'> {

171 }

172};

173

174

175class F80ShadowConfig : public ShadowTypeConfigImpl<'l'> {

178 }

179};

180

181

182class F128ShadowConfig : public ShadowTypeConfigImpl<'q'> {

185 }

186};

187

188

189class PPC128ShadowConfig : public ShadowTypeConfigImpl<'e'> {

192 }

193};

194

195

196std::unique_ptr

197ShadowTypeConfig::fromNsanTypeId(const char TypeId) {

198 switch (TypeId) {

199 case F64ShadowConfig::kNsanTypeId:

200 return std::make_unique();

201 case F80ShadowConfig::kNsanTypeId:

202 return std::make_unique();

203 case F128ShadowConfig::kNsanTypeId:

204 return std::make_unique();

205 case PPC128ShadowConfig::kNsanTypeId:

206 return std::make_unique();

207 }

209}

210

211

212

213enum FTValueType { kFloat, kDouble, kLongDouble, kNumValueTypes };

214

215

216static std::optional ftValueTypeFromType(Type *FT) {

218 return kFloat;

220 return kDouble;

222 return kLongDouble;

223 return {};

224}

225

226

227static Type *typeFromFTValueType(FTValueType VT, LLVMContext &Context) {

228 switch (VT) {

229 case kFloat:

231 case kDouble:

233 case kLongDouble:

235 case kNumValueTypes:

236 return nullptr;

237 }

239}

240

241

242static const char *typeNameFromFTValueType(FTValueType VT) {

243 switch (VT) {

244 case kFloat:

245 return "float";

246 case kDouble:

247 return "double";

248 case kLongDouble:

249 return "longdouble";

250 case kNumValueTypes:

251 return nullptr;

252 }

254}

255

256

257

258class MappingConfig {

259public:

260 explicit MappingConfig(LLVMContext &C) : Context(C) {

263 unsigned ShadowTypeSizeBits[kNumValueTypes];

264 for (int VT = 0; VT < kNumValueTypes; ++VT) {

269 const unsigned AppTypeSize =

270 typeFromFTValueType(static_cast<FTValueType>(VT), Context)

271 ->getScalarSizeInBits();

272 const unsigned ShadowTypeSize =

273 Config->getType(Context)->getScalarSizeInBits();

274

275

276 if (ShadowTypeSize > kShadowScale * AppTypeSize)

278 "->f" + Twine(ShadowTypeSize) +

279 ": The shadow type size should be at most " +

281 " times the application type size");

282 ShadowTypeSizeBits[VT] = ShadowTypeSize;

283 Configs[VT] = std::move(Config);

284 }

285

286

287

288

289

290

291

292 if (ShadowTypeSizeBits[kFloat] > ShadowTypeSizeBits[kDouble] ||

293 ShadowTypeSizeBits[kDouble] > ShadowTypeSizeBits[kLongDouble])

295 Twine(ShadowTypeSizeBits[kFloat]) + "; double->f" +

296 Twine(ShadowTypeSizeBits[kDouble]) +

297 "; long double->f" +

298 Twine(ShadowTypeSizeBits[kLongDouble]) + " }");

299 }

300

301 const ShadowTypeConfig &byValueType(FTValueType VT) const {

302 assert(VT < FTValueType::kNumValueTypes && "invalid value type");

303 return *Configs[VT];

304 }

305

306

307 Type *getExtendedFPType(Type *FT) const {

308 if (const auto VT = ftValueTypeFromType(FT))

309 return Configs[*VT]->getType(Context);

311 auto *VecTy = cast(FT);

312

313 if (VecTy->isScalableTy())

314 return nullptr;

315 Type *ExtendedScalar = getExtendedFPType(VecTy->getElementType());

316 return ExtendedScalar

317 ? VectorType::get(ExtendedScalar, VecTy->getElementCount())

318 : nullptr;

319 }

320 return nullptr;

321 }

322

323private:

325 std::unique_ptr Configs[FTValueType::kNumValueTypes];

326};

327

328

329

330struct MemoryExtents {

333};

334

335static MemoryExtents getMemoryExtentsOrDie(Type *FT) {

336 if (const auto VT = ftValueTypeFromType(FT))

337 return {*VT, 1};

338 if (auto *VecTy = dyn_cast(FT)) {

339 const auto ScalarExtents = getMemoryExtentsOrDie(VecTy->getElementType());

340 return {ScalarExtents.ValueType,

341 ScalarExtents.NumElts * VecTy->getElementCount().getFixedValue()};

342 }

344}

345

346

347class CheckLoc {

348public:

349

351 CheckLoc Result(kStore);

352 Result.Address = Address;

353 return Result;

354 }

356 CheckLoc Result(kLoad);

357 Result.Address = Address;

358 return Result;

359 }

360

361

362 static CheckLoc makeArg(int ArgId) {

363 CheckLoc Result(kArg);

364 Result.ArgId = ArgId;

365 return Result;

366 }

367

368

369 static CheckLoc makeRet() { return CheckLoc(kRet); }

370

371

372 static CheckLoc makeInsert() { return CheckLoc(kInsert); }

373

374

375

377 return ConstantInt::get(Type::getInt32Ty(C), static_cast<int>(CheckTy));

378 }

379

380

381

382

384 switch (CheckTy) {

385 case kUnknown:

387 case kRet:

388 case kInsert:

389 return ConstantInt::get(IntptrTy, 0);

390 case kArg:

391 return ConstantInt::get(IntptrTy, ArgId);

392 case kLoad:

393 case kStore:

395 }

397 }

398

399private:

400

401

402 enum CheckType {

403 kUnknown = 0,

404 kRet,

405 kArg,

406 kLoad,

407 kStore,

408 kInsert,

409 };

410 explicit CheckLoc(CheckType CheckTy) : CheckTy(CheckTy) {}

411

413 const CheckType CheckTy;

414 int ArgId = -1;

415};

416

417

418class ValueToShadowMap {

419public:

420 explicit ValueToShadowMap(const MappingConfig &Config) : Config(Config) {}

421

422 ValueToShadowMap(const ValueToShadowMap &) = delete;

423 ValueToShadowMap &operator=(const ValueToShadowMap &) = delete;

424

425

426

427 void setShadow(Value &V, Value &Shadow) {

428 [[maybe_unused]] const bool Inserted = Map.try_emplace(&V, &Shadow).second;

430 if (!Inserted) {

431 if (auto *I = dyn_cast(&V))

432 errs() << I->getFunction()->getName() << ": ";

433 errs() << "duplicate shadow (" << &V << "): ";

434 V.dump();

435 }

436 });

437 assert(Inserted && "duplicate shadow");

438 }

439

440

441

442 bool hasShadow(Value *V) const {

443 return isa(V) || (Map.find(V) != Map.end());

444 }

445

446

447

449 if (Constant *C = dyn_cast(V))

450 return getShadowConstant(C);

451 return Map.find(V)->second;

452 }

453

454 bool empty() const { return Map.empty(); }

455

456private:

457

459 bool LosesInfo = false;

461 return CV;

462 }

463

464

466 if (UndefValue *U = dyn_cast(C)) {

468 }

469 if (ConstantFP *CFP = dyn_cast(C)) {

470

471 Type *Ty = Config.getExtendedFPType(CFP->getType());

472 return ConstantFP::get(

473 Ty, extendConstantFP(CFP->getValueAPF(), Ty->getFltSemantics()));

474 }

475

476 if (C->getType()->isVectorTy()) {

478 for (int I = 0, E = cast(C->getType())

479 ->getElementCount()

480 .getFixedValue();

481 I < E; ++I)

482 Elements.push_back(getShadowConstant(C->getAggregateElement(I)));

484 }

486 }

487

488 const MappingConfig &Config;

490};

491

492class NsanMemOpFn {

493public:

495 size_t NumArgs);

498

499private:

501 size_t NumSizedFuncs;

502};

503

508 Attr = Attr.addFnAttribute(Ctx, Attribute::NoUnwind);

509 Type *PtrTy = PointerType::getUnqual(Ctx);

511 IntegerType *IntptrTy = M.getDataLayout().getIntPtrType(Ctx);

513

514 NumSizedFuncs = Sized.size();

515

516

517 if (NumArgs == 3) {

518 Funcs.push_back(

519 M.getOrInsertFunction(Fallback, Attr, VoidTy, PtrTy, PtrTy, IntptrTy));

520 SizedFnTy = FunctionType::get(VoidTy, {PtrTy, PtrTy}, false);

521 } else if (NumArgs == 2) {

522 Funcs.push_back(

523 M.getOrInsertFunction(Fallback, Attr, VoidTy, PtrTy, IntptrTy));

524 SizedFnTy = FunctionType::get(VoidTy, {PtrTy}, false);

525 } else {

526 llvm_unreachable("Unexpected value of sized functions arguments");

527 }

528

529 for (size_t i = 0; i < NumSizedFuncs; ++i)

530 Funcs.push_back(M.getOrInsertFunction(Sized[i], SizedFnTy, Attr));

531}

532

534

535

536 assert(NumSizedFuncs >= 3 && "Unexpected number of sized functions");

537

538 size_t Idx =

539 MemOpSize == 4 ? 1 : (MemOpSize == 8 ? 2 : (MemOpSize == 16 ? 3 : 0));

540

541 return Funcs[Idx];

542}

543

544FunctionCallee NsanMemOpFn::getFallback() const { return Funcs[0]; }

545

546

547

548

549

550class NumericalStabilitySanitizer {

551public:

552 NumericalStabilitySanitizer(Module &M);

554

555private:

557 void maybeAddSuffixForNsanInterface(CallBase *CI);

558 bool addrPointsToConstantData(Value *Addr);

560 ValueToShadowMap &Map);

561 Value *createShadowValueWithOperandsAvailable(Instruction &Inst,

563 const ValueToShadowMap &Map);

566 ValueToShadowMap &Map);

567

569 const ValueToShadowMap &Map);

570

572 const ValueToShadowMap &Map);

574 CheckLoc Loc);

576 CheckLoc Loc);

577 void emitFCmpCheck(FCmpInst &FCmp, const ValueToShadowMap &Map);

578

579

583 const ValueToShadowMap &Map, IRBuilder<> &Builder);

586 const ValueToShadowMap &Map,

589 const ValueToShadowMap &Map, IRBuilder<> &Builder);

591 const ValueToShadowMap &Map, IRBuilder<> &Builder);

592

593

594 void propagateFTStore(StoreInst &Store, Type *VT, Type *ExtendedVT,

595 const ValueToShadowMap &Map);

596 void propagateNonFTStore(StoreInst &Store, Type *VT,

597 const ValueToShadowMap &Map);

598

601 MappingConfig Config;

603

604

605 FunctionCallee NsanGetShadowPtrForStore[FTValueType::kNumValueTypes] = {};

606 FunctionCallee NsanGetShadowPtrForLoad[FTValueType::kNumValueTypes] = {};

607 FunctionCallee NsanCheckValue[FTValueType::kNumValueTypes] = {};

608 FunctionCallee NsanFCmpFail[FTValueType::kNumValueTypes] = {};

609

610 NsanMemOpFn NsanCopyFns;

611 NsanMemOpFn NsanSetUnknownFns;

612

616

617 Type *NsanShadowRetType = nullptr;

619

620 GlobalValue *NsanShadowArgsTag = nullptr;

621

622 Type *NsanShadowArgsType = nullptr;

623 GlobalValue *NsanShadowArgsPtr = nullptr;

624

625 std::optional CheckFunctionsFilter;

626};

627}

628

633 {},

634

635

637

638 NumericalStabilitySanitizer Nsan(M);

642

644}

645

647 return dyn_cast(M.getOrInsertGlobal(Name, Ty, [&M, Ty, Name] {

648 return new GlobalVariable(M, Ty, false, GlobalVariable::ExternalLinkage,

649 nullptr, Name, nullptr,

650 GlobalVariable::InitialExecTLSModel);

651 }));

652}

653

654NumericalStabilitySanitizer::NumericalStabilitySanitizer(Module &M)

655 : DL(M.getDataLayout()), Context(M.getContext()), Config(Context),

656 NsanCopyFns(M, {"__nsan_copy_4", "__nsan_copy_8", "__nsan_copy_16"},

657 "__nsan_copy_values", 3),

658 NsanSetUnknownFns(M,

659 {"__nsan_set_value_unknown_4",

660 "__nsan_set_value_unknown_8",

661 "__nsan_set_value_unknown_16"},

662 "__nsan_set_value_unknown", 2) {

663 IntptrTy = DL.getIntPtrType(Context);

664 Type *PtrTy = PointerType::getUnqual(Context);

668

670 Attr = Attr.addFnAttribute(Context, Attribute::NoUnwind);

671

672 for (int I = 0; I < kNumValueTypes; ++I) {

673 const FTValueType VT = static_cast<FTValueType>(I);

674 const char *VTName = typeNameFromFTValueType(VT);

675 Type *VTTy = typeFromFTValueType(VT, Context);

676

677

678 const std::string GetterPrefix =

679 std::string("__nsan_get_shadow_ptr_for_") + VTName;

680 NsanGetShadowPtrForStore[VT] = M.getOrInsertFunction(

681 GetterPrefix + "_store", Attr, PtrTy, PtrTy, IntptrTy);

682 NsanGetShadowPtrForLoad[VT] = M.getOrInsertFunction(

683 GetterPrefix + "_load", Attr, PtrTy, PtrTy, IntptrTy);

684

685

686 const auto &ShadowConfig = Config.byValueType(VT);

687 Type *ShadowTy = ShadowConfig.getType(Context);

688 NsanCheckValue[VT] =

689 M.getOrInsertFunction(std::string("__nsan_internal_check_") + VTName +

690 "_" + ShadowConfig.getNsanTypeId(),

691 Attr, Int32Ty, VTTy, ShadowTy, Int32Ty, IntptrTy);

692 NsanFCmpFail[VT] = M.getOrInsertFunction(

693 std::string("__nsan_fcmp_fail_") + VTName + "_" +

694 ShadowConfig.getNsanTypeId(),

695 Attr, VoidTy, VTTy, VTTy, ShadowTy, ShadowTy, Int32Ty, Int1Ty, Int1Ty);

696 }

697

698

699 NsanGetRawShadowTypePtr = M.getOrInsertFunction(

700 "__nsan_internal_get_raw_shadow_type_ptr", Attr, PtrTy, PtrTy);

701 NsanGetRawShadowPtr = M.getOrInsertFunction(

702 "__nsan_internal_get_raw_shadow_ptr", Attr, PtrTy, PtrTy);

703

704 NsanShadowRetTag = createThreadLocalGV("__nsan_shadow_ret_tag", M, IntptrTy);

705

706 NsanShadowRetType = ArrayType::get(Type::getInt8Ty(Context),

708 NsanShadowRetPtr =

710

711 NsanShadowArgsTag =

713

714 NsanShadowArgsType =

717

718 NsanShadowArgsPtr =

720

723 std::string RegexError;

724 assert(R.isValid(RegexError));

725 CheckFunctionsFilter = std::move(R);

726 }

727}

728

729

730

731bool NumericalStabilitySanitizer::addrPointsToConstantData(Value *Addr) {

732

734 Addr = GEP->getPointerOperand();

735

737 return GV->isConstant();

738 return false;

739}

740

741

742

743

744

745

746

747

748

749

750

751

752

753void NumericalStabilitySanitizer::createShadowArguments(

755 assert(.getIntrinsicID() && "found a definition of an intrinsic");

756

757

759 return Config.getExtendedFPType(Arg.getType()) == nullptr;

760 }))

761 return;

762

763 IRBuilder<> Builder(F.getEntryBlock().getFirstNonPHI());

764

765

766 Value *HasShadowArgs = Builder.CreateICmpEQ(

767 Builder.CreateLoad(IntptrTy, NsanShadowArgsTag, false),

768 Builder.CreatePtrToInt(&F, IntptrTy));

769

770 unsigned ShadowArgsOffsetBytes = 0;

773 Type *ExtendedVT = Config.getExtendedFPType(VT);

774 if (ExtendedVT == nullptr)

775 continue;

776 Value *L = Builder.CreateAlignedLoad(

777 ExtendedVT,

778 Builder.CreateConstGEP2_64(NsanShadowArgsType, NsanShadowArgsPtr, 0,

779 ShadowArgsOffsetBytes),

780 Align(1), false);

781 Value *Shadow = Builder.CreateSelect(HasShadowArgs, L,

782 Builder.CreateFPExt(&Arg, ExtendedVT));

783 Map.setShadow(Arg, *Shadow);

784 TypeSize SlotSize = DL.getTypeStoreSize(ExtendedVT);

786 ShadowArgsOffsetBytes += SlotSize;

787 }

788 Builder.CreateStore(ConstantInt::get(IntptrTy, 0), NsanShadowArgsTag);

789}

790

791

792

794 const std::optional &CheckFunctionsFilter) {

795

797

798 if (CheckFunctionsFilter) {

799

800 if (Fn == nullptr)

801 return false;

802 if (CheckFunctionsFilter->match(Fn->getName()))

803 return true;

804 return false;

805 }

806

807 if (Fn == nullptr)

808 return true;

809

810

812 return false;

813

815 LibFunc LFunc = LibFunc::NumLibFuncs;

816

818 return true;

819

820

821

822

823 if (ID == Intrinsic::fabs || LFunc == LibFunc_fabsf ||

824 LFunc == LibFunc_fabs || LFunc == LibFunc_fabsl)

825 for (const auto &U : CI.users())

826 if (isa(U))

827 return false;

828

829 return true;

830}

831

832

833

834void NumericalStabilitySanitizer::populateShadowStack(

836

838 return;

839

840

842 return Config.getExtendedFPType(Arg->getType()) == nullptr;

843 }))

844 return;

845

848 const bool ShouldCheckArgs = shouldCheckArgs(CI, TLI, CheckFunctionsFilter);

850 if (Config.getExtendedFPType(Arg->getType()) == nullptr)

851 continue;

852 Value *ArgShadow = Map.getShadow(Arg);

853 ArgShadows.push_back(ShouldCheckArgs ? emitCheck(Arg, ArgShadow, Builder,

854 CheckLoc::makeArg(ArgIdx))

855 : ArgShadow);

856 }

857

858

861 if (Fn->isIntrinsic() || TLI.getLibFunc(*Fn, LFunc))

862 return;

863 }

864

865

866 Builder.CreateStore(CI.getCalledOperand(), NsanShadowArgsTag);

868

869 unsigned ShadowArgId = 0;

872 Type *ExtendedVT = Config.getExtendedFPType(VT);

873 if (ExtendedVT == nullptr)

874 continue;

875 Builder.CreateAlignedStore(

876 ArgShadows[ShadowArgId++],

877 Builder.CreateConstGEP2_64(NsanShadowArgsType, NsanShadowArgsPtr, 0,

878 ShadowArgsOffsetBytes),

879 Align(1), false);

880 TypeSize SlotSize = DL.getTypeStoreSize(ExtendedVT);

882 ShadowArgsOffsetBytes += SlotSize;

883 }

884}

885

886

887

888

892};

893

894Value *NumericalStabilitySanitizer::emitCheckInternal(Value *V, Value *ShadowV,

896 CheckLoc Loc) {

897

898 if (isa(V))

899 return ConstantInt::get(

902

903 Type *Ty = V->getType();

904 if (const auto VT = ftValueTypeFromType(Ty))

906 NsanCheckValue[*VT],

907 {V, ShadowV, Loc.getType(Context), Loc.getValue(IntptrTy, Builder)});

908

910 auto *VecTy = cast(Ty);

911

912

913 assert(!VecTy->isScalableTy() &&

914 "Scalable vector types are not supported yet");

915 Value *CheckResult = nullptr;

916 for (int I = 0, E = VecTy->getElementCount().getFixedValue(); I < E; ++I) {

917

918

919

922 Value *ComponentCheckResult =

923 emitCheckInternal(ExtractV, ExtractShadowV, Builder, Loc);

924 CheckResult = CheckResult

925 ? Builder.CreateOr(CheckResult, ComponentCheckResult)

926 : ComponentCheckResult;

927 }

928 return CheckResult;

929 }

931 Value *CheckResult = nullptr;

935 Value *ComponentCheckResult =

936 emitCheckInternal(ExtractV, ExtractShadowV, Builder, Loc);

937 CheckResult = CheckResult

938 ? Builder.CreateOr(CheckResult, ComponentCheckResult)

939 : ComponentCheckResult;

940 }

941 return CheckResult;

942 }

944 Value *CheckResult = nullptr;

947 continue;

950 Value *ComponentCheckResult =

951 emitCheckInternal(ExtractV, ExtractShadowV, Builder, Loc);

952 CheckResult = CheckResult

953 ? Builder.CreateOr(CheckResult, ComponentCheckResult)

954 : ComponentCheckResult;

955 }

956 if (!CheckResult)

957 return ConstantInt::get(

960 return CheckResult;

961 }

962

964}

965

966

967

968

969

970

971

972Value *NumericalStabilitySanitizer::emitCheck(Value *V, Value *ShadowV,

974 CheckLoc Loc) {

975

976 if (isa(V))

977 return ShadowV;

978

979 if (Instruction *Inst = dyn_cast(V)) {

980 Function *F = Inst->getFunction();

981 if (CheckFunctionsFilter && !CheckFunctionsFilter->match(F->getName())) {

982 return ShadowV;

983 }

984 }

985

986 Value *CheckResult = emitCheckInternal(V, ShadowV, Builder, Loc);

988 CheckResult,

989 ConstantInt::get(Builder.getInt32Ty(),

992 ICmpEQ, Builder.CreateFPExt(V, Config.getExtendedFPType(V->getType())),

993 ShadowV);

994}

995

996

997

998void NumericalStabilitySanitizer::emitFCmpCheck(FCmpInst &FCmp,

999 const ValueToShadowMap &Map) {

1001 return;

1002

1004 if (CheckFunctionsFilter && !CheckFunctionsFilter->match(F->getName()))

1005 return;

1006

1009 return;

1011

1012

1013

1016

1020

1021

1023 FCmpBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());

1024 Value *ShadowLHS = Map.getShadow(LHS);

1025 Value *ShadowRHS = Map.getShadow(RHS);

1026

1029 ShadowLHS = FCmpBuilder.CreateFPExt(

1030 FCmpBuilder.CreateFPTrunc(ShadowLHS, LHS->getType()), Ty);

1031 ShadowRHS = FCmpBuilder.CreateFPExt(

1032 FCmpBuilder.CreateFPTrunc(ShadowRHS, RHS->getType()), Ty);

1033 }

1034 Value *ShadowFCmp =

1035 FCmpBuilder.CreateFCmp(FCmp.getPredicate(), ShadowLHS, ShadowRHS);

1036 Value *OriginalAndShadowFcmpMatch =

1037 FCmpBuilder.CreateICmpEQ(&FCmp, ShadowFCmp);

1038

1040

1041

1042

1043 OriginalAndShadowFcmpMatch =

1044 FCmpBuilder.CreateAndReduce(OriginalAndShadowFcmpMatch);

1045 }

1046

1047

1048

1049 FCmpBuilder.CreateCondBr(OriginalAndShadowFcmpMatch, NextBB, FailBB,

1050 MDBuilder(Context).createLikelyBranchWeights());

1051

1052

1054 FailBuilder.SetCurrentDebugLocation(FCmp.getDebugLoc());

1055

1056 const auto EmitFailCall = [this, &FCmp, &FCmpBuilder,

1059 Value *ShadowResult) {

1060 Type *FT = L->getType();

1063 Callee = &(NsanFCmpFail[kFloat]);

1065 Callee = &(NsanFCmpFail[kDouble]);

1067

1068 Callee = &(NsanFCmpFail[kDouble]);

1071 } else {

1073 }

1074 FailBuilder.CreateCall(*Callee, {L, R, ShadowL, ShadowR,

1075 ConstantInt::get(FCmpBuilder.getInt32Ty(),

1077 Result, ShadowResult});

1078 };

1080 for (int I = 0, E = cast(LHS->getType())

1081 ->getElementCount()

1082 .getFixedValue();

1083 I < E; ++I) {

1084 Value *ExtractLHS = FailBuilder.CreateExtractElement(LHS, I);

1085 Value *ExtractRHS = FailBuilder.CreateExtractElement(RHS, I);

1086 Value *ExtractShaodwLHS = FailBuilder.CreateExtractElement(ShadowLHS, I);

1087 Value *ExtractShaodwRHS = FailBuilder.CreateExtractElement(ShadowRHS, I);

1088 Value *ExtractFCmp = FailBuilder.CreateExtractElement(&FCmp, I);

1089 Value *ExtractShadowFCmp =

1090 FailBuilder.CreateExtractElement(ShadowFCmp, I);

1091 EmitFailCall(ExtractLHS, ExtractRHS, ExtractShaodwLHS, ExtractShaodwRHS,

1092 ExtractFCmp, ExtractShadowFCmp);

1093 }

1094 } else {

1095 EmitFailCall(LHS, RHS, ShadowLHS, ShadowRHS, &FCmp, ShadowFCmp);

1096 }

1097 FailBuilder.CreateBr(NextBB);

1098

1099 ++NumInstrumentedFCmp;

1100}

1101

1102

1103PHINode *NumericalStabilitySanitizer::maybeCreateShadowPhi(

1105 Type *VT = Phi.getType();

1106 Type *ExtendedVT = Config.getExtendedFPType(VT);

1107 if (ExtendedVT == nullptr)

1108 return nullptr;

1109

1110

1111

1114 return Shadow;

1115}

1116

1117Value *NumericalStabilitySanitizer::handleLoad(LoadInst &Load, Type *VT,

1118 Type *ExtendedVT) {

1121 if (addrPointsToConstantData(Load.getPointerOperand())) {

1122

1123

1124 return Builder.CreateFPExt(&Load, ExtendedVT);

1125 }

1126

1127

1128

1129

1130

1131

1132

1133

1134

1135

1136

1137

1138

1139

1140

1141 const auto Extents = getMemoryExtentsOrDie(VT);

1143 NsanGetShadowPtrForLoad[Extents.ValueType],

1144 {Load.getPointerOperand(), ConstantInt::get(IntptrTy, Extents.NumElts)});

1145 ++NumInstrumentedFTLoads;

1146

1147

1150

1155

1156

1157

1158 {

1160 IRBuilder<> LoadBBBuilder(LoadBB);

1161 LoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());

1162 LoadBBBuilder.CreateCondBr(LoadBBBuilder.CreateIsNull(ShadowPtr), FExtBB,

1163 ShadowLoadBB);

1164 }

1165

1166

1167 IRBuilder<> ShadowLoadBBBuilder(ShadowLoadBB);

1168 ShadowLoadBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());

1169 Value *ShadowLoad = ShadowLoadBBBuilder.CreateAlignedLoad(

1170 ExtendedVT, ShadowPtr, Align(1), Load.isVolatile());

1172 ShadowLoad = emitCheck(&Load, ShadowLoad, ShadowLoadBBBuilder,

1173 CheckLoc::makeLoad(Load.getPointerOperand()));

1174 }

1175 ShadowLoadBBBuilder.CreateBr(NextBB);

1176

1177

1179 FExtBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());

1180 Value *FExt = FExtBBBuilder.CreateFPExt(&Load, ExtendedVT);

1181 FExtBBBuilder.CreateBr(NextBB);

1182

1183

1185 NextBBBuilder.SetCurrentDebugLocation(Load.getDebugLoc());

1186 PHINode *ShadowPhi = NextBBBuilder.CreatePHI(ExtendedVT, 2);

1187 ShadowPhi->addIncoming(ShadowLoad, ShadowLoadBB);

1189 return ShadowPhi;

1190}

1191

1192Value *NumericalStabilitySanitizer::handleTrunc(const FPTruncInst &Trunc,

1194 const ValueToShadowMap &Map,

1197 Type *OrigSourceTy = OrigSource->getType();

1198 Type *ExtendedSourceTy = Config.getExtendedFPType(OrigSourceTy);

1199

1200

1201

1202

1203

1204

1205

1206

1207

1208

1209

1210

1211

1212

1213

1214

1215

1216

1217

1218

1219

1220

1221

1222

1223

1224

1225

1226

1227

1228

1229

1230

1231 Value *Source = ExtendedSourceTy ? Map.getShadow(OrigSource) : OrigSource;

1232 Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;

1233

1234 if (SourceTy == ExtendedVT)

1236

1237 return Builder.CreateFPTrunc(Source, ExtendedVT);

1238}

1239

1240Value *NumericalStabilitySanitizer::handleExt(const FPExtInst &Ext, Type *VT,

1241 Type *ExtendedVT,

1242 const ValueToShadowMap &Map,

1244 Value *OrigSource = Ext.getOperand(0);

1245 Type *OrigSourceTy = OrigSource->getType();

1246 Type *ExtendedSourceTy = Config.getExtendedFPType(OrigSourceTy);

1247

1248

1249

1250

1251

1252

1253

1254

1255

1256

1257

1258

1259

1260

1261

1262

1263

1264

1265

1266

1267

1268

1269

1270

1271

1272

1273

1274

1275

1276

1277

1278 Value *Source = ExtendedSourceTy ? Map.getShadow(OrigSource) : OrigSource;

1279 Type *SourceTy = ExtendedSourceTy ? ExtendedSourceTy : OrigSourceTy;

1280

1281 if (SourceTy == ExtendedVT)

1283

1284 return Builder.CreateFPExt(Source, ExtendedVT);

1285}

1286

1287namespace {

1288

1289struct KnownIntrinsic {

1290 struct WidenedIntrinsic {

1291 const char *NarrowName;

1294 FnTypeFactory MakeFnTy;

1295 };

1296

1297 static const char *get(LibFunc LFunc);

1298

1299

1300

1301

1302

1303

1304 static const WidenedIntrinsic *widen(StringRef Name);

1305

1306private:

1307 struct LFEntry {

1309 const char *IntrinsicName;

1310 };

1311 static const LFEntry kLibfuncIntrinsics[];

1312

1313 static const WidenedIntrinsic kWidenedIntrinsics[];

1314};

1315}

1316

1319}

1320

1323 false);

1324}

1325

1329}

1330

1334 false);

1335}

1336

1340}

1341

1345 false);

1346}

1347

1349 return FunctionType::get(

1352 false);

1353}

1354

1356 return FunctionType::get(

1359 false);

1360}

1361

1362const KnownIntrinsic::WidenedIntrinsic KnownIntrinsic::kWidenedIntrinsics[] = {

1363

1364

1365

1366

1367

1368

1369

1370

1402

1404

1406

1408

1440 {"llvm.nearbyint.f32", Intrinsic::nearbyint, makeDoubleDouble},

1458};

1459

1460const KnownIntrinsic::LFEntry KnownIntrinsic::kLibfuncIntrinsics[] = {

1461 {LibFunc_sqrtf, "llvm.sqrt.f32"},

1462 {LibFunc_sqrt, "llvm.sqrt.f64"},

1463 {LibFunc_sqrtl, "llvm.sqrt.f80"},

1464 {LibFunc_sinf, "llvm.sin.f32"},

1465 {LibFunc_sin, "llvm.sin.f64"},

1466 {LibFunc_sinl, "llvm.sin.f80"},

1467 {LibFunc_cosf, "llvm.cos.f32"},

1468 {LibFunc_cos, "llvm.cos.f64"},

1469 {LibFunc_cosl, "llvm.cos.f80"},

1470 {LibFunc_powf, "llvm.pow.f32"},

1471 {LibFunc_pow, "llvm.pow.f64"},

1472 {LibFunc_powl, "llvm.pow.f80"},

1473 {LibFunc_expf, "llvm.exp.f32"},

1474 {LibFunc_exp, "llvm.exp.f64"},

1475 {LibFunc_expl, "llvm.exp.f80"},

1476 {LibFunc_exp2f, "llvm.exp2.f32"},

1477 {LibFunc_exp2, "llvm.exp2.f64"},

1478 {LibFunc_exp2l, "llvm.exp2.f80"},

1479 {LibFunc_logf, "llvm.log.f32"},

1480 {LibFunc_log, "llvm.log.f64"},

1481 {LibFunc_logl, "llvm.log.f80"},

1482 {LibFunc_log10f, "llvm.log10.f32"},

1483 {LibFunc_log10, "llvm.log10.f64"},

1484 {LibFunc_log10l, "llvm.log10.f80"},

1485 {LibFunc_log2f, "llvm.log2.f32"},

1486 {LibFunc_log2, "llvm.log2.f64"},

1487 {LibFunc_log2l, "llvm.log2.f80"},

1488 {LibFunc_fabsf, "llvm.fabs.f32"},

1489 {LibFunc_fabs, "llvm.fabs.f64"},

1490 {LibFunc_fabsl, "llvm.fabs.f80"},

1491 {LibFunc_copysignf, "llvm.copysign.f32"},

1492 {LibFunc_copysign, "llvm.copysign.f64"},

1493 {LibFunc_copysignl, "llvm.copysign.f80"},

1494 {LibFunc_floorf, "llvm.floor.f32"},

1495 {LibFunc_floor, "llvm.floor.f64"},

1496 {LibFunc_floorl, "llvm.floor.f80"},

1497 {LibFunc_fmaxf, "llvm.maxnum.f32"},

1498 {LibFunc_fmax, "llvm.maxnum.f64"},

1499 {LibFunc_fmaxl, "llvm.maxnum.f80"},

1500 {LibFunc_fminf, "llvm.minnum.f32"},

1501 {LibFunc_fmin, "llvm.minnum.f64"},

1502 {LibFunc_fminl, "llvm.minnum.f80"},

1503 {LibFunc_ceilf, "llvm.ceil.f32"},

1504 {LibFunc_ceil, "llvm.ceil.f64"},

1505 {LibFunc_ceill, "llvm.ceil.f80"},

1506 {LibFunc_truncf, "llvm.trunc.f32"},

1507 {LibFunc_trunc, "llvm.trunc.f64"},

1508 {LibFunc_truncl, "llvm.trunc.f80"},

1509 {LibFunc_rintf, "llvm.rint.f32"},

1510 {LibFunc_rint, "llvm.rint.f64"},

1511 {LibFunc_rintl, "llvm.rint.f80"},

1512 {LibFunc_nearbyintf, "llvm.nearbyint.f32"},

1513 {LibFunc_nearbyint, "llvm.nearbyint.f64"},

1514 {LibFunc_nearbyintl, "llvm.nearbyint.f80"},

1515 {LibFunc_roundf, "llvm.round.f32"},

1516 {LibFunc_round, "llvm.round.f64"},

1517 {LibFunc_roundl, "llvm.round.f80"},

1518};

1519

1520const char *KnownIntrinsic::get(LibFunc LFunc) {

1521 for (const auto &E : kLibfuncIntrinsics) {

1522 if (E.LFunc == LFunc)

1523 return E.IntrinsicName;

1524 }

1525 return nullptr;

1526}

1527

1528const KnownIntrinsic::WidenedIntrinsic *KnownIntrinsic::widen(StringRef Name) {

1529 for (const auto &E : kWidenedIntrinsics) {

1530 if (E.NarrowName == Name)

1531 return &E;

1532 }

1533 return nullptr;

1534}

1535

1536

1541 return nullptr;

1542

1543 if (const char *Name = KnownIntrinsic::get(LFunc))

1544 return Name;

1545

1547 return nullptr;

1548}

1549

1550

1551Value *NumericalStabilitySanitizer::maybeHandleKnownCallBase(

1553 const ValueToShadowMap &Map, IRBuilder<> &Builder) {

1555 if (Fn == nullptr)

1556 return nullptr;

1557

1561 const auto *Widened = KnownIntrinsic::widen(Fn->getName());

1562 if (Widened) {

1563 WidenedId = Widened->ID;

1564 WidenedFnTy = Widened->MakeFnTy(Context);

1565 } else {

1566

1567

1568

1569 WidenedId = ID;

1571 }

1573

1574

1575 const auto *Widened = KnownIntrinsic::widen(Name);

1576 assert(Widened && "make sure KnownIntrinsic entries are consistent");

1577 WidenedId = Widened->ID;

1578 WidenedFnTy = Widened->MakeFnTy(Context);

1579 } else {

1580

1581 return nullptr;

1582 }

1583

1584

1592 "invalid widened intrinsic");

1593

1594

1596

1597 for (unsigned I = 0, E = Call.getNumOperands() - 1; I < E; ++I) {

1600 Type *IntrinsicArgTy = WidenedFnTy->getParamType(I);

1601 if (OrigArgTy == IntrinsicArgTy) {

1602 Args.push_back(Arg);

1603 continue;

1604 }

1606 assert(ShadowArgTy &&

1607 "don't know how to get the shadow value for a non-FT");

1608 Value *Shadow = Map.getShadow(Arg);

1609 if (ShadowArgTy == IntrinsicArgTy) {

1610

1612 Args.push_back(Shadow);

1613 continue;

1614 }

1615

1617 }

1619 return WidenedFnTy->getReturnType() == ExtendedVT

1620 ? IntrinsicCall

1621 : Builder.CreateFPExt(IntrinsicCall, ExtendedVT);

1622}

1623

1624

1625

1626Value *NumericalStabilitySanitizer::handleCallBase(CallBase &Call, Type *VT,

1627 Type *ExtendedVT,

1629 const ValueToShadowMap &Map,

1631

1632 if (Call.isInlineAsm())

1633 return Builder.CreateFPExt(&Call, ExtendedVT);

1634

1635

1636

1637

1639 maybeHandleKnownCallBase(Call, VT, ExtendedVT, TLI, Map, Builder))

1640 return V;

1641

1642

1643

1645 Builder.CreateLoad(IntptrTy, NsanShadowRetTag, false);

1648

1650 ExtendedVT,

1651 Builder.CreateConstGEP2_64(NsanShadowRetType, NsanShadowRetPtr, 0, 0),

1652 false);

1653 Value *Shadow = Builder.CreateSelect(HasShadowRet, ShadowRetVal,

1654 Builder.CreateFPExt(&Call, ExtendedVT));

1655 ++NumInstrumentedFTCalls;

1656 return Shadow;

1657}

1658

1659

1660

1661Value *NumericalStabilitySanitizer::createShadowValueWithOperandsAvailable(

1663 const ValueToShadowMap &Map) {

1665 Type *ExtendedVT = Config.getExtendedFPType(VT);

1666 assert(ExtendedVT != nullptr && "trying to create a shadow for a non-FT");

1667

1668 if (auto *Load = dyn_cast(&Inst))

1669 return handleLoad(*Load, VT, ExtendedVT);

1670

1671 if (auto *Call = dyn_cast(&Inst)) {

1672

1676 return handleCallBase(*Call, VT, ExtendedVT, TLI, Map, Builder);

1677 }

1678

1679 if (auto *Invoke = dyn_cast(&Inst)) {

1680

1681

1683 BasicBlock *NextBB = Invoke->getNormalDest();

1687

1690 Value *Shadow = handleCallBase(*Invoke, VT, ExtendedVT, TLI, Map, Builder);

1693 return Shadow;

1694 }

1695

1698

1699 if (auto *Trunc = dyn_cast(&Inst))

1700 return handleTrunc(*Trunc, VT, ExtendedVT, Map, Builder);

1701 if (auto *Ext = dyn_cast(&Inst))

1702 return handleExt(*Ext, VT, ExtendedVT, Map, Builder);

1703

1704 if (auto *UnaryOp = dyn_cast(&Inst))

1705 return Builder.CreateUnOp(UnaryOp->getOpcode(),

1706 Map.getShadow(UnaryOp->getOperand(0)));

1707

1708 if (auto *BinOp = dyn_cast(&Inst))

1709 return Builder.CreateBinOp(BinOp->getOpcode(),

1710 Map.getShadow(BinOp->getOperand(0)),

1711 Map.getShadow(BinOp->getOperand(1)));

1712

1713 if (isa(&Inst) || isa(&Inst)) {

1714 auto *Cast = cast(&Inst);

1715 return Builder.CreateCast(Cast->getOpcode(), Cast->getOperand(0),

1716 ExtendedVT);

1717 }

1718

1719 if (auto *S = dyn_cast(&Inst))

1720 return Builder.CreateSelect(S->getCondition(),

1721 Map.getShadow(S->getTrueValue()),

1722 Map.getShadow(S->getFalseValue()));

1723

1724 if (auto *Freeze = dyn_cast(&Inst))

1725 return Builder.CreateFreeze(Map.getShadow(Freeze->getOperand(0)));

1726

1727 if (auto *Extract = dyn_cast(&Inst))

1729 Map.getShadow(Extract->getVectorOperand()), Extract->getIndexOperand());

1730

1731 if (auto *Insert = dyn_cast(&Inst))

1733 Map.getShadow(Insert->getOperand(1)),

1734 Insert->getOperand(2));

1735

1736 if (auto *Shuffle = dyn_cast(&Inst))

1738 Map.getShadow(Shuffle->getOperand(1)),

1739 Shuffle->getShuffleMask());

1740

1741

1742 if (auto *Extract = dyn_cast(&Inst))

1743 return Builder.CreateFPExt(Extract, ExtendedVT);

1744

1745 if (auto *BC = dyn_cast(&Inst))

1746 return Builder.CreateFPExt(BC, ExtendedVT);

1747

1750}

1751

1752

1753

1754

1755

1756void NumericalStabilitySanitizer::maybeCreateShadowValue(

1759 Type *ExtendedVT = Config.getExtendedFPType(VT);

1760 if (ExtendedVT == nullptr)

1761 return;

1762

1763 if (Map.hasShadow(&Root))

1764 return;

1765

1766 assert(!isa(Root) && "phi nodes should already have shadows");

1767

1768 std::vector<Instruction *> DfsStack(1, &Root);

1769 while (!DfsStack.empty()) {

1770

1771

1773

1774

1775 if (Map.hasShadow(I)) {

1776 DfsStack.pop_back();

1777 continue;

1778 }

1779

1780 bool MissingShadow = false;

1781 for (Value *Op : I->operands()) {

1782 Type *VT = Op->getType();

1783 if ( Config .getExtendedFPType(VT))

1784 continue;

1785 if (Map.hasShadow(Op))

1786 continue;

1787 MissingShadow = true;

1788 DfsStack.push_back(cast(Op));

1789 }

1790 if (MissingShadow)

1791 continue;

1792

1793

1794 Value *Shadow = createShadowValueWithOperandsAvailable(*I, TLI, Map);

1795 Map.setShadow(*I, *Shadow);

1796 DfsStack.pop_back();

1797 }

1798}

1799

1800

1801void NumericalStabilitySanitizer::propagateFTStore(

1802 StoreInst &Store, Type *VT, Type *ExtendedVT, const ValueToShadowMap &Map) {

1803 Value *StoredValue = Store.getValueOperand();

1806 const auto Extents = getMemoryExtentsOrDie(VT);

1808 NsanGetShadowPtrForStore[Extents.ValueType],

1809 {Store.getPointerOperand(), ConstantInt::get(IntptrTy, Extents.NumElts)});

1810

1811 Value *StoredShadow = Map.getShadow(StoredValue);

1812 if ( Store .getParent()->getParent()->hasOptNone()) {

1813

1814

1816 StoredShadow = emitCheck(StoredValue, StoredShadow, Builder,

1817 CheckLoc::makeStore(Store.getPointerOperand()));

1818 ++NumInstrumentedFTStores;

1819 }

1820 }

1821

1823 Store.isVolatile());

1824}

1825

1826

1827

1828

1829

1830

1831

1832

1833

1834void NumericalStabilitySanitizer::propagateNonFTStore(

1835 StoreInst &Store, Type *VT, const ValueToShadowMap &Map) {

1836 Value *PtrOp = Store.getPointerOperand();

1839 Value *Dst = PtrOp;

1840 TypeSize SlotSize = DL.getTypeStoreSize(VT);

1842 const auto LoadSizeBytes = SlotSize.getFixedValue();

1844 IntptrTy, APInt(IntptrTy->getPrimitiveSizeInBits(), LoadSizeBytes));

1845

1846 ++NumInstrumentedNonFTStores;

1847 Value *StoredValue = Store.getValueOperand();

1848 if (LoadInst *Load = dyn_cast(StoredValue)) {

1849

1850

1851

1852

1853

1855 Type *ShadowValueIntTy =

1859 Value *LoadSrc = Load->getPointerOperand();

1860

1861

1862

1863

1864

1865 Value *RawShadowType = LoadBuilder.CreateAlignedLoad(

1866 ShadowTypeIntTy,

1867 LoadBuilder.CreateCall(NsanGetRawShadowTypePtr, {LoadSrc}), Align(1),

1868 false);

1869 Value *RawShadowValue = LoadBuilder.CreateAlignedLoad(

1870 ShadowValueIntTy,

1871 LoadBuilder.CreateCall(NsanGetRawShadowPtr, {LoadSrc}), Align(1),

1872 false);

1873

1874

1876 RawShadowType, Builder.CreateCall(NsanGetRawShadowTypePtr, {Dst}),

1878 false);

1880 Builder.CreateCall(NsanGetRawShadowPtr, {Dst}),

1882 false);

1883

1884 ++NumInstrumentedNonFTMemcpyStores;

1885 return;

1886 }

1887

1889 (C = dyn_cast(StoredValue))) {

1890

1891

1892 Type *BitcastTy = nullptr;

1893 if (auto *CInt = dyn_cast(C)) {

1894 switch (CInt->getType()->getScalarSizeInBits()) {

1895 case 32:

1897 break;

1898 case 64:

1900 break;

1901 case 80:

1903 break;

1904 default:

1905 break;

1906 }

1907 } else if (auto *CDV = dyn_cast(C)) {

1908 const int NumElements =

1909 cast(CDV->getType())->getElementCount().getFixedValue();

1910 switch (CDV->getType()->getScalarSizeInBits()) {

1911 case 32:

1912 BitcastTy =

1913 VectorType::get(Type::getFloatTy(Context), NumElements, false);

1914 break;

1915 case 64:

1916 BitcastTy =

1917 VectorType::get(Type::getDoubleTy(Context), NumElements, false);

1918 break;

1919 case 80:

1920 BitcastTy =

1922 break;

1923 default:

1924 break;

1925 }

1926 }

1927 if (BitcastTy) {

1928 const MemoryExtents Extents = getMemoryExtentsOrDie(BitcastTy);

1930 NsanGetShadowPtrForStore[Extents.ValueType],

1931 {PtrOp, ConstantInt::get(IntptrTy, Extents.NumElts)});

1932

1933 Type *ExtVT = Config.getExtendedFPType(BitcastTy);

1937 Store.isVolatile());

1938 return;

1939 }

1940 }

1941

1942 Builder.CreateCall(NsanSetUnknownFns.getFallback(), {Dst, ValueSize});

1943}

1944

1945void NumericalStabilitySanitizer::propagateShadowValues(

1947 const ValueToShadowMap &Map) {

1948 if (auto *Store = dyn_cast(&Inst)) {

1949 Value *StoredValue = Store->getValueOperand();

1951 Type *ExtendedVT = Config.getExtendedFPType(VT);

1952 if (ExtendedVT == nullptr)

1953 return propagateNonFTStore(*Store, VT, Map);

1954 return propagateFTStore(*Store, VT, ExtendedVT, Map);

1955 }

1956

1957 if (auto *FCmp = dyn_cast(&Inst)) {

1958 emitFCmpCheck(*FCmp, Map);

1959 return;

1960 }

1961

1962 if (auto *CB = dyn_cast(&Inst)) {

1963 maybeAddSuffixForNsanInterface(CB);

1964 if (CallInst *CI = dyn_cast(&Inst))

1966 if (MemIntrinsic *MI = dyn_cast(&Inst)) {

1967 instrumentMemIntrinsic(MI);

1968 return;

1969 }

1970 populateShadowStack(*CB, TLI, Map);

1971 return;

1972 }

1973

1974 if (auto *RetInst = dyn_cast(&Inst)) {

1976 return;

1977

1978 Value *RV = RetInst->getReturnValue();

1979 if (RV == nullptr)

1980 return;

1982 Type *ExtendedVT = Config.getExtendedFPType(VT);

1983 if (ExtendedVT == nullptr)

1984 return;

1985 Value *RVShadow = Map.getShadow(RV);

1987

1988 RVShadow = emitCheck(RV, RVShadow, Builder, CheckLoc::makeRet());

1989 ++NumInstrumentedFTRets;

1990

1993 Builder.CreateStore(FnAddr, NsanShadowRetTag);

1994

1995 Value *ShadowRetValPtr =

1996 Builder.CreateConstGEP2_64(NsanShadowRetType, NsanShadowRetPtr, 0, 0);

1997 Builder.CreateStore(RVShadow, ShadowRetValPtr);

1998 return;

1999 }

2000

2001 if (InsertValueInst *Insert = dyn_cast(&Inst)) {

2003 Type *VT = V->getType();

2004 Type *ExtendedVT = Config.getExtendedFPType(VT);

2005 if (ExtendedVT == nullptr)

2006 return;

2008 emitCheck(V, Map.getShadow(V), Builder, CheckLoc::makeInsert());

2009 return;

2010 }

2011}

2012

2013

2014

2015

2017 std::vector<Instruction *> &Instructions) {

2019#define MOVE_FLAG(attr, setter) \

2020 if (F.getFnAttribute(attr).getValueAsString() == "true") { \

2021 F.removeFnAttr(attr); \

2022 FMF.set##setter(); \

2023 }

2025 MOVE_FLAG("no-infs-fp-math", NoInfs)

2026 MOVE_FLAG("no-nans-fp-math", NoNaNs)

2027 MOVE_FLAG("no-signed-zeros-fp-math", NoSignedZeros)

2028#undef MOVE_FLAG

2029

2031 if (isa(I))

2032 I->setFastMathFlags(FMF);

2033}

2034

2035bool NumericalStabilitySanitizer::sanitizeFunction(

2037 if (.hasFnAttribute(Attribute::SanitizeNumericalStability) ||

2038 F.isDeclaration())

2039 return false;

2040

2041

2042

2044 return false;

2047

2048

2049

2050

2051

2052

2053

2054

2055

2056

2057

2058

2059

2060

2061

2062

2063

2064

2065

2066

2067

2068

2069

2070

2071

2072

2073

2074

2075

2076

2077

2078

2079

2080

2081

2082

2083

2084

2085

2086

2087

2088

2089

2090

2091

2092

2093

2094

2095

2096

2097

2098

2099

2100

2101

2102

2103

2104

2105

2106

2107

2108

2109

2110

2111

2112

2113

2114

2115

2116

2117

2118

2119

2120

2121

2122

2123

2124

2125

2126

2127

2128

2129

2130

2131

2132

2133

2134

2135

2136 std::vector<Instruction *> OriginalInstructions;

2140

2142 ValueToShadowMap ValueToShadow(Config);

2143

2144

2145

2146

2147 std::vector<PHINode *> OriginalPhis;

2148 createShadowArguments(F, TLI, ValueToShadow);

2149 for (Instruction *I : OriginalInstructions) {

2150 if (PHINode *Phi = dyn_cast(I)) {

2151 if (PHINode *Shadow = maybeCreateShadowPhi(*Phi, TLI)) {

2152 OriginalPhis.push_back(Phi);

2153 ValueToShadow.setShadow(*Phi, *Shadow);

2154 }

2155 }

2156 }

2157

2158

2160 maybeCreateShadowValue(*I, TLI, ValueToShadow);

2161

2162

2164 propagateShadowValues(*I, TLI, ValueToShadow);

2165

2166

2167 for (PHINode *Phi : OriginalPhis) {

2168 PHINode *ShadowPhi = cast(ValueToShadow.getShadow(Phi));

2169 for (unsigned I : seq(Phi->getNumOperands())) {

2171 Value *Shadow = ValueToShadow.getShadow(V);

2173

2174

2175

2176

2177

2178

2179 ShadowPhi->addIncoming(Shadow, IncomingBB);

2180 }

2181 }

2182

2183 return !ValueToShadow.empty();

2184}

2185

2188 if (Constant *C = dyn_cast(V)) {

2189 auto *CInt = dyn_cast(C);

2190 if (CInt && CInt->getValue().getBitWidth() <= 64)

2191 OpSize = CInt->getValue().getZExtValue();

2192 }

2193

2194 return OpSize;

2195}

2196

2197

2198

2199bool NumericalStabilitySanitizer::instrumentMemIntrinsic(MemIntrinsic *MI) {

2201 if (auto *M = dyn_cast(MI)) {

2203 NsanSetUnknownFns.getFunctionFor(GetMemOpSize(M->getArgOperand(2)));

2205 Builder.CreateCall(SetUnknownFn, {M->getArgOperand(0)});

2206 else

2208 {M->getArgOperand(0),

2210 IntptrTy, false)});

2211

2212 } else if (auto *M = dyn_cast(MI)) {

2214 NsanCopyFns.getFunctionFor(GetMemOpSize(M->getArgOperand(2)));

2215

2217 Builder.CreateCall(CopyFn, {M->getArgOperand(0),

2218 M->getArgOperand(1)});

2219 else

2220 Builder.CreateCall(CopyFn, {M->getArgOperand(0),

2221 M->getArgOperand(1),

2222

2224 IntptrTy, false)});

2225 }

2226 return false;

2227}

2228

2229void NumericalStabilitySanitizer::maybeAddSuffixForNsanInterface(CallBase *CI) {

2231 if (Fn == nullptr)

2232 return;

2233

2235 return;

2236

2237 if (Fn->getName() == "__nsan_dump_shadow_mem") {

2239 "invalid prototype for __nsan_dump_shadow_mem");

2240

2241

2242

2243

2244 const uint64_t shadow_value_type_ids =

2245 (static_cast<size_t>(Config.byValueType(kLongDouble).getNsanTypeId())

2246 << 16) |

2247 (static_cast<size_t>(Config.byValueType(kDouble).getNsanTypeId())

2248 << 8) |

2249 static_cast<size_t>(Config.byValueType(kFloat).getNsanTypeId());

2250 CI->setArgOperand(3, ConstantInt::get(IntptrTy, shadow_value_type_ids));

2251 }

2252}

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

Returns the sub type a function will return at a given Idx Should correspond to the result type of an ExtractValue instruction executed with just that one unsigned Idx

This file defines the DenseMap class.

Module.h This file contains the declarations for the Module class.

static GlobalValue * createThreadLocalGV(const char *Name, Module &M, Type *Ty)

static FunctionType * makeDoubleDouble(LLVMContext &C)

constexpr int kMaxVectorWidth

static cl::opt< bool > ClCheckRet("nsan-check-ret", cl::init(true), cl::desc("Check floating-point return values"), cl::Hidden)

static bool shouldCheckArgs(CallBase &CI, const TargetLibraryInfo &TLI, const std::optional< Regex > &CheckFunctionsFilter)

static cl::opt< bool > ClCheckStores("nsan-check-stores", cl::init(true), cl::desc("Check floating-point stores"), cl::Hidden)

static FunctionType * makeX86FP80X86FP80(LLVMContext &C)

#define MOVE_FLAG(attr, setter)

static FunctionType * makeX86FP80X86FP80I32(LLVMContext &C)

constexpr int kMaxNumArgs

constexpr int kMaxShadowTypeSizeBytes

static const char * getIntrinsicFromLibfunc(Function &Fn, Type *VT, const TargetLibraryInfo &TLI)

static FunctionType * makeX86FP80X86FP80X86FP80(LLVMContext &C)

static FunctionType * makeX86FP80X86FP80X86FP80X86FP80(LLVMContext &C)

static uint64_t GetMemOpSize(Value *V)

constexpr StringLiteral kNsanInitName("__nsan_init")

static FunctionType * makeDoubleDoubleDouble(LLVMContext &C)

constexpr int kShadowScale

static cl::opt< bool > ClCheckLoads("nsan-check-loads", cl::desc("Check floating-point load"), cl::Hidden)

constexpr StringLiteral kNsanModuleCtorName("nsan.module_ctor")

static cl::opt< bool > ClPropagateNonFTConstStoresAsFT("nsan-propagate-non-ft-const-stores-as-ft", cl::desc("Propagate non floating-point const stores as floating point values." "For debugging purposes only"), cl::Hidden)

static cl::opt< std::string > ClShadowMapping("nsan-shadow-type-mapping", cl::init("dqq"), cl::desc("One shadow type id for each of `float`, `double`, `long double`. " "`d`,`l`,`q`,`e` mean double, x86_fp80, fp128 (quad) and " "ppc_fp128 (extended double) respectively. The default is to " "shadow `float` as `double`, and `double` and `x86_fp80` as " "`fp128`"), cl::Hidden)

static FunctionType * makeDoubleDoubleDoubleDouble(LLVMContext &C)

static cl::opt< bool > ClTruncateFCmpEq("nsan-truncate-fcmp-eq", cl::init(true), cl::desc("This flag controls the behaviour of fcmp equality comparisons." "For equality comparisons such as `x == 0.0f`, we can perform the " "shadow check in the shadow (`x_shadow == 0.0) == (x == 0.0f)`) or app " " domain (`(trunc(x_shadow) == 0.0f) == (x == 0.0f)`). This helps " "catch the case when `x_shadow` is accurate enough (and therefore " "close enough to zero) so that `trunc(x_shadow)` is zero even though " "both `x` and `x_shadow` are not"), cl::Hidden)

static cl::opt< std::string > ClCheckFunctionsFilter("check-functions-filter", cl::desc("Only emit checks for arguments of functions " "whose names match the given regular expression"), cl::value_desc("regex"))

static FunctionType * makeDoubleDoubleI32(LLVMContext &C)

static void moveFastMathFlags(Function &F, std::vector< Instruction * > &Instructions)

static cl::opt< bool > ClInstrumentFCmp("nsan-instrument-fcmp", cl::init(true), cl::desc("Instrument floating-point comparisons"), cl::Hidden)

FunctionAnalysisManager FAM

ModuleAnalysisManager MAM

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

This file defines the SmallVector class.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

static SymbolRef::Type getType(const Symbol *Sym)

opStatus convert(const fltSemantics &ToSemantics, roundingMode RM, bool *losesInfo)

Class for arbitrary precision integers.

A container for analyses that lazily runs them and caches their results.

PassT::Result & getResult(IRUnitT &IR, ExtraArgTs... ExtraArgs)

Get the result of an analysis pass for a given IR unit.

This class represents an incoming formal argument to a Function.

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

size_t size() const

size - Get the array size.

AttributeList addFnAttribute(LLVMContext &C, Attribute::AttrKind Kind) const

Add a function attribute to the list.

LLVM Basic Block Representation.

void replaceSuccessorsPhiUsesWith(BasicBlock *Old, BasicBlock *New)

Update all phi nodes in this basic block's successors to refer to basic block New instead of basic bl...

iterator begin()

Instruction iterator methods.

static BasicBlock * Create(LLVMContext &Context, const Twine &Name="", Function *Parent=nullptr, BasicBlock *InsertBefore=nullptr)

Creates a new BasicBlock.

BasicBlock * splitBasicBlock(iterator I, const Twine &BBName="", bool Before=false)

Split the basic block into two basic blocks at the specified instruction.

const Function * getParent() const

Return the enclosing method, or null if none.

InstListType::iterator iterator

Instruction iterators...

const Instruction & back() const

Base class for all callable instructions (InvokeInst and CallInst) Holds everything related to callin...

bool isInlineAsm() const

Check if this call is an inline asm statement.

Function * getCalledFunction() const

Returns the function called, or null if this is an indirect function invocation or the function signa...

Value * getCalledOperand() const

void setArgOperand(unsigned i, Value *v)

unsigned arg_size() const

This class represents a function call, abstracting a target machine's calling convention.

Predicate getPredicate() const

Return the predicate for this instruction.

ConstantFP - Floating Point Values [float, double].

static Constant * get(ArrayRef< Constant * > V)

This is an important base class in LLVM.

static Constant * getIntegerValue(Type *Ty, const APInt &V)

Return the value for an integer or pointer constant, or a vector thereof, with the given scalar value...

This class represents an Operation in the Expression.

A parsed version of the target data layout string in and methods for querying it.

This instruction compares its operands according to the predicate given to the constructor.

static bool isEquality(Predicate Pred)

This class represents an extension of floating point types.

This class represents a truncation of floating point types.

Convenience struct for specifying and reasoning about fast-math flags.

A handy container for a FunctionType+Callee-pointer pair, which can be passed around as a single enti...

FunctionType * getFunctionType()

unsigned getNumParams() const

Return the number of fixed parameters this function type requires.

FunctionType * getFunctionType() const

Returns the FunctionType for me.

Intrinsic::ID getIntrinsicID() const LLVM_READONLY

getIntrinsicID - This method returns the ID number of the specified function, or Intrinsic::not_intri...

an instruction for type-safe pointer arithmetic to access elements of arrays and structs

Value * CreateInsertElement(Type *VecTy, Value *NewElt, Value *Idx, const Twine &Name="")

Value * CreateExtractElement(Value *Vec, Value *Idx, const Twine &Name="")

Value * CreateFPTrunc(Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr)

Value * CreateExtractValue(Value *Agg, ArrayRef< unsigned > Idxs, const Twine &Name="")

Value * CreateSelect(Value *C, Value *True, Value *False, const Twine &Name="", Instruction *MDFrom=nullptr)

Value * CreateConstGEP2_64(Type *Ty, Value *Ptr, uint64_t Idx0, uint64_t Idx1, const Twine &Name="")

BasicBlock::iterator GetInsertPoint() const

Value * CreateFreeze(Value *V, const Twine &Name="")

IntegerType * getInt32Ty()

Fetch the type representing a 32-bit integer.

Value * CreateCast(Instruction::CastOps Op, Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr, FMFSource FMFSource={})

void SetCurrentDebugLocation(DebugLoc L)

Set location information used by debugging information.

Value * CreateUnOp(Instruction::UnaryOps Opc, Value *V, const Twine &Name="", MDNode *FPMathTag=nullptr)

CallInst * CreateIntrinsic(Intrinsic::ID ID, ArrayRef< Type * > Types, ArrayRef< Value * > Args, FMFSource FMFSource={}, const Twine &Name="")

Create a call to intrinsic ID with Args, mangled using Types.

Value * CreateICmpEQ(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateBitCast(Value *V, Type *DestTy, const Twine &Name="")

LoadInst * CreateLoad(Type *Ty, Value *Ptr, const char *Name)

Provided to resolve 'CreateLoad(Ty, Ptr, "...")' correctly, instead of converting the string to 'bool...

Value * CreateShuffleVector(Value *V1, Value *V2, Value *Mask, const Twine &Name="")

StoreInst * CreateStore(Value *Val, Value *Ptr, bool isVolatile=false)

Value * CreatePtrToInt(Value *V, Type *DestTy, const Twine &Name="")

CallInst * CreateCall(FunctionType *FTy, Value *Callee, ArrayRef< Value * > Args={}, const Twine &Name="", MDNode *FPMathTag=nullptr)

Value * CreateOr(Value *LHS, Value *RHS, const Twine &Name="")

Value * CreateBinOp(Instruction::BinaryOps Opc, Value *LHS, Value *RHS, const Twine &Name="", MDNode *FPMathTag=nullptr)

BranchInst * CreateBr(BasicBlock *Dest)

Create an unconditional 'br label X' instruction.

Value * CreateIntCast(Value *V, Type *DestTy, bool isSigned, const Twine &Name="")

StoreInst * CreateAlignedStore(Value *Val, Value *Ptr, MaybeAlign Align, bool isVolatile=false)

Value * CreateFPExt(Value *V, Type *DestTy, const Twine &Name="", MDNode *FPMathTag=nullptr)

This provides a uniform API for creating instructions and inserting them into a basic block: either a...

An analysis over an "outer" IR unit that provides access to an analysis manager over an "inner" IR un...

This instruction inserts a struct field of array element value into an aggregate value.

const DebugLoc & getDebugLoc() const

Return the debug location for this node as a DebugLoc.

InstListType::iterator eraseFromParent()

This method unlinks 'this' from the containing basic block and deletes it.

void replaceSuccessorWith(BasicBlock *OldBB, BasicBlock *NewBB)

Replace specified successor OldBB to point at the provided block.

const Function * getFunction() const

Return the function this instruction belongs to.

const char * getOpcodeName() const

void insertAfter(Instruction *InsertPos)

Insert an unlinked instruction into a basic block immediately after the specified instruction.

Class to represent integer types.

This is an important class for using LLVM in a threaded context.

An instruction for reading from memory.

This is the common base class for memset/memcpy/memmove.

A Module instance is used to store all the information related to an LLVM module.

void addIncoming(Value *V, BasicBlock *BB)

Add an incoming value to the end of the PHI list.

static PHINode * Create(Type *Ty, unsigned NumReservedValues, const Twine &NameStr="", InsertPosition InsertBefore=nullptr)

Constructors - NumReservedValues is a hint for the number of incoming edges that this phi node will h...

A set of analyses that are preserved following a run of a transformation pass.

static PreservedAnalyses none()

Convenience factory function for the empty preserved set.

reference emplace_back(ArgTypes &&... Args)

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

An instruction for storing to memory.

A wrapper around a string literal that serves as a proxy for constructing global tables of StringRefs...

StringRef - Represent a constant reference to a string, i.e.

bool starts_with(StringRef Prefix) const

Check if this string starts with the given Prefix.

Analysis pass providing the TargetLibraryInfo.

Provides information about what library functions are available for the current target.

bool getLibFunc(StringRef funcName, LibFunc &F) const

Searches for a particular function name.

StringRef getName(LibFunc F) const

Twine - A lightweight data structure for efficiently representing the concatenation of temporary valu...

static constexpr TypeSize getFixed(ScalarTy ExactSize)

The instances of the Type class are immutable: once they are created, they are never changed.

Type * getStructElementType(unsigned N) const

static Type * getDoubleTy(LLVMContext &C)

const fltSemantics & getFltSemantics() const

bool isVectorTy() const

True if this is an instance of VectorType.

bool isX86_FP80Ty() const

Return true if this is x86 long double.

bool isArrayTy() const

True if this is an instance of ArrayType.

static Type * getX86_FP80Ty(LLVMContext &C)

static IntegerType * getInt1Ty(LLVMContext &C)

bool isFloatTy() const

Return true if this is 'float', a 32-bit IEEE fp type.

unsigned getStructNumElements() const

uint64_t getArrayNumElements() const

static IntegerType * getIntNTy(LLVMContext &C, unsigned N)

static Type * getVoidTy(LLVMContext &C)

bool isStructTy() const

True if this is an instance of StructType.

static Type * getFP128Ty(LLVMContext &C)

static IntegerType * getInt8Ty(LLVMContext &C)

bool isDoubleTy() const

Return true if this is 'double', a 64-bit IEEE fp type.

static IntegerType * getInt32Ty(LLVMContext &C)

static Type * getFloatTy(LLVMContext &C)

static Type * getPPC_FP128Ty(LLVMContext &C)

'undef' values are things that do not have specified contents.

static UndefValue * get(Type *T)

Static factory methods - Return an 'undef' object of the specified type.

Value * getOperand(unsigned i) const

LLVM Value Representation.

Type * getType() const

All values are typed, get the type of this value.

iterator_range< user_iterator > users()

StringRef getName() const

Return a constant reference to the value's name.

constexpr ScalarTy getFixedValue() const

constexpr bool isScalable() const

Returns whether the quantity is scaled by a runtime quantity (vscale).

const ParentTy * getParent() const

NodeTy * getNextNode()

Get the next node, or nullptr for the list tail.

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

constexpr char Args[]

Key for Kernel::Metadata::mArgs.

@ Fast

Attempts to make calls as fast as possible (e.g.

@ C

The default llvm calling convention, compatible with C.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

MatchIntrinsicTypesResult matchIntrinsicSignature(FunctionType *FTy, ArrayRef< IITDescriptor > &Infos, SmallVectorImpl< Type * > &ArgTys)

Match the specified function type with the type constraints specified by the .td file.

void getIntrinsicInfoTableEntries(ID id, SmallVectorImpl< IITDescriptor > &T)

Return the IIT table descriptor for the specified intrinsic into an array of IITDescriptors.

MatchIntrinsicTypesResult

@ MatchIntrinsicTypes_Match

initializer< Ty > init(const Ty &Val)

NodeAddr< PhiNode * > Phi

This is an optimization pass for GlobalISel generic memory operations.

bool all_of(R &&range, UnaryPredicate P)

Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.

auto enumerate(FirstRange &&First, RestRanges &&...Rest)

Given two or more input ranges, returns a new range whose values are tuples (A, B,...

std::pair< Function *, FunctionCallee > getOrCreateSanitizerCtorAndInitFunctions(Module &M, StringRef CtorName, StringRef InitName, ArrayRef< Type * > InitArgTypes, ArrayRef< Value * > InitArgs, function_ref< void(Function *, FunctionCallee)> FunctionsCreatedCallback, StringRef VersionCheckName=StringRef(), bool Weak=false)

Creates sanitizer constructor function lazily.

decltype(auto) get(const PointerIntPair< PointerTy, IntBits, IntType, PtrTraits, Info > &Pair)

void report_fatal_error(Error Err, bool gen_crash_diag=true)

Report a serious error, calling any installed error handler.

raw_fd_ostream & errs()

This returns a reference to a raw_ostream for standard error.

void appendToGlobalCtors(Module &M, Function *F, int Priority, Constant *Data=nullptr)

Append F to the list of global ctors of module M with the given Priority.

auto seq(T Begin, T End)

Iterate over an integral type from Begin up to - but not including - End.

void maybeMarkSanitizerLibraryCallNoBuiltin(CallInst *CI, const TargetLibraryInfo *TLI)

Given a CallInst, check if it calls a string function known to CodeGen, and mark it with NoBuiltin if...

static constexpr roundingMode rmTowardZero

This struct is a compact representation of a valid (non-zero power of two) alignment.

PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM)