X86SpeculativeLoadHardening.cpp Source File (original) (raw)

54#include

55#include

56#include

58using namespace llvm;

60#define PASS_KEY "x86-slh"

61#define DEBUG_TYPE PASS_KEY

63 STATISTIC(NumCondBranchesTraced, "Number of conditional branches traced");

64 STATISTIC(NumBranchesUntraced, "Number of branches unable to trace");

66 "Number of address mode used registers hardaned");

68 "Number of post-load register values hardened");

70 "Number of calls or jumps requiring extra hardening");

71 STATISTIC(NumInstsInserted, "Number of instructions inserted");

72 STATISTIC(NumLFENCEsInserted, "Number of lfence instructions inserted");

75 "x86-speculative-load-hardening",

76 cl::desc("Force enable speculative load hardening"), cl::init(false),

82 "Use LFENCE along each conditional edge to harden against speculative "

83 "loads rather than conditional movs and poisoned pointers."),

88 cl::desc("Harden the value loaded *after* it is loaded by "

89 "flushing the loaded bits to 1. This is hard to do "

90 "in general but can be done easily for GPRs."),

94 PASS_KEY "-fence-call-and-ret",

95 cl::desc("Use a full speculation fence to harden both call and ret edges "

96 "rather than a lighter weight mitigation."),

101 cl::desc("Harden interprocedurally by passing our state in and out of "

102 "functions in the high bits of the stack pointer."),

104

107 cl::desc("Sanitize loads from memory. When disable, no "

108 "significant security is provided."),

110

113 cl::desc("Harden indirect calls and jumps against using speculatively "

114 "stored attacker controlled addresses. This is designed to "

115 "mitigate Spectre v1.2 style attacks."),

117

118namespace {

119

121public:

123

124 StringRef getPassName() const override {

125 return "X86 speculative load hardening";

126 }

127 bool runOnMachineFunction(MachineFunction &MF) override;

128 void getAnalysisUsage(AnalysisUsage &AU) const override;

129

130

131 static char ID;

132

133private:

134

135

136 struct BlockCondInfo {

137 MachineBasicBlock *MBB;

138

139

140

141 SmallVector<MachineInstr *, 2> CondBrs;

142

143 MachineInstr *UncondBr;

144 };

145

146

147 struct PredState {

150

151 const TargetRegisterClass *RC;

152 MachineSSAUpdater SSA;

153

154 PredState(MachineFunction &MF, const TargetRegisterClass *RC)

155 : RC(RC), SSA(MF) {}

156 };

157

158 const X86Subtarget *Subtarget = nullptr;

159 MachineRegisterInfo *MRI = nullptr;

160 const X86InstrInfo *TII = nullptr;

161 const TargetRegisterInfo *TRI = nullptr;

162

163 std::optional PS;

164

165 void hardenEdgesWithLFENCE(MachineFunction &MF);

166

168

171

172 void unfoldCallAndJumpLoads(MachineFunction &MF);

173

175 tracePredStateThroughIndirectBranches(MachineFunction &MF);

176

177 void tracePredStateThroughBlocksAndHarden(MachineFunction &MF);

178

179 Register saveEFLAGS(MachineBasicBlock &MBB,

182 void restoreEFLAGS(MachineBasicBlock &MBB,

185

186 void mergePredStateIntoSP(MachineBasicBlock &MBB,

189 Register extractPredStateFromSP(MachineBasicBlock &MBB,

192

193 void

194 hardenLoadAddr(MachineInstr &MI, MachineOperand &BaseMO,

195 MachineOperand &IndexMO,

196 SmallDenseMap<Register, Register, 32> &AddrRegToHardenedReg);

197 MachineInstr *

198 sinkPostLoadHardenedInst(MachineInstr &MI,

199 SmallPtrSetImpl<MachineInstr *> &HardenedInstrs);

204 Register hardenPostLoad(MachineInstr &MI);

205 void hardenReturnInstr(MachineInstr &MI);

206 void tracePredStateThroughCall(MachineInstr &MI);

207 void hardenIndirectCallOrJumpInstr(

208 MachineInstr &MI,

209 SmallDenseMap<Register, Register, 32> &AddrRegToHardenedReg);

210};

211

212}

213

214char X86SpeculativeLoadHardeningPass::ID = 0;

215

216void X86SpeculativeLoadHardeningPass::getAnalysisUsage(

219}

220

225 assert(!Succ.isEHPad() && "Shouldn't get edges to EH pads!");

226

228

230

231

232

233

235

236

237 if (Br) {

239 "Didn't start with the right target!");

241

242

243

244

245 if (!UncondBr) {

248 assert(MBB.isSuccessor(&OldLayoutSucc) &&

249 "Without an unconditional branch, the old layout successor should "

250 "be an actual successor!");

251 auto BrBuilder =

253

254 UncondBr = &*BrBuilder;

255 }

256

257

258

262 }

263 } else {

265 "Cannot have a branchless successor and an unconditional branch!");

267 "A non-branch successor must have been a layout successor before "

268 "and now is a layout successor of the new block.");

269 }

270

271

272

273

274 if (SuccCount == 1) {

275 MBB.replaceSuccessor(&Succ, &NewMBB);

276 } else {

277 MBB.splitSuccessor(&Succ, &NewMBB);

278 }

279

280

282

283

285 if (.isPHI())

286 break;

291 assert(OpMBB.isMBB() && "Block operand to a PHI is not a block!");

293 continue;

294

295

296 if (SuccCount == 1) {

297 OpMBB.setMBB(&NewMBB);

298 break;

299 }

300

301

302 MI.addOperand(MF, OpV);

304 break;

305 }

306 }

307

308

309 for (auto &LI : Succ.liveins())

311

312 LLVM_DEBUG(dbgs() << " Split edge from '" << MBB.getName() << "' to '"

313 << Succ.getName() << "'.\n");

314 return NewMBB;

315}

316

317

318

319

320

321

322

323

324

328 for (auto &MBB : MF)

329 for (auto &MI : MBB) {

330 if (.isPHI())

331 break;

332

333

334

335

338 if (!Preds.insert(MI.getOperand(OpIdx + 1).getMBB()).second)

340

341

342

343

344

345

346

347

348

349

350

351

352 while (!DupIndices.empty()) {

354

355

356 MI.removeOperand(OpIdx + 1);

358 }

359

361 }

362}

363

364

365

366

367

368

372

373

374

375 if (MI.getOpcode() == X86::LFENCE)

376 break;

377

378

379 if (.mayLoad())

380 continue;

381

382

383 if (MI.getOpcode() == X86::MFENCE)

384 continue;

385

386

387 return true;

388 }

389 }

390

391

392 return false;

393}

394

395bool X86SpeculativeLoadHardeningPass::runOnMachineFunction(

396 MachineFunction &MF) {

398 << " **********\n");

399

400

401

404 return false;

405

406 Subtarget = &MF.getSubtarget();

410

411

412 PS.emplace(MF, &X86::GR64_NOSPRegClass);

413

415

416 return false;

417

418

420 hardenEdgesWithLFENCE(MF);

421 return true;

422 }

423

424

426

427 MachineBasicBlock &Entry = *MF.begin();

428 auto EntryInsertPt = Entry.SkipPHIsLabelsAndDebug(Entry.begin());

429

430

432

433

434

436

437

438 if (!HasVulnerableLoad && Infos.empty())

439 return true;

440

441

442

443 const int PoisonVal = -1;

444 PS->PoisonReg = MRI->createVirtualRegister(PS->RC);

445 BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::MOV64ri32), PS->PoisonReg)

447 ++NumInstsInserted;

448

449

450

452

453

454

455

456

457

458 BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::LFENCE));

459 ++NumInstsInserted;

460 ++NumLFENCEsInserted;

461 }

462

463

464

466

467 return true;

468

469

471

472

473 PS->InitialReg = extractPredStateFromSP(Entry, EntryInsertPt, Loc);

474 } else {

475

476

477 PS->InitialReg = MRI->createVirtualRegister(PS->RC);

478 Register PredStateSubReg = MRI->createVirtualRegister(&X86::GR32RegClass);

479 auto ZeroI = BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::MOV32r0),

480 PredStateSubReg);

481 ++NumInstsInserted;

482 MachineOperand *ZeroEFLAGSDefOp =

483 ZeroI->findRegisterDefOperand(X86::EFLAGS, nullptr);

485 "Must have an implicit def of EFLAGS!");

486 ZeroEFLAGSDefOp->setIsDead(true);

487 BuildMI(Entry, EntryInsertPt, Loc, TII->get(X86::SUBREG_TO_REG),

488 PS->InitialReg)

490 .addReg(PredStateSubReg)

491 .addImm(X86::sub_32bit);

492 }

493

494

495

496

498

499

500

501 PS->SSA.Initialize(PS->InitialReg);

502 PS->SSA.AddAvailableValue(&Entry, PS->InitialReg);

503

504

505 auto CMovs = tracePredStateThroughCFG(MF, Infos);

506

507

508

509

510

511

512

513

515 for (MachineBasicBlock &MBB : MF) {

520 continue;

521 PS->SSA.AddAvailableValue(

524 }

525 }

526

528

529

530 unfoldCallAndJumpLoads(MF);

531

532

533 auto IndirectBrCMovs = tracePredStateThroughIndirectBranches(MF);

534 CMovs.append(IndirectBrCMovs.begin(), IndirectBrCMovs.end());

535 }

536

537

538

539

540 tracePredStateThroughBlocksAndHarden(MF);

541

542

543

544 for (MachineInstr *CMovI : CMovs)

545 for (MachineOperand &Op : CMovI->operands()) {

546 if (.isReg() || Op.getReg() != PS->InitialReg)

547 continue;

548

549 PS->SSA.RewriteUse(Op);

550 }

551

552 LLVM_DEBUG(dbgs() << "Final speculative load hardened function:\n"; MF.dump();

553 dbgs() << "\n"; MF.verify(this));

554 return true;

555}

556

557

558

559

560

561

562

563void X86SpeculativeLoadHardeningPass::hardenEdgesWithLFENCE(

564 MachineFunction &MF) {

565

566

567 SmallSetVector<MachineBasicBlock *, 8> Blocks;

568 for (MachineBasicBlock &MBB : MF) {

569

571 continue;

572

573

574

576 if (TermIt == MBB.end() || !TermIt->isBranch())

577 continue;

578

579

580

581

582 for (MachineBasicBlock *SuccMBB : MBB.successors())

583 if (!SuccMBB->isEHPad())

584 Blocks.insert(SuccMBB);

585 }

586

587 for (MachineBasicBlock *MBB : Blocks) {

590 ++NumInstsInserted;

591 ++NumLFENCEsInserted;

592 }

593}

594

596X86SpeculativeLoadHardeningPass::collectBlockCondInfo(MachineFunction &MF) {

598

599

600

601 for (MachineBasicBlock &MBB : MF) {

602

604 continue;

605

606

607

608

609

610

611

612

613

614

615

616

617

618

619

620

621

622

623

624

625

626 BlockCondInfo Info = {&MBB, {}, nullptr};

627

628

629

631

632 if (.isTerminator())

633 break;

634

635

636 if (.isBranch()) {

637 Info.CondBrs.clear();

638 break;

639 }

640

641

642

643 if (MI.getOpcode() == X86::JMP_1) {

644 Info.CondBrs.clear();

646 continue;

647 }

648

649

650

651

652

653

654

655

656

657

659 Info.CondBrs.clear();

661 continue;

662 }

663

664

665 Info.CondBrs.push_back(&MI);

666 }

667 if (Info.CondBrs.empty()) {

668 ++NumBranchesUntraced;

669 LLVM_DEBUG(dbgs() << "WARNING: unable to secure successors of block:\n";

671 continue;

672 }

673

675 }

676

677 return Infos;

678}

679

680

681

682

683

684

685

686

688X86SpeculativeLoadHardeningPass::tracePredStateThroughCFG(

690

691

693

694

695

696 for (const BlockCondInfo &Info : Infos) {

697 MachineBasicBlock &MBB = *Info.MBB;

698 const SmallVectorImpl<MachineInstr *> &CondBrs = Info.CondBrs;

699 MachineInstr *UncondBr = Info.UncondBr;

700

702 << "\n");

703 ++NumCondBranchesTraced;

704

705

706

707 MachineBasicBlock *UncondSucc =

708 UncondBr ? (UncondBr->getOpcode() == X86::JMP_1

710 : nullptr)

711 : &*std::next(MachineFunction::iterator(&MBB));

712

713

714 SmallDenseMap<MachineBasicBlock *, int> SuccCounts;

715 if (UncondSucc)

716 ++SuccCounts[UncondSucc];

717 for (auto *CondBr : CondBrs)

718 ++SuccCounts[CondBr->getOperand(0).getMBB()];

719

720

721

722 auto BuildCheckingBlockForSuccAndConds =

723 [&](MachineBasicBlock &MBB, MachineBasicBlock &Succ, int SuccCount,

724 MachineInstr *Br, MachineInstr *&UncondBr,

726

727

728 auto &CheckingMBB =

729 (SuccCount == 1 && Succ.pred_size() == 1)

730 ? Succ

731 : splitEdge(MBB, Succ, SuccCount, Br, UncondBr, *TII);

732

733 bool LiveEFLAGS = Succ.isLiveIn(X86::EFLAGS);

734 if (!LiveEFLAGS)

735 CheckingMBB.addLiveIn(X86::EFLAGS);

736

737

738 auto InsertPt = CheckingMBB.begin();

739 assert((InsertPt == CheckingMBB.end() || !InsertPt->isPHI()) &&

740 "Should never have a PHI in the initial checking block as it "

741 "always has a single predecessor!");

742

743

744

745 Register CurStateReg = PS->InitialReg;

746

747 for (X86::CondCode Cond : Conds) {

748 int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8;

749 auto CMovOp = X86::getCMovOpcode(PredStateSizeInBytes);

750

751 Register UpdatedStateReg = MRI->createVirtualRegister(PS->RC);

752

753

754 auto CMovI = BuildMI(CheckingMBB, InsertPt, DebugLoc(),

755 TII->get(CMovOp), UpdatedStateReg)

756 .addReg(CurStateReg)

757 .addReg(PS->PoisonReg)

758 .addImm(Cond);

759

760

761 if (!LiveEFLAGS && Cond == Conds.back())

762 CMovI->findRegisterUseOperand(X86::EFLAGS, nullptr)

763 ->setIsKill(true);

764

765 ++NumInstsInserted;

766 LLVM_DEBUG(dbgs() << " Inserting cmov: "; CMovI->dump();

767 dbgs() << "\n");

768

769

770

771 if (CurStateReg == PS->InitialReg)

772 CMovs.push_back(&*CMovI);

773

774

775 CurStateReg = UpdatedStateReg;

776 }

777

778

779

780 PS->SSA.AddAvailableValue(&CheckingMBB, CurStateReg);

781 };

782

783 std::vectorX86::CondCode UncondCodeSeq;

784 for (auto *CondBr : CondBrs) {

785 MachineBasicBlock &Succ = *CondBr->getOperand(0).getMBB();

786 int &SuccCount = SuccCounts[&Succ];

787

790 UncondCodeSeq.push_back(Cond);

791

792 BuildCheckingBlockForSuccAndConds(MBB, Succ, SuccCount, CondBr, UncondBr,

793 {InvCond});

794

795

796

797

798

799 --SuccCount;

800 }

801

802

803

804

806

807

808

809

810 if (!UncondSucc)

811

812

813 continue;

814

815 assert(SuccCounts[UncondSucc] == 1 &&

816 "We should never have more than one edge to the unconditional "

817 "successor at this point because every other edge must have been "

818 "split above!");

819

820

822 UncondCodeSeq.erase(llvm::unique(UncondCodeSeq), UncondCodeSeq.end());

823

824

825 BuildCheckingBlockForSuccAndConds(MBB, *UncondSucc, 1,

826 UncondBr, UncondBr, UncondCodeSeq);

827 }

828

829 return CMovs;

830}

831

832

833

834

835

836

837static const TargetRegisterClass *

839 unsigned Index;

840 unsigned UnfoldedOpc = TII.getOpcodeAfterMemoryUnfold(

841 Opcode, true, false, &Index);

843 return TII.getRegClass(MCID, Index);

844}

845

846void X86SpeculativeLoadHardeningPass::unfoldCallAndJumpLoads(

847 MachineFunction &MF) {

848 for (MachineBasicBlock &MBB : MF)

849

850

852

853 if (.isCall() && .isBranch())

854 continue;

855

856 if (.mayLoad())

857 continue;

858

859 switch (MI.getOpcode()) {

860 default: {

862 dbgs() << "ERROR: Found an unexpected loading branch or call "

863 "instruction:\n";

866 }

867

868 case X86::FARCALL16m:

869 case X86::FARCALL32m:

870 case X86::FARCALL64m:

871 case X86::FARJMP16m:

872 case X86::FARJMP32m:

873 case X86::FARJMP64m:

874

875

876 continue;

877

878 case X86::CALL16m:

879 case X86::CALL16m_NT:

880 case X86::CALL32m:

881 case X86::CALL32m_NT:

882 case X86::CALL64m:

883 case X86::CALL64m_NT:

884 case X86::JMP16m:

885 case X86::JMP16m_NT:

886 case X86::JMP32m:

887 case X86::JMP32m_NT:

888 case X86::JMP64m:

889 case X86::JMP64m_NT:

890 case X86::TAILJMPm64:

891 case X86::TAILJMPm64_REX:

892 case X86::TAILJMPm:

893 case X86::TCRETURNmi64:

894 case X86::TCRETURN_WINmi64:

895 case X86::TCRETURNmi: {

896

897

898

900 if (!UnfoldedRC) {

902 << "ERROR: Unable to unfold load from instruction:\n";

905 }

906 Register Reg = MRI->createVirtualRegister(UnfoldedRC);

907 SmallVector<MachineInstr *, 2> NewMIs;

908

909

910 bool Unfolded =

912 false, NewMIs);

913 (void)Unfolded;

915 "Computed unfolded register class but failed to unfold");

916

917 for (auto *NewMI : NewMIs)

919

920

921 if (MI.isCandidateForAdditionalCallInfo())

922 MF.eraseAdditionalCallInfo(&MI);

923

924 MI.eraseFromParent();

926 dbgs() << "Unfolded load successfully into:\n";

927 for (auto *NewMI : NewMIs) {

928 NewMI->dump();

929 dbgs() << "\n";

930 }

931 });

932 continue;

933 }

934 }

936 }

937}

938

939

940

941

942

943

944

945

946

947

948

949

950

951

952

953

954

955

957X86SpeculativeLoadHardeningPass::tracePredStateThroughIndirectBranches(

958 MachineFunction &MF) {

959

960

961

962

963 MachineSSAUpdater TargetAddrSSA(MF);

964 TargetAddrSSA.Initialize(MRI->createVirtualRegister(&X86::GR64RegClass));

965

966

967 SmallPtrSet<MachineBasicBlock *, 4> IndirectTerminatedMBBs;

968

969

970

971

972 SmallPtrSet<MachineBasicBlock *, 4> IndirectTargetMBBs;

973

974

975

976 for (MachineBasicBlock &MBB : MF) {

977

979 while (MII != MBB.instr_rend() && MII->isDebugInstr())

980 ++MII;

982 continue;

983 MachineInstr &TI = *MII;

985

986 continue;

987

989

991 default:

992

993 continue;

994

995 case X86::FARJMP16m:

996 case X86::FARJMP32m:

997 case X86::FARJMP64m:

998

999

1000 continue;

1001

1002 case X86::JMP16m:

1003 case X86::JMP16m_NT:

1004 case X86::JMP32m:

1005 case X86::JMP32m_NT:

1006 case X86::JMP64m:

1007 case X86::JMP64m_NT:

1008

1009 report_fatal_error("Memory operand jumps should have been unfolded!");

1010

1011 case X86::JMP16r:

1013 "Support for 16-bit indirect branches is not implemented.");

1014 case X86::JMP32r:

1016 "Support for 32-bit indirect branches is not implemented.");

1017

1018 case X86::JMP64r:

1020 }

1021

1022

1023

1025 return !OtherTI.isDebugInstr() && &OtherTI != &TI;

1026 })) {

1028 dbgs() << "ERROR: Found other terminators in a block with an indirect "

1029 "branch! This is not yet supported! Terminator sequence:\n";

1032 dbgs() << '\n';

1033 }

1034 });

1036 }

1037

1038

1039 TargetAddrSSA.AddAvailableValue(&MBB, TargetReg);

1040 IndirectTerminatedMBBs.insert(&MBB);

1041

1042

1044 }

1045

1046

1048

1049

1050 if (IndirectTargetMBBs.empty())

1051 return CMovs;

1052

1053

1054

1055

1056 for (MachineBasicBlock &MBB : MF) {

1057

1058 if (!IndirectTargetMBBs.count(&MBB))

1059 continue;

1060

1061

1062

1063

1065 "Unexpected EH pad as target of an indirect branch!");

1066

1067

1068

1069

1070

1071

1073 "Cannot check within a block that already has live-in EFLAGS!");

1074

1075

1076

1078

1079

1080 if (IndirectTerminatedMBBs.count(Pred))

1081 continue;

1082

1083

1084

1085

1086

1087

1088 if ( $llvm::all\_of$ (Pred->successors(), [&](MachineBasicBlock *Succ) {

1089 return Succ->isEHPad() || Succ == &MBB;

1090 })) {

1092 dbgs() << "ERROR: Found conditional entry to target of indirect "

1093 "branch!\n";

1094 Pred->dump();

1096 });

1097 report_fatal_error("Cannot harden a conditional entry to a target of "

1098 "an indirect branch!");

1099 }

1100

1101

1102

1103

1104 auto InsertPt = Pred->getFirstTerminator();

1105 Register TargetReg = MRI->createVirtualRegister(&X86::GR64RegClass);

1108

1110 TII->get(X86::MOV64ri32), TargetReg)

1112 ++NumInstsInserted;

1113 (void)AddrI;

1114 LLVM_DEBUG(dbgs() << " Inserting mov: "; AddrI->dump();

1115 dbgs() << "\n");

1116 } else {

1118 TargetReg)

1119 .addReg( X86::RIP)

1120 .addImm( 1)

1121 .addReg( 0)

1123 .addReg( 0);

1124 ++NumInstsInserted;

1125 (void)AddrI;

1126 LLVM_DEBUG(dbgs() << " Inserting lea: "; AddrI->dump();

1127 dbgs() << "\n");

1128 }

1129

1130 TargetAddrSSA.AddAvailableValue(Pred, TargetReg);

1131 }

1132

1133

1134

1135

1136

1137

1138 Register TargetReg = TargetAddrSSA.GetValueInMiddleOfBlock(&MBB);

1139

1140

1141

1142

1147

1151 ++NumInstsInserted;

1152 (void)CheckI;

1153 LLVM_DEBUG(dbgs() << " Inserting cmp: "; CheckI->dump(); dbgs() << "\n");

1154 } else {

1155

1156 Register AddrReg = MRI->createVirtualRegister(&X86::GR64RegClass);

1157 auto AddrI =

1159 .addReg( X86::RIP)

1160 .addImm( 1)

1161 .addReg( 0)

1163 .addReg( 0);

1164 ++NumInstsInserted;

1165 (void)AddrI;

1166 LLVM_DEBUG(dbgs() << " Inserting lea: "; AddrI->dump(); dbgs() << "\n");

1170 ++NumInstsInserted;

1171 (void)CheckI;

1172 LLVM_DEBUG(dbgs() << " Inserting cmp: "; CheckI->dump(); dbgs() << "\n");

1173 }

1174

1175

1176 int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8;

1178 Register UpdatedStateReg = MRI->createVirtualRegister(PS->RC);

1179 auto CMovI =

1181 .addReg(PS->InitialReg)

1182 .addReg(PS->PoisonReg)

1186 ++NumInstsInserted;

1187 LLVM_DEBUG(dbgs() << " Inserting cmov: "; CMovI->dump(); dbgs() << "\n");

1189

1190

1191

1192 PS->SSA.AddAvailableValue(&MBB, UpdatedStateReg);

1193 }

1194

1195

1196 return CMovs;

1197}

1198

1199

1200

1203 MI.findRegisterDefOperand(X86::EFLAGS, nullptr)) {

1204 return !DefOp->isDead();

1205 }

1206 return false;

1207}

1208

1211

1212

1215 MI.findRegisterDefOperand(X86::EFLAGS, nullptr)) {

1216

1217 if (DefOp->isDead())

1218 return false;

1219

1220

1221 return true;

1222 }

1223

1224

1225 if (MI.killsRegister(X86::EFLAGS, &TRI))

1226 return false;

1227 }

1228

1229

1230

1231 return MBB.isLiveIn(X86::EFLAGS);

1232}

1233

1234

1235

1236

1237

1238

1239

1240

1241

1242

1243

1244

1245

1246

1247

1248

1249

1250

1251

1252

1253

1254

1255

1256

1257

1258

1259

1260

1261void X86SpeculativeLoadHardeningPass::tracePredStateThroughBlocksAndHarden(

1262 MachineFunction &MF) {

1263 SmallPtrSet<MachineInstr *, 16> HardenPostLoad;

1264 SmallPtrSet<MachineInstr *, 16> HardenLoadAddr;

1265

1266 SmallSet<Register, 16> HardenedAddrRegs;

1267

1268 SmallDenseMap<Register, Register, 32> AddrRegToHardenedReg;

1269

1270

1271

1272

1273 SparseBitVector<> LoadDepRegs;

1274

1275 for (MachineBasicBlock &MBB : MF) {

1276

1277

1278

1279

1280

1281

1282

1283

1284

1285

1286

1287

1288

1289

1290

1292 for (MachineInstr &MI : MBB) {

1293

1294

1295

1296

1298 return Op.isReg() && LoadDepRegs.test(Op.getReg().id());

1299 }))

1300 for (MachineOperand &Def : MI.defs())

1301 if (Def.isReg())

1302 LoadDepRegs.set(Def.getReg().id());

1303

1304

1305

1306

1307 if (MI.getOpcode() == X86::LFENCE)

1308 break;

1309

1310

1311 if (.mayLoad())

1312 continue;

1313

1314

1315 if (MI.getOpcode() == X86::MFENCE)

1316 continue;

1317

1318

1320 if (MemRefBeginIdx < 0) {

1322 << "WARNING: unable to harden loading instruction: ";

1324 continue;

1325 }

1326

1327 MachineOperand &BaseMO =

1329 MachineOperand &IndexMO =

1331

1332

1333

1335 if (!BaseMO.isFI() && BaseMO.getReg() != X86::RIP &&

1339 IndexReg = IndexMO.getReg();

1340

1341 if (!BaseReg && !IndexReg)

1342

1343 continue;

1344

1345

1346

1347

1348

1349 if ((BaseReg && LoadDepRegs.test(BaseReg.id())) ||

1350 (IndexReg && LoadDepRegs.test(IndexReg.id())))

1351 continue;

1352

1353

1354

1355

1356

1357

1360 MI.getOperand(0).isReg() &&

1361 canHardenRegister(MI.getOperand(0).getReg()) &&

1362 !HardenedAddrRegs.count(BaseReg) &&

1363 !HardenedAddrRegs.count(IndexReg)) {

1365 HardenedAddrRegs.insert(MI.getOperand(0).getReg());

1366 continue;

1367 }

1368

1369

1370

1372 if (BaseReg)

1373 HardenedAddrRegs.insert(BaseReg);

1374 if (IndexReg)

1375 HardenedAddrRegs.insert(IndexReg);

1376

1377 for (MachineOperand &Def : MI.defs())

1378 if (Def.isReg())

1379 LoadDepRegs.set(Def.getReg().id());

1380 }

1381

1382

1383

1384

1385

1386

1387 for (MachineInstr &MI : MBB) {

1389

1391 "Requested to harden both the address and def of a load!");

1392

1393

1394 if (HardenLoadAddr.erase(&MI)) {

1396 assert(MemRefBeginIdx >= 0 && "Cannot have an invalid index here!");

1397

1398 MachineOperand &BaseMO =

1400 MachineOperand &IndexMO =

1402 hardenLoadAddr(MI, BaseMO, IndexMO, AddrRegToHardenedReg);

1403 continue;

1404 }

1405

1406

1407

1408 if (HardenPostLoad.erase(&MI)) {

1409 assert(.isCall() && "Must not try to post-load harden a call!");

1410

1411

1412

1413

1415

1416

1417 MachineInstr *SunkMI = sinkPostLoadHardenedInst(MI, HardenPostLoad);

1418

1419

1420

1421

1422 if (SunkMI != &MI) {

1423

1424

1425 if (!SunkMI)

1426 continue;

1427

1428

1429 HardenPostLoad.insert(SunkMI);

1430 continue;

1431 }

1432 }

1433

1434 Register HardenedReg = hardenPostLoad(MI);

1435

1436

1437 AddrRegToHardenedReg[HardenedReg] = HardenedReg;

1438

1439 continue;

1440 }

1441

1442

1443

1444

1445

1446

1448 hardenIndirectCallOrJumpInstr(MI, AddrRegToHardenedReg);

1449 }

1450

1451

1452

1454 continue;

1455 if (.isCall() && .isReturn())

1456 continue;

1457

1458

1459

1460 if (MI.isReturn() && .isCall()) {

1461 hardenReturnInstr(MI);

1462 continue;

1463 }

1464

1465

1466

1467

1468 assert(MI.isCall() && "Should only reach here for calls!");

1469 tracePredStateThroughCall(MI);

1470 }

1471

1472 HardenPostLoad.clear();

1473 HardenLoadAddr.clear();

1474 HardenedAddrRegs.clear();

1475 AddrRegToHardenedReg.clear();

1476

1477

1478

1479

1480 LoadDepRegs.clear();

1481 }

1482}

1483

1484

1485

1486

1487

1488

1489

1490Register X86SpeculativeLoadHardeningPass::saveEFLAGS(

1493

1494

1495 Register Reg = MRI->createVirtualRegister(&X86::GR32RegClass);

1496

1497

1499 ++NumInstsInserted;

1500 return Reg;

1501}

1502

1503

1504

1505

1506

1507

1508void X86SpeculativeLoadHardeningPass::restoreEFLAGS(

1512 ++NumInstsInserted;

1513}

1514

1515

1516

1517

1518

1519void X86SpeculativeLoadHardeningPass::mergePredStateIntoSP(

1522 Register TmpReg = MRI->createVirtualRegister(PS->RC);

1523

1524

1525

1526 auto ShiftI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::SHL64ri), TmpReg)

1530 ++NumInstsInserted;

1531 auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::OR64rr), X86::RSP)

1535 ++NumInstsInserted;

1536}

1537

1538

1539Register X86SpeculativeLoadHardeningPass::extractPredStateFromSP(

1542 Register PredStateReg = MRI->createVirtualRegister(PS->RC);

1543 Register TmpReg = MRI->createVirtualRegister(PS->RC);

1544

1545

1546

1547

1548 BuildMI(MBB, InsertPt, Loc, TII->get(TargetOpcode::COPY), TmpReg)

1550 auto ShiftI =

1551 BuildMI(MBB, InsertPt, Loc, TII->get(X86::SAR64ri), PredStateReg)

1553 .addImm(TRI->getRegSizeInBits(*PS->RC) - 1);

1555 ++NumInstsInserted;

1556

1557 return PredStateReg;

1558}

1559

1560void X86SpeculativeLoadHardeningPass::hardenLoadAddr(

1561 MachineInstr &MI, MachineOperand &BaseMO, MachineOperand &IndexMO,

1562 SmallDenseMap<Register, Register, 32> &AddrRegToHardenedReg) {

1563 MachineBasicBlock &MBB = *MI.getParent();

1564 const DebugLoc &Loc = MI.getDebugLoc();

1565

1566

1567

1569

1571

1572 if (BaseMO.isFI()) {

1573

1574

1576 dbgs() << " Skipping hardening base of explicit stack frame load: ";

1578 } else if (BaseMO.getReg() == X86::RSP) {

1579

1580

1581

1582 assert(IndexMO.getReg() == X86::NoRegister &&

1583 "Explicit RSP access with dynamic index!");

1585 dbgs() << " Cannot harden base of explicit RSP offset in a load!");

1586 } else if (BaseMO.getReg() == X86::RIP ||

1587 BaseMO.getReg() == X86::NoRegister) {

1588

1589

1590

1591

1592

1593

1594

1595

1597 dbgs() << " Cannot harden base of "

1598 << (BaseMO.getReg() == X86::RIP ? "RIP-relative" : "no-base")

1599 << " address in a load!");

1600 } else {

1602 "Only allowed to have a frame index or register base.");

1603 HardenOpRegs.push_back(&BaseMO);

1604 }

1605

1606 if (IndexMO.getReg() != X86::NoRegister &&

1607 (HardenOpRegs.empty() ||

1608 HardenOpRegs.front()->getReg() != IndexMO.getReg()))

1609 HardenOpRegs.push_back(&IndexMO);

1610

1611 assert((HardenOpRegs.size() == 1 || HardenOpRegs.size() == 2) &&

1612 "Should have exactly one or two registers to harden!");

1614 HardenOpRegs[0]->getReg() != HardenOpRegs[1]->getReg()) &&

1615 "Should not have two of the same registers!");

1616

1617

1619

1620 auto It = AddrRegToHardenedReg.find(Op->getReg());

1621 if (It == AddrRegToHardenedReg.end())

1622

1623 return false;

1624

1625

1626 Op->setReg(It->second);

1627 return true;

1628 });

1629

1630 if (HardenOpRegs.empty())

1631 return;

1632

1633

1634 Register StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB);

1635

1636 auto InsertPt = MI.getIterator();

1637

1638

1639

1640

1642 if (EFLAGSLive && !Subtarget->hasBMI2()) {

1643 EFLAGSLive = false;

1644 FlagsReg = saveEFLAGS(MBB, InsertPt, Loc);

1645 }

1646

1647 for (MachineOperand *Op : HardenOpRegs) {

1649 auto *OpRC = MRI->getRegClass(OpReg);

1650 Register TmpReg = MRI->createVirtualRegister(OpRC);

1651

1652

1653

1654 if (!Subtarget->hasVLX() && (OpRC->hasSuperClassEq(&X86::VR128RegClass) ||

1655 OpRC->hasSuperClassEq(&X86::VR256RegClass))) {

1656 assert(Subtarget->hasAVX2() && "AVX2-specific register classes!");

1657 bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128RegClass);

1658

1659

1660

1661

1662 Register VStateReg = MRI->createVirtualRegister(&X86::VR128RegClass);

1663 auto MovI =

1664 BuildMI(MBB, InsertPt, Loc, TII->get(X86::VMOV64toPQIrr), VStateReg)

1666 (void)MovI;

1667 ++NumInstsInserted;

1668 LLVM_DEBUG(dbgs() << " Inserting mov: "; MovI->dump(); dbgs() << "\n");

1669

1670

1671 Register VBStateReg = MRI->createVirtualRegister(OpRC);

1672 auto BroadcastI = BuildMI(MBB, InsertPt, Loc,

1673 TII->get(Is128Bit ? X86::VPBROADCASTQrr

1674 : X86::VPBROADCASTQYrr),

1675 VBStateReg)

1677 (void)BroadcastI;

1678 ++NumInstsInserted;

1679 LLVM_DEBUG(dbgs() << " Inserting broadcast: "; BroadcastI->dump();

1680 dbgs() << "\n");

1681

1682

1683 auto OrI =

1685 TII->get(Is128Bit ? X86::VPORrr : X86::VPORYrr), TmpReg)

1688 (void)OrI;

1689 ++NumInstsInserted;

1690 LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");

1691 } else if (OpRC->hasSuperClassEq(&X86::VR128XRegClass) ||

1692 OpRC->hasSuperClassEq(&X86::VR256XRegClass) ||

1693 OpRC->hasSuperClassEq(&X86::VR512RegClass)) {

1694 assert(Subtarget->hasAVX512() && "AVX512-specific register classes!");

1695 bool Is128Bit = OpRC->hasSuperClassEq(&X86::VR128XRegClass);

1696 bool Is256Bit = OpRC->hasSuperClassEq(&X86::VR256XRegClass);

1697 if (Is128Bit || Is256Bit)

1698 assert(Subtarget->hasVLX() && "AVX512VL-specific register classes!");

1699

1700

1701 Register VStateReg = MRI->createVirtualRegister(OpRC);

1702 unsigned BroadcastOp = Is128Bit ? X86::VPBROADCASTQrZ128rr

1703 : Is256Bit ? X86::VPBROADCASTQrZ256rr

1704 : X86::VPBROADCASTQrZrr;

1705 auto BroadcastI =

1706 BuildMI(MBB, InsertPt, Loc, TII->get(BroadcastOp), VStateReg)

1708 (void)BroadcastI;

1709 ++NumInstsInserted;

1710 LLVM_DEBUG(dbgs() << " Inserting broadcast: "; BroadcastI->dump();

1711 dbgs() << "\n");

1712

1713

1714 unsigned OrOp = Is128Bit ? X86::VPORQZ128rr

1715 : Is256Bit ? X86::VPORQZ256rr : X86::VPORQZrr;

1716 auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(OrOp), TmpReg)

1719 (void)OrI;

1720 ++NumInstsInserted;

1721 LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");

1722 } else {

1723

1724 assert(OpRC->hasSuperClassEq(&X86::GR64RegClass) &&

1725 "Not a supported register class for address hardening!");

1726

1727 if (!EFLAGSLive) {

1728

1729 auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(X86::OR64rr), TmpReg)

1733 ++NumInstsInserted;

1734 LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");

1735 } else {

1736

1737

1738 auto ShiftI =

1739 BuildMI(MBB, InsertPt, Loc, TII->get(X86::SHRX64rr), TmpReg)

1742 (void)ShiftI;

1743 ++NumInstsInserted;

1744 LLVM_DEBUG(dbgs() << " Inserting shrx: "; ShiftI->dump();

1745 dbgs() << "\n");

1746 }

1747 }

1748

1749

1750 assert(!AddrRegToHardenedReg.count(Op->getReg()) &&

1751 "Should not have checked this register yet!");

1752 AddrRegToHardenedReg[Op->getReg()] = TmpReg;

1753 Op->setReg(TmpReg);

1754 ++NumAddrRegsHardened;

1755 }

1756

1757

1758 if (FlagsReg)

1759 restoreEFLAGS(MBB, InsertPt, Loc, FlagsReg);

1760}

1761

1762MachineInstr *X86SpeculativeLoadHardeningPass::sinkPostLoadHardenedInst(

1763 MachineInstr &InitialMI, SmallPtrSetImpl<MachineInstr *> &HardenedInstrs) {

1765 "Cannot get here with a non-invariant load!");

1767 "Cannot get here with a data invariant load "

1768 "that interferes with EFLAGS!");

1769

1770

1771 auto SinkCheckToSingleUse =

1772 [&](MachineInstr &MI) -> std::optional<MachineInstr *> {

1773 Register DefReg = MI.getOperand(0).getReg();

1774

1775

1776

1777

1778 MachineInstr *SingleUseMI = nullptr;

1779 for (MachineInstr &UseMI : MRI->use_instructions(DefReg)) {

1780

1781

1784

1785

1786

1788 "Data variant instruction being hardened!");

1789 continue;

1790 }

1791

1792

1793

1795 assert(MemRefBeginIdx >= 0 &&

1796 "Should always have mem references here!");

1797

1798 MachineOperand &BaseMO =

1800 MachineOperand &IndexMO =

1802 if ((BaseMO.isReg() && BaseMO.getReg() == DefReg) ||

1803 (IndexMO.isReg() && IndexMO.getReg() == DefReg))

1804

1805

1806 return {};

1807

1808 continue;

1809 }

1810

1811 if (SingleUseMI)

1812

1813 return {};

1814

1815

1816

1819 return {};

1820

1821

1822

1823 if (UseMI.getDesc().getNumDefs() > 1)

1824 return {};

1825

1826

1827

1828

1830 if (!canHardenRegister(UseDefReg))

1831 return {};

1832

1833 SingleUseMI = &UseMI;

1834 }

1835

1836

1837

1838 return {SingleUseMI};

1839 };

1840

1841 MachineInstr *MI = &InitialMI;

1842 while (std::optional<MachineInstr *> SingleUse = SinkCheckToSingleUse(*MI)) {

1843

1844 MI = *SingleUse;

1845 if ()

1846 break;

1847 }

1848

1849 return MI;

1850}

1851

1852bool X86SpeculativeLoadHardeningPass::canHardenRegister(Register Reg) {

1853

1855 return false;

1856

1857 auto *RC = MRI->getRegClass(Reg);

1858 int RegBytes = TRI->getRegSizeInBits(*RC) / 8;

1859 if (RegBytes > 8)

1860

1861 return false;

1862

1863 unsigned RegIdx = Log2_32(RegBytes);

1864 assert(RegIdx < 4 && "Unsupported register size");

1865

1866

1867

1868

1869

1870

1871

1872

1873 const TargetRegisterClass *NOREXRegClasses[] = {

1874 &X86::GR8_NOREXRegClass, &X86::GR16_NOREXRegClass,

1875 &X86::GR32_NOREXRegClass, &X86::GR64_NOREXRegClass};

1876 if (RC == NOREXRegClasses[RegIdx])

1877 return false;

1878

1879 const TargetRegisterClass *GPRRegClasses[] = {

1880 &X86::GR8RegClass, &X86::GR16RegClass, &X86::GR32RegClass,

1881 &X86::GR64RegClass};

1882 return RC->hasSuperClassEq(GPRRegClasses[RegIdx]);

1883}

1884

1885

1886

1887

1888

1889

1890

1891

1892

1893

1894

1895

1896

1897

1898

1899Register X86SpeculativeLoadHardeningPass::hardenValueInRegister(

1902 assert(canHardenRegister(Reg) && "Cannot harden this register!");

1903

1904 auto *RC = MRI->getRegClass(Reg);

1905 int Bytes = TRI->getRegSizeInBits(*RC) / 8;

1906 Register StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB);

1907 assert((Bytes == 1 || Bytes == 2 || Bytes == 4 || Bytes == 8) &&

1908 "Unknown register size");

1909

1910

1911 if (Bytes != 8) {

1912 unsigned SubRegImms[] = {X86::sub_8bit, X86::sub_16bit, X86::sub_32bit};

1913 unsigned SubRegImm = SubRegImms[Log2_32(Bytes)];

1914 Register NarrowStateReg = MRI->createVirtualRegister(RC);

1915 BuildMI(MBB, InsertPt, Loc, TII->get(TargetOpcode::COPY), NarrowStateReg)

1916 .addReg(StateReg, 0, SubRegImm);

1917 StateReg = NarrowStateReg;

1918 }

1919

1922 FlagsReg = saveEFLAGS(MBB, InsertPt, Loc);

1923

1924 Register NewReg = MRI->createVirtualRegister(RC);

1925 unsigned OrOpCodes[] = {X86::OR8rr, X86::OR16rr, X86::OR32rr, X86::OR64rr};

1926 unsigned OrOpCode = OrOpCodes[Log2_32(Bytes)];

1927 auto OrI = BuildMI(MBB, InsertPt, Loc, TII->get(OrOpCode), NewReg)

1931 ++NumInstsInserted;

1932 LLVM_DEBUG(dbgs() << " Inserting or: "; OrI->dump(); dbgs() << "\n");

1933

1934 if (FlagsReg)

1935 restoreEFLAGS(MBB, InsertPt, Loc, FlagsReg);

1936

1937 return NewReg;

1938}

1939

1940

1941

1942

1943

1944

1945

1946

1947

1948

1949Register X86SpeculativeLoadHardeningPass::hardenPostLoad(MachineInstr &MI) {

1950 MachineBasicBlock &MBB = *MI.getParent();

1951 const DebugLoc &Loc = MI.getDebugLoc();

1952

1953 auto &DefOp = MI.getOperand(0);

1954 Register OldDefReg = DefOp.getReg();

1955 auto *DefRC = MRI->getRegClass(OldDefReg);

1956

1957

1958

1959

1960 Register UnhardenedReg = MRI->createVirtualRegister(DefRC);

1961 DefOp.setReg(UnhardenedReg);

1962

1963

1964

1965

1966 Register HardenedReg = hardenValueInRegister(

1967 UnhardenedReg, MBB, std::next(MI.getIterator()), Loc);

1968

1969

1970

1971 MRI->replaceRegWith( OldDefReg, HardenedReg);

1972

1973 ++NumPostLoadRegsHardened;

1974 return HardenedReg;

1975}

1976

1977

1978

1979

1980

1981

1982

1983

1984

1985

1986

1987

1988

1989

1990

1991

1992

1993

1994

1995

1996

1997

1998

1999

2000void X86SpeculativeLoadHardeningPass::hardenReturnInstr(MachineInstr &MI) {

2001 MachineBasicBlock &MBB = *MI.getParent();

2002 const DebugLoc &Loc = MI.getDebugLoc();

2003 auto InsertPt = MI.getIterator();

2004

2006

2007

2008 return;

2009

2010

2011

2012

2013 mergePredStateIntoSP(MBB, InsertPt, Loc, PS->SSA.GetValueAtEndOfBlock(&MBB));

2014}

2015

2016

2017

2018

2019

2020

2021

2022

2023

2024

2025

2026

2027

2028

2029

2030

2031

2032

2033

2034

2035

2036

2037

2038

2039

2040

2041

2042

2043

2044

2045

2046void X86SpeculativeLoadHardeningPass::tracePredStateThroughCall(

2047 MachineInstr &MI) {

2048 MachineBasicBlock &MBB = *MI.getParent();

2050 auto InsertPt = MI.getIterator();

2051 const DebugLoc &Loc = MI.getDebugLoc();

2052

2054 if (MI.isReturn())

2055

2056

2057 return;

2058

2059

2060

2061

2062

2063 BuildMI(MBB, std::next(InsertPt), Loc, TII->get(X86::LFENCE));

2064 ++NumInstsInserted;

2065 ++NumLFENCEsInserted;

2066 return;

2067 }

2068

2069

2070

2071 Register StateReg = PS->SSA.GetValueAtEndOfBlock(&MBB);

2072 mergePredStateIntoSP(MBB, InsertPt, Loc, StateReg);

2073

2074

2075

2076

2077

2079 return;

2080

2081

2082

2083

2086 true);

2087 MI.setPostInstrSymbol(MF, RetSymbol);

2088

2089 const TargetRegisterClass *AddrRC = &X86::GR64RegClass;

2090 Register ExpectedRetAddrReg;

2091

2092

2093

2094

2097

2098

2099

2100

2101

2102

2103

2104

2105

2106

2107

2108

2109

2110

2111

2112

2113

2114

2115 ExpectedRetAddrReg = MRI->createVirtualRegister(AddrRC);

2118 BuildMI(MBB, InsertPt, Loc, TII->get(X86::MOV64ri32), ExpectedRetAddrReg)

2120 } else {

2121 BuildMI(MBB, InsertPt, Loc, TII->get(X86::LEA64r), ExpectedRetAddrReg)

2122 .addReg( X86::RIP)

2123 .addImm( 1)

2124 .addReg( 0)

2126 .addReg( 0);

2127 }

2128 }

2129

2130

2131 ++InsertPt;

2132

2133

2134

2135

2136

2137 if (!ExpectedRetAddrReg) {

2138 ExpectedRetAddrReg = MRI->createVirtualRegister(AddrRC);

2139 BuildMI(MBB, InsertPt, Loc, TII->get(X86::MOV64rm), ExpectedRetAddrReg)

2140 .addReg( X86::RSP)

2141 .addImm( 1)

2142 .addReg( 0)

2143 .addImm( -8)

2144

2145 .addReg( 0);

2146 }

2147

2148

2149 Register NewStateReg = extractPredStateFromSP(MBB, InsertPt, Loc);

2150

2151

2152

2153

2156

2157

2161 } else {

2162 Register ActualRetAddrReg = MRI->createVirtualRegister(AddrRC);

2163 BuildMI(MBB, InsertPt, Loc, TII->get(X86::LEA64r), ActualRetAddrReg)

2164 .addReg( X86::RIP)

2165 .addImm( 1)

2166 .addReg( 0)

2168 .addReg( 0);

2172 }

2173

2174

2175

2176 int PredStateSizeInBytes = TRI->getRegSizeInBits(*PS->RC) / 8;

2178

2179 Register UpdatedStateReg = MRI->createVirtualRegister(PS->RC);

2180 auto CMovI = BuildMI(MBB, InsertPt, Loc, TII->get(CMovOp), UpdatedStateReg)

2182 .addReg(PS->PoisonReg)

2185 ++NumInstsInserted;

2186 LLVM_DEBUG(dbgs() << " Inserting cmov: "; CMovI->dump(); dbgs() << "\n");

2187

2188 PS->SSA.AddAvailableValue(&MBB, UpdatedStateReg);

2189}

2190

2191

2192

2193

2194

2195

2196

2197

2198

2199

2200

2201

2202

2203

2204

2205

2206void X86SpeculativeLoadHardeningPass::hardenIndirectCallOrJumpInstr(

2207 MachineInstr &MI,

2208 SmallDenseMap<Register, Register, 32> &AddrRegToHardenedReg) {

2209 switch (MI.getOpcode()) {

2210 case X86::FARCALL16m:

2211 case X86::FARCALL32m:

2212 case X86::FARCALL64m:

2213 case X86::FARJMP16m:

2214 case X86::FARJMP32m:

2215 case X86::FARJMP64m:

2216

2217

2218 return;

2219

2220 default:

2221 break;

2222 }

2223

2224

2225

2226 assert(.mayLoad() && "Found a lingering loading instruction!");

2227

2228

2229

2230 if (.getOperand(0).isReg())

2231 return;

2232

2233

2234

2235 auto &TargetOp = MI.getOperand(0);

2236 Register OldTargetReg = TargetOp.getReg();

2237

2238

2239

2240

2241 Register &HardenedTargetReg = AddrRegToHardenedReg[OldTargetReg];

2242

2243

2244

2245

2246

2247

2248

2249

2250 if (!HardenedTargetReg)

2251 HardenedTargetReg = hardenValueInRegister(

2252 OldTargetReg, *MI.getParent(), MI.getIterator(), MI.getDebugLoc());

2253

2254

2255 TargetOp.setReg(HardenedTargetReg);

2256

2257 ++NumCallsOrJumpsHardened;

2258}

2259

2261 "X86 speculative load hardener", false, false)

2264

2266 return new X86SpeculativeLoadHardeningPass();

2267}

unsigned const MachineRegisterInfo * MRI

MachineInstrBuilder & UseMI

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

const TargetInstrInfo & TII

static cl::opt< bool > HardenLoads("aarch64-slh-loads", cl::Hidden, cl::desc("Sanitize loads from memory."), cl::init(true))

Analysis containing CSE Info

This file defines the DenseMap class.

const size_t AbstractManglingParser< Derived, Alloc >::NumOps

This file declares the MachineConstantPool class which is an abstract constant pool to keep track of ...

Promote Memory to Register

MachineInstr unsigned OpIdx

#define INITIALIZE_PASS_END(passName, arg, name, cfg, analysis)

#define INITIALIZE_PASS_BEGIN(passName, arg, name, cfg, analysis)

const SmallVectorImpl< MachineOperand > & Cond

This file defines the SmallPtrSet class.

This file defines the SmallSet class.

This file defines the SmallVector class.

This file defines the SparseBitVector class.

This file defines the 'Statistic' class, which is designed to be an easy way to expose various metric...

#define STATISTIC(VARNAME, DESC)

static MachineBasicBlock & splitEdge(MachineBasicBlock &MBB, MachineBasicBlock &Succ, int SuccCount, MachineInstr *Br, MachineInstr *&UncondBr, const X86InstrInfo &TII)

Definition X86SpeculativeLoadHardening.cpp:221

static cl::opt< bool > HardenLoads(PASS_KEY "-loads", cl::desc("Sanitize loads from memory. When disable, no " "significant security is provided."), cl::init(true), cl::Hidden)

static void canonicalizePHIOperands(MachineFunction &MF)

Removing duplicate PHI operands to leave the PHI in a canonical and predictable form.

Definition X86SpeculativeLoadHardening.cpp:325

static cl::opt< bool > HardenInterprocedurally(PASS_KEY "-ip", cl::desc("Harden interprocedurally by passing our state in and out of " "functions in the high bits of the stack pointer."), cl::init(true), cl::Hidden)

static cl::opt< bool > FenceCallAndRet(PASS_KEY "-fence-call-and-ret", cl::desc("Use a full speculation fence to harden both call and ret edges " "rather than a lighter weight mitigation."), cl::init(false), cl::Hidden)

static cl::opt< bool > EnablePostLoadHardening(PASS_KEY "-post-load", cl::desc("Harden the value loaded *after* it is loaded by " "flushing the loaded bits to 1. This is hard to do " "in general but can be done easily for GPRs."), cl::init(true), cl::Hidden)

static cl::opt< bool > HardenEdgesWithLFENCE(PASS_KEY "-lfence", cl::desc("Use LFENCE along each conditional edge to harden against speculative " "loads rather than conditional movs and poisoned pointers."), cl::init(false), cl::Hidden)

static bool isEFLAGSLive(MachineBasicBlock &MBB, MachineBasicBlock::iterator I, const TargetRegisterInfo &TRI)

Definition X86SpeculativeLoadHardening.cpp:1209

static cl::opt< bool > EnableSpeculativeLoadHardening("x86-speculative-load-hardening", cl::desc("Force enable speculative load hardening"), cl::init(false), cl::Hidden)

static const TargetRegisterClass * getRegClassForUnfoldedLoad(const X86InstrInfo &TII, unsigned Opcode)

Compute the register class for the unfolded load.

Definition X86SpeculativeLoadHardening.cpp:838

static bool hasVulnerableLoad(MachineFunction &MF)

Helper to scan a function for loads vulnerable to misspeculation that we want to harden.

Definition X86SpeculativeLoadHardening.cpp:369

static bool isEFLAGSDefLive(const MachineInstr &MI)

Definition X86SpeculativeLoadHardening.cpp:1201

static cl::opt< bool > HardenIndirectCallsAndJumps(PASS_KEY "-indirect", cl::desc("Harden indirect calls and jumps against using speculatively " "stored attacker controlled addresses. This is designed to " "mitigate Spectre v1.2 style attacks."), cl::init(true), cl::Hidden)

Represent the analysis usage information of a pass.

iterator find(const_arg_type_t< KeyT > Val)

size_type count(const_arg_type_t< KeyT > Val) const

Return 1 if the specified key is in the map, 0 otherwise.

FunctionPass class - This class is used to implement most global optimizations.

bool hasFnAttribute(Attribute::AttrKind Kind) const

Return true if the function has the attribute.

LLVM_ABI MCSymbol * createTempSymbol()

Create a temporary symbol with a unique name.

Describe properties that are true of each instruction in the target description file.

const MCInstrDesc & get(unsigned Opcode) const

Return the machine instruction descriptor that corresponds to the specified instruction opcode.

void normalizeSuccProbs()

Normalize probabilities of all successors so that the sum of them becomes one.

bool isEHPad() const

Returns true if the block is a landing pad.

LLVM_ABI instr_iterator insert(instr_iterator I, MachineInstr *M)

Insert MI into the instruction list before I, possibly inside a bundle.

iterator_range< livein_iterator > liveins() const

reverse_instr_iterator instr_rbegin()

LLVM_ABI iterator SkipPHIsAndLabels(iterator I)

Return the first instruction in MBB after I that is not a PHI or a label.

LLVM_ABI iterator SkipPHIsLabelsAndDebug(iterator I, Register Reg=Register(), bool SkipPseudoOp=true)

Return the first instruction in MBB after I that is not a PHI, label or debug.

bool isEHFuncletEntry() const

Returns true if this is the entry block of an EH funclet.

LLVM_ABI iterator getFirstTerminator()

Returns an iterator to the first terminator instruction of this basic block.

unsigned succ_size() const

LLVM_ABI void dump() const

bool isEHScopeEntry() const

Returns true if this is the entry block of an EH scope, i.e., the block that used to have a catchpad ...

LLVM_ABI void addSuccessor(MachineBasicBlock *Succ, BranchProbability Prob=BranchProbability::getUnknown())

Add Succ as a successor of this MachineBasicBlock.

reverse_instr_iterator instr_rend()

LLVM_ABI bool isLayoutSuccessor(const MachineBasicBlock *MBB) const

Return true if the specified MBB will be emitted immediately after this block, such that if this bloc...

void addLiveIn(MCRegister PhysReg, LaneBitmask LaneMask=LaneBitmask::getAll())

Adds the specified register as a live in.

const MachineFunction * getParent() const

Return the MachineFunction containing this basic block.

iterator_range< iterator > terminators()

iterator_range< succ_iterator > successors()

iterator_range< pred_iterator > predecessors()

MachineInstrBundleIterator< MachineInstr > iterator

LLVM_ABI StringRef getName() const

Return the name of the corresponding LLVM basic block, or an empty string.

void setMachineBlockAddressTaken()

Set this block to indicate that its address is used as something other than the target of a terminato...

LLVM_ABI bool isLiveIn(MCRegister Reg, LaneBitmask LaneMask=LaneBitmask::getAll()) const

Return true if the specified register is in the live in set.

bool isCleanupFuncletEntry() const

Returns true if this is the entry block of a cleanup funclet.

MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...

void getAnalysisUsage(AnalysisUsage &AU) const override

getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.

const TargetSubtargetInfo & getSubtarget() const

getSubtarget - Return the subtarget for which this machine code is being compiled.

StringRef getName() const

getName - Return the name of the corresponding LLVM function.

bool exposesReturnsTwice() const

exposesReturnsTwice - Returns true if the function calls setjmp or any other similar functions with a...

MCContext & getContext() const

MachineRegisterInfo & getRegInfo()

getRegInfo - Return information about the registers currently in use.

Function & getFunction()

Return the LLVM function that this machine code represents.

BasicBlockListType::iterator iterator

MachineBasicBlock * CreateMachineBasicBlock(const BasicBlock *BB=nullptr, std::optional< UniqueBBID > BBID=std::nullopt)

CreateMachineInstr - Allocate a new MachineInstr.

void insert(iterator MBBI, MachineBasicBlock *MBB)

const TargetMachine & getTarget() const

getTarget - Return the target machine this machine code is compiled with

Get the register for the operand index.

const MachineInstrBuilder & addImm(int64_t Val) const

Add a new immediate operand.

const MachineInstrBuilder & addSym(MCSymbol *Sym, unsigned char TargetFlags=0) const

const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const

Add a new virtual register operand.

const MachineInstrBuilder & addMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0) const

Representation of each machine instruction.

unsigned getOpcode() const

Returns the opcode of this MachineInstr.

bool isTerminator(QueryType Type=AnyInBundle) const

Returns true if this instruction part of the terminator for a basic block.

bool isBranch(QueryType Type=AnyInBundle) const

Returns true if this is a conditional, unconditional, or indirect branch.

MachineOperand * findRegisterUseOperand(Register Reg, const TargetRegisterInfo *TRI, bool isKill=false)

Wrapper for findRegisterUseOperandIdx, it returns a pointer to the MachineOperand rather than an inde...

const DebugLoc & getDebugLoc() const

Returns the debug location id of this MachineInstr.

const MachineOperand & getOperand(unsigned i) const

LLVM_ABI bool addRegisterDead(Register Reg, const TargetRegisterInfo *RegInfo, bool AddIfNotFound=false)

We have determined MI defined a register without a use.

MachineOperand class - Representation of each machine instruction operand.

bool isReg() const

isReg - Tests if this is a MO_Register operand.

MachineBasicBlock * getMBB() const

void setIsDead(bool Val=true)

void setIsKill(bool Val=true)

void setMBB(MachineBasicBlock *MBB)

getReg - Returns the register number.

bool isFI() const

isFI - Tests if this is a MO_FrameIndex operand.

static MachineOperand CreateMBB(MachineBasicBlock *MBB, unsigned TargetFlags=0)

bool isMBB() const

isMBB - Tests if this is a MO_MachineBasicBlock operand.

constexpr bool isValid() const

constexpr bool isVirtual() const

Return true if the specified register number is in the virtual register namespace.

constexpr unsigned id() const

bool insert(const value_type &X)

Insert a new element into the SetVector.

bool erase(PtrType Ptr)

Remove pointer from the set.

size_type count(ConstPtrType Ptr) const

count - Return 1 if the specified pointer is in the set, 0 otherwise.

void insert_range(Range &&R)

std::pair< iterator, bool > insert(PtrType Ptr)

Inserts Ptr if and only if there is no element in the container equal to Ptr.

SmallPtrSet - This class implements a set which is optimized for holding SmallSize or less elements.

size_type count(const T &V) const

count - Return 1 if the element is in the set, 0 otherwise.

std::pair< const_iterator, bool > insert(const T &V)

insert - Insert an element into the set if it isn't already there.

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

bool test(unsigned Idx) const

virtual bool unfoldMemoryOperand(MachineFunction &MF, MachineInstr &MI, Register Reg, bool UnfoldLoad, bool UnfoldStore, SmallVectorImpl< MachineInstr * > &NewMIs) const

unfoldMemoryOperand - Separate a single instruction which folded a load or a store or a load and a st...

CodeModel::Model getCodeModel() const

Returns the code model.

TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...

bool has128ByteRedZone(const MachineFunction &MF) const

Return true if the function has a redzone (accessible bytes past the frame of the top of stack functi...

static bool isDataInvariantLoad(MachineInstr &MI)

Returns true if the instruction has no behavior (specified or otherwise) that is based on the value l...

static bool isDataInvariant(MachineInstr &MI)

Returns true if the instruction has no behavior (specified or otherwise) that is based on the value o...

const X86InstrInfo * getInstrInfo() const override

bool isPositionIndependent() const

const X86RegisterInfo * getRegisterInfo() const override

const X86FrameLowering * getFrameLowering() const override

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

@ Kill

The last use of a register.

CondCode getCondFromBranch(const MachineInstr &MI)

int getFirstAddrOperandIdx(const MachineInstr &MI)

Return the index of the instruction's first address operand, if it has a memory reference,...

CondCode GetOppositeBranchCondition(CondCode CC)

GetOppositeBranchCondition - Return the inverse of the specified cond, e.g.

unsigned getCMovOpcode(unsigned RegBytes, bool HasMemoryOperand=false, bool HasNDD=false)

Return a cmov opcode for the given register size in bytes, and operand type.

initializer< Ty > init(const Ty &Val)

NodeAddr< DefNode * > Def

BaseReg

Stack frame base register. Bit 0 of FREInfo.Info.

This is an optimization pass for GlobalISel generic memory operations.

bool all_of(R &&range, UnaryPredicate P)

Provide wrappers to std::all_of which take ranges instead of having to pass begin/end explicitly.

MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)

Builder interface. Specify how to create the initial instruction itself.

iterator_range< T > make_range(T x, T y)

Convenience function for iterating over sub-ranges.

iterator_range< early_inc_iterator_impl< detail::IterOfRange< RangeT > > > make_early_inc_range(RangeT &&Range)

Make a range that does early increment to allow mutation of the underlying range without disrupting i...

auto unique(Range &&R, Predicate P)

bool any_of(R &&range, UnaryPredicate P)

Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.

unsigned Log2_32(uint32_t Value)

Return the floor log base 2 of the specified value, -1 if the value is zero.

FunctionPass * createX86SpeculativeLoadHardeningPass()

Definition X86SpeculativeLoadHardening.cpp:2265

auto reverse(ContainerTy &&C)

void sort(IteratorTy Start, IteratorTy End)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

LLVM_ABI void report_fatal_error(Error Err, bool gen_crash_diag=true)

class LLVM_GSL_OWNER SmallVector

Forward declaration of SmallVector so that calculateSmallVectorDefaultInlinedElements can reference s...

DWARFExpression::Operation Op

ArrayRef(const T &OneElt) -> ArrayRef< T >

void erase_if(Container &C, UnaryPredicate P)

Provide a container algorithm similar to C++ Library Fundamentals v2's erase_if which is equivalent t...