MachineSMEABIPass.cpp Source File (original) (raw)

69using namespace llvm;

71#define DEBUG_TYPE "aarch64-machine-sme-abi"

73namespace {

81 ANY = 0,

84 ACTIVE,

88 ACTIVE_ZT0_SAVED,

92 LOCAL_SAVED,

97 LOCAL_COMMITTED,

101

102

103 OFF,

104

105

106 NUM_ZA_STATE

107};

108

109

110

113 NZCV = 1 << 0,

114 W0 = 1 << 1,

115 W0_HI = 1 << 2,

116 X0 = W0 | W0_HI,

118};

119

120

121struct PhysRegSave {

123 Register StatusFlags = AArch64::NoRegister;

124 Register X0Save = AArch64::NoRegister;

125};

126

127

128

129struct InstInfo {

130 ZAState NeededState{ZAState::ANY};

132 LiveRegs PhysLiveRegs = LiveRegs::None;

133};

134

135

136

137struct BlockInfo {

139 ZAState FixedEntryState{ZAState::ANY};

140 ZAState DesiredIncomingState{ZAState::ANY};

141 ZAState DesiredOutgoingState{ZAState::ANY};

142 LiveRegs PhysLiveRegsAtEntry = LiveRegs::None;

143 LiveRegs PhysLiveRegsAtExit = LiveRegs::None;

144};

145

146

147struct FunctionInfo {

149 std::optionalMachineBasicBlock::iterator AfterSMEProloguePt;

150 LiveRegs PhysLiveRegsAfterSMEPrologue = LiveRegs::None;

151};

152

153

154

155class EmitContext {

156public:

157 EmitContext() = default;

158

159

161 if (TPIDR2BlockFI)

162 return *TPIDR2BlockFI;

165 return *TPIDR2BlockFI;

166 }

167

168

170 if (AgnosticZABufferPtr != AArch64::NoRegister)

171 return AgnosticZABufferPtr;

174 AgnosticZABufferPtr =

175 BufferPtr != AArch64::NoRegister

176 ? BufferPtr

178 return AgnosticZABufferPtr;

179 }

180

182 if (ZT0SaveFI)

183 return *ZT0SaveFI;

186 return *ZT0SaveFI;

187 }

188

189

190

191 bool needsSaveBuffer() const {

192 assert(!(TPIDR2BlockFI && AgnosticZABufferPtr) &&

193 "Cannot have both a TPIDR2 block and agnostic ZA buffer");

194 return TPIDR2BlockFI || AgnosticZABufferPtr != AArch64::NoRegister;

195 }

196

197private:

198 std::optional ZT0SaveFI;

199 std::optional TPIDR2BlockFI;

200 Register AgnosticZABufferPtr = AArch64::NoRegister;

201};

202

203

204

205

206

207

208static bool isLegalEdgeBundleZAState(ZAState State) {

209 switch (State) {

210 case ZAState::ACTIVE:

211 case ZAState::ACTIVE_ZT0_SAVED:

212 case ZAState::LOCAL_SAVED:

213 case ZAState::LOCAL_COMMITTED:

214 return true;

215 default:

216 return false;

217 }

218}

219

220StringRef getZAStateString(ZAState State) {

221#define MAKE_CASE(V) \

222 case V: \

223 return #V;

224 switch (State) {

227 MAKE_CASE(ZAState::ACTIVE_ZT0_SAVED)

229 MAKE_CASE(ZAState::LOCAL_COMMITTED)

232 default:

234 }

235#undef MAKE_CASE

236}

237

241 return false;

243 return AArch64::MPR128RegClass.contains(SR) ||

244 AArch64::ZTRRegClass.contains(SR);

245 });

246}

247

248

249

250static std::pair<ZAState, MachineBasicBlock::iterator>

254

255

256

257

258

259

260 if (MI.getOpcode() == AArch64::InOutZAUsePseudo)

261 return {ZAState::ACTIVE, std::prev(InsertPt)};

262

263

264 if (MI.getOpcode() == AArch64::RequiresZASavePseudo)

265 return {ZAState::LOCAL_SAVED, std::prev(InsertPt)};

266

267

268

269

270

271

272

273

274 if (MI.getOpcode() == AArch64::RequiresZT0SavePseudo) {

275 return {SMEFnAttrs.hasZAState() ? ZAState::ACTIVE_ZT0_SAVED

276 : ZAState::LOCAL_COMMITTED,

277 std::prev(InsertPt)};

278 }

279

280 if (MI.isReturn()) {

282 return {ZAOffAtReturn ? ZAState::OFF : ZAState::ACTIVE, InsertPt};

283 }

284

285 for (auto &MO : MI.operands()) {

286 if (isZAorZTRegOp(TRI, MO))

287 return {ZAState::ACTIVE, InsertPt};

288 }

289

290 return {ZAState::ANY, InsertPt};

291}

292

294 inline static char ID = 0;

295

298

300

301 StringRef getPassName() const override { return "Machine SME ABI pass"; }

302

303 void getAnalysisUsage(AnalysisUsage &AU) const override {

309 }

310

311

312

313 FunctionInfo collectNeededZAStates(SMEAttrs SMEFnAttrs);

314

315

316

318 const FunctionInfo &FnInfo);

319

320

321

322 void insertStateChanges(EmitContext &, const FunctionInfo &FnInfo,

325

326

327

328

329 void propagateDesiredStates(FunctionInfo &FnInfo, bool Forwards = true);

330

333

334

345 bool ClearTPIDR2, bool On);

346

347

351

352

353

356 LiveRegs PhysLiveRegs, bool IsSave);

360

361

362

363

364 std::pair<MachineBasicBlock::iterator, LiveRegs>

369 ZAState To, LiveRegs PhysLiveRegs);

370

371

374 if (AFI->getSMEFnAttrs().hasAgnosticZAInterface())

375 return emitFullZASaveRestore(Context, MBB, MBBI, PhysLiveRegs,

376 true);

377 return emitSetupLazySave(Context, MBB, MBBI);

378 }

381 if (AFI->getSMEFnAttrs().hasAgnosticZAInterface())

382 return emitFullZASaveRestore(Context, MBB, MBBI, PhysLiveRegs,

383 false);

384 return emitRestoreLazySave(Context, MBB, MBBI, PhysLiveRegs);

385 }

389 if (AFI->getSMEFnAttrs().hasAgnosticZAInterface())

390 return emitAllocateFullZASaveBuffer(Context, MBB, MBBI, PhysLiveRegs);

391 return emitAllocateLazySaveBuffer(Context, MBB, MBBI);

392 }

393

394

397

400

401private:

403

411};

412

414 LiveRegs PhysLiveRegs = LiveRegs::None;

415 if (!LiveUnits.available(AArch64::NZCV))

416 PhysLiveRegs |= LiveRegs::NZCV;

417

418

419 if (!LiveUnits.available(AArch64::W0))

420 PhysLiveRegs |= LiveRegs::W0;

421 if (!LiveUnits.available(AArch64::W0_HI))

422 PhysLiveRegs |= LiveRegs::W0_HI;

423 return PhysLiveRegs;

424}

425

427 if (PhysLiveRegs & LiveRegs::NZCV)

428 LiveUnits.addReg(AArch64::NZCV);

429 if (PhysLiveRegs & LiveRegs::W0)

430 LiveUnits.addReg(AArch64::W0);

431 if (PhysLiveRegs & LiveRegs::W0_HI)

432 LiveUnits.addReg(AArch64::W0_HI);

433}

434

435[[maybe_unused]] bool isCallStartOpcode(unsigned Opc) {

436 switch (Opc) {

437 case AArch64::TLSDESC_CALLSEQ:

438 case AArch64::TLSDESC_AUTH_CALLSEQ:

439 case AArch64::ADJCALLSTACKDOWN:

440 return true;

441 default:

442 return false;

443 }

444}

445

446FunctionInfo MachineSMEABI::collectNeededZAStates(SMEAttrs SMEFnAttrs) {

449 "Expected function to have ZA/ZT0 state!");

450

452 LiveRegs PhysLiveRegsAfterSMEPrologue = LiveRegs::None;

453 std::optionalMachineBasicBlock::iterator AfterSMEProloguePt;

454

456 BlockInfo &Block = Blocks[MBB.getNumber()];

457

458 if (MBB.isEntryBlock()) {

459

460 Block.FixedEntryState = ZAState::ENTRY;

461 } else if (MBB.isEHPad()) {

462

463 Block.FixedEntryState = ZAState::LOCAL_COMMITTED;

464 }

465

468

469 Block.PhysLiveRegsAtExit = getPhysLiveRegs(LiveUnits);

470 auto FirstTerminatorInsertPt = MBB.getFirstTerminator();

471 auto FirstNonPhiInsertPt = MBB.getFirstNonPHI();

475 LiveRegs PhysLiveRegs = getPhysLiveRegs(LiveUnits);

476

477

478

479

480 if (MI.getOpcode() == AArch64::SMEStateAllocPseudo) {

481 AfterSMEProloguePt = MBBI;

482 PhysLiveRegsAfterSMEPrologue = PhysLiveRegs;

483 }

484

485 auto [NeededState, InsertPt] = getInstNeededZAState(*TRI, MI, SMEFnAttrs);

486 assert((InsertPt == MBBI || isCallStartOpcode(InsertPt->getOpcode())) &&

487 "Unexpected state change insertion point!");

488

489 if (MBBI == FirstTerminatorInsertPt)

490 Block.PhysLiveRegsAtExit = PhysLiveRegs;

491 if (MBBI == FirstNonPhiInsertPt)

492 Block.PhysLiveRegsAtEntry = PhysLiveRegs;

493 if (NeededState != ZAState::ANY)

494 Block.Insts.push_back({NeededState, InsertPt, PhysLiveRegs});

495 }

496

497

498 std::reverse(Block.Insts.begin(), Block.Insts.end());

499

500

501

502 if ( Block .Insts.empty()) {

503 Block.DesiredIncomingState = Block.Insts.front().NeededState;

504 Block.DesiredOutgoingState = Block.Insts.back().NeededState;

505 }

506 }

507

508 return FunctionInfo{std::move(Blocks), AfterSMEProloguePt,

509 PhysLiveRegsAfterSMEPrologue};

510}

511

512void MachineSMEABI::propagateDesiredStates(FunctionInfo &FnInfo,

513 bool Forwards) {

514

515

516

517 auto GetBlockState = [](BlockInfo &Block, bool Incoming) -> ZAState & {

518 return Incoming ? Block.DesiredIncomingState : Block.DesiredOutgoingState;

519 };

520

522 for (auto [BlockID, BlockInfo] : enumerate(FnInfo.Blocks)) {

523 if (!isLegalEdgeBundleZAState(GetBlockState(BlockInfo, Forwards)))

525 }

526

527 while (!Worklist.empty()) {

529 BlockInfo &Block = FnInfo.Blocks[MBB->getNumber()];

530

531

532

533 int StateCounts[ZAState::NUM_ZA_STATE] = {0};

536 BlockInfo &PredOrSuccBlock = FnInfo.Blocks[PredOrSucc->getNumber()];

537 ZAState ZAState = GetBlockState(PredOrSuccBlock, !Forwards);

538 if (isLegalEdgeBundleZAState(ZAState))

539 StateCounts[ZAState]++;

540 }

541

542 ZAState PropagatedState = ZAState(max_element(StateCounts) - StateCounts);

543 ZAState &CurrentState = GetBlockState(Block, Forwards);

544 if (PropagatedState != CurrentState) {

545 CurrentState = PropagatedState;

546 ZAState &OtherState = GetBlockState(Block, !Forwards);

547

548 if (OtherState == ZAState::ANY)

549 OtherState = PropagatedState;

550

551

554 BlockInfo &SuccOrPredBlock = FnInfo.Blocks[SuccOrPred->getNumber()];

555 if (!isLegalEdgeBundleZAState(GetBlockState(SuccOrPredBlock, Forwards)))

557 }

558 }

559 }

560}

561

562

563

565MachineSMEABI::assignBundleZAStates(const EdgeBundles &Bundles,

566 const FunctionInfo &FnInfo) {

569 LLVM_DEBUG(dbgs() << "Assigning ZA state for edge bundle: " << I << '\n');

570

571

572

573

574 int EdgeStateCounts[ZAState::NUM_ZA_STATE] = {0};

575 for (unsigned BlockID : Bundles.getBlocks(I)) {

577

578 const BlockInfo &Block = FnInfo.Blocks[BlockID];

579 bool InEdge = Bundles.getBundle(BlockID, false) == I;

580 bool OutEdge = Bundles.getBundle(BlockID, true) == I;

581

582 bool LegalInEdge =

583 InEdge && isLegalEdgeBundleZAState(Block.DesiredIncomingState);

584 bool LegalOutEgde =

585 OutEdge && isLegalEdgeBundleZAState(Block.DesiredOutgoingState);

586 if (LegalInEdge) {

588 << getZAStateString(Block.DesiredIncomingState));

589 EdgeStateCounts[Block.DesiredIncomingState]++;

590 }

591 if (LegalOutEgde) {

593 << getZAStateString(Block.DesiredOutgoingState));

594 EdgeStateCounts[Block.DesiredOutgoingState]++;

595 }

596 if (!LegalInEdge && !LegalOutEgde)

599 }

600

601 ZAState BundleState =

602 ZAState(max_element(EdgeStateCounts) - EdgeStateCounts);

603

604 if (BundleState == ZAState::ANY)

605 BundleState = ZAState::ACTIVE;

606

608 dbgs() << "Chosen ZA state: " << getZAStateString(BundleState) << '\n'

609 << "Edge counts:";

610 for (auto [State, Count] : enumerate(EdgeStateCounts))

611 dbgs() << " " << getZAStateString(ZAState(State)) << ": " << Count;

612 dbgs() << "\n\n";

613 });

614

615 BundleStates[I] = BundleState;

616 }

617

618 return BundleStates;

619}

620

621std::pair<MachineBasicBlock::iterator, LiveRegs>

622MachineSMEABI::findStateChangeInsertionPoint(

627 if (Inst != Block.Insts.end()) {

628 InsertPt = Inst->InsertPt;

629 PhysLiveRegs = Inst->PhysLiveRegs;

630 } else {

631 InsertPt = MBB.getFirstTerminator();

632 PhysLiveRegs = Block.PhysLiveRegsAtExit;

633 }

634

635 if (PhysLiveRegs == LiveRegs::None)

636 return {InsertPt, PhysLiveRegs};

637

638

640 if (Inst == Block.Insts.begin()) {

641 PrevStateChangeI = MBB.begin();

642 } else {

643

644

645

646

647 PrevStateChangeI = std::prev(Inst)->InsertPt;

648 }

649

650

652 setPhysLiveRegs(LiveUnits, PhysLiveRegs);

653 auto BestCandidate = std::make_pair(InsertPt, PhysLiveRegs);

655

656 if (I->getOpcode() == TII->getCallFrameDestroyOpcode() || I->isCall())

657 break;

659 LiveRegs CurrentPhysLiveRegs = getPhysLiveRegs(LiveUnits);

660

661

662 if (!(CurrentPhysLiveRegs & LiveRegs::NZCV))

663 BestCandidate = {I, CurrentPhysLiveRegs};

664 if (CurrentPhysLiveRegs == LiveRegs::None)

665 break;

666 }

667 return BestCandidate;

668}

669

670void MachineSMEABI::insertStateChanges(EmitContext &Context,

671 const FunctionInfo &FnInfo,

675 const BlockInfo &Block = FnInfo.Blocks[MBB.getNumber()];

676 ZAState InState = BundleStates[Bundles.getBundle(MBB.getNumber(),

677 false)];

678

679 ZAState CurrentState = Block.FixedEntryState;

680 if (CurrentState == ZAState::ANY)

681 CurrentState = InState;

682

683 for (auto &Inst : Block.Insts) {

684 if (CurrentState != Inst.NeededState) {

685 auto [InsertPt, PhysLiveRegs] =

686 findStateChangeInsertionPoint(MBB, Block, &Inst);

687 emitStateChange(Context, MBB, InsertPt, CurrentState, Inst.NeededState,

688 PhysLiveRegs);

689 CurrentState = Inst.NeededState;

690 }

691 }

692

693 if (MBB.succ_empty())

694 continue;

695

696 ZAState OutState =

697 BundleStates[Bundles.getBundle(MBB.getNumber(), true)];

698 if (CurrentState != OutState) {

699 auto [InsertPt, PhysLiveRegs] =

700 findStateChangeInsertionPoint(MBB, Block, Block.Insts.end());

701 emitStateChange(Context, MBB, InsertPt, CurrentState, OutState,

702 PhysLiveRegs);

703 }

704 }

705}

706

710 return MBBI->getDebugLoc();

712}

713

714void MachineSMEABI::emitSetupLazySave(EmitContext &Context,

718

719

720 Register TPIDR2 = MRI->createVirtualRegister(&AArch64::GPR64spRegClass);

721 Register TPIDR2Ptr = MRI->createVirtualRegister(&AArch64::GPR64RegClass);

728

730 .addImm(AArch64SysReg::TPIDR2_EL0)

732}

733

734PhysRegSave MachineSMEABI::createPhysRegSave(LiveRegs PhysLiveRegs,

738 PhysRegSave RegSave{PhysLiveRegs};

739 if (PhysLiveRegs & LiveRegs::NZCV) {

740 RegSave.StatusFlags = MRI->createVirtualRegister(&AArch64::GPR64RegClass);

742 .addImm(AArch64SysReg::NZCV)

744 }

745

746

747 if (PhysLiveRegs & LiveRegs::W0) {

748 RegSave.X0Save = MRI->createVirtualRegister(PhysLiveRegs & LiveRegs::W0_HI

749 ? &AArch64::GPR64RegClass

750 : &AArch64::GPR32RegClass);

752 .addReg(PhysLiveRegs & LiveRegs::W0_HI ? AArch64::X0 : AArch64::W0);

753 }

754 return RegSave;

755}

756

757void MachineSMEABI::restorePhyRegSave(const PhysRegSave &RegSave,

761 if (RegSave.StatusFlags != AArch64::NoRegister)

763 .addImm(AArch64SysReg::NZCV)

764 .addReg(RegSave.StatusFlags)

766

767 if (RegSave.X0Save != AArch64::NoRegister)

769 RegSave.PhysLiveRegs & LiveRegs::W0_HI ? AArch64::X0 : AArch64::W0)

770 .addReg(RegSave.X0Save);

771}

772

773void MachineSMEABI::emitRestoreLazySave(EmitContext &Context,

779 Register TPIDR2EL0 = MRI->createVirtualRegister(&AArch64::GPR64RegClass);

780 Register TPIDR2 = AArch64::X0;

781

782

783 PhysRegSave RegSave = createPhysRegSave(PhysLiveRegs, MBB, MBBI, DL);

784

785

787 .addImm(AArch64SVCR::SVCRZA)

789

791 .addImm(AArch64SysReg::TPIDR2_EL0);

792

797

801 .addExternalSymbol(TLI->getLibcallName(RTLIB::SMEABI_TPIDR2_RESTORE))

802 .addRegMask(TRI->SMEABISupportRoutinesCallPreservedMaskFromX0());

803

805 .addImm(AArch64SysReg::TPIDR2_EL0)

806 .addReg(AArch64::XZR);

807

808 restorePhyRegSave(RegSave, MBB, MBBI, DL);

809}

810

813 bool ClearTPIDR2, bool On) {

815

816 if (ClearTPIDR2)

818 .addImm(AArch64SysReg::TPIDR2_EL0)

819 .addReg(AArch64::XZR);

820

821

823 .addImm(AArch64SVCR::SVCRZA)

825}

826

827void MachineSMEABI::emitAllocateLazySaveBuffer(

832 Register SP = MRI->createVirtualRegister(&AArch64::GPR64RegClass);

833 Register SVL = MRI->createVirtualRegister(&AArch64::GPR64RegClass);

835

836

838

839

840 if (Buffer == AArch64::NoRegister) {

841

842

843

844

845

846

848 "Lazy ZA save is not yet supported on Windows");

849 Buffer = MRI->createVirtualRegister(&AArch64::GPR64RegClass);

850

852 .addReg(AArch64::SP);

853

860

862 }

863

864

865 {

866

867

870 "TPIDR2 block initialization is not supported on big-endian targets");

871

872

873

879 }

880}

881

883

888

892

893 Register TPIDR2EL0 = MRI->createVirtualRegister(&AArch64::GPR64RegClass);

896 .addImm(AArch64SysReg::TPIDR2_EL0);

897

898

899 auto CommitZASave =

904 .addExternalSymbol(TLI->getLibcallName(RTLIB::SMEABI_TPIDR2_SAVE))

905 .addRegMask(TRI->SMEABISupportRoutinesCallPreservedMaskFromX0());

906 if (ZeroZA)

908 if (ZeroZT0)

910

912 .addImm(AArch64SVCR::SVCRZA)

915 if (ZeroZA)

919 if (ZeroZT0)

921 }

922}

923

924void MachineSMEABI::emitFullZASaveRestore(EmitContext &Context,

927 LiveRegs PhysLiveRegs, bool IsSave) {

930 Register BufferPtr = AArch64::X0;

931

932 PhysRegSave RegSave = createPhysRegSave(PhysLiveRegs, MBB, MBBI, DL);

933

934

936 .addReg(Context.getAgnosticZABufferPtr(*MF));

937

938

942 IsSave ? RTLIB::SMEABI_SME_SAVE : RTLIB::SMEABI_SME_RESTORE))

944 *MF,

946

947 restorePhyRegSave(RegSave, MBB, MBBI, DL);

948}

949

950void MachineSMEABI::emitZT0SaveRestore(EmitContext &Context,

953 bool IsSave) {

955 Register ZT0Save = MRI->createVirtualRegister(&AArch64::GPR64spRegClass);

956

961

962 if (IsSave) {

964 .addReg(AArch64::ZT0)

966 } else {

969 }

970}

971

972void MachineSMEABI::emitAllocateFullZASaveBuffer(

975

977 return;

978

980 Register BufferPtr = Context.getAgnosticZABufferPtr(*MF);

981 Register BufferSize = MRI->createVirtualRegister(&AArch64::GPR64RegClass);

982

983 PhysRegSave RegSave = createPhysRegSave(PhysLiveRegs, MBB, MBBI, DL);

984

985

986 {

990 .addExternalSymbol(TLI->getLibcallName(RTLIB::SMEABI_SME_STATE_SIZE))

996 .addReg(AArch64::X0);

997 }

998

999

1000 {

1003 .addReg(AArch64::SP)

1007 .addReg(AArch64::SP);

1008

1009

1011 }

1012

1013 restorePhyRegSave(RegSave, MBB, MBBI, DL);

1014}

1015

1016struct FromState {

1017 ZAState From;

1018

1019 constexpr uint8_t to(ZAState To) const {

1020 static_assert(NUM_ZA_STATE < 16, "expected ZAState to fit in 4-bits");

1022 }

1023};

1024

1025constexpr FromState transitionFrom(ZAState From) { return FromState{From}; }

1026

1027void MachineSMEABI::emitStateChange(EmitContext &Context,

1030 ZAState From, ZAState To,

1032

1033 if (From == ZAState::ANY || To == ZAState::ANY)

1034 return;

1035

1036

1037

1038 if (From == ZAState::ENTRY && To == ZAState::OFF)

1039 return;

1040

1041

1042

1043 if (From == ZAState::ENTRY) {

1045 "ENTRY state only valid in entry block");

1046 emitSMEPrologue(MBB, MBB.getFirstNonPHI());

1047 if (To == ZAState::ACTIVE)

1048 return;

1049

1050

1051

1052

1053 From = ZAState::ACTIVE;

1054 }

1055

1058 bool HasZT0State = SMEFnAttrs.hasZT0State();

1059 bool HasZAState = IsAgnosticZA || SMEFnAttrs.hasZAState();

1060

1061 switch (transitionFrom(From).to(To)) {

1062

1063 case transitionFrom(ZAState::ACTIVE).to(ZAState::ACTIVE_ZT0_SAVED):

1064 emitZT0SaveRestore(Context, MBB, InsertPt, true);

1065 break;

1066 case transitionFrom(ZAState::ACTIVE_ZT0_SAVED).to(ZAState::ACTIVE):

1067 emitZT0SaveRestore(Context, MBB, InsertPt, false);

1068 break;

1069

1070

1071 case transitionFrom(ZAState::ACTIVE).to(ZAState::LOCAL_SAVED):

1072 case transitionFrom(ZAState::ACTIVE_ZT0_SAVED).to(ZAState::LOCAL_SAVED):

1073 if (HasZT0State && From == ZAState::ACTIVE)

1074 emitZT0SaveRestore(Context, MBB, InsertPt, true);

1075 if (HasZAState)

1076 emitZASave(Context, MBB, InsertPt, PhysLiveRegs);

1077 break;

1078

1079

1080 case transitionFrom(ZAState::ACTIVE).to(ZAState::LOCAL_COMMITTED):

1081

1082

1083 assert(HasZT0State && !HasZAState && "Expect to only have ZT0 state.");

1084 emitZT0SaveRestore(Context, MBB, InsertPt, true);

1085 emitZAMode(MBB, InsertPt, false, false);

1086 break;

1087

1088

1089 case transitionFrom(ZAState::LOCAL_COMMITTED).to(ZAState::OFF):

1090 case transitionFrom(ZAState::LOCAL_COMMITTED).to(ZAState::LOCAL_SAVED):

1091

1092 break;

1093

1094

1095 case transitionFrom(ZAState::LOCAL_COMMITTED).to(ZAState::ACTIVE):

1096 case transitionFrom(ZAState::LOCAL_COMMITTED).to(ZAState::ACTIVE_ZT0_SAVED):

1097 case transitionFrom(ZAState::LOCAL_SAVED).to(ZAState::ACTIVE):

1098 if (HasZAState)

1099 emitZARestore(Context, MBB, InsertPt, PhysLiveRegs);

1100 else

1101 emitZAMode(MBB, InsertPt, false, true);

1102 if (HasZT0State && To == ZAState::ACTIVE)

1103 emitZT0SaveRestore(Context, MBB, InsertPt, false);

1104 break;

1105

1106

1107 case transitionFrom(ZAState::ACTIVE).to(ZAState::OFF):

1108 case transitionFrom(ZAState::ACTIVE_ZT0_SAVED).to(ZAState::OFF):

1109 case transitionFrom(ZAState::LOCAL_SAVED).to(ZAState::OFF):

1111 "Did not expect to turn ZA off in shared/agnostic ZA function");

1112 emitZAMode(MBB, InsertPt, From == ZAState::LOCAL_SAVED,

1113 false);

1114 break;

1115

1116 default:

1117 dbgs() << "Error: Transition from " << getZAStateString(From) << " to "

1118 << getZAStateString(To) << '\n';

1120 }

1121}

1122

1123}

1124

1125 INITIALIZE_PASS(MachineSMEABI, "aarch64-machine-sme-abi", "Machine SME ABI",

1126 false, false)

1127

1130 return false;

1131

1133 SMEAttrs SMEFnAttrs = AFI->getSMEFnAttrs();

1134 if (!SMEFnAttrs.hasZAState() && !SMEFnAttrs.hasZT0State() &&

1135 !SMEFnAttrs.hasAgnosticZAInterface())

1136 return false;

1137

1138 assert(MF.getRegInfo().isSSA() && "Expected to be run on SSA form!");

1139

1140 this->MF = &MF;

1142 TII = Subtarget->getInstrInfo();

1143 TRI = Subtarget->getRegisterInfo();

1144 MRI = &MF.getRegInfo();

1145

1147 getAnalysis().getEdgeBundles();

1148

1149 FunctionInfo FnInfo = collectNeededZAStates(SMEFnAttrs);

1150

1152

1153

1154

1155

1156

1157

1158

1159

1160

1161

1162

1163

1164

1165

1166

1167

1168

1169

1170

1171

1172

1173

1174

1175

1176

1177

1178

1179

1180

1181

1182

1183 for (bool Forwards : {true, false})

1184 propagateDesiredStates(FnInfo, Forwards);

1185 }

1186

1188

1189 EmitContext Context;

1190 insertStateChanges(Context, FnInfo, Bundles, BundleStates);

1191

1192 if (Context.needsSaveBuffer()) {

1193 if (FnInfo.AfterSMEProloguePt) {

1194

1195

1197 emitAllocateZASaveBuffer(Context, *MBBI->getParent(), MBBI,

1198 FnInfo.PhysLiveRegsAfterSMEPrologue);

1199 } else {

1201 emitAllocateZASaveBuffer(

1203 FnInfo.Blocks[EntryBlock.getNumber()].PhysLiveRegsAtEntry);

1204 }

1205 }

1206

1207 return true;

1208}

1209

1211 return new MachineSMEABI(OptLevel);

1212}

unsigned const MachineRegisterInfo * MRI

static constexpr unsigned ZERO_ALL_ZA_MASK

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

MachineBasicBlock MachineBasicBlock::iterator DebugLoc DL

MachineBasicBlock MachineBasicBlock::iterator MBBI

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

const HexagonInstrInfo * TII

This file implements the LivePhysRegs utility for tracking liveness of physical registers.

#define ENTRY(ASMNAME, ENUM)

static DebugLoc getDebugLoc(MachineBasicBlock::instr_iterator FirstMI, MachineBasicBlock::instr_iterator LastMI)

Return the first DebugLoc that has line number information, given a range of instructions.

#define INITIALIZE_PASS(passName, arg, name, cfg, analysis)

This file defines the SmallVector class.

AArch64FunctionInfo - This class is derived from MachineFunctionInfo and contains private AArch64-spe...

SMEAttrs getSMEFnAttrs() const

bool isTargetWindows() const

const AArch64RegisterInfo * getRegisterInfo() const override

const AArch64TargetLowering * getTargetLowering() const override

bool isLittleEndian() const

Represent the analysis usage information of a pass.

AnalysisUsage & addPreservedID(const void *ID)

AnalysisUsage & addRequired()

LLVM_ABI void setPreservesCFG()

This function should be called by the pass, iff they do not:

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

ArrayRef< unsigned > getBlocks(unsigned Bundle) const

getBlocks - Return an array of blocks that are connected to Bundle.

unsigned getBundle(unsigned N, bool Out) const

getBundle - Return the ingoing (Out = false) or outgoing (Out = true) bundle number for basic block N

unsigned getNumBundles() const

getNumBundles - Return the total number of bundles in the CFG.

FunctionPass class - This class is used to implement most global optimizations.

A set of register units used to track register liveness.

bool available(MCRegister Reg) const

Returns true if no part of physical register Reg is live.

void addReg(MCRegister Reg)

Adds register units covered by physical register Reg.

LLVM_ABI void stepBackward(const MachineInstr &MI)

Updates liveness when stepping backwards over the instruction MI.

LLVM_ABI void addLiveOuts(const MachineBasicBlock &MBB)

Adds registers living out of block MBB.

int getNumber() const

MachineBasicBlocks are uniquely numbered at the function level, unless they're not in a MachineFuncti...

LLVM_ABI iterator getFirstNonPHI()

Returns a pointer to the first instruction in this block that is not a PHINode instruction.

MachineInstrBundleIterator< MachineInstr > iterator

The MachineFrameInfo class represents an abstract stack frame until prolog/epilog code is inserted.

LLVM_ABI int CreateStackObject(uint64_t Size, Align Alignment, bool isSpillSlot, const AllocaInst *Alloca=nullptr, uint8_t ID=0)

Create a new statically sized stack object, returning a nonnegative identifier to represent it.

LLVM_ABI int CreateSpillStackObject(uint64_t Size, Align Alignment)

Create a new statically sized stack object that represents a spill slot, returning a nonnegative iden...

LLVM_ABI int CreateVariableSizedObject(Align Alignment, const AllocaInst *Alloca)

Notify the MachineFrameInfo object that a variable sized object has been created.

MachineFunctionPass - This class adapts the FunctionPass interface to allow convenient creation of pa...

void getAnalysisUsage(AnalysisUsage &AU) const override

getAnalysisUsage - Subclasses that override getAnalysisUsage must call this.

MachineFrameInfo & getFrameInfo()

getFrameInfo - Return the frame info object for the current function.

MachineRegisterInfo & getRegInfo()

getRegInfo - Return information about the registers currently in use.

MachineBasicBlock * getBlockNumbered(unsigned N) const

getBlockNumbered - MachineBasicBlocks are automatically numbered when they are inserted into the mach...

unsigned getNumBlockIDs() const

getNumBlockIDs - Return the number of MBB ID's allocated.

Ty * getInfo()

getInfo - Keep track of various per-function pieces of information for backends that would like to do...

const MachineInstrBuilder & addExternalSymbol(const char *FnName, unsigned TargetFlags=0) const

const MachineInstrBuilder & addImm(int64_t Val) const

Add a new immediate operand.

const MachineInstrBuilder & addFrameIndex(int Idx) const

const MachineInstrBuilder & addRegMask(const uint32_t *Mask) const

const MachineInstrBuilder & addReg(Register RegNo, unsigned flags=0, unsigned SubReg=0) const

Add a new virtual register operand.

const MachineInstrBuilder & addDef(Register RegNo, unsigned Flags=0, unsigned SubReg=0) const

Add a virtual register definition operand.

Representation of each machine instruction.

MachineOperand class - Representation of each machine instruction operand.

bool isReg() const

isReg - Tests if this is a MO_Register operand.

getReg - Returns the register number.

MachineRegisterInfo - Keep track of information for virtual and physical registers,...

LLVM_ABI Register createVirtualRegister(const TargetRegisterClass *RegClass, StringRef Name="")

createVirtualRegister - Create and return a new virtual register in the function with the specified r...

Wrapper class representing virtual and physical registers.

constexpr bool isPhysical() const

Return true if the specified register number is in the physical register namespace.

SMEAttrs is a utility class to parse the SME ACLE attributes on functions.

bool hasAgnosticZAInterface() const

bool hasPrivateZAInterface() const

bool hasSharedZAInterface() const

typename SuperClass::const_iterator const_iterator

void push_back(const T &Elt)

This is a 'vector' (really, a variable-sized array), optimized for the case when the array is small.

StringRef - Represent a constant reference to a string, i.e.

TargetInstrInfo - Interface to description of machine instruction set.

TargetRegisterInfo base class - We assume that the target defines a static array of TargetRegisterDes...

#define llvm_unreachable(msg)

Marks that the current location is not supposed to be reachable.

static unsigned getArithExtendImm(AArch64_AM::ShiftExtendType ET, unsigned Imm)

getArithExtendImm - Encode the extend type and shift amount for an arithmetic instruction: imm: 3-bit...

CallingConv Namespace - This namespace contains an enum with a value for the well-known calling conve...

unsigned ID

LLVM IR allows to use arbitrary numbers as calling convention identifiers.

@ AArch64_SME_ABI_Support_Routines_PreserveMost_From_X1

Preserve X1-X15, X19-X29, SP, Z0-Z31, P0-P15.

@ Implicit

Not emitted register (e.g. carry, or temporary result).

@ Define

This is an optimization pass for GlobalISel generic memory operations.

MachineInstrBuilder BuildMI(MachineFunction &MF, const MIMetadata &MIMD, const MCInstrDesc &MCID)

Builder interface. Specify how to create the initial instruction itself.

auto enumerate(FirstRange &&First, RestRanges &&...Rest)

Given two or more input ranges, returns a new range whose values are tuples (A, B,...

auto successors(const MachineBasicBlock *BB)

FunctionPass * createMachineSMEABIPass(CodeGenOptLevel)

Definition MachineSMEABIPass.cpp:1210

LLVM_ABI char & MachineDominatorsID

MachineDominators - This pass is a machine dominators analysis pass.

LLVM_ABI void reportFatalInternalError(Error Err)

Report a fatal error that indicates a bug in LLVM.

LLVM_ABI char & MachineLoopInfoID

MachineLoopInfo - This pass is a loop analysis pass.

bool any_of(R &&range, UnaryPredicate P)

Provide wrappers to std::any_of which take ranges instead of having to pass begin/end explicitly.

auto reverse(ContainerTy &&C)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

FunctionAddr VTableAddr Count

CodeGenOptLevel

Code generation optimization level.

@ LLVM_MARK_AS_BITMASK_ENUM

uint16_t MCPhysReg

An unsigned integer type large enough to represent all physical registers, but not necessarily virtua...

auto max_element(R &&Range)

Provide wrappers to std::max_element which take ranges instead of having to pass begin/end explicitly...

auto predecessors(const MachineBasicBlock *BB)

This struct is a compact representation of a valid (non-zero power of two) alignment.

Incoming for lane maks phi as machine instruction, incoming register Reg and incoming block Block are...