aarch64.h Source File (original) (raw)

13#ifndef LLVM_EXECUTIONENGINE_JITLINK_AARCH64_H

14#define LLVM_EXECUTIONENGINE_JITLINK_AARCH64_H

21namespace llvm {

100

101

102

103

104

106

107

108

109

110

111

112

113

114

115

116

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

184

185

186

187

188

189

190

191

192

193

194

195

197

198

199

200

201

202

203

204

205

206

207

208

209

210

211

212

213

214

215

216

217

218

220

221

222

223

224

225

226

227

228

229

230

231

232

234

235

236

237

238

239

240

241

242

243

244

245

246

247

249

250

251

252

253

254

255

256

257

258

259

260

261

262

263

264

266

267

268

269

270

271

272

273

274

275

276

277

278

279

281

282

283

284

285

286

287

288

289

290

291

292

293

294

295

296

297

298

299

301

302

303

304

305

306

307

308

309

310

311

312

313

314

315

316

317

318

319

321

322

323

324

325

326

327

328

329

330

331

332

333

334

335

336

338

339

340

341

342

343

344

345

346

347

348

349

350

351

352

353

354

355

356

358

359

360

361

362

363

364

365

366

367

368

369

370

371

373

374

375

376

377

378

379

380

381

382

383

384

385

386

388

389

390

391

392

393

394

395

396

397

398

399

400

401

403

404

405

406

407

408

409

410

411

412

413

414

415

416

418};

419

420

421

423

424

426 constexpr uint32_t LoadStoreImm12Mask = 0x3b000000;

427 return (Instr & LoadStoreImm12Mask) == 0x39000000;

428}

429

431 constexpr uint32_t TestAndBranchImm14Mask = 0x7e000000;

432 return (Instr & TestAndBranchImm14Mask) == 0x36000000;

433}

434

436 constexpr uint32_t CondBranchImm19Mask = 0xfe000000;

437 return (Instr & CondBranchImm19Mask) == 0x54000000;

438}

439

441 constexpr uint32_t CompAndBranchImm19Mask = 0x7e000000;

442 return (Instr & CompAndBranchImm19Mask) == 0x34000000;

443}

444

446 constexpr uint32_t ADRMask = 0x9f000000;

447 return (Instr & ADRMask) == 0x10000000;

448}

449

451 constexpr uint32_t LDRLitMask = 0x3b000000;

452 return (Instr & LDRLitMask) == 0x18000000;

453}

454

455

456

457

458

459

460

462 constexpr uint32_t Vec128Mask = 0x04800000;

463

465 uint32_t ImplicitShift = Instr >> 30;

466 if (ImplicitShift == 0)

467 if ((Instr & Vec128Mask) == Vec128Mask)

468 ImplicitShift = 4;

469

470 return ImplicitShift;

471 }

472

473 return 0;

474}

475

476

478 constexpr uint32_t MoveWideImm16Mask = 0x5f9fffe0;

479 return (Instr & MoveWideImm16Mask) == 0x52800000;

480}

481

482

483

484

485

488 uint32_t ImplicitShift = (Instr >> 21) & 0b11;

489 return ImplicitShift << 4;

490 }

491

492 return 0;

493}

494

495

497 const Symbol *GOTSymbol) {

499

500 char *BlockWorkingMem = B.getAlreadyMutableContent().data();

501 char *FixupPtr = BlockWorkingMem + E.getOffset();

503

504 switch (E.getKind()) {

506 uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();

507 *(ulittle64_t *)FixupPtr = Value;

508 break;

509 }

511 uint64_t Value = E.getTarget().getAddress().getValue() + E.getAddend();

512 if (Value > std::numeric_limits<uint32_t>::max())

514 *(ulittle32_t *)FixupPtr = Value;

515 break;

516 }

523 Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();

524 else

525 Value = FixupAddress - E.getTarget().getAddress() + E.getAddend();

526

528 if (Value < std::numeric_limits<int32_t>::min() ||

529 Value > std::numeric_limits<int32_t>::max())

531 *(little32_t *)FixupPtr = Value;

532 } else

533 *(little64_t *)FixupPtr = Value;

534 break;

535 }

538 "Branch-inst is not 32-bit aligned");

539

540 int64_t Value = E.getTarget().getAddress() - FixupAddress + E.getAddend();

541

544 "aligned");

545

546 if (Value < -(1 << 27) || Value > ((1 << 27) - 1))

548

549 uint32_t RawInstr = *(little32_t *)FixupPtr;

550 assert((RawInstr & 0x7fffffff) == 0x14000000 &&

551 "RawInstr isn't a B or BR immediate instruction");

553 uint32_t FixedInstr = RawInstr | Imm;

554 *(little32_t *)FixupPtr = FixedInstr;

555 break;

556 }

559 (E.getTarget().getAddress() + E.getAddend()).getValue();

560

561 uint32_t RawInstr = *(ulittle32_t *)FixupPtr;

563 "RawInstr isn't a MOVK/MOVZ instruction");

564

566 uint32_t Imm = (TargetOffset >> ImmShift) & 0xffff;

567 uint32_t FixedInstr = RawInstr | (Imm << 5);

568 *(ulittle32_t *)FixupPtr = FixedInstr;

569 break;

570 }

572 assert((FixupAddress.getValue() & 0x3) == 0 && "LDR is not 32-bit aligned");

573 uint32_t RawInstr = *(ulittle32_t *)FixupPtr;

575 int64_t Delta = E.getTarget().getAddress() + E.getAddend() - FixupAddress;

576 if (Delta & 0x3)

578 "aligned");

581 uint32_t EncodedImm = ((static_cast<uint32_t>(Delta) >> 2) & 0x7ffff) << 5;

582 uint32_t FixedInstr = RawInstr | EncodedImm;

583 *(ulittle32_t *)FixupPtr = FixedInstr;

584 break;

585 }

587 assert((FixupAddress.getValue() & 0x3) == 0 && "ADR is not 32-bit aligned");

588 uint32_t RawInstr = *(ulittle32_t *)FixupPtr;

589 assert(isADR(RawInstr) && "RawInstr is not an ADR");

590 int64_t Delta = E.getTarget().getAddress() + E.getAddend() - FixupAddress;

593 auto UDelta = static_cast<uint32_t>(Delta);

594 uint32_t EncodedImmHi = ((UDelta >> 2) & 0x7ffff) << 5;

595 uint32_t EncodedImmLo = (UDelta & 0x3) << 29;

596 uint32_t FixedInstr = RawInstr | EncodedImmHi | EncodedImmLo;

597 *(ulittle32_t *)FixupPtr = FixedInstr;

598 break;

599 }

602 "Test and branch is not 32-bit aligned");

603 uint32_t RawInstr = *(ulittle32_t *)FixupPtr;

605 "RawInstr is not a test and branch");

606 int64_t Delta = E.getTarget().getAddress() + E.getAddend() - FixupAddress;

607 if (Delta & 0x3)

609 "Test and branch literal target is not 32-bit aligned");

612 uint32_t EncodedImm = ((static_cast<uint32_t>(Delta) >> 2) & 0x3fff) << 5;

613 uint32_t FixedInstr = RawInstr | EncodedImm;

614 *(ulittle32_t *)FixupPtr = FixedInstr;

615 break;

616 }

619 "Conditional branch is not 32-bit aligned");

620 uint32_t RawInstr = *(ulittle32_t *)FixupPtr;

622 "RawInstr is not a conditional branch");

623 int64_t Delta = E.getTarget().getAddress() + E.getAddend() - FixupAddress;

624 if (Delta & 0x3)

626 "Conditional branch literal target is not 32-bit "

627 "aligned");

630 uint32_t EncodedImm = ((static_cast<uint32_t>(Delta) >> 2) & 0x7ffff) << 5;

631 uint32_t FixedInstr = RawInstr | EncodedImm;

632 *(ulittle32_t *)FixupPtr = FixedInstr;

633 break;

634 }

637 (E.getTarget().getAddress().getValue() + E.getAddend()) &

638 ~static_cast<uint64_t>(4096 - 1);

640 FixupAddress.getValue() & ~static_cast<uint64_t>(4096 - 1);

641

642 int64_t PageDelta = TargetPage - PCPage;

645

646 uint32_t RawInstr = *(ulittle32_t *)FixupPtr;

647 assert((RawInstr & 0xffffffe0) == 0x90000000 &&

648 "RawInstr isn't an ADRP instruction");

649 uint32_t ImmLo = (static_cast<uint64_t>(PageDelta) >> 12) & 0x3;

650 uint32_t ImmHi = (static_cast<uint64_t>(PageDelta) >> 14) & 0x7ffff;

651 uint32_t FixedInstr = RawInstr | (ImmLo << 29) | (ImmHi << 5);

652 *(ulittle32_t *)FixupPtr = FixedInstr;

653 break;

654 }

657 (E.getTarget().getAddress() + E.getAddend()).getValue() & 0xfff;

658

659 uint32_t RawInstr = *(ulittle32_t *)FixupPtr;

661

662 if (TargetOffset & ((1 << ImmShift) - 1))

664

665 uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10;

666 uint32_t FixedInstr = RawInstr | EncodedImm;

667 *(ulittle32_t *)FixupPtr = FixedInstr;

668 break;

669 }

671 assert(GOTSymbol && "No GOT section symbol");

673 (E.getTarget().getAddress() + E.getAddend()).getValue() -

674 (GOTSymbol->getAddress().getValue() & ~static_cast<uint64_t>(4096 - 1));

675 if (TargetOffset > 0x7fff)

677

678 uint32_t RawInstr = *(ulittle32_t *)FixupPtr;

679 const unsigned ImmShift = 3;

680 if (TargetOffset & ((1 << ImmShift) - 1))

682

683 uint32_t EncodedImm = (TargetOffset >> ImmShift) << 10;

684 uint32_t FixedInstr = RawInstr | EncodedImm;

685 *(ulittle32_t *)FixupPtr = FixedInstr;

686 break;

687 }

688 default:

690 "In graph " + G.getName() + ", section " + B.getSection().getName() +

692 }

693

695}

696

697

699

700

702

703

704

705

706

707

708

709

711

712

713

714

715

716

717

718

719

720

721

723 Symbol *InitialTarget = nullptr,

724 uint64_t InitialAddend = 0) {

727 if (InitialTarget)

728 B.addEdge(Pointer64, 0, *InitialTarget, InitialAddend);

729 return G.addAnonymousSymbol(B, 0, 8, false, false);

730}

731

732

733

734

735

736

737

739 Symbol &PointerSymbol) {

742 B.addEdge(Page21, 0, PointerSymbol, 0);

744 return B;

745}

746

747

748

749

750

753 Symbol &PointerSymbol) {

754 return G.addAnonymousSymbol(

757}

758

759

760

761

762

763

764

766

767

769 Section &TrampolineSection,

770 Symbol &ReentrySymbol) {

774 return B;

775}

776

778 Section &TrampolineSection,

779 Symbol &ReentrySymbol) {

780 return G.addAnonymousSymbol(

783}

784

785

787public:

789

791 if ((GOTSection = G.findSectionByName(getSectionName())))

792 registerExistingEntries();

793 }

794

796 Edge::Kind KindToSet = Edge::Invalid;

797 const char *BlockWorkingMem = B->getContent().data();

798 const char *FixupPtr = BlockWorkingMem + E.getOffset();

799

800 switch (E.getKind()) {

804 break;

805 }

810 (void)RawInstr;

811 assert(E.getAddend() == 0 &&

812 "GOTPageOffset12/TLVPageOffset12 with non-zero addend");

813 assert((RawInstr & 0xfffffc00) == 0xf9400000 &&

814 "RawInstr isn't a 64-bit LDR immediate");

815 break;

816 }

820 (void)RawInstr;

821 assert(E.getAddend() == 0 && "GOTPageOffset15 with non-zero addend");

822 assert((RawInstr & 0xfffffc00) == 0xf9400000 &&

823 "RawInstr isn't a 64-bit LDR immediate");

824 break;

825 }

828 break;

829 }

830 default:

831 return false;

832 }

833 assert(KindToSet != Edge::Invalid &&

834 "Fell through switch, but no new kind to set");

836 dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "

837 << B->getFixupAddress(E) << " (" << B->getAddress() << " + "

838 << formatv("{0:x}", E.getOffset()) << ")\n";

839 });

840 E.setKind(KindToSet);

842 return true;

843 }

844

848

849private:

851 if (!GOTSection)

854 return *GOTSection;

855 }

856

857 LLVM_ABI void registerExistingEntries();

858

859 Section *GOTSection = nullptr;

860};

861

862

864public:

866

871

875 dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at "

876 << B->getFixupAddress(E) << " (" << B->getAddress() << " + "

877 << formatv("{0:x}", E.getOffset()) << ")\n";

878 });

880 return true;

881 }

882 return false;

883 }

884

889

890public:

897

899

902};

903

904

906

907

908

909

910

911

912

913

914

916

917

918

919

920

921

922

923

924

926

927}

928}

929}

930

931#endif

assert(UImm &&(UImm !=~static_cast< T >(0)) &&"Invalid immediate!")

static GCRegistry::Add< CoreCLRGC > E("coreclr", "CoreCLR-compatible GC")

static GCRegistry::Add< OcamlGC > B("ocaml", "ocaml 3.10-compatible GC")

#define DEBUG_WITH_TYPE(TYPE,...)

DEBUG_WITH_TYPE macro - This macro should be used by passes to emit debug information.

Lightweight error class with error context and mandatory checking.

static ErrorSuccess success()

Create a success value.

StringRef - Represent a constant reference to a string, i.e.

Target - Wrapper for Target specific information.

LLVM Value Representation.

An Addressable with content and edges.

Represents fixups and constraints in the LinkGraph.

Represents an object file section.

orc::ExecutorAddr getAddress() const

Returns the address of this symbol.

A CRTP base for tables that are built on demand, e.g.

Symbol & getEntryForTarget(LinkGraph &G, Symbol &Target)

Global Offset Table Builder.

Definition aarch64.h:786

GOTTableManager(LinkGraph &G)

Definition aarch64.h:790

Symbol & createEntry(LinkGraph &G, Symbol &Target)

Definition aarch64.h:845

bool visitEdge(LinkGraph &G, Block *B, Edge &E)

Definition aarch64.h:795

static StringRef getSectionName()

Definition aarch64.h:788

static StringRef getSectionName()

Definition aarch64.h:865

PLTTableManager(LinkGraph &G, GOTTableManager &GOT)

Definition aarch64.h:867

GOTTableManager & GOT

Definition aarch64.h:900

bool visitEdge(LinkGraph &G, Block *B, Edge &E)

Definition aarch64.h:872

Symbol & createEntry(LinkGraph &G, Symbol &Target)

Definition aarch64.h:885

LLVM_ABI void registerExistingEntries()

Section * StubsSection

Definition aarch64.h:901

Section & getStubsSection(LinkGraph &G)

Definition aarch64.h:891

Represents an address in the executor process.

uint64_t getValue() const

LLVM_ABI const char NullPointerContent[PointerSize]

AArch64 null pointer content.

bool isLDRLiteral(uint32_t Instr)

Definition aarch64.h:450

bool isADR(uint32_t Instr)

Definition aarch64.h:445

bool isCondBranchImm19(uint32_t Instr)

Definition aarch64.h:435

LLVM_ABI const char ReentryTrampolineContent[8]

AArch64 reentry trampoline.

constexpr uint64_t PointerSize

aarch64 pointer size.

Definition aarch64.h:698

Error applyFixup(LinkGraph &G, Block &B, const Edge &E, const Symbol *GOTSymbol)

Apply fixup expression for edge to block content.

Definition aarch64.h:496

Block & createReentryTrampolineBlock(LinkGraph &G, Section &TrampolineSection, Symbol &ReentrySymbol)

Create a block of N reentry trampolines.

Definition aarch64.h:768

bool isMoveWideImm16(uint32_t Instr)

Definition aarch64.h:477

bool isCompAndBranchImm19(uint32_t Instr)

Definition aarch64.h:440

LLVM_ABI const char PointerJumpStubContent[12]

bool isLoadStoreImm12(uint32_t Instr)

Definition aarch64.h:425

LLVM_ABI Error createEmptyPointerSigningFunction(LinkGraph &G)

Creates a pointer signing function section, block, and symbol to reserve space for a signing function...

LLVM_ABI Error lowerPointer64AuthEdgesToSigningFunction(LinkGraph &G)

Given a LinkGraph containing Pointer64Authenticated edges, transform those edges to Pointer64 and add...

LLVM_ABI const char * getEdgeKindName(Edge::Kind K)

Returns a string name for the given aarch64 edge.

Block & createPointerJumpStubBlock(LinkGraph &G, Section &StubSection, Symbol &PointerSymbol)

Create a jump stub block that jumps via the pointer at the given symbol.

Definition aarch64.h:738

Symbol & createAnonymousPointer(LinkGraph &G, Section &PointerSection, Symbol *InitialTarget=nullptr, uint64_t InitialAddend=0)

Creates a new pointer block in the given section and returns an Anonymous symbol pointing to it.

Definition aarch64.h:722

LLVM_ABI const char * getPointerSigningFunctionSectionName()

Returns the name of the pointer signing function section.

Symbol & createAnonymousReentryTrampoline(LinkGraph &G, Section &TrampolineSection, Symbol &ReentrySymbol)

Definition aarch64.h:777

EdgeKind_aarch64

Represents aarch64 fixups and other aarch64-specific edge kinds.

Definition aarch64.h:26

@ LDRLiteral19

The signed 21-bit delta from the fixup to the target.

Definition aarch64.h:219

@ Pointer64Authenticated

An arm64e authenticated pointer relocation.

Definition aarch64.h:63

@ RequestGOTAndTransformToPageOffset15

A GOT entry getter/constructor, transformed to Pageoffset15 pointing at the GOT entry for the origina...

Definition aarch64.h:337

@ CondBranch19PCRel

A 19-bit PC-relative conditional branch.

Definition aarch64.h:183

@ RequestTLVPAndTransformToPageOffset12

A TLVP entry getter/constructor, transformed to PageOffset12.

Definition aarch64.h:387

@ RequestTLSDescEntryAndTransformToPageOffset12

A TLSDesc entry getter/constructor, transformed to PageOffset12.

Definition aarch64.h:417

@ Page21

The signed 21-bit delta from the fixup page to the page containing the target.

Definition aarch64.h:248

@ Branch26PCRel

A 26-bit PC-relative branch.

Definition aarch64.h:139

@ Delta32

A 32-bit delta.

Definition aarch64.h:96

@ Pointer64

A plain 64-bit pointer value relocation.

Definition aarch64.h:33

@ Delta64

A 64-bit delta.

Definition aarch64.h:83

@ Pointer32

A plain 32-bit pointer value relocation.

Definition aarch64.h:74

@ RequestTLVPAndTransformToPage21

A TLVP entry getter/constructor, transformed to Page21.

Definition aarch64.h:372

@ GotPageOffset15

The 15-bit offset of the GOT entry from the GOT table.

Definition aarch64.h:280

@ MoveWide16

A 16-bit slice of the target address (which slice depends on the instruction at the fixup location).

Definition aarch64.h:196

@ TestAndBranch14PCRel

A 14-bit PC-relative test and branch.

Definition aarch64.h:161

@ RequestGOTAndTransformToPage21

A GOT entry getter/constructor, transformed to Page21 pointing at the GOT entry for the original targ...

Definition aarch64.h:300

@ ADRLiteral21

The signed 21-bit delta from the fixup to the target.

Definition aarch64.h:233

@ RequestGOTAndTransformToPageOffset12

A GOT entry getter/constructor, transformed to Pageoffset12 pointing at the GOT entry for the origina...

Definition aarch64.h:320

@ NegDelta32

A 32-bit negative delta.

Definition aarch64.h:117

@ NegDelta64

A 64-bit negative delta.

Definition aarch64.h:105

@ RequestGOTAndTransformToDelta32

A GOT entry getter/constructor, transformed to Delta32 pointing at the GOT entry for the original tar...

Definition aarch64.h:357

@ PageOffset12

The 12-bit (potentially shifted) offset of the target within its page.

Definition aarch64.h:265

@ RequestTLSDescEntryAndTransformToPage21

A TLSDesc entry getter/constructor, transformed to Page21.

Definition aarch64.h:402

bool isTestAndBranchImm14(uint32_t Instr)

Definition aarch64.h:430

unsigned getPageOffset12Shift(uint32_t Instr)

Definition aarch64.h:461

unsigned getMoveWide16Shift(uint32_t Instr)

Definition aarch64.h:486

Symbol & createAnonymousPointerJumpStub(LinkGraph &G, Section &StubSection, Symbol &PointerSymbol)

Create a jump stub that jumps via the pointer at the given symbol and an anonymous symbol pointing to...

Definition aarch64.h:751

LLVM_ABI Error makeTargetOutOfRangeError(const LinkGraph &G, const Block &B, const Edge &E)

Create an out of range error for the given edge in the given block.

detail::packed_endian_specific_integral< uint32_t, llvm::endianness::little, unaligned > ulittle32_t

This is an optimization pass for GlobalISel generic memory operations.

constexpr bool isInt(int64_t x)

Checks if an integer fits into the given bit width.

auto formatv(bool Validate, const char *Fmt, Ts &&...Vals)

LLVM_ABI raw_ostream & dbgs()

dbgs() - This returns a reference to a raw_ostream for debugging messages.

Error make_error(ArgTs &&... Args)

Make a Error instance representing failure using the given error info type.