AArch64MacroFusion.cpp Source File (original) (raw)

19using namespace llvm;

24 if (SecondMI.getOpcode() != AArch64::Bcc)

25 return false;

28 if (FirstMI == nullptr)

29 return true;

36 return false;

37 }

40 case AArch64::ADDSWri:

41 case AArch64::ADDSWrr:

42 case AArch64::ADDSXri:

43 case AArch64::ADDSXrr:

44 case AArch64::ANDSWri:

45 case AArch64::ANDSWrr:

46 case AArch64::ANDSXri:

47 case AArch64::ANDSXrr:

48 case AArch64::SUBSWri:

49 case AArch64::SUBSWrr:

50 case AArch64::SUBSXri:

51 case AArch64::SUBSXrr:

52 case AArch64::BICSWrr:

53 case AArch64::BICSXrr:

54 return true;

55 case AArch64::ADDSWrs:

56 case AArch64::ADDSXrs:

57 case AArch64::ANDSWrs:

58 case AArch64::ANDSXrs:

59 case AArch64::SUBSWrs:

60 case AArch64::SUBSXrs:

61 case AArch64::BICSWrs:

62 case AArch64::BICSXrs:

64 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);

65 }

67 return false;

68}

73 if (SecondMI.getOpcode() != AArch64::CBZW &&

74 SecondMI.getOpcode() != AArch64::CBZX &&

75 SecondMI.getOpcode() != AArch64::CBNZW &&

76 SecondMI.getOpcode() != AArch64::CBNZX)

77 return false;

80 if (FirstMI == nullptr)

81 return true;

84 case AArch64::ADDWri:

85 case AArch64::ADDWrr:

86 case AArch64::ADDXri:

87 case AArch64::ADDXrr:

88 case AArch64::ANDWri:

89 case AArch64::ANDWrr:

90 case AArch64::ANDXri:

91 case AArch64::ANDXrr:

92 case AArch64::EORWri:

93 case AArch64::EORWrr:

94 case AArch64::EORXri:

95 case AArch64::EORXrr:

96 case AArch64::ORRWri:

97 case AArch64::ORRWrr:

98 case AArch64::ORRXri:

99 case AArch64::ORRXrr:

100 case AArch64::SUBWri:

101 case AArch64::SUBWrr:

102 case AArch64::SUBXri:

103 case AArch64::SUBXrr:

104 return true;

105 case AArch64::ADDWrs:

106 case AArch64::ADDXrs:

107 case AArch64::ANDWrs:

108 case AArch64::ANDXrs:

109 case AArch64::SUBWrs:

110 case AArch64::SUBXrs:

111 case AArch64::BICWrs:

112 case AArch64::BICXrs:

113

114 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);

115 }

116

117 return false;

118}

119

120

123

125

126 case AArch64::AESMCrr:

127 case AArch64::AESMCrrTied:

128 return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESErr;

129

130 case AArch64::AESIMCrr:

131 case AArch64::AESIMCrrTied:

132 return FirstMI == nullptr || FirstMI->getOpcode() == AArch64::AESDrr;

133 }

134

135 return false;

136}

137

138

141 if (SecondMI.getOpcode() != AArch64::EORv16i8)

142 return false;

143

144

145 if (FirstMI == nullptr)

146 return true;

147

149 case AArch64::AESErr:

150 case AArch64::AESDrr:

151 case AArch64::PMULLv16i8:

152 case AArch64::PMULLv8i8:

153 case AArch64::PMULLv1i64:

154 case AArch64::PMULLv2i64:

155 return true;

156 }

157

158 return false;

159}

160

163

164 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::ADRP) &&

165 SecondMI.getOpcode() == AArch64::ADDXri)

166 return true;

167 return false;

168}

169

170

173

174

175 if ((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZWi) &&

176 (SecondMI.getOpcode() == AArch64::MOVKWi &&

178 return true;

179

180

181 if((FirstMI == nullptr || FirstMI->getOpcode() == AArch64::MOVZXi) &&

182 (SecondMI.getOpcode() == AArch64::MOVKXi &&

184 return true;

185

186

187 if ((FirstMI == nullptr ||

188 (FirstMI->getOpcode() == AArch64::MOVKXi &&

190 (SecondMI.getOpcode() == AArch64::MOVKXi &&

192 return true;

193

194 return false;

195}

196

197

201 case AArch64::STRBBui:

202 case AArch64::STRBui:

203 case AArch64::STRDui:

204 case AArch64::STRHHui:

205 case AArch64::STRHui:

206 case AArch64::STRQui:

207 case AArch64::STRSui:

208 case AArch64::STRWui:

209 case AArch64::STRXui:

210 case AArch64::LDRBBui:

211 case AArch64::LDRBui:

212 case AArch64::LDRDui:

213 case AArch64::LDRHHui:

214 case AArch64::LDRHui:

215 case AArch64::LDRQui:

216 case AArch64::LDRSui:

217 case AArch64::LDRWui:

218 case AArch64::LDRXui:

219 case AArch64::LDRSBWui:

220 case AArch64::LDRSBXui:

221 case AArch64::LDRSHWui:

222 case AArch64::LDRSHXui:

223 case AArch64::LDRSWui:

224

225 if (FirstMI == nullptr)

226 return true;

227

229 case AArch64::ADR:

231 case AArch64::ADRP:

232 return true;

233 }

234 }

235

236 return false;

237}

238

239

242

243 if (SecondMI.getOpcode() == AArch64::CSELWr) {

244

245 if (FirstMI == nullptr)

246 return true;

247

248 if (FirstMI->definesRegister(AArch64::WZR, nullptr))

250 case AArch64::SUBSWrs:

251 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);

252 case AArch64::SUBSWrx:

253 return !AArch64InstrInfo::hasExtendedReg(*FirstMI);

254 case AArch64::SUBSWrr:

255 case AArch64::SUBSWri:

256 return true;

257 }

258 }

259

260

261 if (SecondMI.getOpcode() == AArch64::CSELXr) {

262

263 if (FirstMI == nullptr)

264 return true;

265

266 if (FirstMI->definesRegister(AArch64::XZR, nullptr))

268 case AArch64::SUBSXrs:

269 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);

270 case AArch64::SUBSXrx:

271 case AArch64::SUBSXrx64:

272 return !AArch64InstrInfo::hasExtendedReg(*FirstMI);

273 case AArch64::SUBSXrr:

274 case AArch64::SUBSXri:

275 return true;

276 }

277 }

278

279 return false;

280}

281

282

285 if ((SecondMI.getOpcode() == AArch64::CSINCWr &&

288 (SecondMI.getOpcode() == AArch64::CSINCXr &&

291

292 if (FirstMI == nullptr)

293 return true;

294

295 if (FirstMI->definesRegister(AArch64::WZR, nullptr) ||

298 case AArch64::SUBSWrs:

299 case AArch64::SUBSXrs:

300 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);

301 case AArch64::SUBSWrx:

302 case AArch64::SUBSXrx:

303 case AArch64::SUBSXrx64:

304 return !AArch64InstrInfo::hasExtendedReg(*FirstMI);

305 case AArch64::SUBSWri:

306 case AArch64::SUBSWrr:

307 case AArch64::SUBSXri:

308 case AArch64::SUBSXrr:

309 return true;

310 }

311 }

312

313 return false;

314}

315

316

319 if (AArch64InstrInfo::hasShiftedReg(SecondMI))

320 return false;

321

323

324 case AArch64::ADDWrr:

325 case AArch64::ADDXrr:

326 case AArch64::SUBWrr:

327 case AArch64::SUBXrr:

328 case AArch64::ADDWrs:

329 case AArch64::ADDXrs:

330 case AArch64::SUBWrs:

331 case AArch64::SUBXrs:

332

333 case AArch64::ANDWrr:

334 case AArch64::ANDXrr:

335 case AArch64::BICWrr:

336 case AArch64::BICXrr:

337 case AArch64::EONWrr:

338 case AArch64::EONXrr:

339 case AArch64::EORWrr:

340 case AArch64::EORXrr:

341 case AArch64::ORNWrr:

342 case AArch64::ORNXrr:

343 case AArch64::ORRWrr:

344 case AArch64::ORRXrr:

345 case AArch64::ANDWrs:

346 case AArch64::ANDXrs:

347 case AArch64::BICWrs:

348 case AArch64::BICXrs:

349 case AArch64::EONWrs:

350 case AArch64::EONXrs:

351 case AArch64::EORWrs:

352 case AArch64::EORXrs:

353 case AArch64::ORNWrs:

354 case AArch64::ORNXrs:

355 case AArch64::ORRWrs:

356 case AArch64::ORRXrs:

357

358 if (FirstMI == nullptr)

359 return true;

360

361

363 case AArch64::ADDWrr:

364 case AArch64::ADDXrr:

365 case AArch64::ADDSWrr:

366 case AArch64::ADDSXrr:

367 case AArch64::SUBWrr:

368 case AArch64::SUBXrr:

369 case AArch64::SUBSWrr:

370 case AArch64::SUBSXrr:

371 return true;

372 case AArch64::ADDWrs:

373 case AArch64::ADDXrs:

374 case AArch64::ADDSWrs:

375 case AArch64::ADDSXrs:

376 case AArch64::SUBWrs:

377 case AArch64::SUBXrs:

378 case AArch64::SUBSWrs:

379 case AArch64::SUBSXrs:

380 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);

381 }

382 break;

383

384

385 case AArch64::ADDSWrr:

386 case AArch64::ADDSXrr:

387 case AArch64::SUBSWrr:

388 case AArch64::SUBSXrr:

389 case AArch64::ADDSWrs:

390 case AArch64::ADDSXrs:

391 case AArch64::SUBSWrs:

392 case AArch64::SUBSXrs:

393

394 if (FirstMI == nullptr)

395 return true;

396

397

399 case AArch64::ADDWrr:

400 case AArch64::ADDXrr:

401 case AArch64::SUBWrr:

402 case AArch64::SUBXrr:

403 return true;

404 case AArch64::ADDWrs:

405 case AArch64::ADDXrs:

406 case AArch64::SUBWrs:

407 case AArch64::SUBXrs:

408 return !AArch64InstrInfo::hasShiftedReg(*FirstMI);

409 }

410 break;

411 }

412

413 return false;

414}

415

416

419 bool NeedsSubtract = false;

420

421

423 case AArch64::SUBWri:

424 case AArch64::SUBXri:

425 NeedsSubtract = true;

426 [[fallthrough]];

427 case AArch64::ADDWri:

428 case AArch64::ADDXri:

429 break;

430

431 default:

432 return false;

433 }

434

435

437 return false;

438 }

439

440

441 if (FirstMI == nullptr) {

442 return true;

443 }

444

446 case AArch64::SUBWrs:

447 case AArch64::SUBXrs:

448 if (AArch64InstrInfo::hasShiftedReg(*FirstMI))

449 return false;

450 [[fallthrough]];

451 case AArch64::SUBWrr:

452 case AArch64::SUBXrr:

453 if (NeedsSubtract) {

454 return true;

455 }

456 break;

457

458 case AArch64::ADDWrs:

459 case AArch64::ADDXrs:

460 if (AArch64InstrInfo::hasShiftedReg(*FirstMI))

461 return false;

462 [[fallthrough]];

463 case AArch64::ADDWrr:

464 case AArch64::ADDXrr:

465 if (!NeedsSubtract) {

466 return true;

467 }

468 break;

469 }

470

471 return false;

472}

473

474

475

476

482

483

484

485 if (ST.hasCmpBccFusion() || ST.hasArithmeticBccFusion()) {

486 bool CmpOnly = !ST.hasArithmeticBccFusion();

488 return true;

489 }

490 if (ST.hasArithmeticCbzFusion() && isArithmeticCbzPair(FirstMI, SecondMI))

491 return true;

492 if (ST.hasFuseAES() && isAESPair(FirstMI, SecondMI))

493 return true;

494 if (ST.hasFuseCryptoEOR() && isCryptoEORPair(FirstMI, SecondMI))

495 return true;

496 if (ST.hasFuseAdrpAdd() && isAdrpAddPair(FirstMI, SecondMI))

497 return true;

498 if (ST.hasFuseLiterals() && isLiteralsPair(FirstMI, SecondMI))

499 return true;

501 return true;

502 if (ST.hasFuseCmpCSel() && isCmpCSelPair(FirstMI, SecondMI))

503 return true;

504 if (ST.hasFuseCmpCSet() && isCmpCSetPair(FirstMI, SecondMI))

505 return true;

507 return true;

508 if (ST.hasFuseAddSub2RegAndConstOne() &&

510 return true;

511

512 return false;

513}

514

515std::unique_ptr

static bool isAddSub2RegAndConstOnePair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)

Definition AArch64MacroFusion.cpp:417

static bool isCmpCSelPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)

Compare and conditional select.

Definition AArch64MacroFusion.cpp:240

static bool isArithmeticBccPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI, bool CmpOnly)

CMN, CMP, TST followed by Bcc.

Definition AArch64MacroFusion.cpp:22

static bool isAddressLdStPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)

Fuse address generation and loads or stores.

Definition AArch64MacroFusion.cpp:198

static bool isArithmeticCbzPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)

ALU operations followed by CBZ/CBNZ.

Definition AArch64MacroFusion.cpp:71

static bool shouldScheduleAdjacent(const TargetInstrInfo &TII, const TargetSubtargetInfo &TSI, const MachineInstr *FirstMI, const MachineInstr &SecondMI)

Check if the instr pair, FirstMI and SecondMI, should be fused together.

Definition AArch64MacroFusion.cpp:477

static bool isAESPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)

AES crypto encoding or decoding.

Definition AArch64MacroFusion.cpp:121

static bool isCmpCSetPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)

Compare and cset.

Definition AArch64MacroFusion.cpp:283

static bool isAdrpAddPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)

Definition AArch64MacroFusion.cpp:161

static bool isArithmeticLogicPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)

Definition AArch64MacroFusion.cpp:317

static bool isCryptoEORPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)

AESE/AESD/PMULL + EOR.

Definition AArch64MacroFusion.cpp:139

static bool isLiteralsPair(const MachineInstr *FirstMI, const MachineInstr &SecondMI)

Literal generation.

Definition AArch64MacroFusion.cpp:171

const TargetInstrInfo & TII

Representation of each machine instruction.

unsigned getOpcode() const

Returns the opcode of this MachineInstr.

bool definesRegister(Register Reg, const TargetRegisterInfo *TRI) const

Return true if the MachineInstr fully defines the specified register.

const MachineOperand & getOperand(unsigned i) const

bool isReg() const

isReg - Tests if this is a MO_Register operand.

bool isImm() const

isImm - Tests if this is a MO_Immediate operand.

getReg - Returns the register number.

TargetInstrInfo - Interface to description of machine instruction set.

TargetSubtargetInfo - Generic base class for all target subtargets.

This is an optimization pass for GlobalISel generic memory operations.

LLVM_ABI std::unique_ptr< ScheduleDAGMutation > createMacroFusionDAGMutation(ArrayRef< MacroFusionPredTy > Predicates, bool BranchOnly=false)

Create a DAG scheduling mutation to pair instructions back to back for instructions that benefit acco...

std::unique_ptr< ScheduleDAGMutation > createAArch64MacroFusionDAGMutation()

Note that you have to add: DAG.addMutation(createAArch64MacroFusionDAGMutation()); to AArch64TargetMa...

Definition AArch64MacroFusion.cpp:516

static bool shouldScheduleAdjacent(const TargetInstrInfo &TII, const TargetSubtargetInfo &TSI, const MachineInstr *FirstMI, const MachineInstr &SecondMI)

Check if the instr pair, FirstMI and SecondMI, should be fused together.