X86InstrFoldTables.cpp Source File (original) (raw)

16#include

17#include

19using namespace llvm;

25#include "X86GenFoldTables.inc"

30 { X86::VANDNPDZ128rr, X86::VANDNPSZ128rmb, TB_BCAST_SS },

31 { X86::VANDNPDZ256rr, X86::VANDNPSZ256rmb, TB_BCAST_SS },

32 { X86::VANDNPDZrr, X86::VANDNPSZrmb, TB_BCAST_SS },

33 { X86::VANDNPSZ128rr, X86::VANDNPDZ128rmb, TB_BCAST_SD },

34 { X86::VANDNPSZ256rr, X86::VANDNPDZ256rmb, TB_BCAST_SD },

35 { X86::VANDNPSZrr, X86::VANDNPDZrmb, TB_BCAST_SD },

36 { X86::VANDPDZ128rr, X86::VANDPSZ128rmb, TB_BCAST_SS },

37 { X86::VANDPDZ256rr, X86::VANDPSZ256rmb, TB_BCAST_SS },

38 { X86::VANDPDZrr, X86::VANDPSZrmb, TB_BCAST_SS },

39 { X86::VANDPSZ128rr, X86::VANDPDZ128rmb, TB_BCAST_SD },

40 { X86::VANDPSZ256rr, X86::VANDPDZ256rmb, TB_BCAST_SD },

41 { X86::VANDPSZrr, X86::VANDPDZrmb, TB_BCAST_SD },

42 { X86::VORPDZ128rr, X86::VORPSZ128rmb, TB_BCAST_SS },

43 { X86::VORPDZ256rr, X86::VORPSZ256rmb, TB_BCAST_SS },

44 { X86::VORPDZrr, X86::VORPSZrmb, TB_BCAST_SS },

45 { X86::VORPSZ128rr, X86::VORPDZ128rmb, TB_BCAST_SD },

46 { X86::VORPSZ256rr, X86::VORPDZ256rmb, TB_BCAST_SD },

47 { X86::VORPSZrr, X86::VORPDZrmb, TB_BCAST_SD },

48 { X86::VPANDDZ128rr, X86::VPANDQZ128rmb, TB_BCAST_Q },

49 { X86::VPANDDZ256rr, X86::VPANDQZ256rmb, TB_BCAST_Q },

50 { X86::VPANDDZrr, X86::VPANDQZrmb, TB_BCAST_Q },

51 { X86::VPANDNDZ128rr, X86::VPANDNQZ128rmb, TB_BCAST_Q },

52 { X86::VPANDNDZ256rr, X86::VPANDNQZ256rmb, TB_BCAST_Q },

53 { X86::VPANDNDZrr, X86::VPANDNQZrmb, TB_BCAST_Q },

54 { X86::VPANDNQZ128rr, X86::VPANDNDZ128rmb, TB_BCAST_D },

55 { X86::VPANDNQZ256rr, X86::VPANDNDZ256rmb, TB_BCAST_D },

56 { X86::VPANDNQZrr, X86::VPANDNDZrmb, TB_BCAST_D },

57 { X86::VPANDQZ128rr, X86::VPANDDZ128rmb, TB_BCAST_D },

58 { X86::VPANDQZ256rr, X86::VPANDDZ256rmb, TB_BCAST_D },

59 { X86::VPANDQZrr, X86::VPANDDZrmb, TB_BCAST_D },

60 { X86::VPORDZ128rr, X86::VPORQZ128rmb, TB_BCAST_Q },

61 { X86::VPORDZ256rr, X86::VPORQZ256rmb, TB_BCAST_Q },

62 { X86::VPORDZrr, X86::VPORQZrmb, TB_BCAST_Q },

63 { X86::VPORQZ128rr, X86::VPORDZ128rmb, TB_BCAST_D },

64 { X86::VPORQZ256rr, X86::VPORDZ256rmb, TB_BCAST_D },

65 { X86::VPORQZrr, X86::VPORDZrmb, TB_BCAST_D },

66 { X86::VPXORDZ128rr, X86::VPXORQZ128rmb, TB_BCAST_Q },

67 { X86::VPXORDZ256rr, X86::VPXORQZ256rmb, TB_BCAST_Q },

68 { X86::VPXORDZrr, X86::VPXORQZrmb, TB_BCAST_Q },

69 { X86::VPXORQZ128rr, X86::VPXORDZ128rmb, TB_BCAST_D },

70 { X86::VPXORQZ256rr, X86::VPXORDZ256rmb, TB_BCAST_D },

71 { X86::VPXORQZrr, X86::VPXORDZrmb, TB_BCAST_D },

72 { X86::VXORPDZ128rr, X86::VXORPSZ128rmb, TB_BCAST_SS },

73 { X86::VXORPDZ256rr, X86::VXORPSZ256rmb, TB_BCAST_SS },

74 { X86::VXORPDZrr, X86::VXORPSZrmb, TB_BCAST_SS },

75 { X86::VXORPSZ128rr, X86::VXORPDZ128rmb, TB_BCAST_SD },

76 { X86::VXORPSZ256rr, X86::VXORPDZ256rmb, TB_BCAST_SD },

77 { X86::VXORPSZrr, X86::VXORPDZrmb, TB_BCAST_SD },

78};

81 { X86::VPTERNLOGDZ128rri, X86::VPTERNLOGQZ128rmbi, TB_BCAST_Q },

82 { X86::VPTERNLOGDZ256rri, X86::VPTERNLOGQZ256rmbi, TB_BCAST_Q },

83 { X86::VPTERNLOGDZrri, X86::VPTERNLOGQZrmbi, TB_BCAST_Q },

84 { X86::VPTERNLOGQZ128rri, X86::VPTERNLOGDZ128rmbi, TB_BCAST_D },

85 { X86::VPTERNLOGQZ256rri, X86::VPTERNLOGDZ256rmbi, TB_BCAST_D },

86 { X86::VPTERNLOGQZrri, X86::VPTERNLOGDZrmbi, TB_BCAST_D },

87};

91#ifndef NDEBUG

92#define CHECK_SORTED_UNIQUE(TABLE) \

93 assert(llvm::is_sorted(TABLE) && #TABLE " is not sorted"); \

94 assert(std::adjacent_find(std::begin(Table), std::end(Table)) == \

95 std::end(Table) && \

96 #TABLE " is not unique");

99 static std::atomic FoldTablesChecked(false);

100 if (!FoldTablesChecked.load(std::memory_order_relaxed)) {

113 FoldTablesChecked.store(true, std::memory_order_relaxed);

114 }

115#endif

116

118 if (Data != Table.end() && Data->KeyOp == RegOp &&

120 return Data;

121 return nullptr;

122}

123

126}

127

130 if (OpNum == 0)

131 FoldTable = ArrayRef(Table0);

132 else if (OpNum == 1)

133 FoldTable = ArrayRef(Table1);

134 else if (OpNum == 2)

135 FoldTable = ArrayRef(Table2);

136 else if (OpNum == 3)

137 FoldTable = ArrayRef(Table3);

138 else if (OpNum == 4)

139 FoldTable = ArrayRef(Table4);

140 else

141 return nullptr;

142

144}

145

147 unsigned OpNum) {

149 if (OpNum == 1)

150 FoldTable = ArrayRef(BroadcastTable1);

151 else if (OpNum == 2)

152 FoldTable = ArrayRef(BroadcastTable2);

153 else if (OpNum == 3)

154 FoldTable = ArrayRef(BroadcastTable3);

155 else if (OpNum == 4)

156 FoldTable = ArrayRef(BroadcastTable4);

157 else

158 return nullptr;

159

161}

162

163namespace {

164

165

166

167struct X86MemUnfoldTable {

168

169 std::vector Table;

170

171 X86MemUnfoldTable() {

173

175

177

179

181

183

185

187

189

191

193

195

196

198

200

202

204

206

208

210

212

213

215

216

217 assert(std::adjacent_find(Table.begin(), Table.end()) == Table.end() &&

218 "Memory unfolding table is not unique!");

219 }

220

222

224 Table.push_back({Entry.DstOp, Entry.KeyOp,

225 static_cast<uint16_t>(Entry.Flags | ExtraFlags)});

226 }

227};

228}

229

231 static X86MemUnfoldTable MemUnfoldTable;

232 auto &Table = MemUnfoldTable.Table;

234 if (I != Table.end() && I->KeyOp == MemOp)

235 return &*I;

236 return nullptr;

237}

238

239namespace {

240

241

242

243struct X86BroadcastFoldTable {

244

245 std::vector Table;

246

247 X86BroadcastFoldTable() {

248

250 unsigned RegOp = Reg2Bcst.KeyOp;

251 unsigned BcstOp = Reg2Bcst.DstOp;

253 unsigned MemOp = Reg2Mem->DstOp;

256 Table.push_back({MemOp, BcstOp, Flags});

257 }

258 }

260 unsigned RegOp = Reg2Bcst.KeyOp;

261 unsigned BcstOp = Reg2Bcst.DstOp;

263 unsigned MemOp = Reg2Mem->DstOp;

266 Table.push_back({MemOp, BcstOp, Flags});

267 }

268 }

269

271 unsigned RegOp = Reg2Bcst.KeyOp;

272 unsigned BcstOp = Reg2Bcst.DstOp;

274 unsigned MemOp = Reg2Mem->DstOp;

277 Table.push_back({MemOp, BcstOp, Flags});

278 }

279 }

281 unsigned RegOp = Reg2Bcst.KeyOp;

282 unsigned BcstOp = Reg2Bcst.DstOp;

284 unsigned MemOp = Reg2Mem->DstOp;

287 Table.push_back({MemOp, BcstOp, Flags});

288 }

289 }

290

292 unsigned RegOp = Reg2Bcst.KeyOp;

293 unsigned BcstOp = Reg2Bcst.DstOp;

295 unsigned MemOp = Reg2Mem->DstOp;

298 Table.push_back({MemOp, BcstOp, Flags});

299 }

300 }

301

302

304 }

305};

306}

307

309 unsigned BroadcastBits) {

313 return BroadcastBits == 16;

316 return BroadcastBits == 32;

319 return BroadcastBits == 64;

320 }

321 return false;

322}

323

326 static X86BroadcastFoldTable BroadcastFoldTable;

327 auto &Table = BroadcastFoldTable.Table;

329 I != Table.end() && I->KeyOp == MemOp; ++I) {

331 return &*I;

332 }

333 return nullptr;

334}

assert(ImpDefSCC.getReg()==AMDGPU::SCC &&ImpDefSCC.isDef())

static const X86FoldTableEntry BroadcastSizeTable2[]

#define CHECK_SORTED_UNIQUE(TABLE)

static const X86FoldTableEntry BroadcastSizeTable3[]

static const X86FoldTableEntry * lookupFoldTableImpl(ArrayRef< X86FoldTableEntry > Table, unsigned RegOp)

ArrayRef - Represent a constant reference to an array (0 or more elements consecutively in memory),...

This is an optimization pass for GlobalISel generic memory operations.

const X86FoldTableEntry * lookupBroadcastFoldTableBySize(unsigned MemOp, unsigned BroadcastBits)

const X86FoldTableEntry * lookupBroadcastFoldTable(unsigned RegOp, unsigned OpNum)

const X86FoldTableEntry * lookupTwoAddrFoldTable(unsigned RegOp)

auto lower_bound(R &&Range, T &&Value)

Provide wrappers to std::lower_bound which take ranges instead of having to pass begin/end explicitly...

const X86FoldTableEntry * lookupUnfoldTable(unsigned MemOp)

bool matchBroadcastSize(const X86FoldTableEntry &Entry, unsigned BroadcastBits)

void array_pod_sort(IteratorTy Start, IteratorTy End)

array_pod_sort - This sorts an array with the specified start and end extent.

const X86FoldTableEntry * lookupFoldTable(unsigned RegOp, unsigned OpNum)