[VP] Add vector-predicated reduction intrinsics · llvm/llvm-project@f3e9047 (original) (raw)
`@@ -111,6 +111,11 @@ END_REGISTER_VP_SDNODE(SDOPC)
`
111
111
`#define HANDLE_VP_IS_MEMOP(VPID, POINTERPOS, DATAPOS)
`
112
112
`#endif
`
113
113
``
``
114
`+
// Map this VP reduction intrinsic to its reduction operand positions.
`
``
115
`+
#ifndef HANDLE_VP_REDUCTION
`
``
116
`+
#define HANDLE_VP_REDUCTION(ID, STARTPOS, VECTORPOS)
`
``
117
`+
#endif
`
``
118
+
114
119
`/// } Property Macros
`
115
120
``
116
121
`///// Integer Arithmetic {
`
`@@ -231,6 +236,94 @@ END_REGISTER_VP(vp_gather, VP_GATHER)
`
231
236
``
232
237
`///// } Memory Operations
`
233
238
``
``
239
`+
///// Reductions {
`
``
240
+
``
241
`+
// Specialized helper macro for VP reductions (%start, %x, %mask, %evl).
`
``
242
`+
#ifdef HELPER_REGISTER_REDUCTION_VP
`
``
243
`+
#error "The internal helper macro HELPER_REGISTER_REDUCTION_VP is already defined!"
`
``
244
`+
#endif
`
``
245
`+
#define HELPER_REGISTER_REDUCTION_VP(VPINTRIN, SDOPC, INTRIN) \
`
``
246
`+
BEGIN_REGISTER_VP(VPINTRIN, 2, 3, SDOPC, -1) \
`
``
247
`+
HANDLE_VP_TO_INTRIN(INTRIN) \
`
``
248
`+
HANDLE_VP_REDUCTION(VPINTRIN, 0, 1) \
`
``
249
`+
END_REGISTER_VP(VPINTRIN, SDOPC)
`
``
250
+
``
251
`+
// llvm.vp.reduce.add(start,x,mask,vlen)
`
``
252
`+
HELPER_REGISTER_REDUCTION_VP(vp_reduce_add, VP_REDUCE_ADD,
`
``
253
`+
experimental_vector_reduce_add)
`
``
254
+
``
255
`+
// llvm.vp.reduce.mul(start,x,mask,vlen)
`
``
256
`+
HELPER_REGISTER_REDUCTION_VP(vp_reduce_mul, VP_REDUCE_MUL,
`
``
257
`+
experimental_vector_reduce_mul)
`
``
258
+
``
259
`+
// llvm.vp.reduce.and(start,x,mask,vlen)
`
``
260
`+
HELPER_REGISTER_REDUCTION_VP(vp_reduce_and, VP_REDUCE_AND,
`
``
261
`+
experimental_vector_reduce_and)
`
``
262
+
``
263
`+
// llvm.vp.reduce.or(start,x,mask,vlen)
`
``
264
`+
HELPER_REGISTER_REDUCTION_VP(vp_reduce_or, VP_REDUCE_OR,
`
``
265
`+
experimental_vector_reduce_or)
`
``
266
+
``
267
`+
// llvm.vp.reduce.xor(start,x,mask,vlen)
`
``
268
`+
HELPER_REGISTER_REDUCTION_VP(vp_reduce_xor, VP_REDUCE_XOR,
`
``
269
`+
experimental_vector_reduce_xor)
`
``
270
+
``
271
`+
// llvm.vp.reduce.smax(start,x,mask,vlen)
`
``
272
`+
HELPER_REGISTER_REDUCTION_VP(vp_reduce_smax, VP_REDUCE_SMAX,
`
``
273
`+
experimental_vector_reduce_smax)
`
``
274
+
``
275
`+
// llvm.vp.reduce.smin(start,x,mask,vlen)
`
``
276
`+
HELPER_REGISTER_REDUCTION_VP(vp_reduce_smin, VP_REDUCE_SMIN,
`
``
277
`+
experimental_vector_reduce_smin)
`
``
278
+
``
279
`+
// llvm.vp.reduce.umax(start,x,mask,vlen)
`
``
280
`+
HELPER_REGISTER_REDUCTION_VP(vp_reduce_umax, VP_REDUCE_UMAX,
`
``
281
`+
experimental_vector_reduce_umax)
`
``
282
+
``
283
`+
// llvm.vp.reduce.umin(start,x,mask,vlen)
`
``
284
`+
HELPER_REGISTER_REDUCTION_VP(vp_reduce_umin, VP_REDUCE_UMIN,
`
``
285
`+
experimental_vector_reduce_umin)
`
``
286
+
``
287
`+
// llvm.vp.reduce.fmax(start,x,mask,vlen)
`
``
288
`+
HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmax, VP_REDUCE_FMAX,
`
``
289
`+
experimental_vector_reduce_fmax)
`
``
290
+
``
291
`+
// llvm.vp.reduce.fmin(start,x,mask,vlen)
`
``
292
`+
HELPER_REGISTER_REDUCTION_VP(vp_reduce_fmin, VP_REDUCE_FMIN,
`
``
293
`+
experimental_vector_reduce_fmin)
`
``
294
+
``
295
`+
#undef HELPER_REGISTER_REDUCTION_VP
`
``
296
+
``
297
`+
// Specialized helper macro for VP reductions as above but with two forms:
`
``
298
`+
// sequential and reassociative. These manifest as the presence of 'reassoc'
`
``
299
`+
// fast-math flags in the IR and as two distinct ISD opcodes in the
`
``
300
`+
// SelectionDAG.
`
``
301
`+
#ifdef HELPER_REGISTER_REDUCTION_SEQ_VP
`
``
302
`+
#error "The internal helper macro HELPER_REGISTER_REDUCTION_SEQ_VP is already defined!"
`
``
303
`+
#endif
`
``
304
`+
#define HELPER_REGISTER_REDUCTION_SEQ_VP(VPINTRIN, SDOPC, SEQ_SDOPC, INTRIN) \
`
``
305
`+
BEGIN_REGISTER_VP_INTRINSIC(VPINTRIN, 2, 3) \
`
``
306
`+
BEGIN_REGISTER_VP_SDNODE(SDOPC, -1, VPINTRIN, 2, 3) \
`
``
307
`+
END_REGISTER_VP_SDNODE(SDOPC) \
`
``
308
`+
BEGIN_REGISTER_VP_SDNODE(SEQ_SDOPC, -1, VPINTRIN, 2, 3) \
`
``
309
`+
END_REGISTER_VP_SDNODE(SEQ_SDOPC) \
`
``
310
`+
HANDLE_VP_TO_INTRIN(INTRIN) \
`
``
311
`+
HANDLE_VP_REDUCTION(VPINTRIN, 0, 1) \
`
``
312
`+
END_REGISTER_VP_INTRINSIC(VPINTRIN)
`
``
313
+
``
314
`+
// llvm.vp.reduce.fadd(start,x,mask,vlen)
`
``
315
`+
HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fadd, VP_REDUCE_FADD,
`
``
316
`+
VP_REDUCE_SEQ_FADD,
`
``
317
`+
experimental_vector_reduce_fadd)
`
``
318
+
``
319
`+
// llvm.vp.reduce.fmul(start,x,mask,vlen)
`
``
320
`+
HELPER_REGISTER_REDUCTION_SEQ_VP(vp_reduce_fmul, VP_REDUCE_FMUL,
`
``
321
`+
VP_REDUCE_SEQ_FMUL,
`
``
322
`+
experimental_vector_reduce_fmul)
`
``
323
+
``
324
`+
#undef HELPER_REGISTER_REDUCTION_SEQ_VP
`
``
325
+
``
326
`+
///// } Reduction
`
234
327
``
235
328
`#undef BEGIN_REGISTER_VP
`
236
329
`#undef BEGIN_REGISTER_VP_INTRINSIC
`
`@@ -242,3 +335,4 @@ END_REGISTER_VP(vp_gather, VP_GATHER)
`
242
335
`#undef HANDLE_VP_TO_CONSTRAINEDFP
`
243
336
`#undef HANDLE_VP_TO_INTRIN
`
244
337
`#undef HANDLE_VP_IS_MEMOP
`
``
338
`+
#undef HANDLE_VP_REDUCTION
`