Utils.cpp Source File (original) (raw)

30 #include "llvm/Support/LogicalResult.h"

31 #include

33 #define DEBUG_TYPE "affine-utils"

35 using namespace mlir;

36 using namespace affine;

37 using namespace presburger;

39 namespace {

43 class AffineApplyExpander

45 public:

50 : builder(builder), dimValues(dimValues), symbolValues(symbolValues),

51 loc(loc) {}

53 template

55 arith::IntegerOverflowFlags overflowFlags =

56 arith::IntegerOverflowFlags::none) {

59 if (!lhs || !rhs)

60 return nullptr;

61 auto op = builder.create(loc, lhs, rhs, overflowFlags);

63 }

66 return buildBinaryExprarith::AddIOp(expr);

67 }

70 return buildBinaryExprarith::MulIOp(expr,

71 arith::IntegerOverflowFlags::nsw);

72 }

84 if (auto rhsConst = dyn_cast(expr.getRHS())) {

85 if (rhsConst.getValue() <= 0) {

86 emitError(loc, "modulo by non-positive value is not supported");

87 return nullptr;

88 }

89 }

93 assert(lhs && rhs && "unexpected affine expr lowering failure");

95 Value remainder = builder.create arith::RemSIOp(loc, lhs, rhs);

96 Value zeroCst = builder.create arith::ConstantIndexOp(loc, 0);

97 Value isRemainderNegative = builder.create arith::CmpIOp(

98 loc, arith::CmpIPredicate::slt, remainder, zeroCst);

99 Value correctedRemainder =

100 builder.create arith::AddIOp(loc, remainder, rhs);

101 Value result = builder.create arith::SelectOp(

102 loc, isRemainderNegative, correctedRemainder, remainder);

103 return result;

104 }

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

124 if (auto rhsConst = dyn_cast(expr.getRHS())) {

125 if (rhsConst.getValue() <= 0) {

126 emitError(loc, "division by non-positive value is not supported");

127 return nullptr;

128 }

129 }

132 assert(lhs && rhs && "unexpected affine expr lowering failure");

133

134 Value zeroCst = builder.create arith::ConstantIndexOp(loc, 0);

135 Value noneCst = builder.create arith::ConstantIndexOp(loc, -1);

136 Value negative = builder.create arith::CmpIOp(

137 loc, arith::CmpIPredicate::slt, lhs, zeroCst);

138 Value negatedDecremented = builder.create arith::SubIOp(loc, noneCst, lhs);

140 builder.create arith::SelectOp(loc, negative, negatedDecremented, lhs);

141 Value quotient = builder.create arith::DivSIOp(loc, dividend, rhs);

142 Value correctedQuotient =

143 builder.create arith::SubIOp(loc, noneCst, quotient);

144 Value result = builder.create arith::SelectOp(loc, negative,

145 correctedQuotient, quotient);

146 return result;

147 }

148

149

150

151

152

153

154

155

156

157

158

159

160

161

163 if (auto rhsConst = dyn_cast(expr.getRHS())) {

164 if (rhsConst.getValue() <= 0) {

165 emitError(loc, "division by non-positive value is not supported");

166 return nullptr;

167 }

168 }

171 assert(lhs && rhs && "unexpected affine expr lowering failure");

172

173 Value zeroCst = builder.create arith::ConstantIndexOp(loc, 0);

174 Value oneCst = builder.create arith::ConstantIndexOp(loc, 1);

175 Value nonPositive = builder.create arith::CmpIOp(

176 loc, arith::CmpIPredicate::sle, lhs, zeroCst);

177 Value negated = builder.create arith::SubIOp(loc, zeroCst, lhs);

178 Value decremented = builder.create arith::SubIOp(loc, lhs, oneCst);

180 builder.create arith::SelectOp(loc, nonPositive, negated, decremented);

181 Value quotient = builder.create arith::DivSIOp(loc, dividend, rhs);

182 Value negatedQuotient =

183 builder.create arith::SubIOp(loc, zeroCst, quotient);

184 Value incrementedQuotient =

185 builder.create arith::AddIOp(loc, quotient, oneCst);

186 Value result = builder.create arith::SelectOp(

187 loc, nonPositive, negatedQuotient, incrementedQuotient);

188 return result;

189 }

190

192 auto op = builder.create arith::ConstantIndexOp(loc, expr.getValue());

194 }

195

197 assert(expr.getPosition() < dimValues.size() &&

198 "affine dim position out of range");

200 }

201

203 assert(expr.getPosition() < symbolValues.size() &&

204 "symbol dim position out of range");

206 }

207

208 private:

212

214 };

215 }

216

217

218

223 return AffineApplyExpander(builder, dimValues, symbolValues, loc).visit(expr);

224 }

225

226

227

228 std::optional<SmallVector<Value, 8>>

231 auto numDims = affineMap.getNumDims();

232 auto expanded = llvm::to_vector<8>(

233 llvm::map_range(affineMap.getResults(),

234 [numDims, &builder, loc, operands](AffineExpr expr) {

235 return expandAffineExpr(builder, loc, expr,

236 operands.take_front(numDims),

237 operands.drop_front(numDims));

238 }));

239 if (llvm::all_of(expanded, [](Value v) { return v; }))

240 return expanded;

241 return std::nullopt;

242 }

243

244

245

246

248 if (elseBlock)

249 assert(ifOp.hasElse() && "else block expected");

250

251 Block *destBlock = ifOp->getBlock();

252 Block *srcBlock = elseBlock ? ifOp.getElseBlock() : ifOp.getThenBlock();

255 std::prev(srcBlock->end()));

256 ifOp.erase();

257 }

258

259

260

261

262

264

269 if (auto forOp = dyn_cast(parentOp)) {

270 if (llvm::is_contained(ifOperands, forOp.getInductionVar()))

271 break;

272 } else if (auto parallelOp = dyn_cast(parentOp)) {

273 if (llvm::any_of(parallelOp.getIVs(), [&](Value iv) {

274 return llvm::is_contained(ifOperands, iv);

275 }))

276 break;

277 } else if (!isa(parentOp)) {

278

279 break;

280 }

281

282 res = parentOp;

283 }

284 return res;

285 }

286

287

288

289

291

292 if (hoistOverOp == ifOp)

293 return ifOp;

294

295

296

297

298

299

302 auto hoistedIfOp = b.create(ifOp.getLoc(), ifOp.getIntegerSet(),

303 ifOp.getOperands(),

304 true);

305

306

307

308 Operation *hoistOverOpClone = nullptr;

309

310

311 StringAttr idForIfOp = b.getStringAttr("__mlir_if_hoisting");

312 operandMap.clear();

314

315 ifOp->setAttr(idForIfOp, b.getBoolAttr(true));

316 hoistOverOpClone = b.clone(*hoistOverOp, operandMap);

317

318

320

321

322 auto *thenBlock = hoistedIfOp.getThenBlock();

323 thenBlock->getOperations().splice(thenBlock->begin(),

326

327

328 AffineIfOp ifCloneInElse;

329 hoistOverOpClone->walk([&](AffineIfOp ifClone) {

330 if (!ifClone->getAttr(idForIfOp))

332 ifCloneInElse = ifClone;

334 });

335 assert(ifCloneInElse && "if op clone should exist");

336

337

338 if (!ifCloneInElse.hasElse())

339 ifCloneInElse.erase();

340 else

342

343

344 auto *elseBlock = hoistedIfOp.getElseBlock();

345 elseBlock->getOperations().splice(

346 elseBlock->begin(), hoistOverOpClone->getBlock()->getOperations(),

348

349 return hoistedIfOp;

350 }

351

352 LogicalResult

355 AffineParallelOp *resOp) {

356

357 unsigned numReductions = parallelReductions.size();

358 if (numReductions != forOp.getNumIterOperands())

359 return failure();

360

361 Location loc = forOp.getLoc();

363 AffineMap lowerBoundMap = forOp.getLowerBoundMap();

364 ValueRange lowerBoundOperands = forOp.getLowerBoundOperands();

365 AffineMap upperBoundMap = forOp.getUpperBoundMap();

366 ValueRange upperBoundOperands = forOp.getUpperBoundOperands();

367

368

369 auto reducedValues = llvm::to_vector<4>(llvm::map_range(

370 parallelReductions, [](const LoopReduction &red) { return red.value; }));

371 auto reductionKinds = llvm::to_vector<4>(llvm::map_range(

372 parallelReductions, [](const LoopReduction &red) { return red.kind; }));

373 AffineParallelOp newPloop = outsideBuilder.create(

378

380 Operation *yieldOp = &newPloop.getBody()->back();

381

382

383

385 newResults.reserve(numReductions);

386 for (unsigned i = 0; i < numReductions; ++i) {

387 Value init = forOp.getInits()[i];

388

389

390

392 assert(reductionOp && "yielded value is expected to be produced by an op");

394 outsideBuilder.getInsertionPoint(), newPloop.getBody()->getOperations(),

395 reductionOp);

396 reductionOp->setOperands({init, newPloop->getResult(i)});

397 forOp->getResult(i).replaceAllUsesWith(reductionOp->getResult(0));

398 }

399

400

401

402

403

404 unsigned numIVs = 1;

406 newPloop.getBody()->eraseArguments(numIVs, numReductions);

407

408 forOp.erase();

409 if (resOp)

410 *resOp = newPloop;

411 return success();

412 }

413

414

416

417

418 if (ifOp.getNumResults() != 0)

419 return failure();

420

421

422

423

424

425

427 AffineIfOp::getCanonicalizationPatterns(patterns, ifOp.getContext());

429 bool erased;

431 ifOp.getOperation(), frozenPatterns,

433 nullptr, &erased);

434 if (erased) {

435 if (folded)

436 *folded = true;

437 return failure();

438 }

439 if (folded)

440 *folded = false;

441

442

443 assert(llvm::all_of(ifOp.getOperands(),

445 return isTopLevelValue(v) || isAffineInductionVar(v);

446 }) &&

447 "operands not composed");

448

449

450

452

454

455 if (hoistedIfOp == ifOp)

456 return failure();

457

458

459

462 frozenPatterns);

463

464 return success();

465 }

466

467

470 bool positivePath) {

471 if (e == dim)

472 return positivePath ? min : max;

473 if (auto bin = dyn_cast(e)) {

479

480 auto c1 = dyn_cast(bin.getLHS());

481 auto c2 = dyn_cast(bin.getRHS());

482 if (c1 && c1.getValue() < 0)

485 if (c2 && c2.getValue() < 0)

491 }

492 return e;

493 }

494

496

497 if (op.hasMinMaxBounds())

498 return;

499

500 AffineMap lbMap = op.getLowerBoundsMap();

502

503 bool isAlreadyNormalized =

504 llvm::all_of(llvm::zip(steps, lbMap.getResults()), [](auto tuple) {

505 int64_t step = std::get<0>(tuple);

506 auto lbExpr = dyn_cast(std::get<1>(tuple));

507 return lbExpr && lbExpr.getValue() == 0 && step == 1;

508 });

509 if (isAlreadyNormalized)

510 return;

511

514 op.getLowerBoundsValueMap(), &ranges);

516 auto zeroExpr = builder.getAffineConstantExpr(0);

519 for (unsigned i = 0, e = steps.size(); i < e; ++i) {

520 int64_t step = steps[i];

521

522

523 lbExprs.push_back(zeroExpr);

524

525

527 ubExprs.push_back(ubExpr);

528

529

533 auto expr = lbExpr + builder.getAffineDimExpr(nDims) * step;

536

537

538

539 OperandRange lbOperands = op.getLowerBoundsOperands();

540 OperandRange dimOperands = lbOperands.take_front(nDims);

541 OperandRange symbolOperands = lbOperands.drop_front(nDims);

543 applyOperands.push_back(iv);

544 applyOperands.append(symbolOperands.begin(), symbolOperands.end());

545 auto apply = builder.create(op.getLoc(), map, applyOperands);

547 }

548

550 op.setSteps(newSteps);

552 0, 0, lbExprs, op.getContext());

553 op.setLowerBounds({}, newLowerMap);

555 ubExprs, op.getContext());

556 op.setUpperBounds(ranges.getOperands(), newUpperMap);

557 }

558

560 bool promoteSingleIter) {

562 return success();

563

564

565 if (op.hasConstantLowerBound() && (op.getConstantLowerBound() == 0) &&

566 (op.getStep() == 1))

567 return success();

568

569

570

571

572

573 if (op.getLowerBoundMap().getNumResults() != 1)

574 return failure();

575

578 int64_t origLoopStep = op.getStepAsInt();

579

580

581 AffineMap oldLbMap = op.getLowerBoundMap();

582

583

584

585

586

588 op.getLowerBoundMap().getResult(0));

589 AffineValueMap lbMap(oldLbMap, op.getLowerBoundOperands());

592 op.getContext());

593 AffineValueMap paddedLbValueMap(paddedLbMap, op.getLowerBoundOperands());

594 AffineValueMap ubValueMap(op.getUpperBoundMap(), op.getUpperBoundOperands());

596

599

600

601 unsigned numResult = newUbValueMap.getNumResults();

603 for (unsigned i = 0; i < numResult; ++i)

605

606

608 AffineMap::get(numResult, 0, scaleDownExprs, op.getContext());

610

611

612 op.setUpperBound(newUbValueMap.getOperands(), newUbMap);

614 op.setStep(1);

615

616

617

619

625 (void)newIvToOldIvMap.canonicalize();

626 auto newIV = opBuilder.create(

627 loc, newIvToOldIvMap.getAffineMap(), newIvToOldIvMap.getOperands());

628 op.getInductionVar().replaceAllUsesExcept(newIV->getResult(0), newIV);

629 return success();

630 }

631

632

633

634

637

638

641 return false;

642

643 unsigned nsLoops =

648 }

649

650

651

652

654 unsigned minSurroundingLoops) {

657

658

659

660

661

662

668 for (unsigned d = nsLoops + 1; d > minSurroundingLoops; d--) {

670 srcAccess, destAccess, d, &dependenceConstraints,

671 nullptr);

672

673

675 return true;

676 }

677

678 return false;

679 }

680

681

682 return true;

683 }

684

685 template <typename EffectType, typename T>

689

690

691 bool hasSideEffect = false;

692

693

694 Value memref = memOp.getMemRef();

695 std::function<void(Operation *)> checkOperation = [&](Operation *op) {

696

697 if (hasSideEffect)

698 return;

699

700 if (auto memEffect = dyn_cast(op)) {

702 memEffect.getEffects(effects);

703

704 bool opMayHaveEffect = false;

705 for (auto effect : effects) {

706

707

708 if (isa(effect.getEffect())) {

709 if (effect.getValue() && effect.getValue() != memref &&

710 mayAlias (effect.getValue(), memref))

711 continue;

712 opMayHaveEffect = true;

713 break;

714 }

715 }

716

717 if (!opMayHaveEffect)

718 return;

719

720

721

722 if (isa<AffineReadOpInterface, AffineWriteOpInterface>(op)) {

723

724

725

726

727

728

729 unsigned minSurroundingLoops =

731 if (mayHaveEffect(op, memOp, minSurroundingLoops))

732 hasSideEffect = true;

733 return;

734 }

735

736

737

738 hasSideEffect = true;

739 return;

740 }

741

743

744

745 for (Region &region : op->getRegions())

746 for (Block &block : region)

748 checkOperation(&op);

749 return;

750 }

751

752

753

754 hasSideEffect = true;

755 };

756

757

758

760

761

762

763

765 checkOperation(parent);

766 };

767

768

769

772 assert(

774 "Checking for side effect between two operations without a common "

775 "ancestor");

776

777

778

779

782 until(untilOp->getParentOp(), untilOp);

783 return;

784 }

785

786

787

788

789

791 {

792

793

794 for (auto iter = ++from->getIterator(), end = from->getBlock()->end();

795 iter != end && &*iter != untilOp; ++iter) {

796 checkOperation(&*iter);

797 }

798

799

800

801 if (untilOp->getBlock() != from->getBlock())

803 todoBlocks.push_back(succ);

804 }

805

807

808 while (!todoBlocks.empty()) {

809 Block *blk = todoBlocks.pop_back_val();

810 if (done.count(blk))

811 continue;

812 done.insert(blk);

813 for (auto &op : *blk) {

814 if (&op == untilOp)

815 break;

816 checkOperation(&op);

819 todoBlocks.push_back(succ);

820 }

821 }

822 };

823 recur(start, memOp);

824 return !hasSideEffect;

825 }

826

827

828

829

830

831

832

833

838

839

840

841 Operation *lastWriteStoreOp = nullptr;

842

843 for (auto *user : loadOp.getMemRef().getUsers()) {

844 auto storeOp = dyn_cast(user);

845 if (!storeOp)

846 continue;

849

850

851

852

853

854

855

856

857

858 if (srcAccess != destAccess)

859 continue;

860

861

862 if (!domInfo.dominates(storeOp, loadOp))

863 continue;

864

865

866

867

869 continue;

870

871

872

873 if (!affine::hasNoInterveningEffectMemoryEffects::Write(storeOp, loadOp,

875 continue;

876

877

878 assert(lastWriteStoreOp == nullptr &&

879 "multiple simultaneous replacement stores");

880 lastWriteStoreOp = storeOp;

881 }

882

883 if (!lastWriteStoreOp)

884 return;

885

886

888 cast(lastWriteStoreOp).getValueToStore();

889

890

891 if (storeVal.getType() != loadOp.getValue().getType())

892 return;

894

895 memrefsToErase.insert(loadOp.getMemRef());

896

897 loadOpsToErase.push_back(loadOp);

898 }

899

900 template bool

902 affine::AffineReadOpInterface>(

905

906

907

908

909

910

915

917

918 auto writeB = dyn_cast(user);

919 if (!writeB)

920 continue;

921

922

923 if (writeB == writeA)

924 continue;

925

926

927 if (writeB->getParentRegion() != writeA->getParentRegion())

928 continue;

929

930

933

934 if (srcAccess != destAccess)

935 continue;

936

937

938 if (!postDominanceInfo.postDominates(writeB, writeA))

939 continue;

940

941

942

943 if (!affine::hasNoInterveningEffectMemoryEffects::Read(writeA, writeB,

945 continue;

946

947 opsToErase.push_back(writeA);

948 break;

949 }

950 }

951

952

953

954

955

956

957

958 static void loadCSE(AffineReadOpInterface loadA,

963 for (auto *user : loadA.getMemRef().getUsers()) {

964 auto loadB = dyn_cast(user);

965 if (!loadB || loadB == loadA)

966 continue;

967

970

971

972 if (srcAccess != destAccess) {

973 continue;

974 }

975

976

977 if (!domInfo.dominates(loadB, loadA))

978 continue;

979

980

981 if (!affine::hasNoInterveningEffectMemoryEffects::Write(

982 loadB.getOperation(), loadA, mayAlias))

983 continue;

984

985

986

987 if (loadB.getValue().getType() != loadA.getValue().getType())

988 continue;

989

990 loadCandidates.push_back(loadB);

991 }

992

993

994

996 for (AffineReadOpInterface option : loadCandidates) {

997 if (llvm::all_of(loadCandidates, [&](AffineReadOpInterface depStore) {

998 return depStore == option ||

999 domInfo.dominates(option.getOperation(),

1000 depStore.getOperation());

1001 })) {

1002 loadB = option.getValue();

1003 break;

1004 }

1005 }

1006

1007 if (loadB) {

1009

1010 loadOpsToErase.push_back(loadA);

1011 }

1012 }

1013

1014

1015

1016

1017

1018

1019

1020

1021

1022

1023

1024

1025

1026

1027

1028

1029

1030

1031

1032

1033

1034

1035

1036

1037

1038

1042

1044

1045

1047

1049 return !aliasAnalysis.alias(val1, val2).isNo();

1050 };

1051

1052

1053 f.walk([&](AffineReadOpInterface loadOp) {

1055 });

1056 for (auto *op : opsToErase)

1057 op->erase();

1058 opsToErase.clear();

1059

1060

1061 f.walk([&](AffineWriteOpInterface storeOp) {

1063 });

1064 for (auto *op : opsToErase)

1065 op->erase();

1066 opsToErase.clear();

1067

1068

1069

1070

1071

1072 for (auto memref : memrefsToErase) {

1073

1074 Operation *defOp = memref.getDefiningOp();

1075 if (!defOp || !hasSingleEffectMemoryEffects::Allocate(defOp, memref))

1076

1077

1078 continue;

1079 if (llvm::any_of(memref.getUsers(), [&](Operation *ownerOp) {

1080 return !isa(ownerOp) &&

1081 !hasSingleEffectMemoryEffects::Free(ownerOp, memref);

1082 }))

1083 continue;

1084

1085

1086 for (auto *user : llvm::make_early_inc_range(memref.getUsers()))

1089 }

1090

1091

1092

1093

1094 f.walk([&](AffineReadOpInterface loadOp) {

1096 });

1097 for (auto *op : opsToErase)

1098 op->erase();

1099 }

1100

1101

1102

1104 return isa<AffineMapAccessInterface, memref::LoadOp, memref::StoreOp>(op);

1105 }

1106

1107

1112 bool allowNonDereferencingOps) {

1113 unsigned newMemRefRank = cast(newMemRef.getType()).getRank();

1114 (void)newMemRefRank;

1115 unsigned oldMemRefRank = cast(oldMemRef.getType()).getRank();

1116 (void)oldMemRefRank;

1117 if (indexRemap) {

1118 assert(indexRemap.getNumSymbols() == symbolOperands.size() &&

1119 "symbolic operand count mismatch");

1121 extraOperands.size() + oldMemRefRank + symbolOperands.size());

1122 assert(indexRemap.getNumResults() + extraIndices.size() == newMemRefRank);

1123 } else {

1124 assert(oldMemRefRank + extraIndices.size() == newMemRefRank);

1125 }

1126

1127

1128 assert(cast(oldMemRef.getType()).getElementType() ==

1129 cast(newMemRef.getType()).getElementType());

1130

1133 if (opEntry.value() == oldMemRef)

1134 usePositions.push_back(opEntry.index());

1135 }

1136

1137

1138 if (usePositions.empty())

1139 return success();

1140

1141 unsigned memRefOperandPos = usePositions.front();

1142

1144

1145

1147 if (!allowNonDereferencingOps) {

1148

1149

1150

1151 return failure();

1152 }

1153 for (unsigned pos : usePositions)

1155 return success();

1156 }

1157

1158 if (usePositions.size() > 1) {

1159

1160 LLVM_DEBUG(llvm::dbgs()

1161 << "multiple dereferencing uses in a single op not supported");

1162 return failure();

1163 }

1164

1165

1168 unsigned oldMemRefNumIndices = oldMemRefRank;

1169 auto startIdx = op->operand_begin() + memRefOperandPos + 1;

1170 auto affMapAccInterface = dyn_cast(op);

1171 if (affMapAccInterface) {

1172

1173

1174

1176 affMapAccInterface.getAffineMapAttrForMemRef(oldMemRef);

1177 oldMap = cast(oldMapAttrPair.getValue()).getValue();

1178 oldMemRefNumIndices = oldMap.getNumInputs();

1179 }

1180 oldMapOperands.assign(startIdx, startIdx + oldMemRefNumIndices);

1181

1182

1185 oldMemRefOperands.reserve(oldMemRefRank);

1186 if (affMapAccInterface &&

1188 for (auto resultExpr : oldMap.getResults()) {

1191 auto afOp = builder.create(op->getLoc(), singleResMap,

1192 oldMapOperands);

1193 oldMemRefOperands.push_back(afOp);

1194 affineApplyOps.push_back(afOp);

1195 }

1196 } else {

1197 oldMemRefOperands.assign(oldMapOperands.begin(), oldMapOperands.end());

1198 }

1199

1200

1201

1202

1204 remapOperands.reserve(extraOperands.size() + oldMemRefRank +

1205 symbolOperands.size());

1206 remapOperands.append(extraOperands.begin(), extraOperands.end());

1207 remapOperands.append(oldMemRefOperands.begin(), oldMemRefOperands.end());

1208 remapOperands.append(symbolOperands.begin(), symbolOperands.end());

1209

1211 remapOutputs.reserve(oldMemRefRank);

1212 if (indexRemap &&

1214

1215 for (auto resultExpr : indexRemap.getResults()) {

1218 auto afOp = builder.create(op->getLoc(), singleResMap,

1219 remapOperands);

1220 remapOutputs.push_back(afOp);

1221 affineApplyOps.push_back(afOp);

1222 }

1223 } else {

1224

1225 remapOutputs.assign(remapOperands.begin(), remapOperands.end());

1226 }

1228 newMapOperands.reserve(newMemRefRank);

1229

1230

1231 for (Value extraIndex : extraIndices) {

1233 "invalid memory op index");

1234 newMapOperands.push_back(extraIndex);

1235 }

1236

1237

1238 newMapOperands.append(remapOutputs.begin(), remapOutputs.end());

1239

1240

1241 assert(newMapOperands.size() == newMemRefRank);

1246

1247 for (Value value : affineApplyOps)

1248 if (value.use_empty())

1249 value.getDefiningOp()->erase();

1250

1252

1253 state.operands.reserve(op->getNumOperands() + extraIndices.size());

1254

1257

1258 state.operands.push_back(newMemRef);

1259

1260

1261 if (affMapAccInterface) {

1262 state.operands.append(newMapOperands.begin(), newMapOperands.end());

1263 } else {

1264

1265

1266

1267 for (unsigned i = 0; i < newMemRefRank; i++) {

1268 state.operands.push_back(builder.create(

1270 AffineMap::get(newMap.getNumDims(), newMap.getNumSymbols(),

1271 newMap.getResult(i)),

1272 newMapOperands));

1273 }

1274 }

1275

1276

1277 unsigned oldMapNumInputs = oldMapOperands.size();

1278 state.operands.append(op->operand_begin() + memRefOperandPos + 1 +

1279 oldMapNumInputs,

1281

1283 for (auto result : op->getResults())

1284 state.types.push_back(result.getType());

1285

1286

1288 for (auto namedAttr : op->getAttrs()) {

1289 if (affMapAccInterface &&

1290 namedAttr.getName() ==

1291 affMapAccInterface.getAffineMapAttrForMemRef(oldMemRef).getName())

1292 state.attributes.push_back({namedAttr.getName(), newMapAttr});

1293 else

1294 state.attributes.push_back(namedAttr);

1295 }

1296

1297

1298 auto *repOp = builder.create(state);

1301

1302 return success();

1303 }

1304

1309 Operation *postDomOpFilter, bool allowNonDereferencingOps,

1310 bool replaceInDeallocOp) {

1311 unsigned newMemRefRank = cast(newMemRef.getType()).getRank();

1312 (void)newMemRefRank;

1313 unsigned oldMemRefRank = cast(oldMemRef.getType()).getRank();

1314 (void)oldMemRefRank;

1315 if (indexRemap) {

1316 assert(indexRemap.getNumSymbols() == symbolOperands.size() &&

1317 "symbol operand count mismatch");

1319 extraOperands.size() + oldMemRefRank + symbolOperands.size());

1320 assert(indexRemap.getNumResults() + extraIndices.size() == newMemRefRank);

1321 } else {

1322 assert(oldMemRefRank + extraIndices.size() == newMemRefRank);

1323 }

1324

1325

1326 assert(cast(oldMemRef.getType()).getElementType() ==

1327 cast(newMemRef.getType()).getElementType());

1328

1329 std::unique_ptr domInfo;

1330 std::unique_ptr postDomInfo;

1331 if (domOpFilter)

1332 domInfo = std::make_unique(

1334

1335 if (postDomOpFilter)

1336 postDomInfo = std::make_unique(

1337 postDomOpFilter->getParentOfType());

1338

1339

1340

1341

1343 for (auto *op : oldMemRef.getUsers()) {

1344

1345 if (domOpFilter && !domInfo->dominates(domOpFilter, op))

1346 continue;

1347

1348

1349 if (postDomOpFilter && !postDomInfo->postDominates(postDomOpFilter, op))

1350 continue;

1351

1352

1353

1354 if (hasSingleEffectMemoryEffects::Free(op, oldMemRef) &&

1355 !replaceInDeallocOp)

1356 continue;

1357

1358

1359

1360

1361 if (!isa(*op)) {

1362 if (!allowNonDereferencingOps) {

1363 LLVM_DEBUG(llvm::dbgs()

1364 << "Memref replacement failed: non-deferencing memref op: \n"

1365 << *op << '\n');

1366 return failure();

1367 }

1368

1369

1371 LLVM_DEBUG(llvm::dbgs() << "Memref replacement failed: use without a "

1372 "memrefs normalizable trait: \n"

1373 << *op << '\n');

1374 return failure();

1375 }

1376 }

1377

1378

1379

1380 opsToReplace.insert(op);

1381 }

1382

1383 for (auto *op : opsToReplace) {

1385 oldMemRef, newMemRef, op, extraIndices, indexRemap, extraOperands,

1386 symbolOperands, allowNonDereferencingOps)))

1387 llvm_unreachable("memref replacement guaranteed to succeed here");

1388 }

1389

1390 return success();

1391 }

1392

1393

1394

1395

1396

1397

1398

1399

1400

1401

1402

1403

1404

1405

1406

1407

1408

1409

1410

1411

1412

1413

1414

1415

1416

1417

1418

1419

1420

1423

1426 for (auto operand : opInst->getOperands())

1427 if (isa_and_nonnull(operand.getDefiningOp()))

1428 subOperands.push_back(operand);

1429

1430

1433

1434 if (affineApplyOps.empty())

1435 return;

1436

1437

1438

1439 bool localized = true;

1440 for (auto *op : affineApplyOps) {

1441 for (auto result : op->getResults()) {

1442 for (auto *user : result.getUsers()) {

1443 if (user != opInst) {

1444 localized = false;

1445 break;

1446 }

1447 }

1448 }

1449 }

1450 if (localized)

1451 return;

1452

1457

1458

1459 sliceOps->reserve(composedMap.getNumResults());

1460 for (auto resultExpr : composedMap.getResults()) {

1461 auto singleResMap = AffineMap::get(composedMap.getNumDims(),

1462 composedMap.getNumSymbols(), resultExpr);

1463 sliceOps->push_back(builder.create(

1464 opInst->getLoc(), singleResMap, composedOpOperands));

1465 }

1466

1467

1468

1469

1470

1472 for (Value &operand : newOperands) {

1473

1474 unsigned j, f;

1475 for (j = 0, f = subOperands.size(); j < f; j++) {

1476 if (operand == subOperands[j])

1477 break;

1478 }

1479 if (j < subOperands.size())

1480 operand = (*sliceOps)[j];

1481 }

1482 for (unsigned idx = 0, e = newOperands.size(); idx < e; idx++)

1483 opInst->setOperand(idx, newOperands[idx]);

1484 }

1485

1486

1487

1488

1489

1490

1491

1492

1493

1494

1496

1497

1498

1499

1500

1501

1502

1505 SmallVectorImpl<std::tuple<AffineExpr, unsigned, unsigned>> &tileSizePos) {

1506

1507

1508

1509

1510

1512 unsigned pos = 0;

1516 if (isa(binaryExpr.getRHS()))

1517 floordivExprs.emplace_back(

1518 std::make_tuple(binaryExpr.getLHS(), binaryExpr.getRHS(), pos));

1519 }

1520 pos++;

1521 }

1522

1523 if (floordivExprs.empty()) {

1525 return success();

1526 }

1527

1528

1529

1530 for (std::tuple<AffineExpr, AffineExpr, unsigned> fexpr : floordivExprs) {

1531 AffineExpr floordivExprLHS = std::get<0>(fexpr);

1532 AffineExpr floordivExprRHS = std::get<1>(fexpr);

1533 unsigned floordivPos = std::get<2>(fexpr);

1534

1535

1536

1537

1538

1539

1540

1541

1542 bool found = false;

1543 pos = 0;

1545 bool notTiled = false;

1546 if (pos != floordivPos) {

1548 if (e == floordivExprLHS) {

1550 AffineBinaryOpExpr binaryExpr = cast(expr);

1551

1552 if (floordivExprLHS == binaryExpr.getLHS() &&

1553 floordivExprRHS == binaryExpr.getRHS()) {

1554

1555

1556 if (!found) {

1557 tileSizePos.emplace_back(

1558 std::make_tuple(binaryExpr.getRHS(), floordivPos, pos));

1559 found = true;

1560 } else {

1561

1562

1563

1564 notTiled = true;

1565 }

1566 } else {

1567

1568

1569

1570 notTiled = true;

1571 }

1572 } else {

1573

1574

1575

1576 notTiled = true;

1577 }

1578 }

1579 });

1580 }

1581 if (notTiled) {

1583 return success();

1584 }

1585 pos++;

1586 }

1587 }

1588 return success();

1589 }

1590

1591

1592

1593

1594

1595

1596

1597

1598

1599

1600

1601 static bool

1605

1606

1608 return expr

1610 if (isa(e) &&

1611 llvm::any_of(inMemrefTypeDynDims, [&](unsigned dim) {

1613 }))

1616 })

1617 .wasInterrupted();

1618 }

1619

1620

1623

1624

1625

1628 switch (pat) {

1630 binaryExpr = cast(oldMapOutput);

1631 newMapOutput = binaryExpr.getRHS();

1632 break;

1634 binaryExpr = cast(oldMapOutput);

1637 break;

1638 default:

1639 newMapOutput = oldMapOutput;

1640 }

1641 return newMapOutput;

1642 }

1643

1644

1645

1646

1647

1648

1649

1650

1651

1652

1653

1654

1655

1656

1657

1658

1659

1660

1661

1662

1663

1664

1665

1666

1667

1668

1669

1670

1671

1672

1673

1674

1675

1676 template

1678 MemRefType newMemRefType, AffineMap map,

1679 AllocLikeOp allocOp, OpBuilder b,

1681

1684 unsigned dynIdx = 0;

1685 for (unsigned d = 0; d < oldMemRefType.getRank(); ++d) {

1686 if (oldMemRefShape[d] < 0) {

1687

1688 inAffineApply.emplace_back(allocOp.getDynamicSizes()[dynIdx]);

1689 dynIdx++;

1690 } else {

1691

1693 inAffineApply.emplace_back(

1694 b.create arith::ConstantOp(allocOp.getLoc(), constantAttr));

1695 }

1696 }

1697

1698

1699

1700 unsigned newDimIdx = 0;

1705 if (newMemRefShape[newDimIdx] < 0) {

1706

1708 for (auto pos : tileSizePos) {

1709 if (newDimIdx == std::get<1>(pos))

1711 else if (newDimIdx == std::get<2>(pos))

1713 }

1717 Value affineApp =

1718 b.create(allocOp.getLoc(), newMap, inAffineApply);

1719 newDynamicSizes.emplace_back(affineApp);

1720 }

1721 newDimIdx++;

1722 }

1723 }

1724

1725 template

1727 MemRefType memrefType = allocOp.getType();

1729

1730

1731

1733 if (newMemRefType == memrefType)

1734

1735

1736 return failure();

1737

1738 Value oldMemRef = allocOp.getResult();

1739

1741 AffineMap layoutMap = memrefType.getLayout().getAffineMap();

1742 AllocLikeOp newAlloc;

1743

1744

1747 if (newMemRefType.getNumDynamicDims() > 0 && !tileSizePos.empty()) {

1748 auto oldMemRefType = cast(oldMemRef.getType());

1751 newDynamicSizes);

1752

1753 newAlloc =

1754 b.create(allocOp.getLoc(), newMemRefType, newDynamicSizes,

1755 allocOp.getAlignmentAttr());

1756 } else {

1757 newAlloc = b.create(allocOp.getLoc(), newMemRefType,

1758 allocOp.getAlignmentAttr());

1759 }

1760

1762 {},

1763 layoutMap,

1764 {},

1765 symbolOperands,

1766 nullptr,

1767 nullptr,

1768 true))) {

1769

1770 newAlloc.erase();

1771 return failure();

1772 }

1773

1774

1776 return hasSingleEffectMemoryEffects::Free(op, oldMemRef);

1777 }));

1779 allocOp.erase();

1780 return success();

1781 }

1782

1783 LogicalResult

1785 MemRefType memrefType = reinterpretCastOp.getType();

1786 AffineMap oldLayoutMap = memrefType.getLayout().getAffineMap();

1787 Value oldMemRef = reinterpretCastOp.getResult();

1788

1789

1791 return success();

1792

1793

1794

1796 if (newMemRefType == memrefType)

1797

1798 return failure();

1799

1800 uint64_t newRank = newMemRefType.getRank();

1804 Location loc = reinterpretCastOp.getLoc();

1805

1809 ValueRange oldSizes = reinterpretCastOp.getSizes();

1810 unsigned idx = 0;

1812

1813

1814 for (unsigned i = 0, e = memrefType.getRank(); i < e; i++) {

1815 if (memrefType.isDynamicDim(i))

1816 mapOperands[i] =

1817 b.create arith::SubIOp(loc, oldSizes[0].getType(), oldSizes[idx++],

1819 else

1821 }

1822 for (unsigned i = 0, e = oldStrides.size(); i < e; i++)

1823 mapOperands[memrefType.getRank() + i] = oldStrides[i];

1826

1827 for (unsigned i = 0; i < newRank; i++) {

1828 if (!newMemRefType.isDynamicDim(i))

1829 continue;

1830 newSizes.push_back(b.create(

1831 loc,

1834 mapOperands));

1835 }

1836 for (unsigned i = 0, e = newSizes.size(); i < e; i++) {

1837 newSizes[i] =

1838 b.create arith::AddIOp(loc, newSizes[i].getType(), newSizes[i],

1840 }

1841

1842 auto newReinterpretCast = b.create memref::ReinterpretCastOp(

1843 loc, newMemRefType, reinterpretCastOp.getSource(),

1846 newStaticOffsets,

1847 newShape,

1848 newStaticStrides);

1849

1850

1852 newReinterpretCast,

1853 {},

1854 oldLayoutMap,

1855 {},

1856 oldStrides,

1857 nullptr,

1858 nullptr,

1859 true))) {

1860

1861 newReinterpretCast.erase();

1862 return failure();

1863 }

1864

1866 reinterpretCastOp.erase();

1867 return success();

1868 }

1869

1870 template LogicalResult

1871 mlir::affine::normalizeMemRefmemref::AllocaOp(memref::AllocaOp op);

1872 template LogicalResult

1873 mlir::affine::normalizeMemRefmemref::AllocOp(memref::AllocOp op);

1874

1876 unsigned rank = memrefType.getRank();

1877 if (rank == 0)

1878 return memrefType;

1879

1880 if (memrefType.getLayout().isIdentity()) {

1881

1882

1883 return memrefType;

1884 }

1885 AffineMap layoutMap = memrefType.getLayout().getAffineMap();

1886 unsigned numSymbolicOperands = layoutMap.getNumSymbols();

1887

1888

1889

1890

1891

1892

1893

1896 if (memrefType.getNumDynamicDims() > 0 && tileSizePos.empty())

1897 return memrefType;

1898

1899

1900

1902

1905 for (unsigned d = 0; d < rank; ++d) {

1906

1907 if (shape[d] > 0) {

1908 fac.addBound(BoundType::LB, d, 0);

1909 fac.addBound(BoundType::UB, d, shape[d] - 1);

1910 } else {

1911 memrefTypeDynDims.emplace_back(d);

1912 }

1913 }

1914

1915

1918 return memrefType;

1919

1920

1923 MLIRContext *context = memrefType.getContext();

1924 for (unsigned d = 0; d < newRank; ++d) {

1925

1927 newShape[d] = ShapedType::kDynamic;

1928 continue;

1929 }

1930

1931 std::optional<int64_t> ubConst = fac.getConstantBound64(BoundType::UB, d);

1932

1933

1934

1935

1936 if (!ubConst.has_value() || *ubConst < 0) {

1937 LLVM_DEBUG(llvm::dbgs()

1938 << "can't normalize map due to unknown/invalid upper bound");

1939 return memrefType;

1940 }

1941

1942 newShape[d] = *ubConst + 1;

1943 }

1944

1945

1946 auto newMemRefType =

1951 return newMemRefType;

1952 }

1953

1963 return result;

1964 }

1965

1966

1967

1975 }

1976

1977 FailureOr<SmallVector>

1980 if (hasOuterBound)

1981 basis = basis.drop_front();

1982

1983

1986 for (OpFoldResult basisElem : llvm::reverse(basis)) {

1987 FailureOr nextProd =

1989 if (failed(nextProd))

1990 return failure();

1991 basisProd = *nextProd;

1993 }

1994

1996 results.reserve(divisors.size() + 1);

1997 Value residual = linearIndex;

1998 for (Value divisor : llvm::reverse(divisors)) {

2000 results.push_back(divMod.quotient);

2002 }

2003 results.push_back(residual);

2004 return results;

2005 }

2006

2007 FailureOr<SmallVector>

2010 bool hasOuterBound) {

2011 if (hasOuterBound)

2012 basis = basis.drop_front();

2013

2014

2017 for (OpFoldResult basisElem : llvm::reverse(basis)) {

2018 FailureOr nextProd =

2020 if (failed(nextProd))

2021 return failure();

2022 basisProd = *nextProd;

2024 }

2025

2027 results.reserve(divisors.size() + 1);

2028 Value residual = linearIndex;

2029 for (Value divisor : llvm::reverse(divisors)) {

2031 results.push_back(divMod.quotient);

2033 }

2034 results.push_back(residual);

2035 return results;

2036 }

2037

2042 }

2043

2047 assert(multiIndex.size() == basis.size() ||

2048 multiIndex.size() == basis.size() + 1);

2050

2051

2052

2053 if (multiIndex.size() == basis.size() + 1)

2055

2056 for (size_t i = 0; i < basis.size(); ++i) {

2058 }

2059

2062 strides.reserve(stridesAffine.size());

2063 llvm::transform(stridesAffine, std::back_inserter(strides),

2064 [&builder, &basis, loc](AffineExpr strideExpr) {

2066 builder, loc, strideExpr, basis);

2067 });

2068

2069 auto &&[linearIndexExpr, multiIndexAndStrides] = computeLinearIndex(

2072 multiIndexAndStrides);

2073 }

static bool mayHaveEffect(Operation *srcMemOp, Operation *destMemOp, unsigned minSurroundingLoops)

Returns true if srcMemOp may have an effect on destMemOp within the scope of the outermost minSurroun...

static void createNewDynamicSizes(MemRefType oldMemRefType, MemRefType newMemRefType, AffineMap map, AllocLikeOp allocOp, OpBuilder b, SmallVectorImpl< Value > &newDynamicSizes)

Create new maps to calculate each dimension size of newMemRefType, and create newDynamicSizes from th...

static bool isDereferencingOp(Operation *op)

static LogicalResult getTileSizePos(AffineMap map, SmallVectorImpl< std::tuple< AffineExpr, unsigned, unsigned >> &tileSizePos)

Check if map is a tiled layout.

TileExprPattern

Enum to set patterns of affine expr in tiled-layout map.

static void promoteIfBlock(AffineIfOp ifOp, bool elseBlock)

Promotes the then or the else block of ifOp (depending on whether elseBlock is false or true) into if...

static bool isNormalizedMemRefDynamicDim(unsigned dim, AffineMap layoutMap, SmallVectorImpl< unsigned > &inMemrefTypeDynDims)

Check if dim dimension of memrefType with layoutMap becomes dynamic after normalization.

static FailureOr< OpFoldResult > composedAffineMultiply(OpBuilder &b, Location loc, OpFoldResult lhs, OpFoldResult rhs)

Create an affine map that computes lhs * rhs, composing in any other affine maps.

static void loadCSE(AffineReadOpInterface loadA, SmallVectorImpl< Operation * > &loadOpsToErase, DominanceInfo &domInfo, llvm::function_ref< bool(Value, Value)> mayAlias)

static AffineExpr createDimSizeExprForTiledLayout(AffineExpr oldMapOutput, TileExprPattern pat)

Create affine expr to calculate dimension size for a tiled-layout map.

static Operation * getOutermostInvariantForOp(AffineIfOp ifOp)

Returns the outermost affine.for/parallel op that the ifOp is invariant on.

static void findUnusedStore(AffineWriteOpInterface writeA, SmallVectorImpl< Operation * > &opsToErase, PostDominanceInfo &postDominanceInfo, llvm::function_ref< bool(Value, Value)> mayAlias)

static bool mustReachAtInnermost(const MemRefAccess &srcAccess, const MemRefAccess &destAccess)

Returns true if the memory operation of destAccess depends on srcAccess inside of the innermost commo...

static void forwardStoreToLoad(AffineReadOpInterface loadOp, SmallVectorImpl< Operation * > &loadOpsToErase, SmallPtrSetImpl< Value > &memrefsToErase, DominanceInfo &domInfo, llvm::function_ref< bool(Value, Value)> mayAlias)

Attempt to eliminate loadOp by replacing it with a value stored into memory which the load is guarant...

static void visit(Operation *op, DenseSet< Operation * > &visited)

Visits all the pdl.operand(s), pdl.result(s), and pdl.operation(s) connected to the given operation.

static bool mayAlias(Value first, Value second)

Returns true if two values may be referencing aliasing memory.

static Value max(ImplicitLocOpBuilder &builder, Value value, Value bound)

static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound)

Affine binary operation expression.

AffineExpr getLHS() const

AffineExpr getRHS() const

An integer constant appearing in affine expression.

A dimensional identifier appearing in an affine expression.

unsigned getPosition() const

See documentation for AffineExprVisitorBase.

Base type for affine expression.

AffineExpr floorDiv(uint64_t v) const

RetT walk(FnT &&callback) const

Walk all of the AffineExpr's in this expression in postorder.

AffineExpr ceilDiv(uint64_t v) const

A multi-dimensional affine map Affine map's are immutable like Type's, and they are uniqued.

MLIRContext * getContext() const

static AffineMap getMultiDimIdentityMap(unsigned numDims, MLIRContext *context)

Returns an AffineMap with 'numDims' identity result dim exprs.

static AffineMap get(MLIRContext *context)

Returns a zero result affine map with no dimensions or symbols: () -> ().

unsigned getNumSymbols() const

unsigned getNumDims() const

ArrayRef< AffineExpr > getResults() const

unsigned getNumResults() const

unsigned getNumInputs() const

AffineExpr getResult(unsigned idx) const

AffineMap compose(AffineMap map) const

Returns the AffineMap resulting from composing this with map.

bool isIdentity() const

Returns true if this affine map is an identity affine map.

A symbolic identifier appearing in an affine expression.

unsigned getPosition() const

This class represents the main alias analysis interface in MLIR.

AliasResult alias(Value lhs, Value rhs)

Given two values, return their aliasing behavior.

bool isNo() const

Returns if this result indicates no possibility of aliasing.

This class represents an argument of a Block.

Block represents an ordered list of Operations.

OpListType::iterator iterator

SuccessorRange getSuccessors()

Operation * getTerminator()

Get the terminator operation of this block.

OpListType & getOperations()

IntegerAttr getIndexAttr(int64_t value)

IntegerAttr getIntegerAttr(Type type, int64_t value)

AffineMap getMultiDimIdentityMap(unsigned rank)

BoolAttr getBoolAttr(bool value)

StringAttr getStringAttr(const Twine &bytes)

AffineExpr getAffineDimExpr(unsigned position)

AffineMap getConstantAffineMap(int64_t val)

Returns a single constant result affine map with 0 dimensions and 0 symbols.

MLIRContext * getContext() const

A class for computing basic dominance information.

bool dominates(Operation *a, Operation *b) const

Return true if operation A dominates operation B, i.e.

LogicalResult composeMatchingMap(AffineMap other)

Composes an affine map whose dimensions and symbols match one to one with the dimensions and symbols ...

void projectOut(Value val)

Projects out the variable that is associate with Value.

This class represents a frozen set of patterns that can be processed by a pattern applicator.

This class allows control over how the GreedyPatternRewriteDriver works.

GreedyRewriteConfig & setStrictness(GreedyRewriteStrictness mode)

This is a utility class for mapping one set of IR entities to another.

void clear()

Clears all mappings held by the mapper.

ImplicitLocOpBuilder maintains a 'current location', allowing use of the create<> method without spec...

Location getLoc() const

Accessors for the implied location.

This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...

MLIRContext is the top-level object for a collection of MLIR operations.

This is a builder type that keeps local references to arguments.

Builder & setLayout(MemRefLayoutAttrInterface newLayout)

Builder & setShape(ArrayRef< int64_t > newShape)

NamedAttribute represents a combination of a name and an Attribute value.

Attribute getValue() const

Return the value of the attribute.

This class helps build Operations.

static OpBuilder atBlockBegin(Block *block, Listener *listener=nullptr)

Create a builder and set the insertion point to before the first operation in the block but still ins...

Block::iterator getInsertionPoint() const

Returns the current insertion point of the builder.

Operation * clone(Operation &op, IRMapping &mapper)

Creates a deep copy of the specified operation, remapping any operands that use values outside of the...

void setInsertionPointToStart(Block *block)

Sets the insertion point to the start of the specified block.

Operation * create(const OperationState &state)

Creates an operation given the fields represented as an OperationState.

void setInsertionPointAfter(Operation *op)

Sets the insertion point to the node after the specified operation, which will cause subsequent inser...

Block * getInsertionBlock() const

Return the block the current insertion point belongs to.

This class represents a single result from folding an operation.

This trait indicates that the memory effects of an operation includes the effects of operations neste...

This class provides the API for ops that are known to be isolated from above.

This class implements the operand iterators for the Operation class.

Operation is the basic unit of execution within MLIR.

Value getOperand(unsigned idx)

bool hasTrait()

Returns true if the operation was registered with a particular trait, e.g.

void setOperand(unsigned idx, Value value)

operand_iterator operand_begin()

OpResult getResult(unsigned idx)

Get the 'idx'th result of this operation.

Location getLoc()

The source location the operation was defined or derived from.

unsigned getNumOperands()

Operation * getParentOp()

Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...

ArrayRef< NamedAttribute > getAttrs()

Return all of the attributes on this operation.

Block * getBlock()

Returns the operation block that contains this operation.

OpTy getParentOfType()

Return the closest surrounding parent operation that is of type 'OpTy'.

operand_iterator operand_end()

Region & getRegion(unsigned index)

Returns the region held by this operation at position 'index'.

OperationName getName()

The name of an operation is the key identifier for it.

operand_range getOperands()

Returns an iterator on the underlying Value's.

bool isAncestor(Operation *other)

Return true if this operation is an ancestor of the other operation.

void replaceAllUsesWith(ValuesT &&values)

Replace all uses of results of this operation with the provided 'values'.

void setOperands(ValueRange operands)

Replace the current operands of this operation with the ones provided in 'operands'.

user_range getUsers()

Returns a range of all users.

Region * getParentRegion()

Returns the region to which the instruction belongs.

result_range getResults()

void erase()

Remove this operation from its parent block and delete it.

unsigned getNumResults()

Return the number of results held by this operation.

A class for computing basic postdominance information.

bool postDominates(Operation *a, Operation *b) const

Return true if operation A postdominates operation B.

This class contains a list of basic blocks and a link to the parent operation it is attached to.

Region * getParentRegion()

Return the region containing this region or nullptr if the region is attached to a top-level operatio...

bool isAncestor(Region *other)

Return true if this region is ancestor of the other region.

void takeBody(Region &other)

Takes body of another region (that region will have no body after this operation completes).

This class provides an abstraction over the different types of ranges over Values.

type_range getType() const

type_range getTypes() const

This class represents an instance of an SSA value in the MLIR system, representing a computable value...

Type getType() const

Return the type of this value.

void replaceAllUsesExcept(Value newValue, const SmallPtrSetImpl< Operation * > &exceptions)

Replace all uses of 'this' value with 'newValue', updating anything in the IR that uses 'this' to use...

void replaceAllUsesWith(Value newValue)

Replace all uses of 'this' value with the new value, updating anything in the IR that uses 'this' to ...

user_range getUsers() const

Operation * getDefiningOp() const

If this value is the result of an operation, return the operation that defines it.

static WalkResult advance()

static WalkResult interrupt()

An AffineValueMap is an affine map plus its ML value operands and results for analysis purposes.

LogicalResult canonicalize()

Attempts to canonicalize the map and operands.

unsigned getNumSymbols() const

ArrayRef< Value > getOperands() const

unsigned getNumDims() const

AffineExpr getResult(unsigned i)

AffineMap getAffineMap() const

unsigned getNumResults() const

static void difference(const AffineValueMap &a, const AffineValueMap &b, AffineValueMap *res)

Return the value map that is the difference of value maps 'a' and 'b', represented as an affine map a...

FlatAffineValueConstraints is an extension of FlatLinearValueConstraints with helper functions for Af...

LogicalResult addBound(presburger::BoundType type, unsigned pos, AffineMap boundMap, ValueRange operands)

Adds a bound for the variable at the specified position with constraints being drawn from the specifi...

Specialization of arith.constant op that returns an integer of index type.

std::optional< int64_t > getConstantBound64(BoundType type, unsigned pos) const

The same, but casts to int64_t.

unsigned getNumVars() const

unsigned getNumLocalVars() const

std::optional< SmallVector< Value, 8 > > expandAffineMap(OpBuilder &builder, Location loc, AffineMap affineMap, ValueRange operands)

Create a sequence of operations that implement the affineMap applied to the given operands (as it it ...

void fullyComposeAffineMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)

Given an affine map map and its input operands, this method composes into map, maps of AffineApplyOps...

void affineScalarReplace(func::FuncOp f, DominanceInfo &domInfo, PostDominanceInfo &postDomInfo, AliasAnalysis &analysis)

Replace affine store and load accesses by scalars by forwarding stores to loads and eliminate invaria...

LogicalResult promoteIfSingleIteration(AffineForOp forOp)

Promotes the loop body of a AffineForOp to its containing block if the loop was known to have a singl...

bool isValidDim(Value value)

Returns true if the given Value can be used as a dimension id in the region of the closest surroundin...

Value expandAffineExpr(OpBuilder &builder, Location loc, AffineExpr expr, ValueRange dimValues, ValueRange symbolValues)

Emit code that computes the given affine expression using standard arithmetic operations applied to t...

unsigned getNumCommonSurroundingLoops(Operation &a, Operation &b)

Returns the number of surrounding loops common to both A and B.

void normalizeAffineParallel(AffineParallelOp op)

Normalize a affine.parallel op so that lower bounds are 0 and steps are 1.

DependenceResult checkMemrefAccessDependence(const MemRefAccess &srcAccess, const MemRefAccess &dstAccess, unsigned loopDepth, FlatAffineValueConstraints *dependenceConstraints=nullptr, SmallVector< DependenceComponent, 2 > *dependenceComponents=nullptr, bool allowRAR=false)

LogicalResult affineParallelize(AffineForOp forOp, ArrayRef< LoopReduction > parallelReductions={}, AffineParallelOp *resOp=nullptr)

Replaces a parallel affine.for op with a 1-d affine.parallel op.

AffineApplyOp makeComposedAffineApply(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)

Returns a composed AffineApplyOp by composing map and operands with other AffineApplyOps supplying th...

void canonicalizeMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)

Modifies both map and operands in-place so as to:

void getReachableAffineApplyOps(ArrayRef< Value > operands, SmallVectorImpl< Operation * > &affineApplyOps)

Returns in affineApplyOps, the sequence of those AffineApplyOp Operations that are reachable via a se...

LogicalResult normalizeAffineFor(AffineForOp op, bool promoteSingleIter=false)

Normalize an affine.for op.

Region * getAffineAnalysisScope(Operation *op)

Returns the closest region enclosing op that is held by a non-affine operation; nullptr if there is n...

bool isValidSymbol(Value value)

Returns true if the given value can be used as a symbol in the region of the closest surrounding op t...

OpFoldResult makeComposedFoldedAffineApply(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)

Constructs an AffineApplyOp that applies map to operands after composing the map with the maps of any...

bool hasDependence(DependenceResult result)

Utility function that returns true if the provided DependenceResult corresponds to a dependence resul...

MemRefType normalizeMemRefType(MemRefType memrefType)

Normalizes memrefType so that the affine layout map of the memref is transformed to an identity map w...

LogicalResult normalizeMemRef(AllocLikeOp op)

Rewrites the memref defined by alloc or reinterpret_cast op to have an identity layout map and update...

FailureOr< SmallVector< Value > > delinearizeIndex(OpBuilder &b, Location loc, Value linearIndex, ArrayRef< Value > basis, bool hasOuterBound=true)

Generate the IR to delinearize linearIndex given the basis and return the multi-index.

OpFoldResult linearizeIndex(ArrayRef< OpFoldResult > multiIndex, ArrayRef< OpFoldResult > basis, ImplicitLocOpBuilder &builder)

DivModValue getDivMod(OpBuilder &b, Location loc, Value lhs, Value rhs)

Create IR to calculate (div lhs, rhs) and (mod lhs, rhs).

bool hasNoInterveningEffect(Operation *start, T memOp, llvm::function_ref< bool(Value, Value)> mayAlias)

Ensure that all operations that could be executed after start (noninclusive) and prior to memOp (e....

void createAffineComputationSlice(Operation *opInst, SmallVectorImpl< AffineApplyOp > *sliceOps)

Given an operation, inserts one or more single result affine apply operations, results of which are e...

LogicalResult hoistAffineIfOp(AffineIfOp ifOp, bool *folded=nullptr)

Hoists out affine.if/else to as high as possible, i.e., past all invariant affine....

bool noDependence(DependenceResult result)

Returns true if the provided DependenceResult corresponds to the absence of a dependence.

AffineExpr substWithMin(AffineExpr e, AffineExpr dim, AffineExpr min, AffineExpr max, bool positivePath=true)

Traverse e and return an AffineExpr where all occurrences of dim have been replaced by either:

LogicalResult replaceAllMemRefUsesWith(Value oldMemRef, Value newMemRef, ArrayRef< Value > extraIndices={}, AffineMap indexRemap=AffineMap(), ArrayRef< Value > extraOperands={}, ArrayRef< Value > symbolOperands={}, Operation *domOpFilter=nullptr, Operation *postDomOpFilter=nullptr, bool allowNonDereferencingOps=false, bool replaceInDeallocOp=false)

Replaces all "dereferencing" uses of oldMemRef with newMemRef while optionally remapping the old memr...

constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)

Include the generated interface declarations.

AffineMap simplifyAffineMap(AffineMap map)

Simplifies an affine map by simplifying its underlying AffineExpr results.

void bindDims(MLIRContext *ctx, AffineExprTy &...exprs)

Bind a list of AffineExpr references to DimExpr at positions: [0 .

std::pair< AffineExpr, SmallVector< OpFoldResult > > computeLinearIndex(OpFoldResult sourceOffset, ArrayRef< OpFoldResult > strides, ArrayRef< OpFoldResult > indices)

Compute linear index from provided strides and indices, assuming strided layout.

LogicalResult applyPatternsGreedily(Region &region, const FrozenRewritePatternSet &patterns, GreedyRewriteConfig config=GreedyRewriteConfig(), bool *changed=nullptr)

Rewrite ops in the given region, which must be isolated from above, by repeatedly applying the highes...

SmallVector< int64_t > computeStrides(ArrayRef< int64_t > sizes)

InFlightDiagnostic emitError(Location loc)

Utility method to emit an error message using this location.

LogicalResult applyOpPatternsGreedily(ArrayRef< Operation * > ops, const FrozenRewritePatternSet &patterns, GreedyRewriteConfig config=GreedyRewriteConfig(), bool *changed=nullptr, bool *allErased=nullptr)

Rewrite the specified ops by repeatedly applying the highest benefit patterns in a greedy worklist dr...

@ CeilDiv

RHS of ceildiv is always a constant or a symbolic expression.

@ Mod

RHS of mod is always a constant or a symbolic expression with a positive value.

@ FloorDiv

RHS of floordiv is always a constant or a symbolic expression.

AffineExpr getAffineBinaryOpExpr(AffineExprKind kind, AffineExpr lhs, AffineExpr rhs)

const FrozenRewritePatternSet & patterns

void bindSymbols(MLIRContext *ctx, AffineExprTy &...exprs)

Bind a list of AffineExpr references to SymbolExpr at positions: [0 .

Value getValueOrCreateConstantIndexOp(OpBuilder &b, Location loc, OpFoldResult ofr)

Converts an OpFoldResult to a Value.

AffineExpr getAffineConstantExpr(int64_t constant, MLIRContext *context)

auto get(MLIRContext *context, Ts &&...params)

Helper method that injects context only if needed, this helps unify some of the attribute constructio...

@ ExistingOps

Only pre-existing ops are processed.

AffineExpr getAffineDimExpr(unsigned position, MLIRContext *context)

These free functions allow clients of the API to not use classes in detail.

AffineExpr getAffineSymbolExpr(unsigned position, MLIRContext *context)

The following effect indicates that the operation reads from some resource.

This represents an operation in an abstracted form, suitable for use with the builder APIs.

Checks whether two accesses to the same memref access the same element.

Holds the result of (div a, b) and (mod a, b).

A description of a (parallelizable) reduction in an affine loop.

arith::AtomicRMWKind kind

Reduction kind.

Value value

The value being reduced.

Encapsulates a memref load or store access information.

Eliminates variable at the specified position using Fourier-Motzkin variable elimination.