ModuleTranslation.cpp Source File (original) (raw)

34#include "llvm/ADT/STLExtras.h"

35#include "llvm/ADT/StringExtras.h"

36#include "llvm/ADT/TypeSwitch.h"

37#include "llvm/Analysis/TargetFolder.h"

38#include "llvm/Frontend/OpenMP/OMPIRBuilder.h"

39#include "llvm/IR/BasicBlock.h"

40#include "llvm/IR/CFG.h"

41#include "llvm/IR/Constants.h"

42#include "llvm/IR/DerivedTypes.h"

43#include "llvm/IR/IRBuilder.h"

44#include "llvm/IR/InlineAsm.h"

45#include "llvm/IR/LLVMContext.h"

46#include "llvm/IR/MDBuilder.h"

47#include "llvm/IR/Module.h"

48#include "llvm/IR/Verifier.h"

49#include "llvm/Support/Debug.h"

50#include "llvm/Support/ErrorHandling.h"

51#include "llvm/Support/raw_ostream.h"

52#include "llvm/Transforms/Utils/BasicBlockUtils.h"

53#include "llvm/Transforms/Utils/Cloning.h"

54#include "llvm/Transforms/Utils/ModuleUtils.h"

55#include

56#include

58#define DEBUG_TYPE "llvm-dialect-to-llvm-ir"

60using namespace mlir;

64#include "mlir/Dialect/LLVMIR/LLVMConversionEnumsToLLVM.inc"

66namespace {

87class InstructionCapturingInserter : public llvm::IRBuilderCallbackInserter {

88public:

90 InstructionCapturingInserter()

91 : llvm::IRBuilderCallbackInserter([this](llvm::Instruction *instruction) {

92 if (LLVM_LIKELY(enabled))

93 capturedInstructions.push_back(instruction);

94 }) {}

98 return capturedInstructions;

99 }

100

101

102 void clearCapturedInstructions() { capturedInstructions.clear(); }

103

104

105 class CollectionScope {

106 public:

107

108 CollectionScope(llvm::IRBuilderBase &irBuilder, bool isBuilderCapturing);

109

110

111 ~CollectionScope();

112

114 if (!inserter)

115 return {};

116 return inserter->getCapturedInstructions();

117 }

118

119 private:

120

121 InstructionCapturingInserter *inserter = nullptr;

122

123

125

126

127 bool wasEnabled;

128 };

129

130

131 void setEnabled(bool enabled = true) { this->enabled = enabled; }

132

133private:

134

136

137

138 bool enabled = false;

139};

140

141using CapturingIRBuilder =

142 llvm::IRBuilder<llvm::TargetFolder, InstructionCapturingInserter>;

143}

144

145InstructionCapturingInserter::CollectionScope::CollectionScope(

146 llvm::IRBuilderBase &irBuilder, bool isBuilderCapturing) {

147

148 if (!isBuilderCapturing)

149 return;

150

151 auto &capturingIRBuilder = static_cast<CapturingIRBuilder &>(irBuilder);

152 inserter = &capturingIRBuilder.getInserter();

153 wasEnabled = inserter->enabled;

154 if (wasEnabled)

155 previouslyCollectedInstructions.swap(inserter->capturedInstructions);

156 inserter->setEnabled(true);

157}

158

159InstructionCapturingInserter::CollectionScope::~CollectionScope() {

160 if (!inserter)

161 return;

162

163 previouslyCollectedInstructions.swap(inserter->capturedInstructions);

164

165

166 if (wasEnabled) {

167 llvm::append_range(inserter->capturedInstructions,

168 previouslyCollectedInstructions);

169 }

170 inserter->setEnabled(wasEnabled);

171}

172

173

174

175static FailureOrllvm::DataLayout

178 std::optional loc = std::nullopt) {

179 if (!loc)

180 loc = UnknownLoc::get(attribute.getContext());

181

182

183 std::string llvmDataLayout;

184 llvm::raw_string_ostream layoutStream(llvmDataLayout);

185 for (DataLayoutEntryInterface entry : attribute.getEntries()) {

186 auto key = llvm::dyn_cast_if_present(entry.getKey());

187 if (!key)

188 continue;

189 if (key.getValue() == DLTIDialect::kDataLayoutEndiannessKey) {

190 auto value = cast(entry.getValue());

191 bool isLittleEndian =

192 value.getValue() == DLTIDialect::kDataLayoutEndiannessLittle;

193 layoutStream << "-" << (isLittleEndian ? "e" : "E");

194 continue;

195 }

196 if (key.getValue() == DLTIDialect::kDataLayoutManglingModeKey) {

197 auto value = cast(entry.getValue());

198 layoutStream << "-m:" << value.getValue();

199 continue;

200 }

201 if (key.getValue() == DLTIDialect::kDataLayoutProgramMemorySpaceKey) {

202 auto value = cast(entry.getValue());

203 uint64_t space = value.getValue().getZExtValue();

204

205 if (space == 0)

206 continue;

207 layoutStream << "-P" << space;

208 continue;

209 }

210 if (key.getValue() == DLTIDialect::kDataLayoutGlobalMemorySpaceKey) {

211 auto value = cast(entry.getValue());

212 uint64_t space = value.getValue().getZExtValue();

213

214 if (space == 0)

215 continue;

216 layoutStream << "-G" << space;

217 continue;

218 }

219 if (key.getValue() == DLTIDialect::kDataLayoutAllocaMemorySpaceKey) {

220 auto value = cast(entry.getValue());

221 uint64_t space = value.getValue().getZExtValue();

222

223 if (space == 0)

224 continue;

225 layoutStream << "-A" << space;

226 continue;

227 }

228 if (key.getValue() == DLTIDialect::kDataLayoutStackAlignmentKey) {

229 auto value = cast(entry.getValue());

230 uint64_t alignment = value.getValue().getZExtValue();

231

232 if (alignment == 0)

233 continue;

234 layoutStream << "-S" << alignment;

235 continue;

236 }

237 if (key.getValue() == DLTIDialect::kDataLayoutFunctionPointerAlignmentKey) {

238 auto value = cast(entry.getValue());

239 uint64_t alignment = value.getAlignment();

240

241 if (alignment == 0)

242 continue;

243 layoutStream << "-F" << (value.getFunctionDependent() ? "n" : "i")

244 << alignment;

245 continue;

246 }

247 if (key.getValue() == DLTIDialect::kDataLayoutLegalIntWidthsKey) {

248 layoutStream << "-n";

249 llvm::interleave(

250 cast(entry.getValue()).asArrayRef(), layoutStream,

251 [&](int32_t val) { layoutStream << val; }, ":");

252 continue;

253 }

254 emitError(*loc) << "unsupported data layout key " << key;

255 return failure();

256 }

257

258

259

260

261 for (DataLayoutEntryInterface entry : attribute.getEntries()) {

262 auto type = llvm::dyn_cast_if_present(entry.getKey());

263 if (!type)

264 continue;

265

266 if (isa(type))

267 continue;

268 layoutStream << "-";

269 LogicalResult result =

271 .Case<IntegerType, Float16Type, Float32Type, Float64Type,

272 Float80Type, Float128Type>([&](Type type) -> LogicalResult {

273 if (auto intType = dyn_cast(type)) {

274 if (intType.getSignedness() != IntegerType::Signless)

276 << "unsupported data layout for non-signless integer "

277 << intType;

278 layoutStream << "i";

279 } else {

280 layoutStream << "f";

281 }

284 uint64_t preferred =

286 layoutStream << size << ":" << abi;

287 if (abi != preferred)

288 layoutStream << ":" << preferred;

290 })

291 .Case([&](LLVMPointerType type) {

292 layoutStream << "p" << type.getAddressSpace() << ":";

295 uint64_t preferred =

298 layoutStream << size << ":" << abi << ":" << preferred << ":"

301 })

302 .Default([loc](Type type) {

304 << "unsupported type in data layout: " << type;

305 });

307 return failure();

308 }

309 StringRef layoutSpec(llvmDataLayout);

310 layoutSpec.consume_front("-");

311

312 return llvm::DataLayout(layoutSpec);

313}

314

315

316

317

318

319static llvm::Constant *

323 if (shape.empty()) {

324 llvm::Constant *result = constants.front();

325 constants = constants.drop_front();

327 }

328

329 llvm::Type *elementType;

330 if (auto *arrayTy = dyn_castllvm::ArrayType(type)) {

331 elementType = arrayTy->getElementType();

332 } else if (auto *vectorTy = dyn_castllvm::VectorType(type)) {

333 elementType = vectorTy->getElementType();

334 } else {

335 emitError(loc) << "expected sequential LLVM types wrapping a scalar";

336 return nullptr;

337 }

338

340 nested.reserve(shape.front());

341 for (int64_t i = 0; i < shape.front(); ++i) {

343 elementType, loc));

344 if (!nested.back())

345 return nullptr;

346 }

347

348 if (shape.size() == 1 && type->isVectorTy())

349 return llvm::ConstantVector::get(nested);

350 return llvm::ConstantArray::get(

351 llvm::ArrayType::get(elementType, shape.front()), nested);

352}

353

354

356 do {

357 if (auto *arrayTy = dyn_castllvm::ArrayType(type)) {

358 type = arrayTy->getElementType();

359 } else if (auto *vectorTy = dyn_castllvm::VectorType(type)) {

360 type = vectorTy->getElementType();

361 } else {

362 return type;

363 }

364 } while (true);

365}

366

367

368

369

370

371

372

373

374static llvm::Constant *

376 llvm::Type *llvmType,

378 if (!denseElementsAttr)

379 return nullptr;

380

382 if (!llvm::ConstantDataSequential::isElementTypeCompatible(innermostLLVMType))

383 return nullptr;

384

385 ShapedType type = denseElementsAttr.getType();

386 if (type.getNumElements() == 0)

387 return nullptr;

388

389

390

391

392

393

394 int64_t elementByteSize = denseElementsAttr.getRawData().size() /

396 if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits())

397 return nullptr;

398

399

400

401 bool hasVectorElementType = isa(type.getElementType());

404 (hasVectorElementType ? 1

407 if (!hasVectorElementType)

408 outerShape = outerShape.drop_back();

409

410

411 if (denseElementsAttr.isSplat() &&

412 (isa(type) || hasVectorElementType)) {

415 moduleTranslation);

416 llvm::Constant *splatVector =

417 llvm::ConstantDataVector::getSplat(0, splatValue);

421 }

422 if (denseElementsAttr.isSplat())

423 return nullptr;

424

425

426

427 std::function<llvm::Constant *(StringRef)> buildCstData;

428 if (isa(type)) {

429 auto vectorElementType = dyn_cast(type.getElementType());

430 if (vectorElementType && vectorElementType.getRank() == 1) {

431 buildCstData = [&](StringRef data) {

432 return llvm::ConstantDataVector::getRaw(

433 data, vectorElementType.getShape().back(), innermostLLVMType);

434 };

435 } else if (!vectorElementType) {

436 buildCstData = [&](StringRef data) {

437 return llvm::ConstantDataArray::getRaw(data, type.getShape().back(),

438 innermostLLVMType);

439 };

440 }

441 } else if (isa(type)) {

442 buildCstData = [&](StringRef data) {

443 return llvm::ConstantDataVector::getRaw(data, type.getShape().back(),

444 innermostLLVMType);

445 };

446 }

447 if (!buildCstData)

448 return nullptr;

449

450

451

453 int64_t aggregateSize = denseElementsAttr.getType().getShape().back() *

454 (innermostLLVMType->getScalarSizeInBits() / 8);

455 constants.reserve(numAggregates);

456 for (unsigned i = 0; i < numAggregates; ++i) {

457 StringRef data(denseElementsAttr.getRawData().data() + i * aggregateSize,

458 aggregateSize);

459 constants.push_back(buildCstData(data));

460 }

461

464}

465

466

467

468

469

470

471

473 Location loc, DenseResourceElementsAttr denseResourceAttr,

474 llvm::Type *llvmType, const ModuleTranslation &moduleTranslation) {

475 assert(denseResourceAttr && "expected non-null attribute");

476

478 if (!llvm::ConstantDataSequential::isElementTypeCompatible(

479 innermostLLVMType)) {

480 emitError(loc, "no known conversion for innermost element type");

481 return nullptr;

482 }

483

484 ShapedType type = denseResourceAttr.getType();

485 assert(type.getNumElements() > 0 && "Expected non-empty elements attribute");

486

487 AsmResourceBlob *blob = denseResourceAttr.getRawHandle().getBlob();

488 if (!blob) {

489 emitError(loc, "resource does not exist");

490 return nullptr;

491 }

492

494

495

496

497

498

499

500 int64_t numElements = denseResourceAttr.getType().getNumElements();

501 int64_t elementByteSize = rawData.size() / numElements;

502 if (8 * elementByteSize != innermostLLVMType->getScalarSizeInBits()) {

503 emitError(loc, "raw data size does not match element type size");

504 return nullptr;

506

507

508

509 bool hasVectorElementType = isa(type.getElementType());

511 numElements / (hasVectorElementType

512 ? 1

513 : denseResourceAttr.getType().getShape().back());

515 if (!hasVectorElementType)

516 outerShape = outerShape.drop_back();

517

518

519 std::function<llvm::Constant *(StringRef)> buildCstData;

520 if (isa(type)) {

521 auto vectorElementType = dyn_cast(type.getElementType());

522 if (vectorElementType && vectorElementType.getRank() == 1) {

523 buildCstData = [&](StringRef data) {

524 return llvm::ConstantDataVector::getRaw(

525 data, vectorElementType.getShape().back(), innermostLLVMType);

526 };

527 } else if (!vectorElementType) {

528 buildCstData = [&](StringRef data) {

529 return llvm::ConstantDataArray::getRaw(data, type.getShape().back(),

530 innermostLLVMType);

531 };

532 }

533 } else if (isa(type)) {

534 buildCstData = [&](StringRef data) {

535 return llvm::ConstantDataVector::getRaw(data, type.getShape().back(),

536 innermostLLVMType);

537 };

538 }

539 if (!buildCstData) {

540 emitError(loc, "unsupported dense_resource type");

541 return nullptr;

542 }

543

544

545

546 SmallVector<llvm::Constant *> constants;

547 int64_t aggregateSize = denseResourceAttr.getType().getShape().back() *

548 (innermostLLVMType->getScalarSizeInBits() / 8);

549 constants.reserve(numAggregates);

550 for (unsigned i = 0; i < numAggregates; ++i) {

551 StringRef data(rawData.data() + i * aggregateSize, aggregateSize);

552 constants.push_back(buildCstData(data));

553 }

554

555 ArrayRef<llvm::Constant *> constantsRef = constants;

557}

558

559

560

561

562

563

567 if (!attr || isa(attr))

568 return llvm::UndefValue::get(llvmType);

569 if (isa(attr))

570 return llvm::Constant::getNullValue(llvmType);

571 if (auto *structType = dyn_cast<::llvm::StructType>(llvmType)) {

572 auto arrayAttr = dyn_cast(attr);

573 if (!arrayAttr) {

574 emitError(loc, "expected an array attribute for a struct constant");

575 return nullptr;

576 }

578 structElements.reserve(structType->getNumElements());

579 for (auto [elemType, elemAttr] :

580 zip_equal(structType->elements(), arrayAttr)) {

581 llvm::Constant *element =

582 getLLVMConstant(elemType, elemAttr, loc, moduleTranslation);

583 if (!element)

584 return nullptr;

585 structElements.push_back(element);

586 }

587 return llvm::ConstantStruct::get(structType, structElements);

588 }

589

590

591 if (auto intAttr = dyn_cast(attr)) {

592

593

594 auto intTy = dyn_cast(intAttr.getType());

595 APInt value;

596 if (intTy && (intTy.isUnsigned() || intTy.getWidth() == 1))

597 value = intAttr.getValue().zextOrTrunc(llvmType->getIntegerBitWidth());

598 else

599 value = intAttr.getValue().sextOrTrunc(llvmType->getIntegerBitWidth());

600 return llvm::ConstantInt::get(llvmType, value);

601 }

602 if (auto floatAttr = dyn_cast(attr)) {

603 const llvm::fltSemantics &sem = floatAttr.getValue().getSemantics();

604

605

606

607

608 unsigned floatWidth = APFloat::getSizeInBits(sem);

609 if (llvmType->isIntegerTy(floatWidth))

610 return llvm::ConstantInt::get(llvmType,

611 floatAttr.getValue().bitcastToAPInt());

612 if (llvmType !=

613 llvm::Type::getFloatingPointTy(llvmType->getContext(),

614 floatAttr.getValue().getSemantics())) {

615 emitError(loc, "FloatAttr does not match expected type of the constant");

616 return nullptr;

617 }

618 return llvm::ConstantFP::get(llvmType, floatAttr.getValue());

619 }

620 if (auto funcAttr = dyn_cast(attr))

621 return llvm::ConstantExpr::getBitCast(

622 moduleTranslation.lookupFunction(funcAttr.getValue()), llvmType);

623 if (auto splatAttr = dyn_cast(attr)) {

624 llvm::Type *elementType;

625 uint64_t numElements;

626 bool isScalable = false;

627 if (auto *arrayTy = dyn_castllvm::ArrayType(llvmType)) {

628 elementType = arrayTy->getElementType();

629 numElements = arrayTy->getNumElements();

630 } else if (auto *fVectorTy = dyn_castllvm::FixedVectorType(llvmType)) {

631 elementType = fVectorTy->getElementType();

632 numElements = fVectorTy->getNumElements();

633 } else if (auto *sVectorTy = dyn_castllvm::ScalableVectorType(llvmType)) {

634 elementType = sVectorTy->getElementType();

635 numElements = sVectorTy->getMinNumElements();

636 isScalable = true;

637 } else {

638 llvm_unreachable("unrecognized constant vector type");

639 }

640

641

642

643 bool elementTypeSequential =

644 isa<llvm::ArrayType, llvm::VectorType>(elementType);

646 elementType,

647 elementTypeSequential ? splatAttr

648 : splatAttr.getSplatValue<Attribute>(),

649 loc, moduleTranslation);

650 if (!child)

651 return nullptr;

652 if (llvmType->isVectorTy())

653 return llvm::ConstantVector::getSplat(

654 llvm::ElementCount::get(numElements, isScalable), child);

655 if (llvmType->isArrayTy()) {

656 auto *arrayType = llvm::ArrayType::get(elementType, numElements);

657 if (child->isZeroValue() && !elementType->isFPOrFPVectorTy()) {

658 return llvm::ConstantAggregateZero::get(arrayType);

659 }

660 if (llvm::ConstantDataSequential::isElementTypeCompatible(elementType)) {

661

662 if (isallvm::IntegerType(elementType)) {

663 if (llvm::ConstantInt *ci = dyn_castllvm::ConstantInt(child)) {

664 if (ci->getBitWidth() == 8) {

666 return llvm::ConstantDataArray::get(elementType->getContext(),

667 constants);

668 }

669 if (ci->getBitWidth() == 16) {

671 return llvm::ConstantDataArray::get(elementType->getContext(),

672 constants);

673 }

674 if (ci->getBitWidth() == 32) {

676 return llvm::ConstantDataArray::get(elementType->getContext(),

677 constants);

678 }

679 if (ci->getBitWidth() == 64) {

681 return llvm::ConstantDataArray::get(elementType->getContext(),

682 constants);

683 }

684 }

685 }

686 }

687

688

689 std::vector<llvm::Constant *> constants(numElements, child);

690 return llvm::ConstantArray::get(arrayType, constants);

691 }

692 }

693

694

695 if (llvm::Constant *result =

697 llvmType, moduleTranslation)) {

699 }

700

701 if (auto denseResourceAttr = dyn_cast(attr)) {

703 moduleTranslation);

704 }

705

706

707 if (auto elementsAttr = dyn_cast(attr)) {

708 assert(elementsAttr.getShapedType().hasStaticShape());

709 assert(!elementsAttr.getShapedType().getShape().empty() &&

710 "unexpected empty elements attribute shape");

711

713 constants.reserve(elementsAttr.getNumElements());

715 for (auto n : elementsAttr.getValues<Attribute>()) {

716 constants.push_back(

717 getLLVMConstant(innermostType, n, loc, moduleTranslation));

718 if (!constants.back())

719 return nullptr;

720 }

723 constantsRef, elementsAttr.getShapedType().getShape(), llvmType, loc);

724 assert(constantsRef.empty() && "did not consume all elemental constants");

726 }

727

728 if (auto stringAttr = dyn_cast(attr)) {

729 return llvm::ConstantDataArray::get(moduleTranslation.getLLVMContext(),

731 }

732

733

734

735 if (auto arrayAttr = dyn_cast(attr)) {

736 if (auto *arrayTy = dyn_castllvm::ArrayType(llvmType)) {

737 llvm::Type *elementType = arrayTy->getElementType();

739 llvm::Constant *elementCst = nullptr;

741 constants.reserve(arrayTy->getNumElements());

742 for (Attribute elementAttr : arrayAttr) {

743

744

745

746 if (!previousElementAttr || previousElementAttr != elementAttr) {

747 previousElementAttr = elementAttr;

748 elementCst =

749 getLLVMConstant(elementType, elementAttr, loc, moduleTranslation);

750 if (!elementCst)

751 return nullptr;

752 }

753 constants.push_back(elementCst);

754 }

755 return llvm::ConstantArray::get(arrayTy, constants);

756 }

757 }

758

759 emitError(loc, "unsupported constant value");

760 return nullptr;

761}

762

763ModuleTranslation::ModuleTranslation(Operation *module,

764 std::unique_ptrllvm::Module llvmModule)

765 : mlirModule(module), llvmModule(std::move(llvmModule)),

766 debugTranslation(

767 std::make_unique<DebugTranslation>(module, *this->llvmModule)),

769 *this, *this->llvmModule)),

770 typeTranslator(this->llvmModule->getContext()),

773 "mlirModule should honor LLVM's module semantics.");

774}

775

776ModuleTranslation::~ModuleTranslation() {

777 if (ompBuilder && !ompBuilder->isFinalized())

778 ompBuilder->finalize();

779}

780

783 toProcess.push_back(&region);

784 while (!toProcess.empty()) {

785 Region *current = toProcess.pop_back_val();

786 for (Block &block : *current) {

787 blockMapping.erase(&block);

788 for (Value arg : block.getArguments())

789 valueMapping.erase(arg);

791 for (Value value : op.getResults())

792 valueMapping.erase(value);

793 if (op.hasSuccessors())

794 branchMapping.erase(&op);

795 if (isaLLVM::GlobalOp(op))

796 globalsMapping.erase(&op);

797 if (isaLLVM::AliasOp(op))

798 aliasesMapping.erase(&op);

799 if (isaLLVM::IFuncOp(op))

800 ifuncMapping.erase(&op);

801 if (isaLLVM::CallOp(op))

802 callMapping.erase(&op);

803 llvm::append_range(

804 toProcess,

805 llvm::map_range(op.getRegions(), [](Region &r) { return &r; }));

806 }

807 }

808 }

809}

810

811

812

814 unsigned numArguments, unsigned index) {

816 if (isaLLVM::BrOp(terminator))

818

819#ifndef NDEBUG

821 for (unsigned i = 0, e = terminator.getNumSuccessors(); i < e; ++i) {

823 auto branch = cast(terminator);

824 SuccessorOperands successorOperands = branch.getSuccessorOperands(i);

825 assert(

826 (!seenSuccessors.contains(successor) || successorOperands.empty()) &&

827 "successors with arguments in LLVM branches must be different blocks");

828 seenSuccessors.insert(successor);

829 }

830#endif

831

832

833

834 if (auto condBranchOp = dyn_castLLVM::CondBrOp(terminator)) {

835

836

837 return condBranchOp.getSuccessor(0) == current

838 ? condBranchOp.getTrueDestOperands()[index]

839 : condBranchOp.getFalseDestOperands()[index];

840 }

841

842 if (auto switchOp = dyn_castLLVM::SwitchOp(terminator)) {

843

844

845 if (switchOp.getDefaultDestination() == current)

846 return switchOp.getDefaultOperands()[index];

847 for (const auto &i : llvm::enumerate(switchOp.getCaseDestinations()))

848 if (i.value() == current)

849 return switchOp.getCaseOperands(i.index())[index];

850 }

851

852 if (auto indBrOp = dyn_castLLVM::IndirectBrOp(terminator)) {

853

854 for (const auto &i : llvm::enumerate(indBrOp->getSuccessors())) {

855 if (indBrOp->getSuccessor(i.index()) == current)

856 return indBrOp.getSuccessorOperands(i.index())[index];

857 }

858 }

859

860 if (auto invokeOp = dyn_castLLVM::InvokeOp(terminator)) {

861 return invokeOp.getNormalDest() == current

862 ? invokeOp.getNormalDestOperands()[index]

863 : invokeOp.getUnwindDestOperands()[index];

864 }

865

866 llvm_unreachable(

867 "only branch, switch or invoke operations can be terminators "

868 "of a block that has successors");

869}

870

871

874

875

876 for (Block &bb : llvm::drop_begin(region)) {

877 llvm::BasicBlock *llvmBB = state.lookupBlock(&bb);

878 auto phis = llvmBB->phis();

879 auto numArguments = bb.getNumArguments();

880 assert(numArguments == std::distance(phis.begin(), phis.end()));

881 for (auto [index, phiNode] : llvm::enumerate(phis)) {

882 for (auto *pred : bb.getPredecessors()) {

883

884

885

886

887

888 llvm::Instruction *terminator =

890 assert(terminator && "missing the mapping for a terminator");

892 &bb, pred, numArguments, index)),

893 terminator->getParent());

894 }

895 }

896 }

897}

898

900 llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic,

902 return builder.CreateIntrinsic(intrinsic, tys, args);

903}

904

906 llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic,

908 return builder.CreateIntrinsic(retTy, intrinsic, args);

909}

910

912 llvm::IRBuilderBase &builder, ModuleTranslation &moduleTranslation,

913 Operation *intrOp, llvm::Intrinsic::ID intrinsic, unsigned numResults,

917 assert(immArgPositions.size() == immArgAttrNames.size() &&

918 "LLVM `immArgPositions` and MLIR `immArgAttrNames` should have equal "

919 "length");

920

922 size_t numOpBundleOperands = 0;

923 auto opBundleSizesAttr = cast_if_present(

924 intrOp->getAttr(LLVMDialect::getOpBundleSizesAttrName()));

925 auto opBundleTagsAttr = cast_if_present(

926 intrOp->getAttr(LLVMDialect::getOpBundleTagsAttrName()));

927

928 if (opBundleSizesAttr && opBundleTagsAttr) {

929 ArrayRef opBundleSizes = opBundleSizesAttr.asArrayRef();

930 assert(opBundleSizes.size() == opBundleTagsAttr.size() &&

931 "operand bundles and tags do not match");

932

933 numOpBundleOperands = llvm::sum_of(opBundleSizes);

934 assert(numOpBundleOperands <= intrOp->getNumOperands() &&

935 "operand bundle operands is more than the number of operands");

936

938 size_t nextOperandIdx = 0;

939 opBundles.reserve(opBundleSizesAttr.size());

940

941 for (auto [opBundleTagAttr, bundleSize] :

942 llvm::zip(opBundleTagsAttr, opBundleSizes)) {

943 auto bundleTag = cast(opBundleTagAttr).str();

944 auto bundleOperands = moduleTranslation.lookupValues(

945 operands.slice(nextOperandIdx, bundleSize));

946 opBundles.emplace_back(std::move(bundleTag), std::move(bundleOperands));

947 nextOperandIdx += bundleSize;

948 }

949 }

950

951

952 auto opOperands = intrOp->getOperands().drop_back(numOpBundleOperands);

953 auto operands = moduleTranslation.lookupValues(opOperands);

955 for (auto [immArgPos, immArgName] :

956 llvm::zip(immArgPositions, immArgAttrNames)) {

957 auto attr = llvm::cast(intrOp->getAttr(immArgName));

958 assert(attr.getType().isIntOrFloat() && "expected int or float immarg");

959 auto *type = moduleTranslation.convertType(attr.getType());

961 type, attr, intrOp->getLoc(), moduleTranslation);

962 }

963 unsigned opArg = 0;

964 for (auto &arg : args) {

965 if (!arg)

966 arg = operands[opArg++];

967 }

968

969

971 for (unsigned overloadedResultIdx : overloadedResults) {

972 if (numResults > 1) {

973

974 overloadedTypes.push_back(moduleTranslation.convertType(

975 llvm::castLLVM::LLVMStructType(intrOp->getResult(0).getType())

976 .getBody()[overloadedResultIdx]));

977 } else {

978 overloadedTypes.push_back(

980 }

981 }

982 for (unsigned overloadedOperandIdx : overloadedOperands)

983 overloadedTypes.push_back(args[overloadedOperandIdx]->getType());

984 llvm::Module *module = builder.GetInsertBlock()->getModule();

985 llvm::Function *llvmIntr = llvm::Intrinsic::getOrInsertDeclaration(

986 module, intrinsic, overloadedTypes);

987

988 return builder.CreateCall(llvmIntr, args, opBundles);

989}

990

991

992

993LogicalResult ModuleTranslation::convertOperation(Operation &op,

994 llvm::IRBuilderBase &builder,

995 bool recordInsertions) {

997 if (!opIface)

998 return op.emitError("cannot be converted to LLVM IR: missing "

999 "`LLVMTranslationDialectInterface` registration for "

1000 "dialect for op: ")

1002

1003 InstructionCapturingInserter::CollectionScope scope(builder,

1004 recordInsertions);

1006 return op.emitError("LLVM Translation failed for operation: ")

1008

1009 return convertDialectAttributes(&op, scope.getCapturedInstructions());

1010}

1011

1012

1013

1014

1015

1016

1017

1018

1019LogicalResult ModuleTranslation::convertBlockImpl(Block &bb,

1020 bool ignoreArguments,

1021 llvm::IRBuilderBase &builder,

1022 bool recordInsertions) {

1023 builder.SetInsertPoint(lookupBlock(&bb));

1024 auto *subprogram = builder.GetInsertBlock()->getParent()->getSubprogram();

1025

1026

1027

1028

1029

1030

1031

1032 if (!ignoreArguments) {

1034 unsigned numPredecessors =

1035 std::distance(predecessors.begin(), predecessors.end());

1037 auto wrappedType = arg.getType();

1040 "block argument does not have an LLVM type");

1041 builder.SetCurrentDebugLocation(

1042 debugTranslation->translateLoc(arg.getLoc(), subprogram));

1043 llvm::Type *type = convertType(wrappedType);

1044 llvm::PHINode *phi = builder.CreatePHI(type, numPredecessors);

1046 }

1047 }

1048

1049

1050 for (auto &op : bb) {

1051

1052 builder.SetCurrentDebugLocation(

1053 debugTranslation->translateLoc(op.getLoc(), subprogram));

1054

1055 if (failed(convertOperation(op, builder, recordInsertions)))

1056 return failure();

1057

1058

1059 if (auto iface = dyn_cast(op))

1061 }

1062

1064}

1065

1066

1067

1069 return module->getRegion(0).front();

1070}

1071

1072

1073

1074

1075

1076

1078 llvm::Constant *cst) {

1079 return (linkage == llvm::GlobalVariable::ExternalLinkage && !cst) ||

1080 linkage == llvm::GlobalVariable::ExternalWeakLinkage;

1081}

1082

1083

1084

1086 llvm::GlobalValue *gv) {

1087 if (dsoLocalRequested)

1088 gv->setDSOLocal(true);

1089}

1090

1091

1092

1093

1094

1095

1096

1097static FailureOrllvm::Attribute

1099 StringRef value = StringRef()) {

1100 auto kind = llvm::Attribute::getAttrKindFromName(key);

1101 if (kind == llvm::Attribute::None)

1102 return llvm::Attribute::get(ctx, key, value);

1103

1104 if (llvm::Attribute::isIntAttrKind(kind)) {

1105 if (value.empty())

1106 return emitError(loc) << "LLVM attribute '" << key << "' expects a value";

1107

1109 if (!value.getAsInteger(0, result))

1110 return llvm::Attribute::get(ctx, kind, result);

1111 return llvm::Attribute::get(ctx, key, value);

1112 }

1113

1114 if (!value.empty())

1115 return emitError(loc) << "LLVM attribute '" << key

1116 << "' does not expect a value, found '" << value

1117 << "'";

1118

1119 return llvm::Attribute::get(ctx, kind);

1120}

1121

1122

1123

1124

1125

1126

1127

1128

1129

1130static FailureOrllvm::AttrBuilder

1132 ArrayAttr arrayAttr, StringRef arrayAttrName) {

1133 llvm::AttrBuilder attrBuilder(ctx);

1134 if (!arrayAttr)

1135 return attrBuilder;

1136

1137 for (Attribute attr : arrayAttr) {

1138 if (auto stringAttr = dyn_cast(attr)) {

1139 FailureOrllvm::Attribute llvmAttr =

1141 if (failed(llvmAttr))

1142 return failure();

1143 attrBuilder.addAttribute(*llvmAttr);

1144 continue;

1145 }

1146

1147 auto arrayAttr = dyn_cast(attr);

1148 if (!arrayAttr || arrayAttr.size() != 2)

1149 return emitError(loc) << "expected '" << arrayAttrName

1150 << "' to contain string or array attributes";

1151

1152 auto keyAttr = dyn_cast(arrayAttr[0]);

1153 auto valueAttr = dyn_cast(arrayAttr[1]);

1154 if (!keyAttr || !valueAttr)

1155 return emitError(loc) << "expected arrays within '" << arrayAttrName

1156 << "' to contain two strings";

1157

1159 loc, ctx, keyAttr.getValue(), valueAttr.getValue());

1160 if (failed(llvmAttr))

1161 return failure();

1162 attrBuilder.addAttribute(*llvmAttr);

1163 }

1164

1165 return attrBuilder;

1166}

1167

1168LogicalResult ModuleTranslation::convertGlobalsAndAliases() {

1169

1171

1172

1173

1174

1175

1176

1177 for (auto op : getModuleBody(mlirModule).getOpsLLVM::GlobalOp()) {

1178 llvm::Type *type = convertType(op.getType());

1179 llvm::Constant *cst = nullptr;

1180 if (op.getValueOrNull()) {

1181

1182

1183 if (auto strAttr = dyn_cast_or_null(op.getValueOrNull())) {

1184 cst = llvm::ConstantDataArray::getString(

1185 llvmModule->getContext(), strAttr.getValue(), false);

1186 type = cst->getType();

1188 *this))) {

1189 return failure();

1190 }

1191 }

1192

1193 auto linkage = convertLinkageToLLVM(op.getLinkage());

1194

1195

1196

1197

1199 if (!dropInitializer && !cst)

1200 cst = llvm::UndefValue::get(type);

1201 else if (dropInitializer && cst)

1202 cst = nullptr;

1203

1204 auto *var = new llvm::GlobalVariable(

1205 *llvmModule, type, op.getConstant(), linkage, cst, op.getSymName(),

1206 nullptr,

1207 op.getThreadLocal_() ? llvm::GlobalValue::GeneralDynamicTLSModel

1208 : llvm::GlobalValue::NotThreadLocal,

1209 op.getAddrSpace(), op.getExternallyInitialized());

1210

1211 if (std::optionalmlir::SymbolRefAttr comdat = op.getComdat()) {

1212 auto selectorOp = cast(

1214 var->setComdat(comdatMapping.lookup(selectorOp));

1215 }

1216

1217 if (op.getUnnamedAddr().has_value())

1218 var->setUnnamedAddr(convertUnnamedAddrToLLVM(*op.getUnnamedAddr()));

1219

1220 if (op.getSection().has_value())

1221 var->setSection(*op.getSection());

1222

1224

1225 std::optional<uint64_t> alignment = op.getAlignment();

1226 if (alignment.has_value())

1227 var->setAlignment(llvm::MaybeAlign(alignment.value()));

1228

1229 var->setVisibility(convertVisibilityToLLVM(op.getVisibility_()));

1230

1231 globalsMapping.try_emplace(op, var);

1232

1233

1234 if (op.getDbgExprs()) {

1235 for (auto exprAttr :

1236 op.getDbgExprs()->getAsRange()) {

1237 llvm::DIGlobalVariableExpression *diGlobalExpr =

1238 debugTranslation->translateGlobalVariableExpression(exprAttr);

1239 llvm::DIGlobalVariable *diGlobalVar = diGlobalExpr->getVariable();

1240 var->addDebugInfo(diGlobalExpr);

1241

1242

1243

1244

1245

1246

1247

1248

1249

1250

1251

1252

1253

1254

1255

1256

1257

1258

1259 llvm::DIScope *scope = diGlobalVar->getScope();

1260 if (auto *mod = dyn_cast_if_presentllvm::DIModule(scope))

1261 scope = mod->getScope();

1262 else if (auto *cb = dyn_cast_if_presentllvm::DICommonBlock(scope)) {

1263 if (auto *sp =

1264 dyn_cast_if_presentllvm::DISubprogram(cb->getScope()))

1265 scope = sp->getUnit();

1266 } else if (auto *sp = dyn_cast_if_presentllvm::DISubprogram(scope))

1267 scope = sp->getUnit();

1268

1269

1270 if (llvm::DICompileUnit *compileUnit =

1271 dyn_cast_if_presentllvm::DICompileUnit(scope)) {

1272

1273

1274 allGVars[compileUnit].push_back(diGlobalExpr);

1275 }

1276 }

1277 }

1278

1279

1280 FailureOrllvm::AttrBuilder convertedTargetSpecificAttrs =

1282 op.getTargetSpecificAttrsAttr(),

1283 op.getTargetSpecificAttrsAttrName());

1284 if (failed(convertedTargetSpecificAttrs))

1285 return failure();

1286 var->addAttributes(*convertedTargetSpecificAttrs);

1287 }

1288

1289

1290 for (auto op : getModuleBody(mlirModule).getOpsLLVM::AliasOp()) {

1291 llvm::Type *type = convertType(op.getType());

1292 llvm::Constant *cst = nullptr;

1293 llvm::GlobalValue::LinkageTypes linkage =

1294 convertLinkageToLLVM(op.getLinkage());

1295 llvm::Module &llvmMod = *llvmModule;

1296

1297

1298 llvm::GlobalAlias *var = llvm::GlobalAlias::create(

1299 type, op.getAddrSpace(), linkage, op.getSymName(), cst,

1300 &llvmMod);

1301

1302 var->setThreadLocalMode(op.getThreadLocal_()

1303 ? llvm::GlobalAlias::GeneralDynamicTLSModel

1304 : llvm::GlobalAlias::NotThreadLocal);

1305

1306

1307

1308

1309 if (op.getUnnamedAddr().has_value())

1310 var->setUnnamedAddr(convertUnnamedAddrToLLVM(*op.getUnnamedAddr()));

1311

1312 var->setVisibility(convertVisibilityToLLVM(op.getVisibility_()));

1313

1314 aliasesMapping.try_emplace(op, var);

1315 }

1316

1317

1318 for (auto op : getModuleBody(mlirModule).getOpsLLVM::GlobalOp()) {

1319 if (Block *initializer = op.getInitializerBlock()) {

1320 llvm::IRBuilderllvm::TargetFolder builder(

1321 llvmModule->getContext(),

1322 llvm::TargetFolder(llvmModule->getDataLayout()));

1323

1324 [[maybe_unused]] int numConstantsHit = 0;

1325 [[maybe_unused]] int numConstantsErased = 0;

1327

1328 for (auto &op : initializer->without_terminator()) {

1329 if (failed(convertOperation(op, builder)))

1330 return emitError(op.getLoc(), "fail to convert global initializer");

1332 if (!cst)

1333 return emitError(op.getLoc(), "unemittable constant value");

1334

1335

1336

1337

1338

1339

1340

1341

1342 if (auto *agg = dyn_castllvm::ConstantAggregate(cst)) {

1343 numConstantsHit++;

1345 int numUsers = std::distance(result.use_begin(), result.use_end());

1347 constantAggregateUseMap.try_emplace(agg, numUsers);

1349

1350 iterator->second += numUsers;

1351 }

1352 }

1353

1354

1356 auto cst = dyn_castllvm::ConstantAggregate(lookupValue(v));

1357 if (!cst)

1358 continue;

1359 auto iter = constantAggregateUseMap.find(cst);

1360 assert(iter != constantAggregateUseMap.end() && "constant not found");

1361 iter->second--;

1362 if (iter->second == 0) {

1363

1364

1365 if (cst->user_empty()) {

1366 cst->destroyConstant();

1367 numConstantsErased++;

1368 }

1369 constantAggregateUseMap.erase(iter);

1370 }

1371 }

1372 }

1373

1374 ReturnOp ret = cast(initializer->getTerminator());

1375 llvm::Constant *cst =

1376 castllvm::Constant(lookupValue(ret.getOperand(0)));

1377 auto *global = castllvm::GlobalVariable(lookupGlobal(op));

1379 global->setInitializer(cst);

1380

1381

1382

1383 for (auto it : constantAggregateUseMap) {

1384 auto cst = it.first;

1385 cst->removeDeadConstantUsers();

1386 if (cst->user_empty()) {

1387 cst->destroyConstant();

1388 numConstantsErased++;

1389 }

1390 }

1391

1392 LLVM_DEBUG(llvm::dbgs()

1393 << "Convert initializer for " << op.getName() << "\n";

1394 llvm::dbgs() << numConstantsHit << " new constants hit\n";

1395 llvm::dbgs()

1396 << numConstantsErased << " dangling constants erased\n";);

1397 }

1398 }

1399

1400

1401 for (Operation &op : getModuleBody(mlirModule)) {

1402 auto ctorOp = dyn_cast(op);

1403 auto dtorOp = dyn_cast(op);

1404 if (!ctorOp && !dtorOp)

1405 continue;

1406

1407

1408

1409

1410 if ((ctorOp && ctorOp.getCtors().empty()) ||

1411 (dtorOp && dtorOp.getDtors().empty())) {

1412 llvm::IRBuilderllvm::TargetFolder builder(

1413 llvmModule->getContext(),

1414 llvm::TargetFolder(llvmModule->getDataLayout()));

1415 llvm::Type *eltTy = llvm::StructType::get(

1416 builder.getInt32Ty(), builder.getPtrTy(), builder.getPtrTy());

1417 llvm::ArrayType *at = llvm::ArrayType::get(eltTy, 0);

1418 llvm::Constant *zeroInit = llvm::Constant::getNullValue(at);

1419 (void)new llvm::GlobalVariable(

1420 *llvmModule, zeroInit->getType(), false,

1421 llvm::GlobalValue::AppendingLinkage, zeroInit,

1422 ctorOp ? "llvm.global_ctors" : "llvm.global_dtors");

1423 } else {

1424 auto range = ctorOp

1425 ? llvm::zip(ctorOp.getCtors(), ctorOp.getPriorities())

1426 : llvm::zip(dtorOp.getDtors(), dtorOp.getPriorities());

1427 auto appendGlobalFn =

1428 ctorOp ? llvm::appendToGlobalCtors : llvm::appendToGlobalDtors;

1429 for (const auto &[sym, prio] : range) {

1430 llvm::Function *f =

1431 lookupFunction(cast(sym).getValue());

1432 appendGlobalFn(*llvmModule, f, cast(prio).getInt(),

1433 nullptr);

1434 }

1435 }

1436 }

1437

1438 for (auto op : getModuleBody(mlirModule).getOpsLLVM::GlobalOp())

1439 if (failed(convertDialectAttributes(op, {})))

1440 return failure();

1441

1442

1443

1444 for (const auto &[compileUnit, globals] : allGVars) {

1445 compileUnit->replaceGlobalVariables(

1447 }

1448

1449

1450 for (auto op : getModuleBody(mlirModule).getOpsLLVM::AliasOp()) {

1451 Block &initializer = op.getInitializerBlock();

1452 llvm::IRBuilderllvm::TargetFolder builder(

1453 llvmModule->getContext(),

1454 llvm::TargetFolder(llvmModule->getDataLayout()));

1455

1457 if (failed(convertOperation(op, builder)))

1458 return emitError(op.getLoc(), "fail to convert alias initializer");

1460 return emitError(op.getLoc(), "unemittable constant value");

1461 }

1462

1463 auto ret = cast(initializer.getTerminator());

1464 auto *cst = castllvm::Constant(lookupValue(ret.getOperand(0)));

1465 assert(aliasesMapping.count(op));

1466 auto *alias = castllvm::GlobalAlias(aliasesMapping[op]);

1467 alias->setAliasee(cst);

1468 }

1469

1470 for (auto op : getModuleBody(mlirModule).getOpsLLVM::AliasOp())

1471 if (failed(convertDialectAttributes(op, {})))

1472 return failure();

1473

1475}

1476

1477

1479 const llvm::APInt &value) {

1480 llvm::Constant *constant = llvm::ConstantInt::get(context, value);

1481 return llvm::ConstantAsMetadata::get(constant);

1482}

1483

1484

1486 const llvm::APInt &value) {

1488}

1489

1490

1492 llvm::Type *type,

1493 bool isSigned) {

1494 llvm::Metadata *typeMD =

1495 llvm::ConstantAsMetadata::get(llvm::UndefValue::get(type));

1496 llvm::Metadata *isSignedMD =

1498 return llvm::MDNode::get(context, {typeMD, isSignedMD});

1499}

1500

1501

1505 llvm::transform(

1506 values, std::back_inserter(mdValues), [&context](int32_t value) {

1508 });

1509 return llvm::MDNode::get(context, mdValues);

1510}

1511

1512LogicalResult ModuleTranslation::convertOneFunction(LLVMFuncOp func) {

1513

1514

1515 blockMapping.clear();

1516 valueMapping.clear();

1517 branchMapping.clear();

1518 llvm::Function *llvmFunc = lookupFunction(func.getName());

1519

1520

1521 for (auto [mlirArg, llvmArg] :

1522 llvm::zip(func.getArguments(), llvmFunc->args()))

1523 mapValue(mlirArg, &llvmArg);

1524

1525

1526 if (func.getPersonality()) {

1527 llvm::Type *ty = llvm::PointerType::getUnqual(llvmFunc->getContext());

1528 if (llvm::Constant *pfunc = getLLVMConstant(ty, func.getPersonalityAttr(),

1529 func.getLoc(), *this))

1530 llvmFunc->setPersonalityFn(pfunc);

1531 }

1532

1533 if (std::optional section = func.getSection())

1534 llvmFunc->setSection(*section);

1535

1536 if (func.getArmStreaming())

1537 llvmFunc->addFnAttr("aarch64_pstate_sm_enabled");

1538 else if (func.getArmLocallyStreaming())

1539 llvmFunc->addFnAttr("aarch64_pstate_sm_body");

1540 else if (func.getArmStreamingCompatible())

1541 llvmFunc->addFnAttr("aarch64_pstate_sm_compatible");

1542

1543 if (func.getArmNewZa())

1544 llvmFunc->addFnAttr("aarch64_new_za");

1545 else if (func.getArmInZa())

1546 llvmFunc->addFnAttr("aarch64_in_za");

1547 else if (func.getArmOutZa())

1548 llvmFunc->addFnAttr("aarch64_out_za");

1549 else if (func.getArmInoutZa())

1550 llvmFunc->addFnAttr("aarch64_inout_za");

1551 else if (func.getArmPreservesZa())

1552 llvmFunc->addFnAttr("aarch64_preserves_za");

1553

1554 if (auto targetCpu = func.getTargetCpu())

1555 llvmFunc->addFnAttr("target-cpu", *targetCpu);

1556

1557 if (auto tuneCpu = func.getTuneCpu())

1558 llvmFunc->addFnAttr("tune-cpu", *tuneCpu);

1559

1560 if (auto reciprocalEstimates = func.getReciprocalEstimates())

1561 llvmFunc->addFnAttr("reciprocal-estimates", *reciprocalEstimates);

1562

1563 if (auto preferVectorWidth = func.getPreferVectorWidth())

1564 llvmFunc->addFnAttr("prefer-vector-width", *preferVectorWidth);

1565

1566 if (auto attr = func.getVscaleRange())

1567 llvmFunc->addFnAttr(llvm::Attribute::getWithVScaleRangeArgs(

1569 attr->getMaxRange().getInt()));

1570

1571 if (auto noInfsFpMath = func.getNoInfsFpMath())

1572 llvmFunc->addFnAttr("no-infs-fp-math", llvm::toStringRef(*noInfsFpMath));

1573

1574 if (auto noNansFpMath = func.getNoNansFpMath())

1575 llvmFunc->addFnAttr("no-nans-fp-math", llvm::toStringRef(*noNansFpMath));

1576

1577 if (auto noSignedZerosFpMath = func.getNoSignedZerosFpMath())

1578 llvmFunc->addFnAttr("no-signed-zeros-fp-math",

1579 llvm::toStringRef(*noSignedZerosFpMath));

1580

1581 if (auto denormalFpMath = func.getDenormalFpMath())

1582 llvmFunc->addFnAttr("denormal-fp-math", *denormalFpMath);

1583

1584 if (auto denormalFpMathF32 = func.getDenormalFpMathF32())

1585 llvmFunc->addFnAttr("denormal-fp-math-f32", *denormalFpMathF32);

1586

1587 if (auto fpContract = func.getFpContract())

1588 llvmFunc->addFnAttr("fp-contract", *fpContract);

1589

1590 if (auto instrumentFunctionEntry = func.getInstrumentFunctionEntry())

1591 llvmFunc->addFnAttr("instrument-function-entry", *instrumentFunctionEntry);

1592

1593 if (auto instrumentFunctionExit = func.getInstrumentFunctionExit())

1594 llvmFunc->addFnAttr("instrument-function-exit", *instrumentFunctionExit);

1595

1596

1597 llvm::LLVMContext &llvmContext = llvmFunc->getContext();

1598 for (auto &bb : func) {

1599 auto *llvmBB = llvm::BasicBlock::Create(llvmContext);

1600 llvmBB->insertInto(llvmFunc);

1602 }

1603

1604

1605

1607 for (Block *bb : blocks) {

1608 CapturingIRBuilder builder(llvmContext,

1609 llvm::TargetFolder(llvmModule->getDataLayout()));

1610 if (failed(convertBlockImpl(*bb, bb->isEntryBlock(), builder,

1611 true)))

1612 return failure();

1613 }

1614

1615

1616

1618

1619

1620 return convertDialectAttributes(func, {});

1621}

1622

1623LogicalResult ModuleTranslation::convertDialectAttributes(

1624 Operation *op, ArrayRef<llvm::Instruction *> instructions) {

1626 if (failed(iface.amendOperation(op, instructions, attribute, *this)))

1627 return failure();

1629}

1630

1631

1632

1634 llvm::Function *llvmFunc) {

1635 if ( func .getMemoryEffects())

1636 return;

1637

1638 MemoryEffectsAttr memEffects = func.getMemoryEffectsAttr();

1639

1640

1641 llvm::MemoryEffects newMemEffects =

1642 llvm::MemoryEffects(llvm::MemoryEffects::Location::ArgMem,

1643 convertModRefInfoToLLVM(memEffects.getArgMem()));

1644 newMemEffects |= llvm::MemoryEffects(

1645 llvm::MemoryEffects::Location::InaccessibleMem,

1646 convertModRefInfoToLLVM(memEffects.getInaccessibleMem()));

1647 newMemEffects |=

1648 llvm::MemoryEffects(llvm::MemoryEffects::Location::Other,

1649 convertModRefInfoToLLVM(memEffects.getOther()));

1650 newMemEffects |=

1651 llvm::MemoryEffects(llvm::MemoryEffects::Location::ErrnoMem,

1652 convertModRefInfoToLLVM(memEffects.getErrnoMem()));

1653 newMemEffects |=

1654 llvm::MemoryEffects(llvm::MemoryEffects::Location::TargetMem0,

1655 convertModRefInfoToLLVM(memEffects.getTargetMem0()));

1656 newMemEffects |=

1657 llvm::MemoryEffects(llvm::MemoryEffects::Location::TargetMem1,

1658 convertModRefInfoToLLVM(memEffects.getTargetMem1()));

1659 llvmFunc->setMemoryEffects(newMemEffects);

1660}

1661

1662

1664 llvm::Function *llvmFunc) {

1665 if (func.getNoInlineAttr())

1666 llvmFunc->addFnAttr(llvm::Attribute::NoInline);

1667 if (func.getAlwaysInlineAttr())

1668 llvmFunc->addFnAttr(llvm::Attribute::AlwaysInline);

1669 if (func.getInlineHintAttr())

1670 llvmFunc->addFnAttr(llvm::Attribute::InlineHint);

1671 if (func.getOptimizeNoneAttr())

1672 llvmFunc->addFnAttr(llvm::Attribute::OptimizeNone);

1673 if (func.getConvergentAttr())

1674 llvmFunc->addFnAttr(llvm::Attribute::Convergent);

1675 if (func.getNoUnwindAttr())

1676 llvmFunc->addFnAttr(llvm::Attribute::NoUnwind);

1677 if (func.getWillReturnAttr())

1678 llvmFunc->addFnAttr(llvm::Attribute::WillReturn);

1679 if (TargetFeaturesAttr targetFeatAttr = func.getTargetFeaturesAttr())

1680 llvmFunc->addFnAttr("target-features", targetFeatAttr.getFeaturesString());

1681 if (FramePointerKindAttr fpAttr = func.getFramePointerAttr())

1682 llvmFunc->addFnAttr("frame-pointer", stringifyFramePointerKind(

1683 fpAttr.getFramePointerKind()));

1684 if (UWTableKindAttr uwTableKindAttr = func.getUwtableKindAttr())

1685 llvmFunc->setUWTableKind(

1686 convertUWTableKindToLLVM(uwTableKindAttr.getUwtableKind()));

1688}

1689

1690

1692 llvm::Function *llvmFunc,

1694 llvm::LLVMContext &llvmContext = llvmFunc->getContext();

1695

1696 if (VecTypeHintAttr vecTypeHint = func.getVecTypeHintAttr()) {

1697 Type type = vecTypeHint.getHint().getValue();

1698 llvm::Type *llvmType = translation.convertType(type);

1699 bool isSigned = vecTypeHint.getIsSigned();

1700 llvmFunc->setMetadata(

1701 func.getVecTypeHintAttrName(),

1703 }

1704

1706 func.getWorkGroupSizeHint()) {

1707 llvmFunc->setMetadata(

1708 func.getWorkGroupSizeHintAttrName(),

1710 }

1711

1713 func.getReqdWorkGroupSize()) {

1714 llvmFunc->setMetadata(

1715 func.getReqdWorkGroupSizeAttrName(),

1717 }

1718

1719 if (std::optional<uint32_t> intelReqdSubGroupSize =

1720 func.getIntelReqdSubGroupSize()) {

1721 llvmFunc->setMetadata(

1722 func.getIntelReqdSubGroupSizeAttrName(),

1724 llvm::APInt(32, *intelReqdSubGroupSize)));

1725 }

1726}

1727

1729 llvm::Attribute::AttrKind llvmKind,

1734 .Case([&](auto typeAttr) {

1735 attrBuilder.addTypeAttr(

1736 llvmKind, moduleTranslation.convertType(typeAttr.getValue()));

1738 })

1739 .Case([&](auto intAttr) {

1740 attrBuilder.addRawIntAttr(llvmKind, intAttr.getInt());

1742 })

1743 .Case([&](auto) {

1744 attrBuilder.addAttribute(llvmKind);

1746 })

1747 .CaseLLVM::ConstantRangeAttr([&](auto rangeAttr) {

1748 attrBuilder.addConstantRangeAttr(

1749 llvmKind,

1750 llvm::ConstantRange(rangeAttr.getLower(), rangeAttr.getUpper()));

1752 })

1753 .Default([loc](auto) {

1754 return emitError(loc, "unsupported parameter attribute type");

1755 });

1756}

1757

1758FailureOrllvm::AttrBuilder

1759ModuleTranslation::convertParameterAttrs(LLVMFuncOp func, int argIdx,

1760 DictionaryAttr paramAttrs) {

1761 llvm::AttrBuilder attrBuilder(llvmModule->getContext());

1763 Location loc = func.getLoc();

1764

1765 for (auto namedAttr : paramAttrs) {

1766 auto it = attrNameToKindMapping.find(namedAttr.getName());

1767 if (it != attrNameToKindMapping.end()) {

1768 llvm::Attribute::AttrKind llvmKind = it->second;

1770 loc)))

1771 return failure();

1772 } else if (namedAttr.getNameDialect()) {

1773 if (failed(iface.convertParameterAttr(func, argIdx, namedAttr, *this)))

1774 return failure();

1775 }

1776 }

1777

1778 return attrBuilder;

1779}

1780

1782 ArgAndResultAttrsOpInterface attrsOp, llvm::CallBase *call,

1784

1785 if (ArrayAttr argAttrsArray = attrsOp.getArgAttrsAttr()) {

1786 unsigned argAttrIdx = 0;

1787 llvm::SmallDenseSet immArgPositionsSet(immArgPositions.begin(),

1788 immArgPositions.end());

1789 for (unsigned argIdx : llvm::seq(call->arg_size())) {

1790 if (argAttrIdx >= argAttrsArray.size())

1791 break;

1792

1793 if (immArgPositionsSet.contains(argIdx))

1794 continue;

1795

1796 auto argAttrs = cast(argAttrsArray[argAttrIdx++]);

1797 if (argAttrs.empty())

1798 continue;

1799

1800 FailureOrllvm::AttrBuilder attrBuilder =

1801 convertParameterAttrs(attrsOp->getLoc(), argAttrs);

1802 if (failed(attrBuilder))

1803 return failure();

1804 call->addParamAttrs(argIdx, *attrBuilder);

1805 }

1806 }

1807

1808

1809 if (ArrayAttr resAttrsArray = attrsOp.getResAttrsAttr()) {

1810 if (!resAttrsArray.empty()) {

1811 auto resAttrs = cast(resAttrsArray[0]);

1812 FailureOrllvm::AttrBuilder attrBuilder =

1813 convertParameterAttrs(attrsOp->getLoc(), resAttrs);

1814 if (failed(attrBuilder))

1815 return failure();

1816 call->addRetAttrs(*attrBuilder);

1817 }

1818 }

1819

1821}

1822

1823FailureOrllvm::AttrBuilder

1824ModuleTranslation::convertParameterAttrs(Location loc,

1825 DictionaryAttr paramAttrs) {

1826 llvm::AttrBuilder attrBuilder(llvmModule->getContext());

1828

1829 for (auto namedAttr : paramAttrs) {

1830 auto it = attrNameToKindMapping.find(namedAttr.getName());

1831 if (it != attrNameToKindMapping.end()) {

1832 llvm::Attribute::AttrKind llvmKind = it->second;

1834 loc)))

1835 return failure();

1836 }

1837 }

1838

1839 return attrBuilder;

1840}

1841

1842LogicalResult ModuleTranslation::convertFunctionSignatures() {

1843

1844

1845 for (auto function : getModuleBody(mlirModule).getOps()) {

1846 llvm::FunctionCallee llvmFuncCst = llvmModule->getOrInsertFunction(

1847 function.getName(),

1848 castllvm::FunctionType(convertType(function.getFunctionType())));

1849 llvm::Function *llvmFunc = castllvm::Function(llvmFuncCst.getCallee());

1850 llvmFunc->setLinkage(convertLinkageToLLVM(function.getLinkage()));

1851 llvmFunc->setCallingConv(convertCConvToLLVM(function.getCConv()));

1852 mapFunction(function.getName(), llvmFunc);

1854

1855

1857

1858

1860

1861

1862 if (std::optional<uint64_t> entryCount = function.getFunctionEntryCount())

1863 llvmFunc->setEntryCount(entryCount.value());

1864

1865

1866 if (ArrayAttr allResultAttrs = function.getAllResultAttrs()) {

1867 DictionaryAttr resultAttrs = cast(allResultAttrs[0]);

1868 FailureOrllvm::AttrBuilder attrBuilder =

1869 convertParameterAttrs(function, -1, resultAttrs);

1870 if (failed(attrBuilder))

1871 return failure();

1872 llvmFunc->addRetAttrs(*attrBuilder);

1873 }

1874

1875

1876 for (auto [argIdx, llvmArg] : llvm::enumerate(llvmFunc->args())) {

1877 if (DictionaryAttr argAttrs = function.getArgAttrDict(argIdx)) {

1878 FailureOrllvm::AttrBuilder attrBuilder =

1879 convertParameterAttrs(function, argIdx, argAttrs);

1880 if (failed(attrBuilder))

1881 return failure();

1882 llvmArg.addAttrs(*attrBuilder);

1883 }

1884 }

1885

1886

1887 FailureOrllvm::AttrBuilder convertedPassthroughAttrs =

1889 function.getPassthroughAttr(),

1890 function.getPassthroughAttrName());

1891 if (failed(convertedPassthroughAttrs))

1892 return failure();

1893 llvmFunc->addFnAttrs(*convertedPassthroughAttrs);

1894

1895

1896 llvmFunc->setVisibility(convertVisibilityToLLVM(function.getVisibility_()));

1897

1898

1899 if (std::optionalmlir::SymbolRefAttr comdat = function.getComdat()) {

1900 auto selectorOp = cast(

1902 llvmFunc->setComdat(comdatMapping.lookup(selectorOp));

1903 }

1904

1905 if (auto gc = function.getGarbageCollector())

1906 llvmFunc->setGC(gc->str());

1907

1908 if (auto unnamedAddr = function.getUnnamedAddr())

1909 llvmFunc->setUnnamedAddr(convertUnnamedAddrToLLVM(*unnamedAddr));

1910

1911 if (auto alignment = function.getAlignment())

1912 llvmFunc->setAlignment(llvm::MaybeAlign(*alignment));

1913

1914

1915 debugTranslation->translate(function, *llvmFunc);

1916 }

1917

1919}

1920

1921LogicalResult ModuleTranslation::convertFunctions() {

1922

1923 for (auto function : getModuleBody(mlirModule).getOps()) {

1924

1925

1926 if (function.isExternal()) {

1927 if (failed(convertDialectAttributes(function, {})))

1928 return failure();

1929 continue;

1930 }

1931

1932 if (failed(convertOneFunction(function)))

1933 return failure();

1934 }

1935

1937}

1938

1939LogicalResult ModuleTranslation::convertIFuncs() {

1940 for (auto op : getModuleBody(mlirModule).getOps()) {

1941 llvm::Type *type = convertType(op.getIFuncType());

1942 llvm::GlobalValue::LinkageTypes linkage =

1943 convertLinkageToLLVM(op.getLinkage());

1944 llvm::Constant *resolver;

1945 if (auto *resolverFn = lookupFunction(op.getResolver())) {

1946 resolver = castllvm::Constant(resolverFn);

1947 } else {

1949 op.getResolverAttr());

1950 resolver = castllvm::Constant(lookupAlias(aliasOp));

1951 }

1952

1953 auto *ifunc =

1954 llvm::GlobalIFunc::create(type, op.getAddressSpace(), linkage,

1955 op.getSymName(), resolver, llvmModule.get());

1957 ifunc->setUnnamedAddr(convertUnnamedAddrToLLVM(op.getUnnamedAddr()));

1958 ifunc->setVisibility(convertVisibilityToLLVM(op.getVisibility_()));

1959

1960 ifuncMapping.try_emplace(op, ifunc);

1961 }

1962

1964}

1965

1966LogicalResult ModuleTranslation::convertComdats() {

1967 for (auto comdatOp : getModuleBody(mlirModule).getOps()) {

1968 for (auto selectorOp : comdatOp.getOps()) {

1970 if (module->getComdatSymbolTable().contains(selectorOp.getSymName()))

1971 return emitError(selectorOp.getLoc())

1972 << "comdat selection symbols must be unique even in different "

1973 "comdat regions";

1974 llvm::Comdat *comdat = module->getOrInsertComdat(selectorOp.getSymName());

1975 comdat->setSelectionKind(convertComdatToLLVM(selectorOp.getComdat()));

1976 comdatMapping.try_emplace(selectorOp, comdat);

1977 }

1978 }

1980}

1981

1982LogicalResult ModuleTranslation::convertUnresolvedBlockAddress() {

1983 for (auto &[blockAddressOp, llvmCst] : unresolvedBlockAddressMapping) {

1984 BlockAddressAttr blockAddressAttr = blockAddressOp.getBlockAddr();

1986 assert(llvmBlock && "expected LLVM blocks to be already translated");

1987

1988

1989 auto *llvmBlockAddr = llvm::BlockAddress::get(

1990 lookupFunction(blockAddressAttr.getFunction().getValue()), llvmBlock);

1991 llvmCst->replaceAllUsesWith(llvmBlockAddr);

1992 assert(llvmCst->use_empty() && "expected all uses to be replaced");

1993 castllvm::GlobalVariable(llvmCst)->eraseFromParent();

1994 }

1995 unresolvedBlockAddressMapping.clear();

1997}

1998

2000 llvm::Instruction *inst) {

2001 if (llvm::MDNode *node = loopAnnotationTranslation->getAccessGroups(op))

2002 inst->setMetadata(llvm::LLVMContext::MD_access_group, node);

2003}

2004

2005llvm::MDNode *

2007 auto [scopeIt, scopeInserted] =

2008 aliasScopeMetadataMapping.try_emplace(aliasScopeAttr, nullptr);

2009 if (!scopeInserted)

2010 return scopeIt->second;

2011 llvm::LLVMContext &ctx = llvmModule->getContext();

2012 auto dummy = llvm::MDNode::getTemporary(ctx, {});

2013

2014 auto [domainIt, insertedDomain] = aliasDomainMetadataMapping.try_emplace(

2015 aliasScopeAttr.getDomain(), nullptr);

2016 if (insertedDomain) {

2018

2019 operands.push_back(dummy.get());

2020 if (StringAttr description = aliasScopeAttr.getDomain().getDescription())

2021 operands.push_back(llvm::MDString::get(ctx, description));

2022 domainIt->second = llvm::MDNode::get(ctx, operands);

2023

2025 if (auto stringAttr =

2026 dyn_cast(aliasScopeAttr.getDomain().getId()))

2027 replacement = llvm::MDString::get(ctx, stringAttr.getValue());

2028 else

2030 domainIt->second->replaceOperandWith(0, replacement);

2031 }

2032

2033 assert(domainIt->second && "Scope's domain should already be valid");

2035

2036 operands.push_back(dummy.get());

2037 operands.push_back(domainIt->second);

2038 if (StringAttr description = aliasScopeAttr.getDescription())

2039 operands.push_back(llvm::MDString::get(ctx, description));

2040 scopeIt->second = llvm::MDNode::get(ctx, operands);

2041

2043 if (auto stringAttr = dyn_cast(aliasScopeAttr.getId()))

2044 replacement = llvm::MDString::get(ctx, stringAttr.getValue());

2045 else

2047 scopeIt->second->replaceOperandWith(0, replacement);

2048 return scopeIt->second;

2049}

2050

2054 nodes.reserve(aliasScopeAttrs.size());

2055 for (AliasScopeAttr aliasScopeAttr : aliasScopeAttrs)

2058}

2059

2061 llvm::Instruction *inst) {

2062 auto populateScopeMetadata = [&](ArrayAttr aliasScopeAttrs, unsigned kind) {

2063 if (!aliasScopeAttrs || aliasScopeAttrs.empty())

2064 return;

2066 llvm::to_vector(aliasScopeAttrs.getAsRange()));

2067 inst->setMetadata(kind, node);

2068 };

2069

2070 populateScopeMetadata(op.getAliasScopesOrNull(),

2071 llvm::LLVMContext::MD_alias_scope);

2072 populateScopeMetadata(op.getNoAliasScopesOrNull(),

2073 llvm::LLVMContext::MD_noalias);

2074}

2075

2076llvm::MDNode *ModuleTranslation::getTBAANode(TBAATagAttr tbaaAttr) const {

2077 return tbaaMetadataMapping.lookup(tbaaAttr);

2078}

2079

2081 llvm::Instruction *inst) {

2082 ArrayAttr tagRefs = op.getTBAATagsOrNull();

2083 if (!tagRefs || tagRefs.empty())

2084 return;

2085

2086

2087

2088

2089

2090 if (tagRefs.size() > 1) {

2091 op.emitWarning() << "TBAA access tags were not translated, because LLVM "

2092 "IR only supports a single tag per instruction";

2093 return;

2094 }

2095

2096 llvm::MDNode *node = getTBAANode(cast(tagRefs[0]));

2097 inst->setMetadata(llvm::LLVMContext::MD_tbaa, node);

2098}

2099

2101 DereferenceableOpInterface op, llvm::Instruction *inst) {

2102 DereferenceableAttr derefAttr = op.getDereferenceableOrNull();

2103 if (!derefAttr)

2104 return;

2105

2106 llvm::MDNode *derefSizeNode = llvm::MDNode::get(

2108 llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(

2109 llvm::IntegerType::get(getLLVMContext(), 64), derefAttr.getBytes())));

2110 unsigned kindId = derefAttr.getMayBeNull()

2111 ? llvm::LLVMContext::MD_dereferenceable_or_null

2112 : llvm::LLVMContext::MD_dereferenceable;

2113 inst->setMetadata(kindId, derefSizeNode);

2114}

2115

2118 llvm::transform(op.getWeights(), std::back_inserter(weights),

2119 [](int32_t value) { return static_cast<uint32_t>(value); });

2120 if (weights.empty())

2121 return;

2122

2124 assert(inst && "expected the operation to have a mapping to an instruction");

2125 inst->setMetadata(

2126 llvm::LLVMContext::MD_prof,

2127 llvm::MDBuilder(getLLVMContext()).createBranchWeights(weights));

2128}

2129

2130LogicalResult ModuleTranslation::createTBAAMetadata() {

2131 llvm::LLVMContext &ctx = llvmModule->getContext();

2132 llvm::IntegerType *offsetTy = llvm::IntegerType::get(ctx, 64);

2133

2134

2135

2136

2137

2138

2139

2140

2141

2143 walker.addWalk([&](TBAARootAttr root) {

2144 llvm::MDNode *node;

2145 if (StringAttr id = root.getId()) {

2146 node = llvm::MDNode::get(ctx, llvm::MDString::get(ctx, id));

2147 } else {

2148

2149 auto selfRef = llvm::MDNode::getTemporary(ctx, {});

2150 node = llvm::MDNode::get(ctx, {selfRef.get()});

2151 node->replaceOperandWith(0, node);

2152 }

2153 tbaaMetadataMapping.insert({root, node});

2154 });

2155

2156 walker.addWalk([&](TBAATypeDescriptorAttr descriptor) {

2157 SmallVector<llvm::Metadata *> operands;

2158 operands.push_back(llvm::MDString::get(ctx, descriptor.getId()));

2159 for (TBAAMemberAttr member : descriptor.getMembers()) {

2160 operands.push_back(tbaaMetadataMapping.lookup(member.getTypeDesc()));

2161 operands.push_back(llvm::ConstantAsMetadata::get(

2162 llvm::ConstantInt::get(offsetTy, member.getOffset())));

2163 }

2164

2165 tbaaMetadataMapping.insert({descriptor, llvm::MDNode::get(ctx, operands)});

2166 });

2167

2168 walker.addWalk([&](TBAATagAttr tag) {

2169 SmallVector<llvm::Metadata *> operands;

2170

2171 operands.push_back(tbaaMetadataMapping.lookup(tag.getBaseType()));

2172 operands.push_back(tbaaMetadataMapping.lookup(tag.getAccessType()));

2173

2174 operands.push_back(llvm::ConstantAsMetadata::get(

2175 llvm::ConstantInt::get(offsetTy, tag.getOffset())));

2176 if (tag.getConstant())

2177 operands.push_back(

2178 llvm::ConstantAsMetadata::get(llvm::ConstantInt::get(offsetTy, 1)));

2179

2180 tbaaMetadataMapping.insert({tag, llvm::MDNode::get(ctx, operands)});

2181 });

2182

2183 mlirModule->walk([&](AliasAnalysisOpInterface analysisOpInterface) {

2184 if (auto attr = analysisOpInterface.getTBAATagsOrNull())

2185 walker.walk(attr);

2186 });

2187

2189}

2190

2191LogicalResult ModuleTranslation::createIdentMetadata() {

2192 if (auto attr = mlirModule->getAttrOfType(

2193 LLVMDialect::getIdentAttrName())) {

2194 StringRef ident = attr;

2195 llvm::LLVMContext &ctx = llvmModule->getContext();

2196 llvm::NamedMDNode *namedMd =

2197 llvmModule->getOrInsertNamedMetadata(LLVMDialect::getIdentAttrName());

2198 llvm::MDNode *md = llvm::MDNode::get(ctx, llvm::MDString::get(ctx, ident));

2199 namedMd->addOperand(md);

2200 }

2201

2203}

2204

2205LogicalResult ModuleTranslation::createCommandlineMetadata() {

2206 if (auto attr = mlirModule->getAttrOfType(

2207 LLVMDialect::getCommandlineAttrName())) {

2208 StringRef cmdLine = attr;

2209 llvm::LLVMContext &ctx = llvmModule->getContext();

2210 llvm::NamedMDNode *nmd = llvmModule->getOrInsertNamedMetadata(

2211 LLVMDialect::getCommandlineAttrName());

2212 llvm::MDNode *md =

2213 llvm::MDNode::get(ctx, llvm::MDString::get(ctx, cmdLine));

2214 nmd->addOperand(md);

2215 }

2216

2218}

2219

2220LogicalResult ModuleTranslation::createDependentLibrariesMetadata() {

2221 if (auto dependentLibrariesAttr = mlirModule->getDiscardableAttr(

2222 LLVM::LLVMDialect::getDependentLibrariesAttrName())) {

2223 auto *nmd =

2224 llvmModule->getOrInsertNamedMetadata("llvm.dependent-libraries");

2225 llvm::LLVMContext &ctx = llvmModule->getContext();

2226 for (auto libAttr :

2227 cast(dependentLibrariesAttr).getAsRange()) {

2228 auto *md =

2229 llvm::MDNode::get(ctx, llvm::MDString::get(ctx, libAttr.getValue()));

2230 nmd->addOperand(md);

2231 }

2232 }

2234}

2235

2237 llvm::Instruction *inst) {

2238 LoopAnnotationAttr attr =

2240 .Case<LLVM::BrOp, LLVM::CondBrOp>(

2241 [](auto branchOp) { return branchOp.getLoopAnnotationAttr(); });

2242 if (!attr)

2243 return;

2244 llvm::MDNode *loopMD =

2245 loopAnnotationTranslation->translateLoopAnnotation(attr, op);

2246 inst->setMetadata(llvm::LLVMContext::MD_loop, loopMD);

2247}

2248

2250 auto iface = cast(op);

2251

2252 if (auto disjointInst = dyn_castllvm::PossiblyDisjointInst(value))

2253 disjointInst->setIsDisjoint(iface.getIsDisjoint());

2254}

2255

2257 return typeTranslator.translateType(type);

2258}

2259

2260

2263 remapped.reserve(values.size());

2264 for (Value v : values)

2266 return remapped;

2267}

2268

2270 if (!ompBuilder) {

2271 ompBuilder = std::make_uniquellvm::OpenMPIRBuilder(*llvmModule);

2272

2273

2274

2275

2276 llvm::OpenMPIRBuilderConfig config(

2277 false, false,

2278 false,

2279 false,

2280 false,

2281 false,

2282 false);

2283 unsigned int defaultAS =

2284 llvmModule->getDataLayout().getProgramAddressSpace();

2285 config.setDefaultTargetAS(defaultAS);

2286 config.setRuntimeCC(llvmModule->getTargetTriple().isSPIRV()

2287 ? llvm::CallingConv::SPIR_FUNC

2288 : llvm::CallingConv::C);

2289 ompBuilder->setConfig(std::move(config));

2290 ompBuilder->initialize();

2291 }

2292 return ompBuilder.get();

2293}

2294

2296 llvm::DILocalScope *scope) {

2297 return debugTranslation->translateLoc(loc, scope);

2298}

2299

2300llvm::DIExpression *

2302 return debugTranslation->translateExpression(attr);

2303}

2304

2305llvm::DIGlobalVariableExpression *

2307 LLVM::DIGlobalVariableExpressionAttr attr) {

2308 return debugTranslation->translateGlobalVariableExpression(attr);

2309}

2310

2312 return debugTranslation->translate(attr);

2313}

2314

2315llvm::RoundingMode

2317 return convertRoundingModeToLLVM(rounding);

2318}

2319

2321 LLVM::FPExceptionBehavior exceptionBehavior) {

2322 return convertFPExceptionBehaviorToLLVM(exceptionBehavior);

2323}

2324

2325llvm::NamedMDNode *

2327 return llvmModule->getOrInsertNamedMetadata(name);

2328}

2329

2330static std::unique_ptrllvm::Module

2332 StringRef name) {

2334 auto llvmModule = std::make_uniquellvm::Module(name, llvmContext);

2335 if (auto dataLayoutAttr =

2336 m->getDiscardableAttr(LLVM::LLVMDialect::getDataLayoutAttrName())) {

2337 llvmModule->setDataLayout(cast(dataLayoutAttr).getValue());

2338 } else {

2339 FailureOrllvm::DataLayout llvmDataLayout(llvm::DataLayout(""));

2340 if (auto iface = dyn_cast(m)) {

2341 if (DataLayoutSpecInterface spec = iface.getDataLayoutSpec()) {

2342 llvmDataLayout =

2344 }

2345 } else if (auto mod = dyn_cast(m)) {

2346 if (DataLayoutSpecInterface spec = mod.getDataLayoutSpec()) {

2347 llvmDataLayout =

2349 }

2350 }

2351 if (failed(llvmDataLayout))

2352 return nullptr;

2353 llvmModule->setDataLayout(*llvmDataLayout);

2354 }

2355 if (auto targetTripleAttr =

2356 m->getDiscardableAttr(LLVM::LLVMDialect::getTargetTripleAttrName()))

2357 llvmModule->setTargetTriple(

2358 llvm::Triple(cast(targetTripleAttr).getValue()));

2359

2361 LLVM::LLVMDialect::getModuleLevelAsmAttrName())) {

2362 auto asmArrayAttr = dyn_cast(asmAttr);

2363 if (!asmArrayAttr) {

2364 m->emitError("expected an array attribute for a module level asm");

2365 return nullptr;

2366 }

2367

2368 for (Attribute elt : asmArrayAttr) {

2369 auto asmStrAttr = dyn_cast(elt);

2370 if (!asmStrAttr) {

2372 "expected a string attribute for each entry of a module level asm");

2373 return nullptr;

2374 }

2375 llvmModule->appendModuleInlineAsm(asmStrAttr.getValue());

2376 }

2377 }

2378

2379 return llvmModule;

2380}

2381

2382std::unique_ptrllvm::Module

2384 StringRef name, bool disableVerification) {

2386 module->emitOpError("can not be translated to an LLVMIR module");

2387 return nullptr;

2388 }

2389

2390 std::unique_ptrllvm::Module llvmModule =

2392 if (!llvmModule)

2393 return nullptr;

2394

2397

2399 llvm::IRBuilderllvm::TargetFolder llvmBuilder(

2400 llvmContext,

2401 llvm::TargetFolder(translator.getLLVMModule()->getDataLayout()));

2402

2403

2404

2405

2406

2407 if (failed(translator.convertOperation(*module, llvmBuilder)))

2408 return nullptr;

2409

2410 if (failed(translator.convertComdats()))

2411 return nullptr;

2412 if (failed(translator.convertFunctionSignatures()))

2413 return nullptr;

2414 if (failed(translator.convertGlobalsAndAliases()))

2415 return nullptr;

2416 if (failed(translator.convertIFuncs()))

2417 return nullptr;

2418 if (failed(translator.createTBAAMetadata()))

2419 return nullptr;

2420 if (failed(translator.createIdentMetadata()))

2421 return nullptr;

2422 if (failed(translator.createCommandlineMetadata()))

2423 return nullptr;

2424 if (failed(translator.createDependentLibrariesMetadata()))

2425 return nullptr;

2426

2427

2428 for (Operation &o : getModuleBody(module).getOperations()) {

2429 if (!isa<LLVM::LLVMFuncOp, LLVM::AliasOp, LLVM::GlobalOp,

2430 LLVM::GlobalCtorsOp, LLVM::GlobalDtorsOp, LLVM::ComdatOp,

2431 LLVM::IFuncOp>(&o) &&

2432 !o.hasTraitOpTrait::IsTerminator() &&

2433 failed(translator.convertOperation(o, llvmBuilder))) {

2434 return nullptr;

2435 }

2436 }

2437

2438

2439

2440

2441 if (failed(translator.convertFunctions()))

2442 return nullptr;

2443

2444

2445

2446 if (failed(translator.convertUnresolvedBlockAddress()))

2447 return nullptr;

2448

2449

2450

2451 translator.debugTranslation->addModuleFlagsIfNotPresent();

2452

2453

2454 if (auto *ompBuilder = translator.getOpenMPBuilder())

2455 ompBuilder->finalize();

2456

2457 if (!disableVerification &&

2458 llvm::verifyModule(*translator.llvmModule, &llvm::errs()))

2459 return nullptr;

2460

2461 return std::move(translator.llvmModule);

2462}

*if copies could not be generated due to yet unimplemented cases *copyInPlacementStart and copyOutPlacementStart in copyPlacementBlock *specify the insertion points where the incoming copies and outgoing should be inserted(the insertion happens right before the *insertion point). Since `begin` can itself be invalidated due to the memref *rewriting done from this method

*if copies could not be generated due to yet unimplemented cases *copyInPlacementStart and copyOutPlacementStart in copyPlacementBlock *specify the insertion points where the incoming copies and outgoing should be the output argument nBegin is set to its * replacement(set to `begin` if no invalidation happens). Since outgoing *copies could have been inserted at `end`

static Value getPHISourceValue(Block *current, Block *pred, unsigned numArguments, unsigned index)

Get the SSA value passed to the current block from the terminator operation of its predecessor.

Definition ModuleTranslation.cpp:813

static llvm::Type * getInnermostElementType(llvm::Type *type)

Returns the first non-sequential type nested in sequential types.

Definition ModuleTranslation.cpp:355

static void addRuntimePreemptionSpecifier(bool dsoLocalRequested, llvm::GlobalValue *gv)

Sets the runtime preemption specifier of gv to dso_local if dsoLocalRequested is true,...

Definition ModuleTranslation.cpp:1085

static Block & getModuleBody(Operation *module)

A helper method to get the single Block in an operation honoring LLVM's module requirements.

Definition ModuleTranslation.cpp:1068

static llvm::MDNode * convertIntegerArrayToMDNode(llvm::LLVMContext &context, ArrayRef< int32_t > values)

Return an MDNode with a tuple given by the values in values.

Definition ModuleTranslation.cpp:1502

static void convertFunctionAttributes(LLVMFuncOp func, llvm::Function *llvmFunc)

Converts function attributes from func and attaches them to llvmFunc.

Definition ModuleTranslation.cpp:1663

static bool shouldDropGlobalInitializer(llvm::GlobalValue::LinkageTypes linkage, llvm::Constant *cst)

A helper method to decide if a constant must not be set as a global variable initializer.

Definition ModuleTranslation.cpp:1077

static llvm::MDNode * convertIntegerToMDNode(llvm::LLVMContext &context, const llvm::APInt &value)

Return a representation of value as an MDNode.

Definition ModuleTranslation.cpp:1485

static llvm::Metadata * convertIntegerToMetadata(llvm::LLVMContext &context, const llvm::APInt &value)

Return a representation of value as metadata.

Definition ModuleTranslation.cpp:1478

static FailureOr< llvm::Attribute > convertMLIRAttributeToLLVM(Location loc, llvm::LLVMContext &ctx, StringRef key, StringRef value=StringRef())

Attempts to translate an MLIR attribute identified by key, optionally with the given value,...

Definition ModuleTranslation.cpp:1098

static void convertFunctionKernelAttributes(LLVMFuncOp func, llvm::Function *llvmFunc, ModuleTranslation &translation)

Converts function attributes from func and attaches them to llvmFunc.

Definition ModuleTranslation.cpp:1691

static LogicalResult convertParameterAttr(llvm::AttrBuilder &attrBuilder, llvm::Attribute::AttrKind llvmKind, NamedAttribute namedAttr, ModuleTranslation &moduleTranslation, Location loc)

Definition ModuleTranslation.cpp:1728

static llvm::Constant * buildSequentialConstant(ArrayRef< llvm::Constant * > &constants, ArrayRef< int64_t > shape, llvm::Type *type, Location loc)

Builds a constant of a sequential LLVM type type, potentially containing other sequential types recur...

Definition ModuleTranslation.cpp:320

static FailureOr< llvm::AttrBuilder > convertMLIRAttributesToLLVM(Location loc, llvm::LLVMContext &ctx, ArrayAttr arrayAttr, StringRef arrayAttrName)

Converts the MLIR attributes listed in the given array attribute into LLVM attributes.

Definition ModuleTranslation.cpp:1131

static void convertFunctionMemoryAttributes(LLVMFuncOp func, llvm::Function *llvmFunc)

Converts memory effect attributes from func and attaches them to llvmFunc.

Definition ModuleTranslation.cpp:1633

static llvm::Constant * convertDenseResourceElementsAttr(Location loc, DenseResourceElementsAttr denseResourceAttr, llvm::Type *llvmType, const ModuleTranslation &moduleTranslation)

Convert a dense resource elements attribute to an LLVM IR constant using its raw data storage if poss...

Definition ModuleTranslation.cpp:472

static llvm::MDNode * convertVecTypeHintToMDNode(llvm::LLVMContext &context, llvm::Type *type, bool isSigned)

Return an MDNode encoding vec_type_hint metadata.

Definition ModuleTranslation.cpp:1491

static llvm::Constant * convertDenseElementsAttr(Location loc, DenseElementsAttr denseElementsAttr, llvm::Type *llvmType, const ModuleTranslation &moduleTranslation)

Convert a dense elements attribute to an LLVM IR constant using its raw data storage if possible.

Definition ModuleTranslation.cpp:375

static std::unique_ptr< llvm::Module > prepareLLVMModule(Operation *m, llvm::LLVMContext &llvmContext, StringRef name)

Definition ModuleTranslation.cpp:2331

static ArrayRef< int64_t > getShape(Type type)

Returns the shape of the given type.

This class represents a processed binary blob of data.

ArrayRef< char > getData() const

Return the raw underlying data of this blob.

void addWalk(WalkFn< Attribute > &&fn)

WalkResult walk(T element)

Walk the given attribute/type, and recursively walk any sub elements.

Attributes are known-constant values of operations.

MLIRContext * getContext() const

Return the context this attribute belongs to.

Block represents an ordered list of Operations.

iterator_range< pred_iterator > getPredecessors()

Operation * getTerminator()

Get the terminator operation of this block.

BlockArgListType getArguments()

iterator_range< iterator > without_terminator()

Return an iterator range over the operation within this block excluding the terminator operation at t...

The main mechanism for performing data layout queries.

std::optional< uint64_t > getTypeIndexBitwidth(Type t) const

Returns the bitwidth that should be used when performing index computations for the given pointer-lik...

uint64_t getTypePreferredAlignment(Type t) const

Returns the preferred of the given type in the current scope.

uint64_t getTypeABIAlignment(Type t) const

Returns the required alignment of the given type in the current scope.

llvm::TypeSize getTypeSizeInBits(Type t) const

Returns the size in bits of the given type in the current scope.

An attribute that represents a reference to a dense vector or tensor object.

int64_t getNumElements() const

Returns the number of elements held by this attribute.

std::enable_if_t<!std::is_base_of< Attribute, T >::value||std::is_same< Attribute, T >::value, T > getSplatValue() const

Return the splat value for this attribute.

bool isSplat() const

Returns true if this attribute corresponds to a splat, i.e.

ArrayRef< char > getRawData() const

Return the raw storage data held by this attribute.

ShapedType getType() const

Return the type of this ElementsAttr, guaranteed to be a vector or tensor with static shape.

const InterfaceType * getInterfaceFor(Object *obj) const

Get the interface for a given object, or null if one is not registered.

Base class for dialect interfaces providing translation to LLVM IR.

virtual LogicalResult convertOperation(Operation *op, llvm::IRBuilderBase &builder, LLVM::ModuleTranslation &moduleTranslation) const

Hook for derived dialect interface to provide translation of the operations to LLVM IR.

This class represents the base attribute for all debug info attributes.

Implementation class for module translation.

llvm::fp::ExceptionBehavior translateFPExceptionBehavior(LLVM::FPExceptionBehavior exceptionBehavior)

Translates the given LLVM FP exception behavior metadata.

Definition ModuleTranslation.cpp:2320

llvm::CallInst * lookupCall(Operation *op) const

Finds an LLVM call instruction that corresponds to the given MLIR call operation.

llvm::BasicBlock * lookupBlock(Block *block) const

Finds an LLVM IR basic block that corresponds to the given MLIR block.

llvm::DIGlobalVariableExpression * translateGlobalVariableExpression(LLVM::DIGlobalVariableExpressionAttr attr)

Translates the given LLVM global variable expression metadata.

Definition ModuleTranslation.cpp:2306

llvm::NamedMDNode * getOrInsertNamedModuleMetadata(StringRef name)

Gets the named metadata in the LLVM IR module being constructed, creating it if it does not exist.

Definition ModuleTranslation.cpp:2326

SmallVector< llvm::Value * > lookupValues(ValueRange values)

Looks up remapped a list of remapped values.

Definition ModuleTranslation.cpp:2261

void mapFunction(StringRef name, llvm::Function *func)

Stores the mapping between a function name and its LLVM IR representation.

llvm::DILocation * translateLoc(Location loc, llvm::DILocalScope *scope)

Translates the given location.

Definition ModuleTranslation.cpp:2295

void setDereferenceableMetadata(DereferenceableOpInterface op, llvm::Instruction *inst)

Sets LLVM dereferenceable metadata for operations that have dereferenceable attributes.

Definition ModuleTranslation.cpp:2100

void setBranchWeightsMetadata(WeightedBranchOpInterface op)

Sets LLVM profiling metadata for operations that have branch weights.

Definition ModuleTranslation.cpp:2116

llvm::Instruction * lookupBranch(Operation *op) const

Finds an LLVM IR instruction that corresponds to the given MLIR operation with successors.

llvm::Value * lookupValue(Value value) const

Finds an LLVM IR value corresponding to the given MLIR value.

LogicalResult convertArgAndResultAttrs(ArgAndResultAttrsOpInterface attrsOp, llvm::CallBase *call, ArrayRef< unsigned > immArgPositions={})

Converts argument and result attributes from attrsOp to LLVM IR attributes on the call instruction.

Definition ModuleTranslation.cpp:1781

SymbolTableCollection & symbolTable()

llvm::Type * convertType(Type type)

Converts the type from MLIR LLVM dialect to LLVM.

Definition ModuleTranslation.cpp:2256

llvm::RoundingMode translateRoundingMode(LLVM::RoundingMode rounding)

Translates the given LLVM rounding mode metadata.

Definition ModuleTranslation.cpp:2316

void setTBAAMetadata(AliasAnalysisOpInterface op, llvm::Instruction *inst)

Sets LLVM TBAA metadata for memory operations that have TBAA attributes.

Definition ModuleTranslation.cpp:2080

llvm::DIExpression * translateExpression(LLVM::DIExpressionAttr attr)

Translates the given LLVM DWARF expression metadata.

Definition ModuleTranslation.cpp:2301

llvm::OpenMPIRBuilder * getOpenMPBuilder()

Returns the OpenMP IR builder associated with the LLVM IR module being constructed.

Definition ModuleTranslation.cpp:2269

llvm::GlobalValue * lookupGlobal(Operation *op)

Finds an LLVM IR global value that corresponds to the given MLIR operation defining a global value.

llvm::BasicBlock * lookupBlockAddress(BlockAddressAttr attr) const

Finds the LLVM basic block that corresponds to the given BlockAddressAttr.

llvm::Metadata * translateDebugInfo(LLVM::DINodeAttr attr)

Translates the given LLVM debug info metadata.

Definition ModuleTranslation.cpp:2311

void setDisjointFlag(Operation *op, llvm::Value *value)

Sets the disjoint flag attribute for the exported instruction value given the original operation op.

Definition ModuleTranslation.cpp:2249

llvm::GlobalValue * lookupAlias(Operation *op)

Finds an LLVM IR global value that corresponds to the given MLIR operation defining a global alias va...

llvm::Function * lookupFunction(StringRef name) const

Finds an LLVM IR function by its name.

llvm::MDNode * getOrCreateAliasScopes(ArrayRef< AliasScopeAttr > aliasScopeAttrs)

Returns the LLVM metadata corresponding to an array of mlir LLVM dialect alias scope attributes.

Definition ModuleTranslation.cpp:2051

void mapBlock(Block *mlir, llvm::BasicBlock *llvm)

Stores the mapping between an MLIR block and LLVM IR basic block.

llvm::MDNode * getOrCreateAliasScope(AliasScopeAttr aliasScopeAttr)

Returns the LLVM metadata corresponding to a mlir LLVM dialect alias scope attribute.

Definition ModuleTranslation.cpp:2006

llvm::Module * getLLVMModule()

Returns the LLVM module in which the IR is being constructed.

void forgetMapping(Region &region)

Removes the mapping for blocks contained in the region and values defined in these blocks.

Definition ModuleTranslation.cpp:781

void setAliasScopeMetadata(AliasAnalysisOpInterface op, llvm::Instruction *inst)

Definition ModuleTranslation.cpp:2060

void setAccessGroupsMetadata(AccessGroupOpInterface op, llvm::Instruction *inst)

Definition ModuleTranslation.cpp:1999

void mapValue(Value mlir, llvm::Value *llvm)

Stores the mapping between an MLIR value and its LLVM IR counterpart.

llvm::LLVMContext & getLLVMContext() const

Returns the LLVM context in which the IR is being constructed.

void setLoopMetadata(Operation *op, llvm::Instruction *inst)

Sets LLVM loop metadata for branch operations that have a loop annotation attribute.

Definition ModuleTranslation.cpp:2236

A helper class that converts LoopAnnotationAttrs and AccessGroupAttrs into corresponding llvm::MDNode...

This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...

T * getOrLoadDialect()

Get (or create) a dialect for the given derived dialect type.

NamedAttribute represents a combination of a name and an Attribute value.

Attribute getValue() const

Return the value of the attribute.

Operation is the basic unit of execution within MLIR.

Attribute getDiscardableAttr(StringRef name)

Access a discardable attribute by name, returns a null Attribute if the discardable attribute does no...

Value getOperand(unsigned idx)

Attribute getAttr(StringAttr name)

Return the specified attribute if present, null otherwise.

unsigned getNumSuccessors()

OpResult getResult(unsigned idx)

Get the 'idx'th result of this operation.

Location getLoc()

The source location the operation was defined or derived from.

InFlightDiagnostic emitError(const Twine &message={})

Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...

OperationName getName()

The name of an operation is the key identifier for it.

dialect_attr_range getDialectAttrs()

Return a range corresponding to the dialect attributes for this operation.

operand_range getOperands()

Returns an iterator on the underlying Value's.

Block * getSuccessor(unsigned index)

MLIRContext * getContext()

Return the context this operation is associated with.

This class contains a list of basic blocks and a link to the parent operation it is attached to.

This class models how operands are forwarded to block arguments in control flow.

bool empty() const

Returns true if there are no successor operands.

virtual Operation * lookupSymbolIn(Operation *symbolTableOp, StringAttr symbol)

Look up a symbol with the specified name within the specified symbol table operation,...

static Operation * lookupNearestSymbolFrom(Operation *from, StringAttr symbol)

Returns the operation registered with the given symbol name within the closest parent operation of,...

Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...

This class provides an abstraction over the different types of ranges over Values.

This class represents an instance of an SSA value in the MLIR system, representing a computable value...

Type getType() const

Return the type of this value.

void connectPHINodes(Region &region, const ModuleTranslation &state)

For all blocks in the region that were converted to LLVM IR using the given ModuleTranslation,...

Definition ModuleTranslation.cpp:872

llvm::CallInst * createIntrinsicCall(llvm::IRBuilderBase &builder, llvm::Intrinsic::ID intrinsic, ArrayRef< llvm::Value * > args={}, ArrayRef< llvm::Type * > tys={})

Creates a call to an LLVM IR intrinsic function with the given arguments.

Definition ModuleTranslation.cpp:899

static llvm::DenseMap< llvm::StringRef, llvm::Attribute::AttrKind > getAttrNameToKindMapping()

Returns a dense map from LLVM attribute name to their kind in LLVM IR dialect.

llvm::Constant * getLLVMConstant(llvm::Type *llvmType, Attribute attr, Location loc, const ModuleTranslation &moduleTranslation)

Create an LLVM IR constant of llvmType from the MLIR attribute attr.

Definition ModuleTranslation.cpp:564

Operation * parentLLVMModule(Operation *op)

Lookup parent Module satisfying LLVM conditions on the Module Operation.

bool satisfiesLLVMModule(Operation *op)

LLVM requires some operations to be inside of a Module operation.

void legalizeDIExpressionsRecursively(Operation *op)

bool isCompatibleType(Type type)

Returns true if the given type is compatible with the LLVM dialect.

void ensureDistinctSuccessors(Operation *op)

Make argument-taking successors of each block distinct.

Include the generated interface declarations.

SetVector< Block * > getBlocksSortedByDominance(Region &region)

Gets a list of blocks that is sorted according to dominance.

DataLayoutSpecInterface translateDataLayout(const llvm::DataLayout &dataLayout, MLIRContext *context)

Translate the given LLVM data layout into an MLIR equivalent using the DLTI dialect.

Type getType(OpFoldResult ofr)

Returns the int type of the integer in ofr.

const FrozenRewritePatternSet GreedyRewriteConfig config

InFlightDiagnostic emitError(Location loc)

Utility method to emit an error message using this location.

std::unique_ptr< llvm::Module > translateModuleToLLVMIR(Operation *module, llvm::LLVMContext &llvmContext, llvm::StringRef name="LLVMDialectModule", bool disableVerification=false)

Translates a given LLVM dialect module into an LLVM IR module living in the given context.

llvm::TypeSwitch< T, ResultT > TypeSwitch

llvm::DenseMap< KeyT, ValueT, KeyInfoT, BucketT > DenseMap