rust@cc9a9ec (original) (raw)

`@@ -10,13 +10,12 @@ use rustc_hir::def_id::DefId;

10

`use rustc_index::Idx;

11

`use rustc_index::bit_set::BitSet;

12

`use rustc_middle::bug;

13

use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs};

13

use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs;

14

`use rustc_middle::mir::visit::*;

15

`use rustc_middle::mir::*;

16

`use rustc_middle::ty::{self, Instance, InstanceKind, Ty, TyCtxt, TypeFlags, TypeVisitableExt};

17

`use rustc_session::config::{DebugInfo, OptLevel};

18

`use rustc_span::source_map::Spanned;

19

use rustc_span::sym;

20

19

`use tracing::{debug, instrument, trace, trace_span};

21

20

22

21

`use crate::cost_checker::CostChecker;

`@@ -120,7 +119,6 @@ trait Inliner<'tcx> {

120

119

`callsite: &CallSite<'tcx>,

121

120

`callee_body: &Body<'tcx>,

122

121

`callee_attrs: &CodegenFnAttrs,

123

cross_crate_inlinable: bool,

124

122

`) -> Result<(), &'static str>;

125

123

126

124

`// How many callsites in a body are we allowed to inline? We need to limit this in order

`@@ -196,7 +194,6 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {

196

194

` _: &CallSite<'tcx>,

197

195

`callee_body: &Body<'tcx>,

198

196

`callee_attrs: &CodegenFnAttrs,

199

_: bool,

200

197

`) -> Result<(), &'static str> {

201

198

`if callee_body.tainted_by_errors.is_some() {

202

199

`return Err("body has errors");

`@@ -215,14 +212,6 @@ impl<'tcx> Inliner<'tcx> for ForceInliner<'tcx> {

215

212

`// inline-asm is detected. LLVM will still possibly do an inline later on

216

213

`// if the no-attribute function ends up with the same instruction set anyway.

217

214

`Err("cannot move inline-asm across instruction sets")

218

} else if callee_body

219

.basic_blocks

220

.iter()

221

.any(|bb| matches!(bb.terminator().kind, TerminatorKind::TailCall { .. }))

222

{

223

// FIXME(explicit_tail_calls): figure out how exactly functions containing tail

224

// calls can be inlined (and if they even should)

225

Err("can't inline functions with tail calls")

226

215

`} else {

227

216

`Ok(())

228

217

`@@ -348,7 +337,6 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {

348

337

`callsite: &CallSite<'tcx>,

349

338

`callee_body: &Body<'tcx>,

350

339

`callee_attrs: &CodegenFnAttrs,

351

cross_crate_inlinable: bool,

352

340

`) -> Result<(), &'static str> {

353

341

`let tcx = self.tcx();

354

342

`@@ -358,7 +346,7 @@ impl<'tcx> Inliner<'tcx> for NormalInliner<'tcx> {

358

346

359

347

`let mut threshold = if self.caller_is_inline_forwarder {

360

348

` tcx.sess.opts.unstable_opts.inline_mir_forwarder_threshold.unwrap_or(30)

361

} else if cross_crate_inlinable {

349

} else if tcx.cross_crate_inlinable(callsite.callee.def_id()) {

362

350

` tcx.sess.opts.unstable_opts.inline_mir_hint_threshold.unwrap_or(100)

363

351

`} else {

364

352

` tcx.sess.opts.unstable_opts.inline_mir_threshold.unwrap_or(50)

`@@ -587,16 +575,8 @@ fn try_inlining<'tcx, I: Inliner<'tcx>>(

587

575

`check_mir_is_available(inliner, caller_body, callsite.callee)?;

588

576

589

577

`let callee_attrs = tcx.codegen_fn_attrs(callsite.callee.def_id());

590

let cross_crate_inlinable = tcx.cross_crate_inlinable(callsite.callee.def_id());

591

check_codegen_attributes(inliner, callsite, callee_attrs, cross_crate_inlinable)?;

592

-

593

// Intrinsic fallback bodies are automatically made cross-crate inlineable,

594

// but at this stage we don't know whether codegen knows the intrinsic,

595

// so just conservatively don't inline it. This also ensures that we do not

596

// accidentally inline the body of an intrinsic that must be overridden.

597

if tcx.has_attr(callsite.callee.def_id(), sym::rustc_intrinsic) {

598

return Err("callee is an intrinsic");

599

}

578

rustc_mir_build::check_inline::is_inline_valid_on_fn(tcx, callsite.callee.def_id())?;

579

check_codegen_attributes(inliner, callsite, callee_attrs)?;

600

580

601

581

`let terminator = caller_body[callsite.block].terminator.as_ref().unwrap();

602

582

`let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() };

`@@ -610,7 +590,8 @@ fn try_inlining<'tcx, I: Inliner<'tcx>>(

610

590

611

591

612

592

`let callee_body = try_instance_mir(tcx, callsite.callee.def)?;

613

inliner.check_callee_mir_body(callsite, callee_body, callee_attrs, cross_crate_inlinable)?;

593

rustc_mir_build::check_inline::is_inline_valid_on_body(tcx, callee_body)?;

594

inliner.check_callee_mir_body(callsite, callee_body, callee_attrs)?;

614

595

615

596

`let Ok(callee_body) = callsite.callee.try_instantiate_mir_and_normalize_erasing_regions(

616

597

` tcx,

`@@ -775,38 +756,19 @@ fn check_codegen_attributes<'tcx, I: Inliner<'tcx>>(

775

756

`inliner: &I,

776

757

`callsite: &CallSite<'tcx>,

777

758

`callee_attrs: &CodegenFnAttrs,

778

cross_crate_inlinable: bool,

779

759

`) -> Result<(), &'static str> {

780

760

`let tcx = inliner.tcx();

781

if tcx.has_attr(callsite.callee.def_id(), sym::rustc_no_mir_inline) {

782

return Err("#[rustc_no_mir_inline]");

783

}

784

-

785

761

`if let InlineAttr::Never = callee_attrs.inline {

786

762

`return Err("never inline attribute");

787

763

788

764

789

// FIXME(#127234): Coverage instrumentation currently doesn't handle inlined

790

// MIR correctly when Modified Condition/Decision Coverage is enabled.

791

if tcx.sess.instrument_coverage_mcdc() {

792

return Err("incompatible with MC/DC coverage");

793

}

794

-

795

765

`// Reachability pass defines which functions are eligible for inlining. Generally inlining

796

766

`// other functions is incorrect because they could reference symbols that aren't exported.

797

767

`let is_generic = callsite.callee.args.non_erasable_generics().next().is_some();

798

if !is_generic && !cross_crate_inlinable {

768

if !is_generic && !tcx.cross_crate_inlinable(callsite.callee.def_id()) {

799

769

`return Err("not exported");

800

770

801

771

802

if callsite.fn_sig.c_variadic() {

803

return Err("C variadic");

804

}

805

-

806

if callee_attrs.flags.contains(CodegenFnAttrFlags::COLD) {

807

return Err("cold");

808

}

809

-

810

772

`let codegen_fn_attrs = tcx.codegen_fn_attrs(inliner.caller_def_id());

811

773

`if callee_attrs.no_sanitize != codegen_fn_attrs.no_sanitize {

812

774

`return Err("incompatible sanitizer set");