Make SSA aggregates without needing an alloca · rust-lang/rust@c38f75c (original) (raw)

`@@ -8,14 +8,16 @@ use crate::traits::*;

8

`use crate::MemFlags;

9

10

`use rustc_hir as hir;

11

use rustc_middle::mir::{self, AggregateKind, Operand};

11

use rustc_middle::mir;

12

`use rustc_middle::ty::cast::{CastTy, IntTy};

13

`use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};

14

`use rustc_middle::ty::{self, adjustment::PointerCoercion, Instance, Ty, TyCtxt};

15

`use rustc_middle::{bug, span_bug};

16

`use rustc_session::config::OptLevel;

17

`use rustc_span::{Span, DUMMY_SP};

18

use rustc_target::abi::{self, FIRST_VARIANT};

18

use rustc_target::abi::{self, FieldIdx, FIRST_VARIANT};

19

+

20

use arrayvec::ArrayVec;

19

21

20

22

`impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {

21

23

`#[instrument(level = "trace", skip(self, bx))]

`@@ -581,7 +583,9 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {

581

583

`self.codegen_place_to_pointer(bx, place, mk_ref)

582

584

583

585

584

mir::Rvalue::CopyForDeref(place) => self.codegen_operand(bx, &Operand::Copy(place)),

586

mir::Rvalue::CopyForDeref(place) => {

587

self.codegen_operand(bx, &mir::Operand::Copy(place))

588

}

585

589

` mir::Rvalue::AddressOf(mutability, place) => {

586

590

`let mk_ptr =

587

591

`move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| Ty::new_ptr(tcx, ty, mutability);

`@@ -739,11 +743,40 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {

739

743

740

744

741

745

` mir::Rvalue::Repeat(..) => bug!("{rvalue:?} in codegen_rvalue_operand"),

742

mir::Rvalue::Aggregate(..) => {

743

`` -

// According to rvalue_creates_operand, only ZST

744

// aggregate rvalues are allowed to be operands.

746

mir::Rvalue::Aggregate(_, ref fields) => {

745

747

`let ty = rvalue.ty(self.mir, self.cx.tcx());

746

OperandRef::zero_sized(self.cx.layout_of(self.monomorphize(ty)))

748

let ty = self.monomorphize(ty);

749

let layout = self.cx.layout_of(ty);

750

+

751

`` +

// rvalue_creates_operand has arranged that we only get here if

752

// we can build the aggregate immediate from the field immediates.

753

let mut inputs = ArrayVec::<Bx::Value, 2>::new();

754

let mut input_scalars = ArrayVec::<abi::Scalar, 2>::new();

755

for field_idx in layout.fields.index_by_increasing_offset() {

756

let field_idx = FieldIdx::from_usize(field_idx);

757

let op = self.codegen_operand(bx, &fields[field_idx]);

758

let values = op.val.immediates_or_place().left_or_else(|p| {

759

bug!("Field {field_idx:?} is {p:?} making {layout:?}");

760

});

761

inputs.extend(values);

762

let scalars = self.value_kind(op.layout).scalars().unwrap();

763

input_scalars.extend(scalars);

764

}

765

+

766

let output_scalars = self.value_kind(layout).scalars().unwrap();

767

itertools::izip!(&mut inputs, input_scalars, output_scalars).for_each(

768

|(v, in_s, out_s)| {

769

if in_s != out_s {

770

// We have to be really careful about bool here, because

771

`` +

// (bool,) stays i1 but Cell<bool> becomes i8.

772

*v = bx.from_immediate(*v);

773

*v = bx.to_immediate_scalar(*v, out_s);

774

}

775

776

);

777

+

778

let val = OperandValue::from_immediates(inputs);

779

OperandRef { val, layout }

747

780

748

781

` mir::Rvalue::ShallowInitBox(ref operand, content_ty) => {

749

782

`let operand = self.codegen_operand(bx, operand);

`@@ -1051,16 +1084,29 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {

1051

1084

` mir::Rvalue::ThreadLocalRef(_) |

1052

1085

` mir::Rvalue::Use(..) => // (*)

1053

1086

`true,

1054

`` -

// This always produces a ty::RawPtr, so will be Immediate or Pair

1055

mir::Rvalue::Aggregate(box AggregateKind::RawPtr(..), ..) => true,

1056

1087

`// Arrays are always aggregates, so it's not worth checking anything here.

1057

1088

`` // (If it's really [(); N] or [T; 0] and we use the place path, fine.)

1058

1089

` mir::Rvalue::Repeat(..) => false,

1059

mir::Rvalue::Aggregate(..) => {

1090

mir::Rvalue::Aggregate(ref kind, _) => {

1091

let allowed_kind = match **kind {

1092

`` +

// This always produces a ty::RawPtr, so will be Immediate or Pair

1093

mir::AggregateKind::RawPtr(..) => true,

1094

mir::AggregateKind::Array(..) => false,

1095

mir::AggregateKind::Tuple => true,

1096

mir::AggregateKind::Adt(def_id, ..) => {

1097

let adt_def = self.cx.tcx().adt_def(def_id);

1098

adt_def.is_struct() && !adt_def.repr().simd()

1099

}

1100

mir::AggregateKind::Closure(..) => true,

1101

// FIXME: Can we do this for simple coroutines too?

1102

mir::AggregateKind::Coroutine(..) | mir::AggregateKind::CoroutineClosure(..) => false,

1103

};

1104

allowed_kind && {

1060

1105

`let ty = rvalue.ty(self.mir, self.cx.tcx());

1061

1106

`let ty = self.monomorphize(ty);

1062

`` -

// For ZST this can be OperandValueKind::ZeroSized.

1063

self.cx.spanned_layout_of(ty, span).is_zst()

1107

let layout = self.cx.spanned_layout_of(ty, span);

1108

!self.cx.is_backend_ref(layout)

1109

}

1064

1110

1065

1111

1066

1112

`@@ -1102,3 +1148,14 @@ enum OperandValueKind {

1102

1148

`Pair(abi::Scalar, abi::Scalar),

1103

1149

`ZeroSized,

1104

1150

1151

+

1152

impl OperandValueKind {

1153

fn scalars(self) -> Option<ArrayVec<abi::Scalar, 2>> {

1154

Some(match self {

1155

OperandValueKind::ZeroSized => ArrayVec::new(),

1156

OperandValueKind::Immediate(a) => ArrayVec::from_iter([a]),

1157

OperandValueKind::Pair(a, b) => [a, b].into(),

1158

OperandValueKind::Ref => return None,

1159

})

1160

}

1161

}