Add `f16` inline ASM support for RISC-V · rust-lang/rust@771e44e (original) (raw)

`@@ -13,7 +13,7 @@ use rustc_codegen_ssa::traits::*;

13

`use rustc_data_structures::fx::FxHashMap;

14

`use rustc_middle::ty::layout::TyAndLayout;

15

`use rustc_middle::{bug, span_bug, ty::Instance};

16

use rustc_span::{Pos, Span};

16

use rustc_span::{sym, Pos, Span, Symbol};

17

`use rustc_target::abi::*;

18

`use rustc_target::asm::*;

19

`use tracing::debug;

`@@ -64,7 +64,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {

64

`let mut layout = None;

65

`let ty = if let Some(ref place) = place {

66

` layout = Some(&place.layout);

67

llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout)

67

llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout, instance)

68

`} else if matches!(

69

` reg.reg_class(),

70

`InlineAsmRegClass::X86(

`@@ -112,7 +112,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {

112

`// so we just use the type of the input.

113

`&in_value.layout

114

`};

115

let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout);

115

let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout, instance);

116

` output_types.push(ty);

117

` op_idx.insert(idx, constraints.len());

118

`let prefix = if late { "=" } else { "=&" };

`@@ -127,8 +127,13 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {

127

`for (idx, op) in operands.iter().enumerate() {

128

`match *op {

129

`InlineAsmOperandRef::In { reg, value } => {

130

let llval =

131

llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout);

130

let llval = llvm_fixup_input(

131

self,

132

value.immediate(),

133

reg.reg_class(),

134

&value.layout,

135

instance,

136

);

132

137

` inputs.push(llval);

133

138

` op_idx.insert(idx, constraints.len());

134

139

` constraints.push(reg_to_llvm(reg, Some(&value.layout)));

`@@ -139,6 +144,7 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {

139

144

` in_value.immediate(),

140

145

` reg.reg_class(),

141

146

`&in_value.layout,

147

instance,

142

148

`);

143

149

` inputs.push(value);

144

150

`@@ -341,7 +347,8 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {

341

347

`} else {

342

348

`self.extract_value(result, op_idx[&idx] as u64)

343

349

`};

344

let value = llvm_fixup_output(self, value, reg.reg_class(), &place.layout);

350

let value =

351

llvm_fixup_output(self, value, reg.reg_class(), &place.layout, instance);

345

352

`OperandValue::Immediate(value).store(self, place);

346

353

347

354

`@@ -913,12 +920,22 @@ fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Ty

913

920

914

921

915

922

923

fn any_target_feature_enabled(

924

cx: &CodegenCx<'_, '_>,

925

instance: Instance<'_>,

926

features: &[Symbol],

927

) -> bool {

928

let enabled = cx.tcx.asm_target_features(instance.def_id());

929

features.iter().any(|feat| enabled.contains(feat))

930

}

931

+

916

932

`/// Fix up an input value to work around LLVM bugs.

917

933

`fn llvm_fixup_input<'ll, 'tcx>(

918

934

`bx: &mut Builder<'_, 'll, 'tcx>,

919

935

`mut value: &'ll Value,

920

936

`reg: InlineAsmRegClass,

921

937

`layout: &TyAndLayout<'tcx>,

938

instance: Instance<'_>,

922

939

`) -> &'ll Value {

923

940

`let dl = &bx.tcx.data_layout;

924

941

`match (reg, layout.abi) {

`@@ -1029,6 +1046,16 @@ fn llvm_fixup_input<'ll, 'tcx>(

1029

1046

` _ => value,

1030

1047

1031

1048

1049

(InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s))

1050

if s.primitive() == Primitive::Float(Float::F16)

1051

&& !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>

1052

{

1053

// Smaller floats are always "NaN-boxed" inside larger floats on RISC-V.

1054

let value = bx.bitcast(value, bx.type_i16());

1055

let value = bx.zext(value, bx.type_i32());

1056

let value = bx.or(value, bx.const_u32(0xFFFF_0000));

1057

bx.bitcast(value, bx.type_f32())

1058

}

1032

1059

` _ => value,

1033

1060

1034

1061

`@@ -1039,6 +1066,7 @@ fn llvm_fixup_output<'ll, 'tcx>(

1039

1066

`mut value: &'ll Value,

1040

1067

`reg: InlineAsmRegClass,

1041

1068

`layout: &TyAndLayout<'tcx>,

1069

instance: Instance<'_>,

1042

1070

`) -> &'ll Value {

1043

1071

`match (reg, layout.abi) {

1044

1072

`(InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {

`@@ -1140,6 +1168,14 @@ fn llvm_fixup_output<'ll, 'tcx>(

1140

1168

` _ => value,

1141

1169

1142

1170

1171

(InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s))

1172

if s.primitive() == Primitive::Float(Float::F16)

1173

&& !any_target_feature_enabled(bx, instance, &[sym::zfhmin, sym::zfh]) =>

1174

{

1175

let value = bx.bitcast(value, bx.type_i32());

1176

let value = bx.trunc(value, bx.type_i16());

1177

bx.bitcast(value, bx.type_f16())

1178

}

1143

1179

` _ => value,

1144

1180

1145

1181

`@@ -1149,6 +1185,7 @@ fn llvm_fixup_output_type<'ll, 'tcx>(

1149

1185

`cx: &CodegenCx<'ll, 'tcx>,

1150

1186

`reg: InlineAsmRegClass,

1151

1187

`layout: &TyAndLayout<'tcx>,

1188

instance: Instance<'_>,

1152

1189

`) -> &'ll Type {

1153

1190

`match (reg, layout.abi) {

1154

1191

`(InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => {

`@@ -1242,6 +1279,12 @@ fn llvm_fixup_output_type<'ll, 'tcx>(

1242

1279

` _ => layout.llvm_type(cx),

1243

1280

1244

1281

1282

(InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg), Abi::Scalar(s))

1283

if s.primitive() == Primitive::Float(Float::F16)

1284

&& !any_target_feature_enabled(cx, instance, &[sym::zfhmin, sym::zfh]) =>

1285

{

1286

cx.type_f32()

1287

}

1245

1288

` _ => layout.llvm_type(cx),

1246

1289

1247

1290

Add f16 inline ASM support for RISC-V · rust-lang/rust@771e44e (original) (raw)

Add `f16` inline ASM support for RISC-V · rust-lang/rust@771e44e (original) (raw)