Skip to content

Commit 2bd9656

Browse files
borsbors
committedFeb 27, 2022
Auto merge of #94221 - erikdesjardins:addattr, r=nikic
Add LLVM attributes in batches instead of individually This should improve performance. ~r? `@ghost` (blocked on #94127)~
·
1.85.11.61.0
2 parents 9323028 + 0d0cc4f commit 2bd9656

File tree

11 files changed

+420
-439
lines changed

11 files changed

+420
-439
lines changed
 

‎compiler/rustc_codegen_llvm/src/abi.rs

+82-129
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
1+
use crate::attributes;
12
use crate::builder::Builder;
23
use crate::context::CodegenCx;
3-
use crate::llvm::{self, AttributePlace};
4+
use crate::llvm::{self, Attribute, AttributePlace};
45
use crate::type_::Type;
56
use crate::type_of::LayoutLlvmExt;
67
use crate::value::Value;
@@ -20,6 +21,7 @@ use rustc_target::abi::{self, HasDataLayout, Int};
2021
pub use rustc_target::spec::abi::Abi;
2122

2223
use libc::c_uint;
24+
use smallvec::SmallVec;
2325

2426
pub trait ArgAttributesExt {
2527
fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value);
@@ -38,57 +40,65 @@ fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool {
3840
cx.tcx.sess.opts.debugging_opts.mutable_noalias.unwrap_or(true)
3941
}
4042

41-
const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::Attribute); 1] =
42-
[(ArgAttribute::InReg, llvm::Attribute::InReg)];
43+
const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] =
44+
[(ArgAttribute::InReg, llvm::AttributeKind::InReg)];
4345

44-
const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::Attribute); 5] = [
45-
(ArgAttribute::NoAlias, llvm::Attribute::NoAlias),
46-
(ArgAttribute::NoCapture, llvm::Attribute::NoCapture),
47-
(ArgAttribute::NonNull, llvm::Attribute::NonNull),
48-
(ArgAttribute::ReadOnly, llvm::Attribute::ReadOnly),
49-
(ArgAttribute::NoUndef, llvm::Attribute::NoUndef),
46+
const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 5] = [
47+
(ArgAttribute::NoAlias, llvm::AttributeKind::NoAlias),
48+
(ArgAttribute::NoCapture, llvm::AttributeKind::NoCapture),
49+
(ArgAttribute::NonNull, llvm::AttributeKind::NonNull),
50+
(ArgAttribute::ReadOnly, llvm::AttributeKind::ReadOnly),
51+
(ArgAttribute::NoUndef, llvm::AttributeKind::NoUndef),
5052
];
5153

52-
impl ArgAttributesExt for ArgAttributes {
53-
fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) {
54-
let mut regular = self.regular;
55-
unsafe {
56-
// ABI-affecting attributes must always be applied
57-
for (attr, llattr) in ABI_AFFECTING_ATTRIBUTES {
58-
if regular.contains(attr) {
59-
llattr.apply_llfn(idx, llfn);
60-
}
61-
}
62-
if let Some(align) = self.pointee_align {
63-
llvm::LLVMRustAddAlignmentAttr(llfn, idx.as_uint(), align.bytes() as u32);
64-
}
65-
match self.arg_ext {
66-
ArgExtension::None => {}
67-
ArgExtension::Zext => llvm::Attribute::ZExt.apply_llfn(idx, llfn),
68-
ArgExtension::Sext => llvm::Attribute::SExt.apply_llfn(idx, llfn),
69-
}
70-
// Only apply remaining attributes when optimizing
71-
if cx.sess().opts.optimize == config::OptLevel::No {
72-
return;
73-
}
74-
let deref = self.pointee_size.bytes();
75-
if deref != 0 {
76-
if regular.contains(ArgAttribute::NonNull) {
77-
llvm::LLVMRustAddDereferenceableAttr(llfn, idx.as_uint(), deref);
78-
} else {
79-
llvm::LLVMRustAddDereferenceableOrNullAttr(llfn, idx.as_uint(), deref);
80-
}
81-
regular -= ArgAttribute::NonNull;
82-
}
83-
for (attr, llattr) in OPTIMIZATION_ATTRIBUTES {
84-
if regular.contains(attr) {
85-
llattr.apply_llfn(idx, llfn);
86-
}
54+
fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 8]> {
55+
let mut regular = this.regular;
56+
57+
let mut attrs = SmallVec::new();
58+
59+
// ABI-affecting attributes must always be applied
60+
for (attr, llattr) in ABI_AFFECTING_ATTRIBUTES {
61+
if regular.contains(attr) {
62+
attrs.push(llattr.create_attr(cx.llcx));
63+
}
64+
}
65+
if let Some(align) = this.pointee_align {
66+
attrs.push(llvm::CreateAlignmentAttr(cx.llcx, align.bytes()));
67+
}
68+
match this.arg_ext {
69+
ArgExtension::None => {}
70+
ArgExtension::Zext => attrs.push(llvm::AttributeKind::ZExt.create_attr(cx.llcx)),
71+
ArgExtension::Sext => attrs.push(llvm::AttributeKind::SExt.create_attr(cx.llcx)),
72+
}
73+
74+
// Only apply remaining attributes when optimizing
75+
if cx.sess().opts.optimize != config::OptLevel::No {
76+
let deref = this.pointee_size.bytes();
77+
if deref != 0 {
78+
if regular.contains(ArgAttribute::NonNull) {
79+
attrs.push(llvm::CreateDereferenceableAttr(cx.llcx, deref));
80+
} else {
81+
attrs.push(llvm::CreateDereferenceableOrNullAttr(cx.llcx, deref));
8782
}
88-
if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) {
89-
llvm::Attribute::NoAlias.apply_llfn(idx, llfn);
83+
regular -= ArgAttribute::NonNull;
84+
}
85+
for (attr, llattr) in OPTIMIZATION_ATTRIBUTES {
86+
if regular.contains(attr) {
87+
attrs.push(llattr.create_attr(cx.llcx));
9088
}
9189
}
90+
if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) {
91+
attrs.push(llvm::AttributeKind::NoAlias.create_attr(cx.llcx));
92+
}
93+
}
94+
95+
attrs
96+
}
97+
98+
impl ArgAttributesExt for ArgAttributes {
99+
fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) {
100+
let attrs = get_attrs(self, cx);
101+
attributes::apply_to_llfn(llfn, idx, &attrs);
92102
}
93103

94104
fn apply_attrs_to_callsite(
@@ -97,52 +107,8 @@ impl ArgAttributesExt for ArgAttributes {
97107
cx: &CodegenCx<'_, '_>,
98108
callsite: &Value,
99109
) {
100-
let mut regular = self.regular;
101-
unsafe {
102-
// ABI-affecting attributes must always be applied
103-
for (attr, llattr) in ABI_AFFECTING_ATTRIBUTES {
104-
if regular.contains(attr) {
105-
llattr.apply_callsite(idx, callsite);
106-
}
107-
}
108-
if let Some(align) = self.pointee_align {
109-
llvm::LLVMRustAddAlignmentCallSiteAttr(
110-
callsite,
111-
idx.as_uint(),
112-
align.bytes() as u32,
113-
);
114-
}
115-
match self.arg_ext {
116-
ArgExtension::None => {}
117-
ArgExtension::Zext => llvm::Attribute::ZExt.apply_callsite(idx, callsite),
118-
ArgExtension::Sext => llvm::Attribute::SExt.apply_callsite(idx, callsite),
119-
}
120-
// Only apply remaining attributes when optimizing
121-
if cx.sess().opts.optimize == config::OptLevel::No {
122-
return;
123-
}
124-
let deref = self.pointee_size.bytes();
125-
if deref != 0 {
126-
if regular.contains(ArgAttribute::NonNull) {
127-
llvm::LLVMRustAddDereferenceableCallSiteAttr(callsite, idx.as_uint(), deref);
128-
} else {
129-
llvm::LLVMRustAddDereferenceableOrNullCallSiteAttr(
130-
callsite,
131-
idx.as_uint(),
132-
deref,
133-
);
134-
}
135-
regular -= ArgAttribute::NonNull;
136-
}
137-
for (attr, llattr) in OPTIMIZATION_ATTRIBUTES {
138-
if regular.contains(attr) {
139-
llattr.apply_callsite(idx, callsite);
140-
}
141-
}
142-
if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) {
143-
llvm::Attribute::NoAlias.apply_callsite(idx, callsite);
144-
}
145-
}
110+
let attrs = get_attrs(self, cx);
111+
attributes::apply_to_callsite(callsite, idx, &attrs);
146112
}
147113
}
148114

@@ -444,15 +410,14 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
444410
}
445411

446412
fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) {
447-
// FIXME(eddyb) can this also be applied to callsites?
413+
let mut func_attrs = SmallVec::<[_; 2]>::new();
448414
if self.ret.layout.abi.is_uninhabited() {
449-
llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn);
415+
func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(cx.llcx));
450416
}
451-
452-
// FIXME(eddyb, wesleywiser): apply this to callsites as well?
453417
if !self.can_unwind {
454-
llvm::Attribute::NoUnwind.apply_llfn(llvm::AttributePlace::Function, llfn);
418+
func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(cx.llcx));
455419
}
420+
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &{ func_attrs });
456421

457422
let mut i = 0;
458423
let mut apply = |attrs: &ArgAttributes| {
@@ -467,13 +432,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
467432
PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => {
468433
assert!(!on_stack);
469434
let i = apply(attrs);
470-
unsafe {
471-
llvm::LLVMRustAddStructRetAttr(
472-
llfn,
473-
llvm::AttributePlace::Argument(i).as_uint(),
474-
self.ret.layout.llvm_type(cx),
475-
);
476-
}
435+
let sret = llvm::CreateStructRetAttr(cx.llcx, self.ret.layout.llvm_type(cx));
436+
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]);
477437
}
478438
PassMode::Cast(cast) => {
479439
cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn);
@@ -488,13 +448,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
488448
PassMode::Ignore => {}
489449
PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => {
490450
let i = apply(attrs);
491-
unsafe {
492-
llvm::LLVMRustAddByValAttr(
493-
llfn,
494-
llvm::AttributePlace::Argument(i).as_uint(),
495-
arg.layout.llvm_type(cx),
496-
);
497-
}
451+
let byval = llvm::CreateByValAttr(cx.llcx, arg.layout.llvm_type(cx));
452+
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]);
498453
}
499454
PassMode::Direct(ref attrs)
500455
| PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => {
@@ -517,12 +472,14 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
517472
}
518473

519474
fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) {
475+
let mut func_attrs = SmallVec::<[_; 2]>::new();
520476
if self.ret.layout.abi.is_uninhabited() {
521-
llvm::Attribute::NoReturn.apply_callsite(llvm::AttributePlace::Function, callsite);
477+
func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx));
522478
}
523479
if !self.can_unwind {
524-
llvm::Attribute::NoUnwind.apply_callsite(llvm::AttributePlace::Function, callsite);
480+
func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(bx.cx.llcx));
525481
}
482+
attributes::apply_to_callsite(callsite, llvm::AttributePlace::Function, &{ func_attrs });
526483

527484
let mut i = 0;
528485
let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| {
@@ -537,13 +494,8 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
537494
PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => {
538495
assert!(!on_stack);
539496
let i = apply(bx.cx, attrs);
540-
unsafe {
541-
llvm::LLVMRustAddStructRetCallSiteAttr(
542-
callsite,
543-
llvm::AttributePlace::Argument(i).as_uint(),
544-
self.ret.layout.llvm_type(bx),
545-
);
546-
}
497+
let sret = llvm::CreateStructRetAttr(bx.cx.llcx, self.ret.layout.llvm_type(bx));
498+
attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]);
547499
}
548500
PassMode::Cast(cast) => {
549501
cast.attrs.apply_attrs_to_callsite(
@@ -572,13 +524,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
572524
PassMode::Ignore => {}
573525
PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => {
574526
let i = apply(bx.cx, attrs);
575-
unsafe {
576-
llvm::LLVMRustAddByValCallSiteAttr(
577-
callsite,
578-
llvm::AttributePlace::Argument(i).as_uint(),
579-
arg.layout.llvm_type(bx),
580-
);
581-
}
527+
let byval = llvm::CreateByValAttr(bx.cx.llcx, arg.layout.llvm_type(bx));
528+
attributes::apply_to_callsite(
529+
callsite,
530+
llvm::AttributePlace::Argument(i),
531+
&[byval],
532+
);
582533
}
583534
PassMode::Direct(ref attrs)
584535
| PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => {
@@ -610,10 +561,12 @@ impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> {
610561
if self.conv == Conv::CCmseNonSecureCall {
611562
// This will probably get ignored on all targets but those supporting the TrustZone-M
612563
// extension (thumbv8m targets).
613-
llvm::AddCallSiteAttrString(
564+
let cmse_nonsecure_call =
565+
llvm::CreateAttrString(bx.cx.llcx, cstr::cstr!("cmse_nonsecure_call"));
566+
attributes::apply_to_callsite(
614567
callsite,
615568
llvm::AttributePlace::Function,
616-
cstr::cstr!("cmse_nonsecure_call"),
569+
&[cmse_nonsecure_call],
617570
);
618571
}
619572
}

‎compiler/rustc_codegen_llvm/src/allocator.rs

+7-4
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,8 @@ pub(crate) unsafe fn codegen(
6464
llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden);
6565
}
6666
if tcx.sess.must_emit_unwind_tables() {
67-
attributes::emit_uwtable(llfn);
67+
let uwtable = attributes::uwtable_attr(llcx);
68+
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]);
6869
}
6970

7071
let callee = kind.fn_name(method.name);
@@ -105,20 +106,22 @@ pub(crate) unsafe fn codegen(
105106
let name = "__rust_alloc_error_handler";
106107
let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty);
107108
// -> ! DIFlagNoReturn
108-
llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, llfn);
109+
let no_return = llvm::AttributeKind::NoReturn.create_attr(llcx);
110+
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[no_return]);
109111

110112
if tcx.sess.target.default_hidden_visibility {
111113
llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden);
112114
}
113115
if tcx.sess.must_emit_unwind_tables() {
114-
attributes::emit_uwtable(llfn);
116+
let uwtable = attributes::uwtable_attr(llcx);
117+
attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]);
115118
}
116119

117120
let kind = if has_alloc_error_handler { AllocatorKind::Global } else { AllocatorKind::Default };
118121
let callee = kind.fn_name(sym::oom);
119122
let callee = llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty);
120123
// -> ! DIFlagNoReturn
121-
llvm::Attribute::NoReturn.apply_llfn(llvm::AttributePlace::Function, callee);
124+
attributes::apply_to_llfn(callee, llvm::AttributePlace::Function, &[no_return]);
122125
llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden);
123126

124127
let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast());

‎compiler/rustc_codegen_llvm/src/asm.rs

+8-5
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
use crate::attributes;
12
use crate::builder::Builder;
23
use crate::common::Funclet;
34
use crate::context::CodegenCx;
@@ -18,6 +19,7 @@ use rustc_target::abi::*;
1819
use rustc_target::asm::*;
1920

2021
use libc::{c_char, c_uint};
22+
use smallvec::SmallVec;
2123
use tracing::debug;
2224

2325
impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
@@ -273,19 +275,20 @@ impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
273275
)
274276
.unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
275277

278+
let mut attrs = SmallVec::<[_; 2]>::new();
276279
if options.contains(InlineAsmOptions::PURE) {
277280
if options.contains(InlineAsmOptions::NOMEM) {
278-
llvm::Attribute::ReadNone.apply_callsite(llvm::AttributePlace::Function, result);
281+
attrs.push(llvm::AttributeKind::ReadNone.create_attr(self.cx.llcx));
279282
} else if options.contains(InlineAsmOptions::READONLY) {
280-
llvm::Attribute::ReadOnly.apply_callsite(llvm::AttributePlace::Function, result);
283+
attrs.push(llvm::AttributeKind::ReadOnly.create_attr(self.cx.llcx));
281284
}
282-
llvm::Attribute::WillReturn.apply_callsite(llvm::AttributePlace::Function, result);
285+
attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx));
283286
} else if options.contains(InlineAsmOptions::NOMEM) {
284-
llvm::Attribute::InaccessibleMemOnly
285-
.apply_callsite(llvm::AttributePlace::Function, result);
287+
attrs.push(llvm::AttributeKind::InaccessibleMemOnly.create_attr(self.cx.llcx));
286288
} else {
287289
// LLVM doesn't have an attribute to represent ReadOnly + SideEffect
288290
}
291+
attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs });
289292

290293
// Write results to outputs
291294
for (idx, op) in operands.iter().enumerate() {

0 commit comments

Comments
 (0)
Please sign in to comment.