Skip to content

Commit 6d4499c

Browse files
borsbors
committed
auto merge of #5424 : luqmana/rust/inline-rt, r=brson
As per #2521. Inlining seems to improve performance slightly: Inlined Not Inlined x86: 13.5482 14.4112 x86_64: 17.4712 18.0696 (Average of 5 runs timed with `time`) ```Rust fn foo() -> int { int::from_str(~"28098").unwrap() } fn main() { for 1000000.times { foo(); foo(); foo(); foo(); foo(); } } ``` All run on: Linux 3.2.0-0.bpo.4-amd64 #1 SMP Debian 3.2.35-2~bpo60+1 x86_64 GNU/Linux The MIPS and ARM bits I didn't inline since I'm not as familiar with them and I also can't test them. All green on try.
2 parents 6f42738 + a692777 commit 6d4499c

File tree

8 files changed

+217
-124
lines changed

8 files changed

+217
-124
lines changed

‎src/rt/arch/arm/sp.h

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2+
// file at the top-level directory of this distribution and at
3+
// http://rust-lang.org/COPYRIGHT.
4+
//
5+
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6+
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7+
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8+
// option. This file may not be copied, modified, or distributed
9+
// except according to those terms.
10+
11+
// Getting the stack pointer and getting/setting sp limit.
12+
13+
#ifndef SP_H
14+
#define SP_H
15+
16+
#include "../../rust_globals.h"
17+
18+
// Gets a pointer to the vicinity of the current stack pointer
19+
extern "C" uintptr_t get_sp();
20+
21+
// Gets the pointer to the end of the Rust stack from a platform-
22+
// specific location in the thread control block
23+
extern "C" CDECL uintptr_t get_sp_limit();
24+
25+
// Records the pointer to the end of the Rust stack in a platform-
26+
// specific location in the thread control block
27+
extern "C" CDECL void record_sp_limit(void *limit);
28+
29+
#endif

‎src/rt/arch/i386/record_sp.S

-60
Original file line numberDiff line numberDiff line change
@@ -1,60 +0,0 @@
1-
.text
2-
3-
#if defined(__APPLE__) || defined(_WIN32)
4-
#define RECORD_SP_LIMIT _record_sp_limit
5-
#define GET_SP_LIMIT _get_sp_limit
6-
#define GET_SP _get_sp
7-
#else
8-
#define RECORD_SP_LIMIT record_sp_limit
9-
#define GET_SP_LIMIT get_sp_limit
10-
#define GET_SP get_sp
11-
#endif
12-
13-
.globl RECORD_SP_LIMIT
14-
.globl GET_SP_LIMIT
15-
.globl GET_SP
16-
17-
#if defined(__linux__) || defined(__FreeBSD__)
18-
RECORD_SP_LIMIT:
19-
movl 4(%esp), %eax
20-
movl %eax, %gs:48
21-
ret
22-
#endif
23-
24-
#if defined(__APPLE__)
25-
RECORD_SP_LIMIT:
26-
movl $0x48+90*4, %eax
27-
movl 4(%esp), %ecx
28-
movl %ecx, %gs:(%eax)
29-
ret
30-
#endif
31-
32-
#if defined(_WIN32)
33-
RECORD_SP_LIMIT:
34-
movl 4(%esp), %eax
35-
movl %eax, %fs:0x14
36-
ret
37-
#endif
38-
39-
#if defined(__linux__) || defined(__FreeBSD__)
40-
GET_SP_LIMIT:
41-
movl %gs:48, %eax
42-
ret
43-
#endif
44-
45-
#if defined(__APPLE__)
46-
GET_SP_LIMIT:
47-
movl $0x48+90*4, %ecx
48-
movl %gs:(%ecx), %eax
49-
ret
50-
#endif
51-
52-
#if defined(_WIN32)
53-
GET_SP_LIMIT:
54-
movl %fs:0x14, %eax
55-
ret
56-
#endif
57-
58-
GET_SP:
59-
movl %esp, %eax
60-
ret

‎src/rt/arch/i386/sp.h

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2+
// file at the top-level directory of this distribution and at
3+
// http://rust-lang.org/COPYRIGHT.
4+
//
5+
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6+
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7+
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8+
// option. This file may not be copied, modified, or distributed
9+
// except according to those terms.
10+
11+
// Getting the stack pointer and getting/setting sp limit.
12+
13+
#ifndef SP_H
14+
#define SP_H
15+
16+
#include "../../rust_globals.h"
17+
18+
// Gets a pointer to the vicinity of the current stack pointer
19+
extern "C" ALWAYS_INLINE uintptr_t get_sp() {
20+
uintptr_t sp;
21+
asm volatile (
22+
"movl %%esp, %0"
23+
: "=m"(sp));
24+
return sp;
25+
}
26+
27+
// Gets the pointer to the end of the Rust stack from a platform-
28+
// specific location in the thread control block
29+
extern "C" CDECL ALWAYS_INLINE uintptr_t get_sp_limit() {
30+
uintptr_t limit;
31+
32+
#if defined(__linux__) || defined(__FreeBSD__)
33+
asm volatile (
34+
"movl %%gs:48, %0"
35+
: "=r"(limit));
36+
#elif defined(__APPLE__)
37+
asm volatile (
38+
"movl $0x48+90*4, %%ecx\n\t"
39+
"movl %%gs:(%%ecx), %0"
40+
: "=r"(limit)
41+
:: "ecx");
42+
#elif defined(_WIN32)
43+
asm volatile (
44+
"movl %%fs:0x14, %0"
45+
: "=r"(limit));
46+
#endif
47+
48+
return limit;
49+
}
50+
51+
// Records the pointer to the end of the Rust stack in a platform-
52+
// specific location in the thread control block
53+
extern "C" CDECL ALWAYS_INLINE void record_sp_limit(void *limit) {
54+
#if defined(__linux__) || defined(__FreeBSD__)
55+
asm volatile (
56+
"movl %0, %%gs:48"
57+
:: "r"(limit));
58+
#elif defined(__APPLE__)
59+
asm volatile (
60+
"movl $0x48+90*4, %%eax\n\t"
61+
"movl %0, %%gs:(%%eax)"
62+
:: "r"(limit)
63+
: "eax");
64+
#elif defined(_WIN32)
65+
asm volatile (
66+
"movl %0, %%fs:0x14"
67+
:: "r"(limit));
68+
#endif
69+
}
70+
71+
#endif

‎src/rt/arch/mips/sp.h

+29
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2+
// file at the top-level directory of this distribution and at
3+
// http://rust-lang.org/COPYRIGHT.
4+
//
5+
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6+
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7+
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8+
// option. This file may not be copied, modified, or distributed
9+
// except according to those terms.
10+
11+
// Getting the stack pointer and getting/setting sp limit.
12+
13+
#ifndef SP_H
14+
#define SP_H
15+
16+
#include "../../rust_globals.h"
17+
18+
// Gets a pointer to the vicinity of the current stack pointer
19+
extern "C" uintptr_t get_sp();
20+
21+
// Gets the pointer to the end of the Rust stack from a platform-
22+
// specific location in the thread control block
23+
extern "C" CDECL uintptr_t get_sp_limit();
24+
25+
// Records the pointer to the end of the Rust stack in a platform-
26+
// specific location in the thread control block
27+
extern "C" CDECL void record_sp_limit(void *limit);
28+
29+
#endif

‎src/rt/arch/x86_64/record_sp.S

-52
Original file line numberDiff line numberDiff line change
@@ -1,52 +0,0 @@
1-
.text
2-
3-
#if defined(__APPLE__) || defined(_WIN32)
4-
#define RECORD_SP_LIMIT _record_sp_limit
5-
#define GET_SP_LIMIT _get_sp_limit
6-
#define GET_SP _get_sp
7-
#else
8-
#define RECORD_SP_LIMIT record_sp_limit
9-
#define GET_SP_LIMIT get_sp_limit
10-
#define GET_SP get_sp
11-
#endif
12-
13-
.globl RECORD_SP_LIMIT
14-
.globl GET_SP_LIMIT
15-
.globl GET_SP
16-
17-
#if defined(__linux__)
18-
RECORD_SP_LIMIT:
19-
movq %rdi, %fs:112
20-
ret
21-
#elif defined(__APPLE__)
22-
RECORD_SP_LIMIT:
23-
movq $0x60+90*8, %rsi
24-
movq %rdi, %gs:(%rsi)
25-
ret
26-
#elif defined(__FreeBSD__)
27-
RECORD_SP_LIMIT:
28-
movq %rdi, %fs:24
29-
ret
30-
#else
31-
RECORD_SP_LIMIT:
32-
ret
33-
#endif
34-
35-
#if defined(__linux__)
36-
GET_SP_LIMIT:
37-
movq %fs:112, %rax
38-
ret
39-
#elif defined(__APPLE__)
40-
GET_SP_LIMIT:
41-
movq $0x60+90*8, %rsi
42-
movq %gs:(%rsi), %rax
43-
ret
44-
#elif defined(__FreeBSD__)
45-
GET_SP_LIMIT:
46-
movq %fs:24, %rax
47-
ret
48-
#endif
49-
50-
GET_SP:
51-
movq %rsp, %rax
52-
ret

‎src/rt/arch/x86_64/sp.h

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
// Copyright 2012 The Rust Project Developers. See the COPYRIGHT
2+
// file at the top-level directory of this distribution and at
3+
// http://rust-lang.org/COPYRIGHT.
4+
//
5+
// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
6+
// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
7+
// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
8+
// option. This file may not be copied, modified, or distributed
9+
// except according to those terms.
10+
11+
// Getting the stack pointer and getting/setting sp limit.
12+
13+
#ifndef SP_H
14+
#define SP_H
15+
16+
#include "../../rust_globals.h"
17+
18+
// Gets a pointer to the vicinity of the current stack pointer
19+
extern "C" ALWAYS_INLINE uintptr_t get_sp() {
20+
uintptr_t sp;
21+
asm volatile (
22+
"movq %%rsp, %0"
23+
: "=m"(sp));
24+
return sp;
25+
}
26+
27+
// Gets the pointer to the end of the Rust stack from a platform-
28+
// specific location in the thread control block
29+
extern "C" CDECL ALWAYS_INLINE uintptr_t get_sp_limit() {
30+
uintptr_t limit;
31+
32+
#if defined(__linux__)
33+
asm volatile (
34+
"movq %%fs:112, %0"
35+
: "=r"(limit));
36+
#elif defined(__APPLE__)
37+
asm volatile (
38+
"movq $0x60+90*8, %%rsi\n\t"
39+
"movq %%gs:(%%rsi), %0"
40+
: "=r"(limit)
41+
:: "rsi");
42+
#elif defined(__FreeBSD__)
43+
asm volatile (
44+
"movq %%fs:24, %0"
45+
: "=r"(limit));
46+
#endif
47+
48+
return limit;
49+
}
50+
51+
// Records the pointer to the end of the Rust stack in a platform-
52+
// specific location in the thread control block
53+
extern "C" CDECL ALWAYS_INLINE void record_sp_limit(void *limit) {
54+
#if defined(__linux__)
55+
asm volatile (
56+
"movq %0, %%fs:112"
57+
:: "r"(limit));
58+
#elif defined(__APPLE__)
59+
asm volatile (
60+
"movq $0x60+90*8, %%rsi\n\t"
61+
"movq %0, %%gs:(%%rsi)"
62+
:: "r"(limit)
63+
: "rsi");
64+
#elif defined(__FreeBSD__)
65+
asm volatile (
66+
"movq %0, %%fs:24"
67+
:: "r"(limit));
68+
#endif
69+
}
70+
71+
#endif

‎src/rt/rust_globals.h

+16
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,22 @@
1111
#ifndef RUST_GLOBALS_H
1212
#define RUST_GLOBALS_H
1313

14+
#if defined(__cplusplus)
15+
#define INLINE inline
16+
#elif defined(_MSC_VER) || defined(__GNUC__)
17+
#define INLINE __inline__
18+
#else
19+
#define INLINE inline
20+
#endif
21+
22+
#if defined(__GNUC__)
23+
#define ALWAYS_INLINE __attribute((always_inline)) INLINE
24+
#elif defined(_MSC_VER)
25+
#define ALWAYS_INLINE __forceinline
26+
#else
27+
#define ALWAYS_INLINE INLINE
28+
#endif
29+
1430
#ifndef __STDC_LIMIT_MACROS
1531
#define __STDC_LIMIT_MACROS 1
1632
#endif

‎src/rt/rust_task.h

+1-12
Original file line numberDiff line numberDiff line change
@@ -118,6 +118,7 @@
118118
#include "rust_stack.h"
119119
#include "rust_type.h"
120120
#include "rust_sched_loop.h"
121+
#include "sp.h"
121122

122123
// The amount of extra space at the end of each stack segment, available
123124
// to the rt, compiler and dynamic linker for running small functions
@@ -419,15 +420,6 @@ template <typename T> struct task_owned {
419420
}
420421
};
421422

422-
// This stuff is on the stack-switching fast path
423-
424-
// Records the pointer to the end of the Rust stack in a platform-
425-
// specific location in the thread control block
426-
extern "C" CDECL void record_sp_limit(void *limit);
427-
extern "C" CDECL uintptr_t get_sp_limit();
428-
// Gets a pointer to the vicinity of the current stack pointer
429-
extern "C" uintptr_t get_sp();
430-
431423
// This is the function that switches between the C and the Rust stack by
432424
// calling another function with a single void* argument while changing the
433425
// stack pointer. It has a funny name because gdb doesn't normally like to
@@ -600,9 +592,6 @@ rust_task::prev_stack() {
600592
record_stack_limit();
601593
}
602594

603-
extern "C" CDECL void
604-
record_sp_limit(void *limit);
605-
606595
// The LLVM-generated segmented-stack function prolog compares the amount of
607596
// stack needed for each frame to the end-of-stack pointer stored in the
608597
// TCB. As an optimization, when the frame size is less than 256 bytes, it

0 commit comments

Comments
 (0)