diff options
author | Denys Vlasenko <dvlasenk@redhat.com> | 2015-03-09 19:39:21 +0100 |
---|---|---|
committer | Ingo Molnar <mingo@kernel.org> | 2015-03-10 13:56:10 +0100 |
commit | 29722cd4ef666705b2eda1c3ba44435488e509eb (patch) | |
tree | 27fcc3f8f267ae426ce9e383422828cea79c6f59 /arch/x86/kernel/entry_64.S | |
parent | 3e1aa7cb59aff4b245b45e326fcdba1bf7f105c6 (diff) |
x86/asm/entry/64: Save R11 into pt_regs->flags on SYSCALL64 fastpath
Before this patch, R11 was saved in pt_regs->r11.
Which looks natural, but requires messy shuffling to/from iret
frame whenever ptrace or e.g. sys_iopl() wants to modify flags -
because that's how this register is used by SYSCALL/SYSRET.
This patch saves R11 in pt_regs->flags, and uses that value for
the SYSRET64 instruction. Shuffling is eliminated.
FIXUP/RESTORE_TOP_OF_STACK are simplified.
stub_iopl is no longer needed: pt_regs->flags needs no fixing up.
Testing shows that syscall fast path is ~54.3 ns before
and after the patch (on 2.7 GHz Sandy Bridge CPU).
Signed-off-by: Denys Vlasenko <dvlasenk@redhat.com>
Cc: Alexei Starovoitov <ast@plumgrid.com>
Cc: Andy Lutomirski <luto@amacapital.net>
Cc: Borislav Petkov <bp@alien8.de>
Cc: Frederic Weisbecker <fweisbec@gmail.com>
Cc: H. Peter Anvin <hpa@zytor.com>
Cc: Kees Cook <keescook@chromium.org>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Oleg Nesterov <oleg@redhat.com>
Cc: Steven Rostedt <rostedt@goodmis.org>
Cc: Will Drewry <wad@chromium.org>
Link: http://lkml.kernel.org/r/1425926364-9526-2-git-send-email-dvlasenk@redhat.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Diffstat (limited to 'arch/x86/kernel/entry_64.S')
-rw-r--r-- | arch/x86/kernel/entry_64.S | 24 |
1 files changed, 11 insertions, 13 deletions
diff --git a/arch/x86/kernel/entry_64.S b/arch/x86/kernel/entry_64.S index 5117a2baefe9..324200aca431 100644 --- a/arch/x86/kernel/entry_64.S +++ b/arch/x86/kernel/entry_64.S @@ -121,14 +121,12 @@ ENDPROC(native_usergs_sysret64) #endif /* - * C code is not supposed to know about undefined top of stack. Every time - * a C function with an pt_regs argument is called from the SYSCALL based - * fast path FIXUP_TOP_OF_STACK is needed. + * C code is not supposed to know that the iret frame is not populated. + * Every time a C function with an pt_regs argument is called from + * the SYSCALL based fast path FIXUP_TOP_OF_STACK is needed. * RESTORE_TOP_OF_STACK syncs the syscall state after any possible ptregs * manipulation. */ - - /* %rsp:at FRAMEEND */ .macro FIXUP_TOP_OF_STACK tmp offset=0 movq PER_CPU_VAR(old_rsp),\tmp movq \tmp,RSP+\offset(%rsp) @@ -136,15 +134,13 @@ ENDPROC(native_usergs_sysret64) movq $__USER_CS,CS+\offset(%rsp) movq RIP+\offset(%rsp),\tmp /* get rip */ movq \tmp,RCX+\offset(%rsp) /* copy it to rcx as sysret would do */ - movq R11+\offset(%rsp),\tmp /* get eflags */ - movq \tmp,EFLAGS+\offset(%rsp) + movq EFLAGS+\offset(%rsp),\tmp /* ditto for rflags->r11 */ + movq \tmp,R11+\offset(%rsp) .endm .macro RESTORE_TOP_OF_STACK tmp offset=0 movq RSP+\offset(%rsp),\tmp movq \tmp,PER_CPU_VAR(old_rsp) - movq EFLAGS+\offset(%rsp),\tmp - movq \tmp,R11+\offset(%rsp) .endm /* @@ -257,9 +253,10 @@ GLOBAL(system_call_after_swapgs) */ ENABLE_INTERRUPTS(CLBR_NONE) ALLOC_PT_GPREGS_ON_STACK 8 /* +8: space for orig_ax */ - SAVE_C_REGS_EXCEPT_RAX_RCX + SAVE_C_REGS_EXCEPT_RAX_RCX_R11 movq $-ENOSYS,RAX(%rsp) movq_cfi rax,ORIG_RAX + movq %r11,EFLAGS(%rsp) movq %rcx,RIP(%rsp) CFI_REL_OFFSET rip,RIP testl $_TIF_WORK_SYSCALL_ENTRY,TI_flags+THREAD_INFO(%rsp,RIP) @@ -277,7 +274,7 @@ system_call_fastpath: movq %rax,RAX(%rsp) /* * Syscall return path ending with SYSRET (fast path) - * Has incomplete stack frame and undefined top of stack. + * Has incompletely filled pt_regs, iret frame is also incomplete. */ ret_from_sys_call: testl $_TIF_ALLWORK_MASK,TI_flags+THREAD_INFO(%rsp,RIP) @@ -291,9 +288,10 @@ ret_from_sys_call: * sysretq will re-enable interrupts: */ TRACE_IRQS_ON - RESTORE_C_REGS_EXCEPT_RCX - movq RIP(%rsp),%rcx + RESTORE_C_REGS_EXCEPT_RCX_R11 + movq RIP(%rsp),%rcx CFI_REGISTER rip,rcx + movq EFLAGS(%rsp),%r11 /*CFI_REGISTER rflags,r11*/ movq PER_CPU_VAR(old_rsp), %rsp /* |