This is the 4.9.75 stable release
-----BEGIN PGP SIGNATURE----- iQIzBAABCAAdFiEEZH8oZUiU471FcZm+ONu9yGCSaT4FAlpPj9UACgkQONu9yGCS aT6W6A/9HYNDiavWc0w5W96jQKBATh/AbvqULeT7F37Gwp9XRihYqUkHYJlCqtLr QiY67DrN08MwNYxq5iXWlG++UJJcG7hE0nWDIuF1WJwsbBVBFuaqeiQWyudkpEXB aTF+pmKaVx9crlbWHtwYZEPM4qwT22zxZEEka+DYggTuF7IwvfSYC0GQ3oHvSGcq e+iChj72ELtymyo4NxWNndm2wxRkVRdiwRmmFriV7vT4B7QkpmNMYnCODBylyJ1J iWD6q4Hv8PsNa9z/Es+v09x0DvTJgexGNsLh6kSCJ+ajiLhw9aEAgqBSlEZANUVm p4dNg3POgsry3JCFGb4R9WDSpD7I98xQCeVKccZr7yKAo1gw+h274wV9VnsReoUI 91MT9VQZDqNnWQFLTC9+k3Cv1UibrDz5hI/m3/foNN3NP8z8xXdsCMMwLVRwSlOP ZAGV9yVgA0Hko6457FenUTWfCDVyMBf8monEZ0YWxucAW8ihh/EovmmJMNIxhlWX oJsuNwf/zYOxoYIm1Z551RtOxvH84axSHtrPrS47bLggK09IWomfeDwOZv4ZQrIK L2LXc5uFlMS7z359paNK8NyvraTFrZ7j5X8RvegZRgKq/T3Qp0jwqu6lxyY8q5hH os13s63QBvqdPZm1G3nc5/89oR976k+yW9hhZqwG7qi1XCPuU/k= =vqC6 -----END PGP SIGNATURE----- Merge 4.9.75 into android-4.9 Changes in 4.9.75 tcp_bbr: reset full pipe detection on loss recovery undo tcp_bbr: reset long-term bandwidth sampling on loss recovery undo x86/boot: Add early cmdline parsing for options with arguments KAISER: Kernel Address Isolation kaiser: merged update kaiser: do not set _PAGE_NX on pgd_none kaiser: stack map PAGE_SIZE at THREAD_SIZE-PAGE_SIZE kaiser: fix build and FIXME in alloc_ldt_struct() kaiser: KAISER depends on SMP kaiser: fix regs to do_nmi() ifndef CONFIG_KAISER kaiser: fix perf crashes kaiser: ENOMEM if kaiser_pagetable_walk() NULL kaiser: tidied up asm/kaiser.h somewhat kaiser: tidied up kaiser_add/remove_mapping slightly kaiser: align addition to x86/mm/Makefile kaiser: cleanups while trying for gold link kaiser: name that 0x1000 KAISER_SHADOW_PGD_OFFSET kaiser: delete KAISER_REAL_SWITCH option kaiser: vmstat show NR_KAISERTABLE as nr_overhead kaiser: enhanced by kernel and user PCIDs kaiser: load_new_mm_cr3() let SWITCH_USER_CR3 flush user kaiser: PCID 0 for kernel and 128 for user kaiser: x86_cr3_pcid_noflush and x86_cr3_pcid_user kaiser: paranoid_entry pass cr3 need to paranoid_exit kaiser: kaiser_remove_mapping() move along the pgd kaiser: fix unlikely error in alloc_ldt_struct() kaiser: add "nokaiser" boot option, using ALTERNATIVE x86/kaiser: Rename and simplify X86_FEATURE_KAISER handling x86/kaiser: Check boottime cmdline params kaiser: use ALTERNATIVE instead of x86_cr3_pcid_noflush kaiser: drop is_atomic arg to kaiser_pagetable_walk() kaiser: asm/tlbflush.h handle noPGE at lower level kaiser: kaiser_flush_tlb_on_return_to_user() check PCID x86/paravirt: Dont patch flush_tlb_single x86/kaiser: Reenable PARAVIRT kaiser: disabled on Xen PV x86/kaiser: Move feature detection up KPTI: Rename to PAGE_TABLE_ISOLATION KPTI: Report when enabled kaiser: Set _PAGE_NX only if supported Linux 4.9.75 Signed-off-by: Greg Kroah-Hartman <gregkh@google.com>
This commit is contained in:
Greg Kroah-Hartman
commit
bc7ff9b998
46 changed files with 1382 additions and 100 deletions
|
|
@ -2775,6 +2775,8 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
|
|||
|
||||
nojitter [IA-64] Disables jitter checking for ITC timers.
|
||||
|
||||
nopti [X86-64] Disable KAISER isolation of kernel from user.
|
||||
|
||||
no-kvmclock [X86,KVM] Disable paravirtualized KVM clock driver
|
||||
|
||||
no-kvmapf [X86,KVM] Disable paravirtualized asynchronous page
|
||||
|
|
@ -3337,6 +3339,12 @@ bytes respectively. Such letter suffixes can also be entirely omitted.
|
|||
pt. [PARIDE]
|
||||
See Documentation/blockdev/paride.txt.
|
||||
|
||||
pti= [X86_64]
|
||||
Control KAISER user/kernel address space isolation:
|
||||
on - enable
|
||||
off - disable
|
||||
auto - default setting
|
||||
|
||||
pty.legacy_count=
|
||||
[KNL] Number of legacy pty's. Overwrites compiled-in
|
||||
default number.
|
||||
|
|
|
|||
2
Makefile
2
Makefile
|
|
@ -1,6 +1,6 @@
|
|||
VERSION = 4
|
||||
PATCHLEVEL = 9
|
||||
SUBLEVEL = 74
|
||||
SUBLEVEL = 75
|
||||
EXTRAVERSION =
|
||||
NAME = Roaring Lionus
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@
|
|||
*/
|
||||
#undef CONFIG_PARAVIRT
|
||||
#undef CONFIG_PARAVIRT_SPINLOCKS
|
||||
#undef CONFIG_PAGE_TABLE_ISOLATION
|
||||
#undef CONFIG_KASAN
|
||||
|
||||
#include <linux/linkage.h>
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@
|
|||
#include <asm/smap.h>
|
||||
#include <asm/pgtable_types.h>
|
||||
#include <asm/export.h>
|
||||
#include <asm/kaiser.h>
|
||||
#include <linux/err.h>
|
||||
|
||||
/* Avoid __ASSEMBLER__'ifying <linux/audit.h> just for this. */
|
||||
|
|
@ -146,6 +147,7 @@ ENTRY(entry_SYSCALL_64)
|
|||
* it is too small to ever cause noticeable irq latency.
|
||||
*/
|
||||
SWAPGS_UNSAFE_STACK
|
||||
SWITCH_KERNEL_CR3_NO_STACK
|
||||
/*
|
||||
* A hypervisor implementation might want to use a label
|
||||
* after the swapgs, so that it can do the swapgs
|
||||
|
|
@ -228,6 +230,14 @@ entry_SYSCALL_64_fastpath:
|
|||
movq RIP(%rsp), %rcx
|
||||
movq EFLAGS(%rsp), %r11
|
||||
RESTORE_C_REGS_EXCEPT_RCX_R11
|
||||
/*
|
||||
* This opens a window where we have a user CR3, but are
|
||||
* running in the kernel. This makes using the CS
|
||||
* register useless for telling whether or not we need to
|
||||
* switch CR3 in NMIs. Normal interrupts are OK because
|
||||
* they are off here.
|
||||
*/
|
||||
SWITCH_USER_CR3
|
||||
movq RSP(%rsp), %rsp
|
||||
USERGS_SYSRET64
|
||||
|
||||
|
|
@ -323,10 +333,26 @@ return_from_SYSCALL_64:
|
|||
syscall_return_via_sysret:
|
||||
/* rcx and r11 are already restored (see code above) */
|
||||
RESTORE_C_REGS_EXCEPT_RCX_R11
|
||||
/*
|
||||
* This opens a window where we have a user CR3, but are
|
||||
* running in the kernel. This makes using the CS
|
||||
* register useless for telling whether or not we need to
|
||||
* switch CR3 in NMIs. Normal interrupts are OK because
|
||||
* they are off here.
|
||||
*/
|
||||
SWITCH_USER_CR3
|
||||
movq RSP(%rsp), %rsp
|
||||
USERGS_SYSRET64
|
||||
|
||||
opportunistic_sysret_failed:
|
||||
/*
|
||||
* This opens a window where we have a user CR3, but are
|
||||
* running in the kernel. This makes using the CS
|
||||
* register useless for telling whether or not we need to
|
||||
* switch CR3 in NMIs. Normal interrupts are OK because
|
||||
* they are off here.
|
||||
*/
|
||||
SWITCH_USER_CR3
|
||||
SWAPGS
|
||||
jmp restore_c_regs_and_iret
|
||||
END(entry_SYSCALL_64)
|
||||
|
|
@ -424,6 +450,7 @@ ENTRY(ret_from_fork)
|
|||
movq %rsp, %rdi
|
||||
call syscall_return_slowpath /* returns with IRQs disabled */
|
||||
TRACE_IRQS_ON /* user mode is traced as IRQS on */
|
||||
SWITCH_USER_CR3
|
||||
SWAPGS
|
||||
jmp restore_regs_and_iret
|
||||
|
||||
|
|
@ -478,6 +505,7 @@ END(irq_entries_start)
|
|||
* tracking that we're in kernel mode.
|
||||
*/
|
||||
SWAPGS
|
||||
SWITCH_KERNEL_CR3
|
||||
|
||||
/*
|
||||
* We need to tell lockdep that IRQs are off. We can't do this until
|
||||
|
|
@ -535,6 +563,7 @@ GLOBAL(retint_user)
|
|||
mov %rsp,%rdi
|
||||
call prepare_exit_to_usermode
|
||||
TRACE_IRQS_IRETQ
|
||||
SWITCH_USER_CR3
|
||||
SWAPGS
|
||||
jmp restore_regs_and_iret
|
||||
|
||||
|
|
@ -612,6 +641,7 @@ native_irq_return_ldt:
|
|||
|
||||
pushq %rdi /* Stash user RDI */
|
||||
SWAPGS
|
||||
SWITCH_KERNEL_CR3
|
||||
movq PER_CPU_VAR(espfix_waddr), %rdi
|
||||
movq %rax, (0*8)(%rdi) /* user RAX */
|
||||
movq (1*8)(%rsp), %rax /* user RIP */
|
||||
|
|
@ -638,6 +668,7 @@ native_irq_return_ldt:
|
|||
* still points to an RO alias of the ESPFIX stack.
|
||||
*/
|
||||
orq PER_CPU_VAR(espfix_stack), %rax
|
||||
SWITCH_USER_CR3
|
||||
SWAPGS
|
||||
movq %rax, %rsp
|
||||
|
||||
|
|
@ -1017,7 +1048,11 @@ idtentry machine_check has_error_code=0 paranoid=1 do_sym=*machine_check_vec
|
|||
/*
|
||||
* Save all registers in pt_regs, and switch gs if needed.
|
||||
* Use slow, but surefire "are we in kernel?" check.
|
||||
* Return: ebx=0: need swapgs on exit, ebx=1: otherwise
|
||||
*
|
||||
* Return: ebx=0: needs swapgs but not SWITCH_USER_CR3 in paranoid_exit
|
||||
* ebx=1: needs neither swapgs nor SWITCH_USER_CR3 in paranoid_exit
|
||||
* ebx=2: needs both swapgs and SWITCH_USER_CR3 in paranoid_exit
|
||||
* ebx=3: needs SWITCH_USER_CR3 but not swapgs in paranoid_exit
|
||||
*/
|
||||
ENTRY(paranoid_entry)
|
||||
cld
|
||||
|
|
@ -1030,7 +1065,26 @@ ENTRY(paranoid_entry)
|
|||
js 1f /* negative -> in kernel */
|
||||
SWAPGS
|
||||
xorl %ebx, %ebx
|
||||
1: ret
|
||||
1:
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
/*
|
||||
* We might have come in between a swapgs and a SWITCH_KERNEL_CR3
|
||||
* on entry, or between a SWITCH_USER_CR3 and a swapgs on exit.
|
||||
* Do a conditional SWITCH_KERNEL_CR3: this could safely be done
|
||||
* unconditionally, but we need to find out whether the reverse
|
||||
* should be done on return (conveyed to paranoid_exit in %ebx).
|
||||
*/
|
||||
ALTERNATIVE "jmp 2f", "movq %cr3, %rax", X86_FEATURE_KAISER
|
||||
testl $KAISER_SHADOW_PGD_OFFSET, %eax
|
||||
jz 2f
|
||||
orl $2, %ebx
|
||||
andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax
|
||||
/* If PCID enabled, set X86_CR3_PCID_NOFLUSH_BIT */
|
||||
ALTERNATIVE "", "bts $63, %rax", X86_FEATURE_PCID
|
||||
movq %rax, %cr3
|
||||
2:
|
||||
#endif
|
||||
ret
|
||||
END(paranoid_entry)
|
||||
|
||||
/*
|
||||
|
|
@ -1043,19 +1097,26 @@ END(paranoid_entry)
|
|||
* be complicated. Fortunately, we there's no good reason
|
||||
* to try to handle preemption here.
|
||||
*
|
||||
* On entry, ebx is "no swapgs" flag (1: don't need swapgs, 0: need it)
|
||||
* On entry: ebx=0: needs swapgs but not SWITCH_USER_CR3
|
||||
* ebx=1: needs neither swapgs nor SWITCH_USER_CR3
|
||||
* ebx=2: needs both swapgs and SWITCH_USER_CR3
|
||||
* ebx=3: needs SWITCH_USER_CR3 but not swapgs
|
||||
*/
|
||||
ENTRY(paranoid_exit)
|
||||
DISABLE_INTERRUPTS(CLBR_NONE)
|
||||
TRACE_IRQS_OFF_DEBUG
|
||||
testl %ebx, %ebx /* swapgs needed? */
|
||||
jnz paranoid_exit_no_swapgs
|
||||
TRACE_IRQS_IRETQ
|
||||
SWAPGS_UNSAFE_STACK
|
||||
jmp paranoid_exit_restore
|
||||
paranoid_exit_no_swapgs:
|
||||
TRACE_IRQS_IRETQ_DEBUG
|
||||
paranoid_exit_restore:
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
/* No ALTERNATIVE for X86_FEATURE_KAISER: paranoid_entry sets %ebx */
|
||||
testl $2, %ebx /* SWITCH_USER_CR3 needed? */
|
||||
jz paranoid_exit_no_switch
|
||||
SWITCH_USER_CR3
|
||||
paranoid_exit_no_switch:
|
||||
#endif
|
||||
testl $1, %ebx /* swapgs needed? */
|
||||
jnz paranoid_exit_no_swapgs
|
||||
SWAPGS_UNSAFE_STACK
|
||||
paranoid_exit_no_swapgs:
|
||||
RESTORE_EXTRA_REGS
|
||||
RESTORE_C_REGS
|
||||
REMOVE_PT_GPREGS_FROM_STACK 8
|
||||
|
|
@ -1070,6 +1131,13 @@ ENTRY(error_entry)
|
|||
cld
|
||||
SAVE_C_REGS 8
|
||||
SAVE_EXTRA_REGS 8
|
||||
/*
|
||||
* error_entry() always returns with a kernel gsbase and
|
||||
* CR3. We must also have a kernel CR3/gsbase before
|
||||
* calling TRACE_IRQS_*. Just unconditionally switch to
|
||||
* the kernel CR3 here.
|
||||
*/
|
||||
SWITCH_KERNEL_CR3
|
||||
xorl %ebx, %ebx
|
||||
testb $3, CS+8(%rsp)
|
||||
jz .Lerror_kernelspace
|
||||
|
|
@ -1230,6 +1298,10 @@ ENTRY(nmi)
|
|||
*/
|
||||
|
||||
SWAPGS_UNSAFE_STACK
|
||||
/*
|
||||
* percpu variables are mapped with user CR3, so no need
|
||||
* to switch CR3 here.
|
||||
*/
|
||||
cld
|
||||
movq %rsp, %rdx
|
||||
movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
|
||||
|
|
@ -1263,12 +1335,34 @@ ENTRY(nmi)
|
|||
|
||||
movq %rsp, %rdi
|
||||
movq $-1, %rsi
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
/* Unconditionally use kernel CR3 for do_nmi() */
|
||||
/* %rax is saved above, so OK to clobber here */
|
||||
ALTERNATIVE "jmp 2f", "movq %cr3, %rax", X86_FEATURE_KAISER
|
||||
/* If PCID enabled, NOFLUSH now and NOFLUSH on return */
|
||||
ALTERNATIVE "", "bts $63, %rax", X86_FEATURE_PCID
|
||||
pushq %rax
|
||||
/* mask off "user" bit of pgd address and 12 PCID bits: */
|
||||
andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax
|
||||
movq %rax, %cr3
|
||||
2:
|
||||
#endif
|
||||
call do_nmi
|
||||
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
/*
|
||||
* Unconditionally restore CR3. I know we return to
|
||||
* kernel code that needs user CR3, but do we ever return
|
||||
* to "user mode" where we need the kernel CR3?
|
||||
*/
|
||||
ALTERNATIVE "", "popq %rax; movq %rax, %cr3", X86_FEATURE_KAISER
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Return back to user mode. We must *not* do the normal exit
|
||||
* work, because we don't want to enable interrupts. Fortunately,
|
||||
* do_nmi doesn't modify pt_regs.
|
||||
* work, because we don't want to enable interrupts. Do not
|
||||
* switch to user CR3: we might be going back to kernel code
|
||||
* that had a user CR3 set.
|
||||
*/
|
||||
SWAPGS
|
||||
jmp restore_c_regs_and_iret
|
||||
|
|
@ -1465,22 +1559,55 @@ end_repeat_nmi:
|
|||
ALLOC_PT_GPREGS_ON_STACK
|
||||
|
||||
/*
|
||||
* Use paranoid_entry to handle SWAPGS, but no need to use paranoid_exit
|
||||
* as we should not be calling schedule in NMI context.
|
||||
* Even with normal interrupts enabled. An NMI should not be
|
||||
* setting NEED_RESCHED or anything that normal interrupts and
|
||||
* exceptions might do.
|
||||
* Use the same approach as paranoid_entry to handle SWAPGS, but
|
||||
* without CR3 handling since we do that differently in NMIs. No
|
||||
* need to use paranoid_exit as we should not be calling schedule
|
||||
* in NMI context. Even with normal interrupts enabled. An NMI
|
||||
* should not be setting NEED_RESCHED or anything that normal
|
||||
* interrupts and exceptions might do.
|
||||
*/
|
||||
call paranoid_entry
|
||||
|
||||
/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
|
||||
cld
|
||||
SAVE_C_REGS
|
||||
SAVE_EXTRA_REGS
|
||||
movl $1, %ebx
|
||||
movl $MSR_GS_BASE, %ecx
|
||||
rdmsr
|
||||
testl %edx, %edx
|
||||
js 1f /* negative -> in kernel */
|
||||
SWAPGS
|
||||
xorl %ebx, %ebx
|
||||
1:
|
||||
movq %rsp, %rdi
|
||||
movq $-1, %rsi
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
/* Unconditionally use kernel CR3 for do_nmi() */
|
||||
/* %rax is saved above, so OK to clobber here */
|
||||
ALTERNATIVE "jmp 2f", "movq %cr3, %rax", X86_FEATURE_KAISER
|
||||
/* If PCID enabled, NOFLUSH now and NOFLUSH on return */
|
||||
ALTERNATIVE "", "bts $63, %rax", X86_FEATURE_PCID
|
||||
pushq %rax
|
||||
/* mask off "user" bit of pgd address and 12 PCID bits: */
|
||||
andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), %rax
|
||||
movq %rax, %cr3
|
||||
2:
|
||||
#endif
|
||||
|
||||
/* paranoidentry do_nmi, 0; without TRACE_IRQS_OFF */
|
||||
call do_nmi
|
||||
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
/*
|
||||
* Unconditionally restore CR3. We might be returning to
|
||||
* kernel code that needs user CR3, like just just before
|
||||
* a sysret.
|
||||
*/
|
||||
ALTERNATIVE "", "popq %rax; movq %rax, %cr3", X86_FEATURE_KAISER
|
||||
#endif
|
||||
|
||||
testl %ebx, %ebx /* swapgs needed? */
|
||||
jnz nmi_restore
|
||||
nmi_swapgs:
|
||||
/* We fixed up CR3 above, so no need to switch it here */
|
||||
SWAPGS_UNSAFE_STACK
|
||||
nmi_restore:
|
||||
RESTORE_EXTRA_REGS
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@
|
|||
#include <asm/irqflags.h>
|
||||
#include <asm/asm.h>
|
||||
#include <asm/smap.h>
|
||||
#include <asm/pgtable_types.h>
|
||||
#include <asm/kaiser.h>
|
||||
#include <linux/linkage.h>
|
||||
#include <linux/err.h>
|
||||
|
||||
|
|
@ -48,6 +50,7 @@
|
|||
ENTRY(entry_SYSENTER_compat)
|
||||
/* Interrupts are off on entry. */
|
||||
SWAPGS_UNSAFE_STACK
|
||||
SWITCH_KERNEL_CR3_NO_STACK
|
||||
movq PER_CPU_VAR(cpu_current_top_of_stack), %rsp
|
||||
|
||||
/*
|
||||
|
|
@ -184,6 +187,7 @@ ENDPROC(entry_SYSENTER_compat)
|
|||
ENTRY(entry_SYSCALL_compat)
|
||||
/* Interrupts are off on entry. */
|
||||
SWAPGS_UNSAFE_STACK
|
||||
SWITCH_KERNEL_CR3_NO_STACK
|
||||
|
||||
/* Stash user ESP and switch to the kernel stack. */
|
||||
movl %esp, %r8d
|
||||
|
|
@ -259,6 +263,7 @@ sysret32_from_system_call:
|
|||
xorq %r8, %r8
|
||||
xorq %r9, %r9
|
||||
xorq %r10, %r10
|
||||
SWITCH_USER_CR3
|
||||
movq RSP-ORIG_RAX(%rsp), %rsp
|
||||
swapgs
|
||||
sysretl
|
||||
|
|
@ -297,7 +302,7 @@ ENTRY(entry_INT80_compat)
|
|||
PARAVIRT_ADJUST_EXCEPTION_FRAME
|
||||
ASM_CLAC /* Do this early to minimize exposure */
|
||||
SWAPGS
|
||||
|
||||
SWITCH_KERNEL_CR3_NO_STACK
|
||||
/*
|
||||
* User tracing code (ptrace or signal handlers) might assume that
|
||||
* the saved RAX contains a 32-bit number when we're invoking a 32-bit
|
||||
|
|
@ -338,6 +343,7 @@ ENTRY(entry_INT80_compat)
|
|||
|
||||
/* Go back to user mode. */
|
||||
TRACE_IRQS_ON
|
||||
SWITCH_USER_CR3
|
||||
SWAPGS
|
||||
jmp restore_regs_and_iret
|
||||
END(entry_INT80_compat)
|
||||
|
|
|
|||
|
|
@ -2,11 +2,15 @@
|
|||
#include <linux/types.h>
|
||||
#include <linux/slab.h>
|
||||
|
||||
#include <asm/kaiser.h>
|
||||
#include <asm/perf_event.h>
|
||||
#include <asm/insn.h>
|
||||
|
||||
#include "../perf_event.h"
|
||||
|
||||
static
|
||||
DEFINE_PER_CPU_SHARED_ALIGNED_USER_MAPPED(struct debug_store, cpu_debug_store);
|
||||
|
||||
/* The size of a BTS record in bytes: */
|
||||
#define BTS_RECORD_SIZE 24
|
||||
|
||||
|
|
@ -268,6 +272,39 @@ void fini_debug_store_on_cpu(int cpu)
|
|||
|
||||
static DEFINE_PER_CPU(void *, insn_buffer);
|
||||
|
||||
static void *dsalloc(size_t size, gfp_t flags, int node)
|
||||
{
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
unsigned int order = get_order(size);
|
||||
struct page *page;
|
||||
unsigned long addr;
|
||||
|
||||
page = __alloc_pages_node(node, flags | __GFP_ZERO, order);
|
||||
if (!page)
|
||||
return NULL;
|
||||
addr = (unsigned long)page_address(page);
|
||||
if (kaiser_add_mapping(addr, size, __PAGE_KERNEL) < 0) {
|
||||
__free_pages(page, order);
|
||||
addr = 0;
|
||||
}
|
||||
return (void *)addr;
|
||||
#else
|
||||
return kmalloc_node(size, flags | __GFP_ZERO, node);
|
||||
#endif
|
||||
}
|
||||
|
||||
static void dsfree(const void *buffer, size_t size)
|
||||
{
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
if (!buffer)
|
||||
return;
|
||||
kaiser_remove_mapping((unsigned long)buffer, size);
|
||||
free_pages((unsigned long)buffer, get_order(size));
|
||||
#else
|
||||
kfree(buffer);
|
||||
#endif
|
||||
}
|
||||
|
||||
static int alloc_pebs_buffer(int cpu)
|
||||
{
|
||||
struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
|
||||
|
|
@ -278,7 +315,7 @@ static int alloc_pebs_buffer(int cpu)
|
|||
if (!x86_pmu.pebs)
|
||||
return 0;
|
||||
|
||||
buffer = kzalloc_node(x86_pmu.pebs_buffer_size, GFP_KERNEL, node);
|
||||
buffer = dsalloc(x86_pmu.pebs_buffer_size, GFP_KERNEL, node);
|
||||
if (unlikely(!buffer))
|
||||
return -ENOMEM;
|
||||
|
||||
|
|
@ -289,7 +326,7 @@ static int alloc_pebs_buffer(int cpu)
|
|||
if (x86_pmu.intel_cap.pebs_format < 2) {
|
||||
ibuffer = kzalloc_node(PEBS_FIXUP_SIZE, GFP_KERNEL, node);
|
||||
if (!ibuffer) {
|
||||
kfree(buffer);
|
||||
dsfree(buffer, x86_pmu.pebs_buffer_size);
|
||||
return -ENOMEM;
|
||||
}
|
||||
per_cpu(insn_buffer, cpu) = ibuffer;
|
||||
|
|
@ -315,7 +352,8 @@ static void release_pebs_buffer(int cpu)
|
|||
kfree(per_cpu(insn_buffer, cpu));
|
||||
per_cpu(insn_buffer, cpu) = NULL;
|
||||
|
||||
kfree((void *)(unsigned long)ds->pebs_buffer_base);
|
||||
dsfree((void *)(unsigned long)ds->pebs_buffer_base,
|
||||
x86_pmu.pebs_buffer_size);
|
||||
ds->pebs_buffer_base = 0;
|
||||
}
|
||||
|
||||
|
|
@ -329,7 +367,7 @@ static int alloc_bts_buffer(int cpu)
|
|||
if (!x86_pmu.bts)
|
||||
return 0;
|
||||
|
||||
buffer = kzalloc_node(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_NOWARN, node);
|
||||
buffer = dsalloc(BTS_BUFFER_SIZE, GFP_KERNEL | __GFP_NOWARN, node);
|
||||
if (unlikely(!buffer)) {
|
||||
WARN_ONCE(1, "%s: BTS buffer allocation failure\n", __func__);
|
||||
return -ENOMEM;
|
||||
|
|
@ -355,19 +393,15 @@ static void release_bts_buffer(int cpu)
|
|||
if (!ds || !x86_pmu.bts)
|
||||
return;
|
||||
|
||||
kfree((void *)(unsigned long)ds->bts_buffer_base);
|
||||
dsfree((void *)(unsigned long)ds->bts_buffer_base, BTS_BUFFER_SIZE);
|
||||
ds->bts_buffer_base = 0;
|
||||
}
|
||||
|
||||
static int alloc_ds_buffer(int cpu)
|
||||
{
|
||||
int node = cpu_to_node(cpu);
|
||||
struct debug_store *ds;
|
||||
|
||||
ds = kzalloc_node(sizeof(*ds), GFP_KERNEL, node);
|
||||
if (unlikely(!ds))
|
||||
return -ENOMEM;
|
||||
struct debug_store *ds = per_cpu_ptr(&cpu_debug_store, cpu);
|
||||
|
||||
memset(ds, 0, sizeof(*ds));
|
||||
per_cpu(cpu_hw_events, cpu).ds = ds;
|
||||
|
||||
return 0;
|
||||
|
|
@ -381,7 +415,6 @@ static void release_ds_buffer(int cpu)
|
|||
return;
|
||||
|
||||
per_cpu(cpu_hw_events, cpu).ds = NULL;
|
||||
kfree(ds);
|
||||
}
|
||||
|
||||
void release_ds_buffers(void)
|
||||
|
|
|
|||
|
|
@ -2,5 +2,7 @@
|
|||
#define _ASM_X86_CMDLINE_H
|
||||
|
||||
int cmdline_find_option_bool(const char *cmdline_ptr, const char *option);
|
||||
int cmdline_find_option(const char *cmdline_ptr, const char *option,
|
||||
char *buffer, int bufsize);
|
||||
|
||||
#endif /* _ASM_X86_CMDLINE_H */
|
||||
|
|
|
|||
|
|
@ -189,6 +189,7 @@
|
|||
|
||||
#define X86_FEATURE_CPB ( 7*32+ 2) /* AMD Core Performance Boost */
|
||||
#define X86_FEATURE_EPB ( 7*32+ 3) /* IA32_ENERGY_PERF_BIAS support */
|
||||
#define X86_FEATURE_INVPCID_SINGLE ( 7*32+ 4) /* Effectively INVPCID && CR4.PCIDE=1 */
|
||||
|
||||
#define X86_FEATURE_HW_PSTATE ( 7*32+ 8) /* AMD HW-PState */
|
||||
#define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */
|
||||
|
|
@ -197,6 +198,9 @@
|
|||
#define X86_FEATURE_AVX512_4VNNIW (7*32+16) /* AVX-512 Neural Network Instructions */
|
||||
#define X86_FEATURE_AVX512_4FMAPS (7*32+17) /* AVX-512 Multiply Accumulation Single precision */
|
||||
|
||||
/* Because the ALTERNATIVE scheme is for members of the X86_FEATURE club... */
|
||||
#define X86_FEATURE_KAISER ( 7*32+31) /* CONFIG_PAGE_TABLE_ISOLATION w/o nokaiser */
|
||||
|
||||
/* Virtualization flags: Linux defined, word 8 */
|
||||
#define X86_FEATURE_TPR_SHADOW ( 8*32+ 0) /* Intel TPR Shadow */
|
||||
#define X86_FEATURE_VNMI ( 8*32+ 1) /* Intel Virtual NMI */
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ struct gdt_page {
|
|||
struct desc_struct gdt[GDT_ENTRIES];
|
||||
} __attribute__((aligned(PAGE_SIZE)));
|
||||
|
||||
DECLARE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page);
|
||||
DECLARE_PER_CPU_PAGE_ALIGNED_USER_MAPPED(struct gdt_page, gdt_page);
|
||||
|
||||
static inline struct desc_struct *get_cpu_gdt_table(unsigned int cpu)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -178,7 +178,7 @@ extern char irq_entries_start[];
|
|||
#define VECTOR_RETRIGGERED ((void *)~0UL)
|
||||
|
||||
typedef struct irq_desc* vector_irq_t[NR_VECTORS];
|
||||
DECLARE_PER_CPU(vector_irq_t, vector_irq);
|
||||
DECLARE_PER_CPU_USER_MAPPED(vector_irq_t, vector_irq);
|
||||
|
||||
#endif /* !ASSEMBLY_ */
|
||||
|
||||
|
|
|
|||
141
arch/x86/include/asm/kaiser.h
Normal file
141
arch/x86/include/asm/kaiser.h
Normal file
|
|
@ -0,0 +1,141 @@
|
|||
#ifndef _ASM_X86_KAISER_H
|
||||
#define _ASM_X86_KAISER_H
|
||||
|
||||
#include <uapi/asm/processor-flags.h> /* For PCID constants */
|
||||
|
||||
/*
|
||||
* This file includes the definitions for the KAISER feature.
|
||||
* KAISER is a counter measure against x86_64 side channel attacks on
|
||||
* the kernel virtual memory. It has a shadow pgd for every process: the
|
||||
* shadow pgd has a minimalistic kernel-set mapped, but includes the whole
|
||||
* user memory. Within a kernel context switch, or when an interrupt is handled,
|
||||
* the pgd is switched to the normal one. When the system switches to user mode,
|
||||
* the shadow pgd is enabled. By this, the virtual memory caches are freed,
|
||||
* and the user may not attack the whole kernel memory.
|
||||
*
|
||||
* A minimalistic kernel mapping holds the parts needed to be mapped in user
|
||||
* mode, such as the entry/exit functions of the user space, or the stacks.
|
||||
*/
|
||||
|
||||
#define KAISER_SHADOW_PGD_OFFSET 0x1000
|
||||
|
||||
#ifdef __ASSEMBLY__
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
|
||||
.macro _SWITCH_TO_KERNEL_CR3 reg
|
||||
movq %cr3, \reg
|
||||
andq $(~(X86_CR3_PCID_ASID_MASK | KAISER_SHADOW_PGD_OFFSET)), \reg
|
||||
/* If PCID enabled, set X86_CR3_PCID_NOFLUSH_BIT */
|
||||
ALTERNATIVE "", "bts $63, \reg", X86_FEATURE_PCID
|
||||
movq \reg, %cr3
|
||||
.endm
|
||||
|
||||
.macro _SWITCH_TO_USER_CR3 reg regb
|
||||
/*
|
||||
* regb must be the low byte portion of reg: because we have arranged
|
||||
* for the low byte of the user PCID to serve as the high byte of NOFLUSH
|
||||
* (0x80 for each when PCID is enabled, or 0x00 when PCID and NOFLUSH are
|
||||
* not enabled): so that the one register can update both memory and cr3.
|
||||
*/
|
||||
movq %cr3, \reg
|
||||
orq PER_CPU_VAR(x86_cr3_pcid_user), \reg
|
||||
js 9f
|
||||
/* If PCID enabled, FLUSH this time, reset to NOFLUSH for next time */
|
||||
movb \regb, PER_CPU_VAR(x86_cr3_pcid_user+7)
|
||||
9:
|
||||
movq \reg, %cr3
|
||||
.endm
|
||||
|
||||
.macro SWITCH_KERNEL_CR3
|
||||
ALTERNATIVE "jmp 8f", "pushq %rax", X86_FEATURE_KAISER
|
||||
_SWITCH_TO_KERNEL_CR3 %rax
|
||||
popq %rax
|
||||
8:
|
||||
.endm
|
||||
|
||||
.macro SWITCH_USER_CR3
|
||||
ALTERNATIVE "jmp 8f", "pushq %rax", X86_FEATURE_KAISER
|
||||
_SWITCH_TO_USER_CR3 %rax %al
|
||||
popq %rax
|
||||
8:
|
||||
.endm
|
||||
|
||||
.macro SWITCH_KERNEL_CR3_NO_STACK
|
||||
ALTERNATIVE "jmp 8f", \
|
||||
__stringify(movq %rax, PER_CPU_VAR(unsafe_stack_register_backup)), \
|
||||
X86_FEATURE_KAISER
|
||||
_SWITCH_TO_KERNEL_CR3 %rax
|
||||
movq PER_CPU_VAR(unsafe_stack_register_backup), %rax
|
||||
8:
|
||||
.endm
|
||||
|
||||
#else /* CONFIG_PAGE_TABLE_ISOLATION */
|
||||
|
||||
.macro SWITCH_KERNEL_CR3
|
||||
.endm
|
||||
.macro SWITCH_USER_CR3
|
||||
.endm
|
||||
.macro SWITCH_KERNEL_CR3_NO_STACK
|
||||
.endm
|
||||
|
||||
#endif /* CONFIG_PAGE_TABLE_ISOLATION */
|
||||
|
||||
#else /* __ASSEMBLY__ */
|
||||
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
/*
|
||||
* Upon kernel/user mode switch, it may happen that the address
|
||||
* space has to be switched before the registers have been
|
||||
* stored. To change the address space, another register is
|
||||
* needed. A register therefore has to be stored/restored.
|
||||
*/
|
||||
DECLARE_PER_CPU_USER_MAPPED(unsigned long, unsafe_stack_register_backup);
|
||||
|
||||
DECLARE_PER_CPU(unsigned long, x86_cr3_pcid_user);
|
||||
|
||||
extern char __per_cpu_user_mapped_start[], __per_cpu_user_mapped_end[];
|
||||
|
||||
extern int kaiser_enabled;
|
||||
extern void __init kaiser_check_boottime_disable(void);
|
||||
#else
|
||||
#define kaiser_enabled 0
|
||||
static inline void __init kaiser_check_boottime_disable(void) {}
|
||||
#endif /* CONFIG_PAGE_TABLE_ISOLATION */
|
||||
|
||||
/*
|
||||
* Kaiser function prototypes are needed even when CONFIG_PAGE_TABLE_ISOLATION is not set,
|
||||
* so as to build with tests on kaiser_enabled instead of #ifdefs.
|
||||
*/
|
||||
|
||||
/**
|
||||
* kaiser_add_mapping - map a virtual memory part to the shadow (user) mapping
|
||||
* @addr: the start address of the range
|
||||
* @size: the size of the range
|
||||
* @flags: The mapping flags of the pages
|
||||
*
|
||||
* The mapping is done on a global scope, so no bigger
|
||||
* synchronization has to be done. the pages have to be
|
||||
* manually unmapped again when they are not needed any longer.
|
||||
*/
|
||||
extern int kaiser_add_mapping(unsigned long addr, unsigned long size, unsigned long flags);
|
||||
|
||||
/**
|
||||
* kaiser_remove_mapping - unmap a virtual memory part of the shadow mapping
|
||||
* @addr: the start address of the range
|
||||
* @size: the size of the range
|
||||
*/
|
||||
extern void kaiser_remove_mapping(unsigned long start, unsigned long size);
|
||||
|
||||
/**
|
||||
* kaiser_init - Initialize the shadow mapping
|
||||
*
|
||||
* Most parts of the shadow mapping can be mapped upon boot
|
||||
* time. Only per-process things like the thread stacks
|
||||
* or a new LDT have to be mapped at runtime. These boot-
|
||||
* time mappings are permanent and never unmapped.
|
||||
*/
|
||||
extern void kaiser_init(void);
|
||||
|
||||
#endif /* __ASSEMBLY */
|
||||
|
||||
#endif /* _ASM_X86_KAISER_H */
|
||||
|
|
@ -18,6 +18,12 @@
|
|||
#ifndef __ASSEMBLY__
|
||||
#include <asm/x86_init.h>
|
||||
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
extern int kaiser_enabled;
|
||||
#else
|
||||
#define kaiser_enabled 0
|
||||
#endif
|
||||
|
||||
void ptdump_walk_pgd_level(struct seq_file *m, pgd_t *pgd);
|
||||
void ptdump_walk_pgd_level_checkwx(void);
|
||||
|
||||
|
|
@ -690,7 +696,17 @@ static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
|
|||
|
||||
static inline int pgd_bad(pgd_t pgd)
|
||||
{
|
||||
return (pgd_flags(pgd) & ~_PAGE_USER) != _KERNPG_TABLE;
|
||||
pgdval_t ignore_flags = _PAGE_USER;
|
||||
/*
|
||||
* We set NX on KAISER pgds that map userspace memory so
|
||||
* that userspace can not meaningfully use the kernel
|
||||
* page table by accident; it will fault on the first
|
||||
* instruction it tries to run. See native_set_pgd().
|
||||
*/
|
||||
if (kaiser_enabled)
|
||||
ignore_flags |= _PAGE_NX;
|
||||
|
||||
return (pgd_flags(pgd) & ~ignore_flags) != _KERNPG_TABLE;
|
||||
}
|
||||
|
||||
static inline int pgd_none(pgd_t pgd)
|
||||
|
|
@ -903,7 +919,15 @@ static inline void pmdp_set_wrprotect(struct mm_struct *mm,
|
|||
*/
|
||||
static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count)
|
||||
{
|
||||
memcpy(dst, src, count * sizeof(pgd_t));
|
||||
memcpy(dst, src, count * sizeof(pgd_t));
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
if (kaiser_enabled) {
|
||||
/* Clone the shadow pgd part as well */
|
||||
memcpy(native_get_shadow_pgd(dst),
|
||||
native_get_shadow_pgd(src),
|
||||
count * sizeof(pgd_t));
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
#define PTE_SHIFT ilog2(PTRS_PER_PTE)
|
||||
|
|
|
|||
|
|
@ -106,9 +106,32 @@ static inline void native_pud_clear(pud_t *pud)
|
|||
native_set_pud(pud, native_make_pud(0));
|
||||
}
|
||||
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
extern pgd_t kaiser_set_shadow_pgd(pgd_t *pgdp, pgd_t pgd);
|
||||
|
||||
static inline pgd_t *native_get_shadow_pgd(pgd_t *pgdp)
|
||||
{
|
||||
#ifdef CONFIG_DEBUG_VM
|
||||
/* linux/mmdebug.h may not have been included at this point */
|
||||
BUG_ON(!kaiser_enabled);
|
||||
#endif
|
||||
return (pgd_t *)((unsigned long)pgdp | (unsigned long)PAGE_SIZE);
|
||||
}
|
||||
#else
|
||||
static inline pgd_t kaiser_set_shadow_pgd(pgd_t *pgdp, pgd_t pgd)
|
||||
{
|
||||
return pgd;
|
||||
}
|
||||
static inline pgd_t *native_get_shadow_pgd(pgd_t *pgdp)
|
||||
{
|
||||
BUILD_BUG_ON(1);
|
||||
return NULL;
|
||||
}
|
||||
#endif /* CONFIG_PAGE_TABLE_ISOLATION */
|
||||
|
||||
static inline void native_set_pgd(pgd_t *pgdp, pgd_t pgd)
|
||||
{
|
||||
*pgdp = pgd;
|
||||
*pgdp = kaiser_set_shadow_pgd(pgdp, pgd);
|
||||
}
|
||||
|
||||
static inline void native_pgd_clear(pgd_t *pgd)
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@
|
|||
#define _PAGE_DEVMAP (_AT(pteval_t, 0))
|
||||
#endif
|
||||
|
||||
#define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE)
|
||||
#define _PAGE_PROTNONE (_AT(pteval_t, 1) << _PAGE_BIT_PROTNONE)
|
||||
|
||||
#define _PAGE_TABLE (_PAGE_PRESENT | _PAGE_RW | _PAGE_USER | \
|
||||
_PAGE_ACCESSED | _PAGE_DIRTY)
|
||||
|
|
@ -137,6 +137,33 @@
|
|||
_PAGE_SOFT_DIRTY)
|
||||
#define _HPAGE_CHG_MASK (_PAGE_CHG_MASK | _PAGE_PSE)
|
||||
|
||||
/* The ASID is the lower 12 bits of CR3 */
|
||||
#define X86_CR3_PCID_ASID_MASK (_AC((1<<12)-1,UL))
|
||||
|
||||
/* Mask for all the PCID-related bits in CR3: */
|
||||
#define X86_CR3_PCID_MASK (X86_CR3_PCID_NOFLUSH | X86_CR3_PCID_ASID_MASK)
|
||||
#define X86_CR3_PCID_ASID_KERN (_AC(0x0,UL))
|
||||
|
||||
#if defined(CONFIG_PAGE_TABLE_ISOLATION) && defined(CONFIG_X86_64)
|
||||
/* Let X86_CR3_PCID_ASID_USER be usable for the X86_CR3_PCID_NOFLUSH bit */
|
||||
#define X86_CR3_PCID_ASID_USER (_AC(0x80,UL))
|
||||
|
||||
#define X86_CR3_PCID_KERN_FLUSH (X86_CR3_PCID_ASID_KERN)
|
||||
#define X86_CR3_PCID_USER_FLUSH (X86_CR3_PCID_ASID_USER)
|
||||
#define X86_CR3_PCID_KERN_NOFLUSH (X86_CR3_PCID_NOFLUSH | X86_CR3_PCID_ASID_KERN)
|
||||
#define X86_CR3_PCID_USER_NOFLUSH (X86_CR3_PCID_NOFLUSH | X86_CR3_PCID_ASID_USER)
|
||||
#else
|
||||
#define X86_CR3_PCID_ASID_USER (_AC(0x0,UL))
|
||||
/*
|
||||
* PCIDs are unsupported on 32-bit and none of these bits can be
|
||||
* set in CR3:
|
||||
*/
|
||||
#define X86_CR3_PCID_KERN_FLUSH (0)
|
||||
#define X86_CR3_PCID_USER_FLUSH (0)
|
||||
#define X86_CR3_PCID_KERN_NOFLUSH (0)
|
||||
#define X86_CR3_PCID_USER_NOFLUSH (0)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* The cache modes defined here are used to translate between pure SW usage
|
||||
* and the HW defined cache mode bits and/or PAT entries.
|
||||
|
|
|
|||
|
|
@ -308,7 +308,7 @@ struct tss_struct {
|
|||
|
||||
} ____cacheline_aligned;
|
||||
|
||||
DECLARE_PER_CPU_SHARED_ALIGNED(struct tss_struct, cpu_tss);
|
||||
DECLARE_PER_CPU_SHARED_ALIGNED_USER_MAPPED(struct tss_struct, cpu_tss);
|
||||
|
||||
#ifdef CONFIG_X86_32
|
||||
DECLARE_PER_CPU(unsigned long, cpu_current_top_of_stack);
|
||||
|
|
|
|||
|
|
@ -132,6 +132,24 @@ static inline void cr4_set_bits_and_update_boot(unsigned long mask)
|
|||
cr4_set_bits(mask);
|
||||
}
|
||||
|
||||
/*
|
||||
* Declare a couple of kaiser interfaces here for convenience,
|
||||
* to avoid the need for asm/kaiser.h in unexpected places.
|
||||
*/
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
extern int kaiser_enabled;
|
||||
extern void kaiser_setup_pcid(void);
|
||||
extern void kaiser_flush_tlb_on_return_to_user(void);
|
||||
#else
|
||||
#define kaiser_enabled 0
|
||||
static inline void kaiser_setup_pcid(void)
|
||||
{
|
||||
}
|
||||
static inline void kaiser_flush_tlb_on_return_to_user(void)
|
||||
{
|
||||
}
|
||||
#endif
|
||||
|
||||
static inline void __native_flush_tlb(void)
|
||||
{
|
||||
/*
|
||||
|
|
@ -140,6 +158,8 @@ static inline void __native_flush_tlb(void)
|
|||
* back:
|
||||
*/
|
||||
preempt_disable();
|
||||
if (kaiser_enabled)
|
||||
kaiser_flush_tlb_on_return_to_user();
|
||||
native_write_cr3(native_read_cr3());
|
||||
preempt_enable();
|
||||
}
|
||||
|
|
@ -149,20 +169,27 @@ static inline void __native_flush_tlb_global_irq_disabled(void)
|
|||
unsigned long cr4;
|
||||
|
||||
cr4 = this_cpu_read(cpu_tlbstate.cr4);
|
||||
/* clear PGE */
|
||||
native_write_cr4(cr4 & ~X86_CR4_PGE);
|
||||
/* write old PGE again and flush TLBs */
|
||||
native_write_cr4(cr4);
|
||||
if (cr4 & X86_CR4_PGE) {
|
||||
/* clear PGE and flush TLB of all entries */
|
||||
native_write_cr4(cr4 & ~X86_CR4_PGE);
|
||||
/* restore PGE as it was before */
|
||||
native_write_cr4(cr4);
|
||||
} else {
|
||||
/* do it with cr3, letting kaiser flush user PCID */
|
||||
__native_flush_tlb();
|
||||
}
|
||||
}
|
||||
|
||||
static inline void __native_flush_tlb_global(void)
|
||||
{
|
||||
unsigned long flags;
|
||||
|
||||
if (static_cpu_has(X86_FEATURE_INVPCID)) {
|
||||
if (this_cpu_has(X86_FEATURE_INVPCID)) {
|
||||
/*
|
||||
* Using INVPCID is considerably faster than a pair of writes
|
||||
* to CR4 sandwiched inside an IRQ flag save/restore.
|
||||
*
|
||||
* Note, this works with CR4.PCIDE=0 or 1.
|
||||
*/
|
||||
invpcid_flush_all();
|
||||
return;
|
||||
|
|
@ -174,24 +201,45 @@ static inline void __native_flush_tlb_global(void)
|
|||
* be called from deep inside debugging code.)
|
||||
*/
|
||||
raw_local_irq_save(flags);
|
||||
|
||||
__native_flush_tlb_global_irq_disabled();
|
||||
|
||||
raw_local_irq_restore(flags);
|
||||
}
|
||||
|
||||
static inline void __native_flush_tlb_single(unsigned long addr)
|
||||
{
|
||||
asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
|
||||
/*
|
||||
* SIMICS #GP's if you run INVPCID with type 2/3
|
||||
* and X86_CR4_PCIDE clear. Shame!
|
||||
*
|
||||
* The ASIDs used below are hard-coded. But, we must not
|
||||
* call invpcid(type=1/2) before CR4.PCIDE=1. Just call
|
||||
* invlpg in the case we are called early.
|
||||
*/
|
||||
|
||||
if (!this_cpu_has(X86_FEATURE_INVPCID_SINGLE)) {
|
||||
if (kaiser_enabled)
|
||||
kaiser_flush_tlb_on_return_to_user();
|
||||
asm volatile("invlpg (%0)" ::"r" (addr) : "memory");
|
||||
return;
|
||||
}
|
||||
/* Flush the address out of both PCIDs. */
|
||||
/*
|
||||
* An optimization here might be to determine addresses
|
||||
* that are only kernel-mapped and only flush the kernel
|
||||
* ASID. But, userspace flushes are probably much more
|
||||
* important performance-wise.
|
||||
*
|
||||
* Make sure to do only a single invpcid when KAISER is
|
||||
* disabled and we have only a single ASID.
|
||||
*/
|
||||
if (kaiser_enabled)
|
||||
invpcid_flush_one(X86_CR3_PCID_ASID_USER, addr);
|
||||
invpcid_flush_one(X86_CR3_PCID_ASID_KERN, addr);
|
||||
}
|
||||
|
||||
static inline void __flush_tlb_all(void)
|
||||
{
|
||||
if (boot_cpu_has(X86_FEATURE_PGE))
|
||||
__flush_tlb_global();
|
||||
else
|
||||
__flush_tlb();
|
||||
|
||||
__flush_tlb_global();
|
||||
/*
|
||||
* Note: if we somehow had PCID but not PGE, then this wouldn't work --
|
||||
* we'd end up flushing kernel translations for the current ASID but
|
||||
|
|
|
|||
|
|
@ -77,7 +77,8 @@
|
|||
#define X86_CR3_PWT _BITUL(X86_CR3_PWT_BIT)
|
||||
#define X86_CR3_PCD_BIT 4 /* Page Cache Disable */
|
||||
#define X86_CR3_PCD _BITUL(X86_CR3_PCD_BIT)
|
||||
#define X86_CR3_PCID_MASK _AC(0x00000fff,UL) /* PCID Mask */
|
||||
#define X86_CR3_PCID_NOFLUSH_BIT 63 /* Preserve old PCID */
|
||||
#define X86_CR3_PCID_NOFLUSH _BITULL(X86_CR3_PCID_NOFLUSH_BIT)
|
||||
|
||||
/*
|
||||
* Intel CPU features in CR4
|
||||
|
|
|
|||
|
|
@ -93,7 +93,7 @@ static const struct cpu_dev default_cpu = {
|
|||
|
||||
static const struct cpu_dev *this_cpu = &default_cpu;
|
||||
|
||||
DEFINE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page) = { .gdt = {
|
||||
DEFINE_PER_CPU_PAGE_ALIGNED_USER_MAPPED(struct gdt_page, gdt_page) = { .gdt = {
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* We need valid kernel segments for data and code in long mode too
|
||||
|
|
@ -327,8 +327,21 @@ static __always_inline void setup_smap(struct cpuinfo_x86 *c)
|
|||
static void setup_pcid(struct cpuinfo_x86 *c)
|
||||
{
|
||||
if (cpu_has(c, X86_FEATURE_PCID)) {
|
||||
if (cpu_has(c, X86_FEATURE_PGE)) {
|
||||
if (cpu_has(c, X86_FEATURE_PGE) || kaiser_enabled) {
|
||||
cr4_set_bits(X86_CR4_PCIDE);
|
||||
/*
|
||||
* INVPCID has two "groups" of types:
|
||||
* 1/2: Invalidate an individual address
|
||||
* 3/4: Invalidate all contexts
|
||||
*
|
||||
* 1/2 take a PCID, but 3/4 do not. So, 3/4
|
||||
* ignore the PCID argument in the descriptor.
|
||||
* But, we have to be careful not to call 1/2
|
||||
* with an actual non-zero PCID in them before
|
||||
* we do the above cr4_set_bits().
|
||||
*/
|
||||
if (cpu_has(c, X86_FEATURE_INVPCID))
|
||||
set_cpu_cap(c, X86_FEATURE_INVPCID_SINGLE);
|
||||
} else {
|
||||
/*
|
||||
* flush_tlb_all(), as currently implemented, won't
|
||||
|
|
@ -341,6 +354,7 @@ static void setup_pcid(struct cpuinfo_x86 *c)
|
|||
clear_cpu_cap(c, X86_FEATURE_PCID);
|
||||
}
|
||||
}
|
||||
kaiser_setup_pcid();
|
||||
}
|
||||
|
||||
/*
|
||||
|
|
@ -1365,7 +1379,7 @@ static const unsigned int exception_stack_sizes[N_EXCEPTION_STACKS] = {
|
|||
[DEBUG_STACK - 1] = DEBUG_STKSZ
|
||||
};
|
||||
|
||||
static DEFINE_PER_CPU_PAGE_ALIGNED(char, exception_stacks
|
||||
DEFINE_PER_CPU_PAGE_ALIGNED_USER_MAPPED(char, exception_stacks
|
||||
[(N_EXCEPTION_STACKS - 1) * EXCEPTION_STKSZ + DEBUG_STKSZ]);
|
||||
|
||||
/* May not be marked __init: used by software suspend */
|
||||
|
|
@ -1523,6 +1537,14 @@ void cpu_init(void)
|
|||
* try to read it.
|
||||
*/
|
||||
cr4_init_shadow();
|
||||
if (!kaiser_enabled) {
|
||||
/*
|
||||
* secondary_startup_64() deferred setting PGE in cr4:
|
||||
* probe_page_size_mask() sets it on the boot cpu,
|
||||
* but it needs to be set on each secondary cpu.
|
||||
*/
|
||||
cr4_set_bits(X86_CR4_PGE);
|
||||
}
|
||||
|
||||
/*
|
||||
* Load microcode on this cpu if a valid microcode is available.
|
||||
|
|
|
|||
|
|
@ -41,6 +41,7 @@
|
|||
#include <asm/pgalloc.h>
|
||||
#include <asm/setup.h>
|
||||
#include <asm/espfix.h>
|
||||
#include <asm/kaiser.h>
|
||||
|
||||
/*
|
||||
* Note: we only need 6*8 = 48 bytes for the espfix stack, but round
|
||||
|
|
@ -126,6 +127,15 @@ void __init init_espfix_bsp(void)
|
|||
/* Install the espfix pud into the kernel page directory */
|
||||
pgd_p = &init_level4_pgt[pgd_index(ESPFIX_BASE_ADDR)];
|
||||
pgd_populate(&init_mm, pgd_p, (pud_t *)espfix_pud_page);
|
||||
/*
|
||||
* Just copy the top-level PGD that is mapping the espfix
|
||||
* area to ensure it is mapped into the shadow user page
|
||||
* tables.
|
||||
*/
|
||||
if (kaiser_enabled) {
|
||||
set_pgd(native_get_shadow_pgd(pgd_p),
|
||||
__pgd(_KERNPG_TABLE | __pa((pud_t *)espfix_pud_page)));
|
||||
}
|
||||
|
||||
/* Randomize the locations */
|
||||
init_espfix_random();
|
||||
|
|
|
|||
|
|
@ -190,8 +190,8 @@ ENTRY(secondary_startup_64)
|
|||
movq $(init_level4_pgt - __START_KERNEL_map), %rax
|
||||
1:
|
||||
|
||||
/* Enable PAE mode and PGE */
|
||||
movl $(X86_CR4_PAE | X86_CR4_PGE), %ecx
|
||||
/* Enable PAE and PSE, but defer PGE until kaiser_enabled is decided */
|
||||
movl $(X86_CR4_PAE | X86_CR4_PSE), %ecx
|
||||
movq %rcx, %cr4
|
||||
|
||||
/* Setup early boot stage 4 level pagetables. */
|
||||
|
|
@ -405,6 +405,27 @@ GLOBAL(early_recursion_flag)
|
|||
.balign PAGE_SIZE; \
|
||||
GLOBAL(name)
|
||||
|
||||
#ifdef CONFIG_PAGE_TABLE_ISOLATION
|
||||
/*
|
||||
* Each PGD needs to be 8k long and 8k aligned. We do not
|
||||
* ever go out to userspace with these, so we do not
|
||||
* strictly *need* the second page, but this allows us to
|
||||
* have a single set_pgd() implementation that does not
|
||||
* need to worry about whether it has 4k or 8k to work
|
||||
* with.
|
||||
*
|
||||
* This ensures PGDs are 8k long:
|
||||
*/
|
||||
#define KAISER_USER_PGD_FILL 512
|
||||
/* This ensures they are 8k-aligned: */
|
||||
#define NEXT_PGD_PAGE(name) \
|
||||
.balign 2 * PAGE_SIZE; \
|
||||
GLOBAL(name)
|
||||
#else
|
||||
#define NEXT_PGD_PAGE(name) NEXT_PAGE(name)
|
||||
#define KAISER_USER_PGD_FILL 0
|
||||
#endif
|
||||
|
||||
/* Automate the creation of 1 to 1 mapping pmd entries */
|
||||
#define PMDS(START, PERM, COUNT) \
|
||||
i = 0 ; \
|
||||
|
|
@ -414,9 +435,10 @@ GLOBAL(name)
|
|||
.endr
|
||||
|
||||
__INITDATA
|
||||
NEXT_PAGE(early_level4_pgt)
|
||||
NEXT_PGD_PAGE(early_level4_pgt)
|
||||
.fill 511,8,0
|
||||
.quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE
|
||||
.fill KAISER_USER_PGD_FILL,8,0
|
||||
|
||||
NEXT_PAGE(early_dynamic_pgts)
|
||||
.fill 512*EARLY_DYNAMIC_PAGE_TABLES,8,0
|
||||
|
|
@ -424,16 +446,18 @@ NEXT_PAGE(early_dynamic_pgts)
|
|||
.data
|
||||
|
||||
#ifndef CONFIG_XEN
|
||||
NEXT_PAGE(init_level4_pgt)
|
||||
NEXT_PGD_PAGE(init_level4_pgt)
|
||||
.fill 512,8,0
|
||||
.fill KAISER_USER_PGD_FILL,8,0
|
||||
#else
|
||||
NEXT_PAGE(init_level4_pgt)
|
||||
NEXT_PGD_PAGE(init_level4_pgt)
|
||||
.quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
|
||||
.org init_level4_pgt + L4_PAGE_OFFSET*8, 0
|
||||
.quad level3_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
|
||||
.org init_level4_pgt + L4_START_KERNEL*8, 0
|
||||
/* (2^48-(2*1024*1024*1024))/(2^39) = 511 */
|
||||
.quad level3_kernel_pgt - __START_KERNEL_map + _PAGE_TABLE
|
||||
.fill KAISER_USER_PGD_FILL,8,0
|
||||
|
||||
NEXT_PAGE(level3_ident_pgt)
|
||||
.quad level2_ident_pgt - __START_KERNEL_map + _KERNPG_TABLE
|
||||
|
|
@ -444,6 +468,7 @@ NEXT_PAGE(level2_ident_pgt)
|
|||
*/
|
||||
PMDS(0, __PAGE_KERNEL_IDENT_LARGE_EXEC, PTRS_PER_PMD)
|
||||
#endif
|
||||
.fill KAISER_USER_PGD_FILL,8,0
|
||||
|
||||
NEXT_PAGE(level3_kernel_pgt)
|
||||
.fill L3_START_KERNEL,8,0
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue