svn commit: r335570 - in stable/11/sys/amd64: amd64 include
Konstantin Belousov
kib at FreeBSD.org
Fri Jun 22 23:52:23 UTC 2018
Author: kib
Date: Fri Jun 22 23:52:20 2018
New Revision: 335570
URL: https://svnweb.freebsd.org/changeset/base/335570
Log:
MFC r333059 (by tychon):
Expand the checks for UCR3 == PMAP_NO_CR3 to enable processes to be
excluded from PTI.
Modified:
stable/11/sys/amd64/amd64/cpu_switch.S
stable/11/sys/amd64/amd64/exception.S
stable/11/sys/amd64/amd64/genassym.c
stable/11/sys/amd64/amd64/machdep.c
stable/11/sys/amd64/amd64/mp_machdep.c
stable/11/sys/amd64/amd64/pmap.c
stable/11/sys/amd64/amd64/trap.c
stable/11/sys/amd64/include/asmacros.h
stable/11/sys/amd64/include/pcpu.h
Directory Properties:
stable/11/ (props changed)
Modified: stable/11/sys/amd64/amd64/cpu_switch.S
==============================================================================
--- stable/11/sys/amd64/amd64/cpu_switch.S Fri Jun 22 21:59:30 2018 (r335569)
+++ stable/11/sys/amd64/amd64/cpu_switch.S Fri Jun 22 23:52:20 2018 (r335570)
@@ -210,10 +210,13 @@ done_tss:
movq %r8,PCPU(RSP0)
movq %r8,PCPU(CURPCB)
/* Update the TSS_RSP0 pointer for the next interrupt */
- cmpb $0,pti(%rip)
- jne 1f
- movq %r8,TSS_RSP0(%rdx)
-1: movq %r12,PCPU(CURTHREAD) /* into next thread */
+ cmpq $~0,PCPU(UCR3)
+ je 1f
+ movq PCPU(PTI_RSP0),%rax
+ movq %rax,TSS_RSP0(%rdx)
+ jmp 2f
+1: movq %r8,TSS_RSP0(%rdx)
+2: movq %r12,PCPU(CURTHREAD) /* into next thread */
/* Test if debug registers should be restored. */
testl $PCB_DBREGS,PCB_FLAGS(%r8)
@@ -292,12 +295,7 @@ do_tss: movq %rdx,PCPU(TSSP)
shrq $8,%rcx
movl %ecx,8(%rax)
movb $0x89,5(%rax) /* unset busy */
- cmpb $0,pti(%rip)
- je 1f
- movq PCPU(PRVSPACE),%rax
- addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8,%rax
- movq %rax,TSS_RSP0(%rdx)
-1: movl $TSSSEL,%eax
+ movl $TSSSEL,%eax
ltr %ax
jmp done_tss
Modified: stable/11/sys/amd64/amd64/exception.S
==============================================================================
--- stable/11/sys/amd64/amd64/exception.S Fri Jun 22 21:59:30 2018 (r335569)
+++ stable/11/sys/amd64/amd64/exception.S Fri Jun 22 23:52:20 2018 (r335570)
@@ -298,11 +298,15 @@ IDTVEC(page_pti)
jz Xpage
swapgs
pushq %rax
- pushq %rdx
movq %cr3,%rax
movq %rax,PCPU(SAVED_UCR3)
+ cmpq $~0,PCPU(UCR3)
+ jne 1f
+ popq %rax
+ jmp 2f
+1: pushq %rdx
PTI_UUENTRY has_err=1
- subq $TF_ERR,%rsp
+2: subq $TF_ERR,%rsp
movq %rdi,TF_RDI(%rsp)
movq %rax,TF_RAX(%rsp)
movq %rdx,TF_RDX(%rsp)
@@ -347,9 +351,11 @@ page_cr2:
*/
.macro PROTF_ENTRY name,trapno
\name\()_pti_doreti:
+ swapgs
+ cmpq $~0,PCPU(UCR3)
+ je 1f
pushq %rax
pushq %rdx
- swapgs
movq PCPU(KCR3),%rax
movq %rax,%cr3
movq PCPU(RSP0),%rax
@@ -362,7 +368,7 @@ page_cr2:
movq %rax,%rsp
popq %rdx
popq %rax
- swapgs
+1: swapgs
jmp X\name
IDTVEC(\name\()_pti)
cmpq $doreti_iret,PTI_RIP-2*8(%rsp)
@@ -438,6 +444,8 @@ prot_addrf:
IDTVEC(fast_syscall_pti)
swapgs
movq %rax,PCPU(SCRATCH_RAX)
+ cmpq $~0,PCPU(UCR3)
+ je fast_syscall_common
movq PCPU(KCR3),%rax
movq %rax,%cr3
jmp fast_syscall_common
@@ -503,7 +511,7 @@ fast_syscall_common:
movq TF_RFLAGS(%rsp),%r11 /* original %rflags */
movq TF_RIP(%rsp),%rcx /* original %rip */
movq TF_RSP(%rsp),%rsp /* user stack pointer */
- cmpb $0,pti
+ cmpq $~0,PCPU(UCR3)
je 2f
movq PCPU(UCR3),%r9
movq %r9,%cr3
@@ -1126,11 +1134,11 @@ ld_regs:
jz 2f /* keep running with kernel GS.base */
cli
call handle_ibrs_exit_rs
- cmpb $0,pti
+ cmpq $~0,PCPU(UCR3)
je 1f
pushq %rdx
- movq PCPU(PRVSPACE),%rdx
- addq $PC_PTI_STACK+PC_PTI_STACK_SZ*8-PTI_SIZE,%rdx
+ movq PCPU(PTI_RSP0),%rdx
+ subq $PTI_SIZE,%rdx
movq %rax,PTI_RAX(%rdx)
popq %rax
movq %rax,PTI_RDX(%rdx)
Modified: stable/11/sys/amd64/amd64/genassym.c
==============================================================================
--- stable/11/sys/amd64/amd64/genassym.c Fri Jun 22 21:59:30 2018 (r335569)
+++ stable/11/sys/amd64/amd64/genassym.c Fri Jun 22 23:52:20 2018 (r335570)
@@ -229,6 +229,7 @@ ASSYM(PC_UCR3, offsetof(struct pcpu, pc_ucr3));
ASSYM(PC_SAVED_UCR3, offsetof(struct pcpu, pc_saved_ucr3));
ASSYM(PC_PTI_STACK, offsetof(struct pcpu, pc_pti_stack));
ASSYM(PC_PTI_STACK_SZ, PC_PTI_STACK_SZ);
+ASSYM(PC_PTI_RSP0, offsetof(struct pcpu, pc_pti_rsp0));
ASSYM(PC_IBPB_SET, offsetof(struct pcpu, pc_ibpb_set));
ASSYM(LA_EOI, LAPIC_EOI * LAPIC_MEM_MUL);
Modified: stable/11/sys/amd64/amd64/machdep.c
==============================================================================
--- stable/11/sys/amd64/amd64/machdep.c Fri Jun 22 21:59:30 2018 (r335569)
+++ stable/11/sys/amd64/amd64/machdep.c Fri Jun 22 23:52:20 2018 (r335570)
@@ -1817,9 +1817,10 @@ hammer_time(u_int64_t modulep, u_int64_t physfree)
rsp0 = (vm_offset_t)thread0.td_pcb;
/* Ensure the stack is aligned to 16 bytes */
rsp0 &= ~0xFul;
- common_tss[0].tss_rsp0 = pti ? ((vm_offset_t)PCPU_PTR(pti_stack) +
- PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful : rsp0;
+ common_tss[0].tss_rsp0 = rsp0;
PCPU_SET(rsp0, rsp0);
+ PCPU_SET(pti_rsp0, ((vm_offset_t)PCPU_PTR(pti_stack) +
+ PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful);
PCPU_SET(curpcb, thread0.td_pcb);
/* transfer to user mode */
Modified: stable/11/sys/amd64/amd64/mp_machdep.c
==============================================================================
--- stable/11/sys/amd64/amd64/mp_machdep.c Fri Jun 22 21:59:30 2018 (r335569)
+++ stable/11/sys/amd64/amd64/mp_machdep.c Fri Jun 22 23:52:20 2018 (r335570)
@@ -255,6 +255,8 @@ init_secondary(void)
pc->pc_tssp = &common_tss[cpu];
pc->pc_commontssp = &common_tss[cpu];
pc->pc_rsp0 = 0;
+ pc->pc_pti_rsp0 = ((vm_offset_t)&pc->pc_pti_stack +
+ PC_PTI_STACK_SZ * sizeof(uint64_t) & ~0xful);
pc->pc_tss = (struct system_segment_descriptor *)&gdt[NGDT * cpu +
GPROC0_SEL];
pc->pc_fs32p = &gdt[NGDT * cpu + GUFS32_SEL];
@@ -264,8 +266,7 @@ init_secondary(void)
pc->pc_curpmap = kernel_pmap;
pc->pc_pcid_gen = 1;
pc->pc_pcid_next = PMAP_PCID_KERN + 1;
- common_tss[cpu].tss_rsp0 = pti ? ((vm_offset_t)&pc->pc_pti_stack +
- PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful : 0;
+ common_tss[cpu].tss_rsp0 = 0;
/* Save the per-cpu pointer for use by the NMI handler. */
np = ((struct nmi_pcpu *) &nmi_stack[PAGE_SIZE]) - 1;
Modified: stable/11/sys/amd64/amd64/pmap.c
==============================================================================
--- stable/11/sys/amd64/amd64/pmap.c Fri Jun 22 21:59:30 2018 (r335569)
+++ stable/11/sys/amd64/amd64/pmap.c Fri Jun 22 23:52:20 2018 (r335570)
@@ -2560,8 +2560,10 @@ pmap_pinit0(pmap_t pmap)
CPU_FOREACH(i) {
pmap->pm_pcids[i].pm_pcid = PMAP_PCID_NONE;
pmap->pm_pcids[i].pm_gen = 0;
- if (!pti)
+ if (!pti) {
__pcpu[i].pc_kcr3 = PMAP_NO_CR3;
+ __pcpu[i].pc_ucr3 = PMAP_NO_CR3;
+ }
}
PCPU_SET(curpmap, kernel_pmap);
pmap_activate(curthread);
@@ -2736,7 +2738,8 @@ _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, str
* the kernel-mode page table active on return
* to user space.
*/
- *pml4 |= pg_nx;
+ if (pmap->pm_ucr3 != PMAP_NO_CR3)
+ *pml4 |= pg_nx;
pml4u = &pmap->pm_pml4u[pml4index];
*pml4u = VM_PAGE_TO_PHYS(m) | PG_U | PG_RW | PG_V |
@@ -7337,9 +7340,10 @@ pmap_activate_sw(struct thread *td)
{
pmap_t oldpmap, pmap;
struct invpcid_descr d;
- uint64_t cached, cr3, kcr3, kern_pti_cached, ucr3;
+ uint64_t cached, cr3, kcr3, kern_pti_cached, rsp0, ucr3;
register_t rflags;
u_int cpuid;
+ struct amd64tss *tssp;
oldpmap = PCPU_GET(curpmap);
pmap = vmspace_pmap(td->td_proc->p_vmspace);
@@ -7429,6 +7433,12 @@ pmap_activate_sw(struct thread *td)
PCPU_SET(kcr3, pmap->pm_cr3);
PCPU_SET(ucr3, pmap->pm_ucr3);
}
+ }
+ if (pmap->pm_ucr3 != PMAP_NO_CR3) {
+ rsp0 = ((vm_offset_t)PCPU_PTR(pti_stack) +
+ PC_PTI_STACK_SZ * sizeof(uint64_t)) & ~0xful;
+ tssp = PCPU_GET(tssp);
+ tssp->tss_rsp0 = rsp0;
}
#ifdef SMP
CPU_CLR_ATOMIC(cpuid, &oldpmap->pm_active);
Modified: stable/11/sys/amd64/amd64/trap.c
==============================================================================
--- stable/11/sys/amd64/amd64/trap.c Fri Jun 22 21:59:30 2018 (r335569)
+++ stable/11/sys/amd64/amd64/trap.c Fri Jun 22 23:52:20 2018 (r335570)
@@ -463,11 +463,13 @@ trap(struct trapframe *frame)
*/
if (frame->tf_rip == (long)doreti_iret) {
frame->tf_rip = (long)doreti_iret_fault;
- if (pti && frame->tf_rsp == (uintptr_t)PCPU_PTR(
- pti_stack) + (PC_PTI_STACK_SZ - 5) *
- sizeof(register_t))
+ if ((PCPU_GET(curpmap)->pm_ucr3 !=
+ PMAP_NO_CR3) &&
+ (frame->tf_rsp == (uintptr_t)PCPU_GET(
+ pti_rsp0) - 5 * sizeof(register_t))) {
frame->tf_rsp = PCPU_GET(rsp0) - 5 *
sizeof(register_t);
+ }
return;
}
if (frame->tf_rip == (long)ld_ds) {
Modified: stable/11/sys/amd64/include/asmacros.h
==============================================================================
--- stable/11/sys/amd64/include/asmacros.h Fri Jun 22 21:59:30 2018 (r335569)
+++ stable/11/sys/amd64/include/asmacros.h Fri Jun 22 23:52:20 2018 (r335570)
@@ -194,9 +194,12 @@
.macro PTI_UENTRY has_err
swapgs
+ cmpq $~0,PCPU(UCR3)
+ je 1f
pushq %rax
pushq %rdx
PTI_UUENTRY \has_err
+1:
.endm
.macro PTI_ENTRY name, cont, has_err=0
Modified: stable/11/sys/amd64/include/pcpu.h
==============================================================================
--- stable/11/sys/amd64/include/pcpu.h Fri Jun 22 21:59:30 2018 (r335569)
+++ stable/11/sys/amd64/include/pcpu.h Fri Jun 22 23:52:20 2018 (r335570)
@@ -67,13 +67,14 @@
u_int pc_cmci_mask; /* MCx banks for CMCI */ \
uint64_t pc_dbreg[16]; /* ddb debugging regs */ \
uint64_t pc_pti_stack[PC_PTI_STACK_SZ]; \
+ register_t pc_pti_rsp0; \
int pc_dbreg_cmd; /* ddb debugging reg cmd */ \
u_int pc_vcpu_id; /* Xen vCPU ID */ \
uint32_t pc_pcid_next; \
uint32_t pc_pcid_gen; \
uint32_t pc_smp_tlb_done; /* TLB op acknowledgement */ \
uint32_t pc_ibpb_set; \
- char __pad[96] /* be divisor of PAGE_SIZE \
+ char __pad[88] /* be divisor of PAGE_SIZE \
after cache alignment */
#define PC_DBREG_CMD_NONE 0
More information about the svn-src-stable-11
mailing list