pvm: Remove MSR_PVM_SUPERVISOR_RSP and PVCS:rsp

When supervisor event delivering is changed to use PVCS instead of the current
stack, and a bit is added in event_vector to protect the supervisor's access
to the PVCS from async events reentrance, %rsp doesn't require to be supervisor
RSP in the process and %rsp can be keept as the value before the event, so
MSR_PVM_SUPERVISOR_RSP and PVCS:rsp are unneeded.

Signed-off-by: Lai Jiangshan <jiangshan.ljs@antgroup.com>

Author: Lai Jiangshan

arch/x86/entry/entry_64_pvm.S

@@ -9,15 +9,29 @@
 
 /* Construct struct pt_regs on stack */
 .macro PUSH_IRET_FRAME_FROM_PVCS user:req
+	movq	%rsp,	%r11
 	.if \user == 1
+		movq	PER_CPU_VAR(pcpu_hot + X86_top_of_stack), %rsp
+
 		movl	PER_CPU_VAR(pvm_vcpu_struct + PVCS_user_ss), %ecx
 		andl	$0xFFFF, %ecx
 		pushq	%rcx				/* pt_regs->ss */
 	.else
+		andq	$-16, %rsp
+
+		/*
+		 * Reserve a fixed-size area in the current stack during an
+		 * event from supervisor mode. This is for the int3 handler
+		 * to emulate a call instruction.
+		 */
+		subq	$16, %rsp
+
+		/* TODO: check stack overflow */
+
 		pushq	$__KERNEL_DS
 	.endif
 
-	pushq	PER_CPU_VAR(pvm_vcpu_struct + PVCS_rsp) /* pt_regs->sp */
+	pushq	%r11					/* pt_regs->sp */
 	movl	PER_CPU_VAR(pvm_vcpu_struct + PVCS_eflags), %ecx
 	pushq	%rcx					/* pt_regs->flags */
 
@@ -67,10 +81,10 @@
 	movw	%cx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_user_cs)
 	movq	3*8(%rsp), %rcx		/* RFLAGS */
 	movl	%ecx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_eflags)
-	movq	4*8(%rsp), %rcx		/* RSP */
-	movq	%rcx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_rsp)
 	movq	5*8(%rsp), %rcx		/* SS */
 	movw	%cx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_user_ss)
+
+	movq	4*8(%rsp), %rsp		/* RSP */
 .endm
 
 .code64
@@ -143,15 +157,8 @@ SYM_CODE_START(pvm_user_event_entry)
 	PUSH_REGS_AND_HANDLE_EVENT handler=pvm_event
 
 SYM_INNER_LABEL(pvm_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
+	STACKLEAK_ERASE
 	POP_REGS_AND_LOAD_IRET_FRAME_INTO_PVCS
-
-	/*
-	 * We are on the trampoline stack.  All regs are live.
-	 * We can do future final exit work right here.
-	 */
-	STACKLEAK_ERASE_NOCLOBBER
-
-	addq	$6*8, %rsp
 SYM_INNER_LABEL(pvm_retu_rip, SYM_L_GLOBAL)
 	ANNOTATE_NOENDBR
 	syscall
@@ -166,19 +173,9 @@ SYM_CODE_START(pvm_kernel_event_entry)
 	UNWIND_HINT_ENTRY
 	ENDBR
 
-	/*
-	 * Reserve a fixed-size area in the current stack during an event from
-	 * supervisor mode. This is for the int3 handler to emulate a call instruction.
-	 */
-	subq	$16, %rsp
-
-	/* TODO: check stack overflow */
-
 	PUSH_IRET_FRAME_FROM_PVCS user=0
 	PUSH_REGS_AND_HANDLE_EVENT handler=pvm_event
 	POP_REGS_AND_LOAD_IRET_FRAME_INTO_PVCS
-
-	addq	$(6*8+16), %rsp
 SYM_INNER_LABEL(pvm_rets_rip, SYM_L_GLOBAL)
 	ANNOTATE_NOENDBR
 	syscall
@@ -194,7 +191,6 @@ SYM_CODE_START_NOALIGN(pvm_early_kernel_event_entry)
 	PUSH_REGS_AND_HANDLE_EVENT handler=pvm_early_event
 	POP_REGS_AND_LOAD_IRET_FRAME_INTO_PVCS
 	decl	early_recursion_flag(%rip)
-	addq	$6*8, %rsp
 	jmp	pvm_rets_rip
 SYM_CODE_END(pvm_early_kernel_event_entry)
 .popsection

arch/x86/entry/entry_64_switcher.S

@@ -147,17 +147,13 @@ SYM_INNER_LABEL(entry_SYSCALL_64_switcher_safe_stack, SYM_L_GLOBAL)
 	movq	%rcx, PVCS_rip(%rdi)
 	movq	%rcx, PVCS_rcx(%rdi)
 	movq	%r11, PVCS_r11(%rdi)
-	movq	RSP-ORIG_RAX(%rsp), %rcx
-	movq	%rcx, PVCS_rsp(%rdi)
 
 	/* switch umod to smod (switch_flags & cr3) */
 	xorb	$SWITCH_FLAGS_MOD_TOGGLE, TSS_extra(switch_flags)
 	movq	TSS_extra(smod_cr3), %rcx
 	movq	%rcx, %cr3
 
 	/* load smod registers from TSS_extra to sp0 stack or %r11 */
-	movq	TSS_extra(smod_rsp), %rcx
-	movq	%rcx, RSP-ORIG_RAX(%rsp)
 	movq	TSS_extra(smod_entry), %rcx
 	movq	%rcx, RIP-ORIG_RAX(%rsp)
 	movq	TSS_extra(smod_gsbase), %r11
@@ -203,10 +199,6 @@ SYM_INNER_LABEL(entry_SYSRETQ_switcher_unsafe_stack, SYM_L_GLOBAL)
 	cmpl	$((__USER_DS << 16) | __USER_CS), PVCS_user_cs(%rdi)
 	jne	.L_switcher_return_to_hypervisor
 
-	/* Switcher and the PVM specification requires the smod RSP to be saved */
-	movq	RSP-ORIG_RAX(%rsp), %rcx
-	movq	%rcx, TSS_extra(smod_rsp)
-
 	/* switch smod to umod (switch_flags & cr3) */
 	xorb	$SWITCH_FLAGS_MOD_TOGGLE, TSS_extra(switch_flags)
 	movq	TSS_extra(umod_cr3), %rcx
@@ -220,9 +212,7 @@ SYM_INNER_LABEL(entry_SYSRETQ_switcher_unsafe_stack, SYM_L_GLOBAL)
 	canonical_rcx
 	wrgsbase %rcx
 
-	/* load sp, flags, ip to sp0 stack and cx, r11, rdi to registers */
-	movq	PVCS_rsp(%rdi), %rcx
-	movq	%rcx, RSP-ORIG_RAX(%rsp)
+	/* load flags, ip to sp0 stack and cx, r11, rdi to registers */
 	movl	PVCS_eflags(%rdi), %r11d
 	movq	%r11, EFLAGS-ORIG_RAX(%rsp)
 	movq	PVCS_rip(%rdi), %rcx

arch/x86/include/asm/switcher.h

@@ -105,7 +105,6 @@ struct tss_extra {
 	unsigned long retu_rip;
 	unsigned long smod_entry;
 	unsigned long smod_gsbase;
-	unsigned long smod_rsp;
 } ____cacheline_aligned;
 
 extern struct pt_regs *switcher_enter_guest(void);

arch/x86/include/uapi/asm/pvm_para.h

@@ -37,7 +37,7 @@
 #define MSR_PVM_LINEAR_ADDRESS_RANGE	0x4b564df0
 #define MSR_PVM_VCPU_STRUCT		0x4b564df1
 #define MSR_PVM_SWITCH_CR3		0x4b564df2
-#define MSR_PVM_SUPERVISOR_RSP		0x4b564df3
+// #define MSR_PVM_SUPERVISOR_RSP	0x4b564df3 // deprecated, FIXME: reordering when sending v2
 #define MSR_PVM_EVENT_ENTRY		0x4b564df4
 #define MSR_PVM_RETU_RIP		0x4b564df5
 #define MSR_PVM_RETS_RIP		0x4b564df6
@@ -136,14 +136,13 @@ struct pvm_vcpu_struct {
 	u16 user_cs, user_ss;
 	u16 event_errcode;
 	u16 event_vector;
-	u64 reserved2;
 	u64 user_gsbase;
 	u32 eflags;
 	u32 pkru;
 	u64 rip;
-	u64 rsp;
 	u64 rcx;
 	u64 r11;
+	u64 reserved1[2];
 };
 
 #endif /* __ASSEMBLY__ */

arch/x86/kernel/asm-offsets_64.c

@@ -73,7 +73,6 @@ int main(void)
 	ENTRY(retu_rip);
 	ENTRY(smod_entry);
 	ENTRY(smod_gsbase);
-	ENTRY(smod_rsp);
 	BLANK();
 #undef ENTRY
 
@@ -84,7 +83,6 @@ int main(void)
 	ENTRY(user_cs);
 	ENTRY(user_ss);
 	ENTRY(user_gsbase);
-	ENTRY(rsp);
 	ENTRY(eflags);
 	ENTRY(rip);
 	ENTRY(rcx);

arch/x86/kvm/pvm/pvm.c

@@ -342,7 +342,6 @@ static inline void switch_to_smod(struct kvm_vcpu *vcpu)
 	swap(pvm->msr_switch_cr3, vcpu->arch.cr3);
 
 	pvm_write_guest_gs_base(pvm, pvm->msr_kernel_gs_base);
-	kvm_rsp_write(vcpu, pvm->msr_supervisor_rsp);
 
 	pvm->hw_cs = __USER_CS;
 	pvm->hw_ss = __USER_DS;
@@ -352,8 +351,6 @@ static inline void switch_to_umod(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_pvm *pvm = to_pvm(vcpu);
 
-	pvm->msr_supervisor_rsp = kvm_rsp_read(vcpu);
-
 	pvm_switch_flags_toggle_mod(pvm);
 	kvm_mmu_new_pgd(vcpu, pvm->msr_switch_cr3);
 	swap(pvm->msr_switch_cr3, vcpu->arch.cr3);
@@ -1101,9 +1098,6 @@ static int pvm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_PVM_VCPU_STRUCT:
 		msr_info->data = pvm->msr_vcpu_struct;
 		break;
-	case MSR_PVM_SUPERVISOR_RSP:
-		msr_info->data = pvm->msr_supervisor_rsp;
-		break;
 	case MSR_PVM_EVENT_ENTRY:
 		msr_info->data = pvm->msr_event_entry;
 		break;
@@ -1251,9 +1245,6 @@ static int pvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 				kvm_make_request(KVM_REQ_GPC_REFRESH, vcpu);
 		}
 		break;
-	case MSR_PVM_SUPERVISOR_RSP:
-		pvm->msr_supervisor_rsp = msr_info->data;
-		break;
 	case MSR_PVM_EVENT_ENTRY:
 		if (is_noncanonical_address(data, vcpu) ||
 		    is_noncanonical_address(data + 256, vcpu) ||
@@ -1565,7 +1556,6 @@ static int __do_pvm_event(struct kvm_vcpu *vcpu, bool user, int vector,
 			  bool has_err_code, u64 err_code)
 {
 	struct vcpu_pvm *pvm = to_pvm(vcpu);
-	unsigned long rsp = kvm_rsp_read(vcpu);
 	unsigned long entry;
 	struct pvm_vcpu_struct *pvcs;
 
@@ -1622,7 +1612,6 @@ static int __do_pvm_event(struct kvm_vcpu *vcpu, bool user, int vector,
 
 	pvcs->eflags = kvm_get_rflags(vcpu);
 	pvcs->rip = kvm_rip_read(vcpu);
-	pvcs->rsp = rsp;
 	pvcs->rcx = kvm_rcx_read(vcpu);
 	pvcs->r11 = kvm_r11_read(vcpu);
 
@@ -1643,8 +1632,6 @@ static int __do_pvm_event(struct kvm_vcpu *vcpu, bool user, int vector,
 
 	if (user)
 		switch_to_smod(vcpu);
-	else
-		kvm_rsp_write(vcpu, rsp & ~15UL);
 
 	if (vector == PVM_SYSCALL_VECTOR)
 		entry = pvm->msr_lstar;
@@ -1839,7 +1826,6 @@ static int handle_synthetic_instruction_return(struct kvm_vcpu *vcpu, bool user)
 		pvcs->event_vector = 0;
 
 	kvm_rip_write(vcpu, pvcs->rip);
-	kvm_rsp_write(vcpu, pvcs->rsp);
 	kvm_rcx_write(vcpu, pvcs->rcx);
 	kvm_r11_write(vcpu, pvcs->r11);
 	rflags = pvcs->eflags;
@@ -2656,18 +2642,15 @@ static noinstr void pvm_vcpu_run_noinstr(struct kvm_vcpu *vcpu)
 	tss_ex->retu_rip = pvm->msr_retu_rip_plus2;
 	tss_ex->smod_entry = pvm->msr_lstar;
 	tss_ex->smod_gsbase = pvm->msr_kernel_gs_base;
-	tss_ex->smod_rsp = pvm->msr_supervisor_rsp;
 
 	if (unlikely(pvm->guest_dr7 & DR7_BP_EN_MASK))
 		set_debugreg(pvm_eff_dr7(vcpu), 7);
 
 	// Call into switcher and enter guest.
 	ret_regs = switcher_enter_guest();
 
-	// Get the resulted mode and PVM MSRs which might be changed
-	// when direct switching.
+	// Get the resulted mode
 	pvm->switch_flags = tss_ex->switch_flags;
-	pvm->msr_supervisor_rsp = tss_ex->smod_rsp;
 
 	// Get the guest registers from the host sp0 stack.
 	save_regs(vcpu, ret_regs);
@@ -2901,7 +2884,6 @@ static void pvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
 	memset(&pvm->tls_array[0], 0, sizeof(pvm->tls_array));
 
 	pvm->msr_vcpu_struct = 0;
-	pvm->msr_supervisor_rsp = 0;
 	pvm->msr_event_entry = 0;
 	pvm->msr_retu_rip_plus2 = 0;
 	pvm->msr_rets_rip_plus2 = 0;

arch/x86/kvm/pvm/pvm.h

@@ -134,7 +134,6 @@ struct vcpu_pvm {
 
 	// PVM paravirt MSRs
 	unsigned long msr_vcpu_struct;
-	unsigned long msr_supervisor_rsp;
 	unsigned long msr_event_entry;
 	unsigned long msr_retu_rip_plus2;
 	unsigned long msr_rets_rip_plus2;

arch/x86/kvm/x86.c

@@ -1529,7 +1529,6 @@ static const u32 emulated_msrs_all[] = {
 	MSR_PVM_LINEAR_ADDRESS_RANGE,
 	MSR_PVM_VCPU_STRUCT,
 	MSR_PVM_SWITCH_CR3,
-	MSR_PVM_SUPERVISOR_RSP,
 	MSR_PVM_EVENT_ENTRY,
 	MSR_PVM_RETU_RIP,
 	MSR_PVM_RETS_RIP,