PVM: remove synthetic RETS instruction

and remove MSR_PVM_RETS_RIP

Signed-off-by: Lai Jiangshan <jiangshan.ljs@antgroup.com>

Author: Lai Jiangshan

arch/x86/entry/entry_64_pvm.S

@@ -7,6 +7,8 @@
 
 #include "calling.h"
 
+#define REDZONE_SIZE	16
+
 /* Construct struct pt_regs on stack */
 .macro PUSH_IRET_FRAME_FROM_PVCS user:req
 	movq	%rsp,	%r11
@@ -24,7 +26,7 @@
 		 * event from supervisor mode. This is for the int3 handler
 		 * to emulate a call instruction.
 		 */
-		subq	$16, %rsp
+		subq	$REDZONE_SIZE, %rsp
 
 		/* TODO: check stack overflow */
 
@@ -174,11 +176,67 @@ SYM_CODE_START(pvm_kernel_event_entry)
 	ENDBR
 
 	PUSH_IRET_FRAME_FROM_PVCS user=0
+
+	// if RIP < .L_pvm_supervisor_iret_irq_enabled_start ==> skip fixup
+	leaq	.L_pvm_supervisor_iret_irq_enabled_start(%rip), %rcx
+	cmpq	RIP-RIP(%rsp), %rcx
+	ja	.L_pvm_supervisor_iret_irq_fixed
+
+	// if RIP >= .L_pvm_supervisor_iret_irq_enabled_end ==> skip fixup
+	leaq	.L_pvm_supervisor_iret_irq_enabled_end(%rip), %rcx
+	cmpq	RIP-RIP(%rsp), %rcx
+	jbe	.L_pvm_supervisor_iret_irq_fixed
+
+	// fixup to Previous IRET Frame
+	//	Previous IRET Frame
+	//	RAX
+	//	RCX
+	//	R11
+	//	HW_ring3 IRET Frame	<-- origin rsp
+	//	padding for alignment (8 bytes)
+	//	RED_ZONE (16 bytes)
+	//	Current IRET Frame
+	movq	RSP-RIP(%rsp), %rcx
+	movq	(16+40)(%rcx), %rax
+	movq	( 8+40)(%rcx), %r11
+	movq	%r11, PER_CPU_VAR(pvm_vcpu_struct + PVCS_rcx)
+	movq	( 0+40)(%rcx), %r11
+	leaq	(24+40)(%rcx), %rsp
+.L_pvm_supervisor_iret_irq_fixed:
+	movq	PER_CPU_VAR(pvm_vcpu_struct + PVCS_rcx), %rcx
+
 	PUSH_REGS_AND_HANDLE_EVENT handler=pvm_event
-	POP_REGS_AND_LOAD_IRET_FRAME_INTO_PVCS
-SYM_INNER_LABEL(pvm_rets_rip, SYM_L_GLOBAL)
-	ANNOTATE_NOENDBR
+
+.L_pvm_supervisor_restore_regs_and_return:
+	POP_REGS
+	movq	%rax, (%rsp)
+	pushq	%rcx
+	pushq	%r11
+	mov	%ss, %rax
+	pushq	%rax		/* current hw SS for supervisor_iret */
+	pushq	8*7(%rsp)	/* RSP */
+	pushq	8*7(%rsp)	/* RFLAGS */
+	mov	%cs, %rax
+	pushq	%rax		/* current hw CS for supervisor_iret */
+	pushq	8*7(%rsp)	/* RIP */
+	testl	$X86_EFLAGS_IF, EFLAGS-RIP(%rsp)
+	jz	.L_pvm_supervisor_iret
+	orq	$X86_EFLAGS_IF, PER_CPU_VAR(pvm_vcpu_struct + PVCS_event_flags)
+.L_pvm_supervisor_iret_irq_enabled_start:
+	btq	$PVM_EVENT_FLAGS_IP_BIT, PER_CPU_VAR(pvm_vcpu_struct + PVCS_event_flags)
+	jc	.L_pvm_supervisor_iret_irq_pending
+.L_pvm_supervisor_iret:
+	movq	8*7(%rsp), %rax
+	iretq
+.L_pvm_supervisor_iret_irq_pending:
+	/* handle pending IRQ */
+	movq	$PVM_HC_IRQ_WIN, %rax
 	syscall
+	// normally don't reach here, but it can only happend when there is actually no IRQ pending
+	movq	8*5(%rsp), %r11
+	movq	8*6(%rsp), %rcx
+	jmp	.L_pvm_supervisor_iret
+.L_pvm_supervisor_iret_irq_enabled_end:
 SYM_CODE_END(pvm_kernel_event_entry)
 
 .pushsection .head.text, "ax"
@@ -189,8 +247,7 @@ SYM_CODE_START_NOALIGN(pvm_early_kernel_event_entry)
 	incl	early_recursion_flag(%rip)
 	PUSH_IRET_FRAME_FROM_PVCS user=0
 	PUSH_REGS_AND_HANDLE_EVENT handler=pvm_early_event
-	POP_REGS_AND_LOAD_IRET_FRAME_INTO_PVCS
 	decl	early_recursion_flag(%rip)
-	jmp	pvm_rets_rip
+	jmp	.L_pvm_supervisor_restore_regs_and_return
 SYM_CODE_END(pvm_early_kernel_event_entry)
 .popsection

arch/x86/include/asm/pvm_para.h

@@ -91,7 +91,6 @@ void entry_SYSCALL_64_pvm(void);
 void pvm_user_event_entry(void);
 void pvm_hypercall(void);
 void pvm_retu_rip(void);
-void pvm_rets_rip(void);
 void pvm_save_fl(void);
 void pvm_irq_disable(void);
 void pvm_irq_enable(void);

arch/x86/include/uapi/asm/pvm_para.h

@@ -34,7 +34,7 @@
 // #define MSR_PVM_SUPERVISOR_RSP	0x4b564df3 // deprecated, FIXME: reordering when sending v2
 #define MSR_PVM_EVENT_ENTRY		0x4b564df4
 #define MSR_PVM_RETU_RIP		0x4b564df5
-#define MSR_PVM_RETS_RIP		0x4b564df6
+// #define MSR_PVM_RETS_RIP		0x4b564df6
 
 #define PVM_HC_SPECIAL_MAX_NR		(256)
 #define PVM_HC_SPECIAL_BASE		(0x17088200)

arch/x86/kernel/pvm.c

@@ -496,7 +496,6 @@ void __init pvm_early_setup(void)
 
 	wrmsrl(MSR_PVM_VCPU_STRUCT, __pa(this_cpu_ptr(&pvm_vcpu_struct)));
 	wrmsrl(MSR_PVM_EVENT_ENTRY, (unsigned long)(void *)pvm_early_kernel_event_entry - 512);
-	wrmsrl(MSR_PVM_RETS_RIP, (unsigned long)(void *)pvm_rets_rip);
 
 	pvm_early_patch();
 }
@@ -509,7 +508,6 @@ void pvm_setup_event_handling(void)
 		wrmsrl(MSR_PVM_VCPU_STRUCT, xpa);
 		wrmsrl(MSR_PVM_EVENT_ENTRY, (unsigned long)(void *)pvm_user_event_entry);
 		wrmsrl(MSR_PVM_RETU_RIP, (unsigned long)(void *)pvm_retu_rip);
-		wrmsrl(MSR_PVM_RETS_RIP, (unsigned long)(void *)pvm_rets_rip);
 
 		/*
 		 * PVM spec requires the hypervisor-maintained

arch/x86/kvm/pvm/pvm.c

@@ -1084,9 +1084,6 @@ static int pvm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_PVM_RETU_RIP:
 		msr_info->data = pvm->msr_retu_rip_plus2 - 2;
 		break;
-	case MSR_PVM_RETS_RIP:
-		msr_info->data = pvm->msr_rets_rip_plus2 - 2;
-		break;
 	case MSR_PVM_LINEAR_ADDRESS_RANGE:
 		msr_info->data = pvm->msr_linear_address_range;
 		break;
@@ -1234,9 +1231,6 @@ static int pvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_PVM_RETU_RIP:
 		pvm->msr_retu_rip_plus2 = msr_info->data + 2;
 		break;
-	case MSR_PVM_RETS_RIP:
-		pvm->msr_rets_rip_plus2 = msr_info->data + 2;
-		break;
 	case MSR_PVM_LINEAR_ADDRESS_RANGE:
 		if (!pvm_check_and_set_msr_linear_address_range(pvm, msr_info->data))
 			return 1;
@@ -1573,7 +1567,7 @@ static int __do_pvm_event(struct kvm_vcpu *vcpu, bool user, int vector,
 		 * the hypervisor to meet the requirement stipulated above in
 		 * case it is on the path to ERETU.
 		 *
-		 * When forced back to handle_synthetic_instruction_return(),
+		 * When forced back to handle_synthetic_instruction_return_user(),
 		 * SWITCH_FLAGS_IRQ_WIN will be cleared in kvm_set_rflags() or
 		 * unhandled NMI/MCE will be reinjected.
 		 */
@@ -1776,16 +1770,15 @@ static void pvm_setup_mce(struct kvm_vcpu *vcpu)
 {
 }
 
-static int handle_synthetic_instruction_return(struct kvm_vcpu *vcpu, bool user)
+static int handle_synthetic_instruction_return_user(struct kvm_vcpu *vcpu)
 {
 	struct vcpu_pvm *pvm = to_pvm(vcpu);
 	struct pvm_vcpu_struct *pvcs;
 	unsigned long rflags;
 	u32 pending_async_exceptions;
 
 	/* switch to user mode before rsp changed. */
-	if (user)
-		switch_to_umod(vcpu);
+	switch_to_umod(vcpu);
 
 	pvcs = pvm_get_vcpu_struct(pvm);
 	if (!pvcs) {
@@ -1794,21 +1787,16 @@ static int handle_synthetic_instruction_return(struct kvm_vcpu *vcpu, bool user)
 	}
 
 	pending_async_exceptions = pvcs->event_vector;
-	if (user)
-		pvcs->event_vector = PVM_PVCS_EVENT_VECTOR_STD;
-	else
-		pvcs->event_vector = 0;
+	pvcs->event_vector = PVM_PVCS_EVENT_VECTOR_STD; // Clear other bits
 
 	kvm_rip_write(vcpu, pvcs->rip);
 	kvm_rcx_write(vcpu, pvcs->rcx);
 	kvm_r11_write(vcpu, pvcs->r11);
 	rflags = pvcs->eflags;
 
-	if (user) {
-		pvm->hw_cs = pvcs->user_cs | USER_RPL;
-		pvm->hw_ss = pvcs->user_ss | USER_RPL;
-		pvm_write_guest_gs_base(pvm, pvcs->user_gsbase);
-	}
+	pvm->hw_cs = pvcs->user_cs | USER_RPL;
+	pvm->hw_ss = pvcs->user_ss | USER_RPL;
+	pvm_write_guest_gs_base(pvm, pvcs->user_gsbase);
 
 	pvm_put_vcpu_struct(pvm, true);
 
@@ -1819,9 +1807,9 @@ static int handle_synthetic_instruction_return(struct kvm_vcpu *vcpu, bool user)
 	kvm_set_rflags(vcpu, rflags);
 
 	if (pending_async_exceptions & PVM_PVCS_EVENT_VECTOR_MCE)
-		__do_pvm_event(vcpu, user, MC_VECTOR, false, 0);
+		do_pvm_event(vcpu, MC_VECTOR, false, 0);
 	if (pending_async_exceptions & PVM_PVCS_EVENT_VECTOR_NMI)
-		__do_pvm_event(vcpu, user, NMI_VECTOR, false, 0);
+		do_pvm_event(vcpu, NMI_VECTOR, false, 0);
 
 	return 1;
 }
@@ -2094,9 +2082,7 @@ static int handle_exit_syscall(struct kvm_vcpu *vcpu)
 		return __do_pvm_event(vcpu, true, PVM_SYSCALL_VECTOR, false, 0);
 
 	if (rip == pvm->msr_retu_rip_plus2)
-		return handle_synthetic_instruction_return(vcpu, true);
-	if (rip == pvm->msr_rets_rip_plus2)
-		return handle_synthetic_instruction_return(vcpu, false);
+		return handle_synthetic_instruction_return_user(vcpu);
 
 	a0 = kvm_rbx_read(vcpu);
 	a1 = kvm_r10_read(vcpu);
@@ -2398,8 +2384,6 @@ static u32 pvm_get_syscall_exit_reason(struct kvm_vcpu *vcpu)
 	if (is_smod(pvm)) {
 		if (rip == pvm->msr_retu_rip_plus2)
 			return PVM_EXIT_REASONS_ERETU;
-		else if (rip == pvm->msr_rets_rip_plus2)
-			return PVM_EXIT_REASONS_ERETS;
 		else
 			return PVM_EXIT_REASONS_HYPERCALL;
 	}
@@ -2853,7 +2837,6 @@ static void pvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
 	pvm->msr_vcpu_struct = 0;
 	pvm->msr_event_entry = 0;
 	pvm->msr_retu_rip_plus2 = 0;
-	pvm->msr_rets_rip_plus2 = 0;
 	pvm_set_default_msr_linear_address_range(pvm);
 }
 

arch/x86/kvm/pvm/pvm.h

@@ -29,9 +29,8 @@
 #define PVM_EXIT_REASONS_SYSCALL	(1UL << PVM_EXIT_REASONS_SHIFT)
 #define PVM_EXIT_REASONS_HYPERCALL	(2UL << PVM_EXIT_REASONS_SHIFT)
 #define PVM_EXIT_REASONS_ERETU		(3UL << PVM_EXIT_REASONS_SHIFT)
-#define PVM_EXIT_REASONS_ERETS		(4UL << PVM_EXIT_REASONS_SHIFT)
-#define PVM_EXIT_REASONS_INTERRUPT	(5UL << PVM_EXIT_REASONS_SHIFT)
-#define PVM_EXIT_REASONS_INT80		(6UL << PVM_EXIT_REASONS_SHIFT)
+#define PVM_EXIT_REASONS_INTERRUPT	(4UL << PVM_EXIT_REASONS_SHIFT)
+#define PVM_EXIT_REASONS_INT80		(5UL << PVM_EXIT_REASONS_SHIFT)
 
 #define PVM_EXIT_REASONS		\
 	{ DE_VECTOR, "DE excp" },	\
@@ -55,7 +54,6 @@
 	{ PVM_EXIT_REASONS_SYSCALL, "SYSCALL" },	\
 	{ PVM_EXIT_REASONS_HYPERCALL, "HYPERCALL" },	\
 	{ PVM_EXIT_REASONS_ERETU, "ERETU" },		\
-	{ PVM_EXIT_REASONS_ERETS, "ERETS" },		\
 	{ PVM_EXIT_REASONS_INTERRUPT, "INTERRUPT" },	\
 	{ PVM_EXIT_REASONS_INT80, "INT80" },		\
 	{ PVM_FAILED_VMENTRY_VECTOR, "FAILED_VMENTRY" }
@@ -136,7 +134,6 @@ struct vcpu_pvm {
 	unsigned long msr_vcpu_struct;
 	unsigned long msr_event_entry;
 	unsigned long msr_retu_rip_plus2;
-	unsigned long msr_rets_rip_plus2;
 	unsigned long msr_linear_address_range;
 
 	u64 l4_range_start;

arch/x86/kvm/x86.c

@@ -1530,7 +1530,6 @@ static const u32 emulated_msrs_all[] = {
 	MSR_PVM_VCPU_STRUCT,
 	MSR_PVM_EVENT_ENTRY,
 	MSR_PVM_RETU_RIP,
-	MSR_PVM_RETS_RIP,
 
 	MSR_IA32_TSC_ADJUST,
 	MSR_IA32_TSC_DEADLINE,