pvm: ABI: Convert dual CR3 to single CR3

PVM is a pagetable-based virtualization system where kernel and user
space separation is implemented via separate page tables. Currently,
a process’s address space is defined by the CR3 register and
MSR_SWITCH_CR3 (a PVM virtual MSR). Address space switching is performed
by the hypercall PVM_HC_LOAD_PGTBL, which loads both CR3 and
MSR_SWITCH_CR3, and by swapping CR3 and MSR_SWITCH_CR3 when switching
between kernel and user modes.

However, this ABI deviates from native x86 architecture and should be
converted to use a single CR3, as is standard. Using a single CR3 does
not eliminate separation — the hypervisor manages two underlying shadow
page tables to maintain proper kernel/user isolation. This change also
removes MSR_PVM_SWITCH_CR3 and the user_pgd argument from
PVM_HC_LOAD_PGTBL.

Benefits of the current dual-CR3 design:
  - The guest explicitly manages which pages belong to kernel CR3 and
    which to user CR3, taking responsibility for proper separation.
  - It allows reuse of the existing Linux kernel KPTI (Kernel Page Table
    Isolation) logic inside the PVM guest — the main reason why the
    current dual-CR3 implementation is relatively simple.

Drawbacks of dual-CR3:
  - It deviates from the native x86 architecture, making the ABI less
    clear.
  - Future kernels may remove KPTI once CPUs affected by the Meltdown
    bug are obsolete (possibly in 10–20 years), making this approach
    unsustainable long-term.
  - Wastes an extra 4 KB root page table per process in the guest.

After adopting a single-CR3 model:
Pros:
  - Clear, native x86-compliant ABI.

Cons:
  - More complex logic required in the hypervisor to carefully manage
    shadow page tables that distinguish between kernel and user mappings.
  - The new implementation must go beyond simple KPTI and fully emulate
    native x86 behavior.

Signed-off-by: Lai Jiangshan <jiangshan.ljs@antgroup.com>
Link: #19

Author: Lai Jiangshan

arch/x86/Kconfig

@@ -854,7 +854,6 @@ config KVM_GUEST
 config PVM_GUEST
 	bool "PVM Guest support"
 	depends on X86_64 && KVM_GUEST && X86_PIE && !KASAN
-	select PAGE_TABLE_ISOLATION
 	select PARAVIRT_XXL
 	select RANDOMIZE_MEMORY
 	select RELOCATABLE_UNCOMPRESSED_KERNEL

arch/x86/include/uapi/asm/pvm_para.h

@@ -30,7 +30,7 @@
 
 #define MSR_PVM_LINEAR_ADDRESS_RANGE	0x4b564df0
 #define MSR_PVM_VCPU_STRUCT		0x4b564df1
-#define MSR_PVM_SWITCH_CR3		0x4b564df2
+// #define MSR_PVM_SWITCH_CR3		0x4b564df2 // deprecated, FIXME: reordering when sending v2
 // #define MSR_PVM_SUPERVISOR_RSP	0x4b564df3 // deprecated, FIXME: reordering when sending v2
 #define MSR_PVM_EVENT_ENTRY		0x4b564df4
 #define MSR_PVM_RETU_RIP		0x4b564df5

arch/x86/kernel/pvm.c

@@ -183,11 +183,6 @@ static unsigned long pvm_read_cr3(void)
 	return this_cpu_read(pvm_guest_cr3);
 }
 
-static unsigned long pvm_user_pgd(unsigned long pgd)
-{
-	return pgd | BIT(PTI_PGTABLE_SWITCH_BIT) | BIT(X86_CR3_PTI_PCID_USER_BIT);
-}
-
 static void pvm_write_cr3(unsigned long val)
 {
 	/* Convert CR3_NO_FLUSH bit to hypercall flags. */
@@ -197,7 +192,7 @@ static void pvm_write_cr3(unsigned long val)
 	if (pgtable_l5_enabled())
 		flags |= PVM_LOAD_PGTBL_FLAGS_LA57;
 	this_cpu_write(pvm_guest_cr3, pgd);
-	pvm_hypercall3(PVM_HC_LOAD_PGTBL, flags, pgd, pvm_user_pgd(pgd));
+	pvm_hypercall2(PVM_HC_LOAD_PGTBL, flags, pgd);
 }
 
 static void pvm_flush_tlb_user(void)

arch/x86/kvm/mmu/mmu.c

@@ -2466,6 +2466,18 @@ static void __link_shadow_page(struct kvm *kvm,
 
 	spte = make_nonleaf_spte(sp->spt, sp_ad_disabled(sp));
 
+	/* for PVM, if the host has NX, force guest SMEP */
+	if (kvm->arch.host_mmu_root_pgd && cpu_feature_enabled(X86_FEATURE_NX)) {
+		struct kvm_mmu_page *parent = sptep_to_sp(sptep);
+
+		/*
+		 * validate_pvm_indirect_access() enables user sp linked beneath
+		 * kernel sp.
+		 */
+		if (!(parent->role.access & ACC_USER_MASK) && (sp->role.access & ACC_USER_MASK))
+			spte |= shadow_nx_mask;
+	}
+
 	mmu_spte_set(sptep, spte);
 
 	mmu_page_add_parent_pte(cache, sp, sptep);
@@ -2489,6 +2501,40 @@ static void link_shadow_page(struct kvm_vcpu *vcpu, u64 *sptep,
 	__link_shadow_page(vcpu->kvm, &vcpu->arch.mmu_pte_list_desc_cache, sptep, sp, true);
 }
 
+static unsigned validate_pvm_indirect_access(struct kvm_vcpu *vcpu, u64 *sptep,
+					     unsigned access, unsigned leaf_access)
+{
+	/*
+	 * return directly when non-pvm or it is going to create user sp/spte
+	 * which is allowed under both kernel and user sp
+	 */
+	if (!vcpu->kvm->arch.host_mmu_root_pgd || (leaf_access & ACC_USER_MASK))
+		return access;
+
+	access &= ~ACC_USER_MASK;
+
+	if (is_shadow_present_pte(*sptep) && !is_large_pte(*sptep)) {
+		struct kvm_mmu_page *child;
+
+		/*
+		 * For the pvm indirect sp, if the previous linked child
+		 * is for user pagetable, no kernel sp/page should be
+		 * mapped under the child, so the child should be updated
+		 * if it is the case.  It is not possible the case for
+		 * current Linux PVM guest, but this check has to be examed
+		 * for correctness.
+		 */
+		child = spte_to_child_sp(*sptep);
+		if (!(child->role.access & ACC_USER_MASK))
+			return access;
+
+		drop_parent_pte(vcpu->kvm, child, sptep);
+		kvm_flush_remote_tlbs_sptep(vcpu->kvm, sptep);
+	}
+
+	return access;
+}
+
 static void validate_direct_spte(struct kvm_vcpu *vcpu, u64 *sptep,
 				   unsigned direct_access)
 {
@@ -5282,9 +5328,13 @@ static void kvm_init_shadow_mmu(struct kvm_vcpu *vcpu,
 	root_role.level = max_t(u32, root_role.level, PT32E_ROOT_LEVEL);
 
 	/* Shadow MMU level should be the same as host for PVM */
-	if (vcpu->kvm->arch.host_mmu_root_pgd && root_role.level != HOST_ROOT_LEVEL) {
-		root_role.level = HOST_ROOT_LEVEL;
-		root_role.passthrough = 1;
+	if (vcpu->kvm->arch.host_mmu_root_pgd) {
+		if (root_role.level != HOST_ROOT_LEVEL) {
+			root_role.level = HOST_ROOT_LEVEL;
+			root_role.passthrough = 1;
+		}
+		if (static_call(kvm_x86_get_cpl)(vcpu) == 0)
+			root_role.access &= ~ACC_USER_MASK;
 	}
 
 	/*

arch/x86/kvm/mmu/paging_tmpl.h

@@ -674,6 +674,7 @@ static int FNAME(fetch)(struct kvm_vcpu *vcpu, struct kvm_page_fault *fault,
 
 		table_gfn = gw->table_gfn[it.level - 2];
 		access = gw->pt_access[it.level - 2];
+		access = validate_pvm_indirect_access(vcpu, it.sptep, access, direct_access);
 		sp = kvm_mmu_get_child_sp(vcpu, it.sptep, table_gfn,
 					  false, access);
 

arch/x86/kvm/pvm/pvm.c

@@ -336,8 +336,8 @@ static inline void switch_to_smod(struct kvm_vcpu *vcpu)
 	struct vcpu_pvm *pvm = to_pvm(vcpu);
 
 	pvm_switch_flags_toggle_mod(pvm);
-	kvm_mmu_new_pgd(vcpu, pvm->msr_switch_cr3);
-	swap(pvm->msr_switch_cr3, vcpu->arch.cr3);
+	vcpu->arch.mmu->root_role.access &= ~ACC_USER_MASK;
+	kvm_mmu_new_pgd(vcpu, vcpu->arch.cr3);
 
 	pvm_write_guest_gs_base(pvm, pvm->msr_kernel_gs_base);
 
@@ -350,8 +350,8 @@ static inline void switch_to_umod(struct kvm_vcpu *vcpu)
 	struct vcpu_pvm *pvm = to_pvm(vcpu);
 
 	pvm_switch_flags_toggle_mod(pvm);
-	kvm_mmu_new_pgd(vcpu, pvm->msr_switch_cr3);
-	swap(pvm->msr_switch_cr3, vcpu->arch.cr3);
+	vcpu->arch.mmu->root_role.access |= ACC_USER_MASK;
+	kvm_mmu_new_pgd(vcpu, vcpu->arch.cr3);
 }
 
 /*
@@ -745,11 +745,12 @@ static void pvm_flush_hwtlb_gva(struct kvm_vcpu *vcpu, gva_t addr)
 static u64 get_switch_hw_cr3(struct vcpu_pvm *pvm)
 {
 	struct kvm_mmu *mmu = pvm->vcpu.arch.mmu;
-	u64 cr3 = is_smod(pvm) ? pvm->vcpu.arch.cr3 : pvm->msr_switch_cr3;
+	union kvm_mmu_page_role switch_role = mmu->root_role;
 	int i;
 
+	switch_role.access ^= ACC_USER_MASK;
 	for (i = 0; i < KVM_MMU_NUM_PREV_ROOTS; i++) {
-		if (is_root_usable(&mmu->prev_roots[i], cr3, mmu->root_role)) {
+		if (is_root_usable(&mmu->prev_roots[i], pvm->vcpu.arch.cr3, switch_role)) {
 			if (i != 0)
 				swap(mmu->prev_roots[0], mmu->prev_roots[i]);
 			return mmu->prev_roots[0].hpa;
@@ -1086,9 +1087,6 @@ static int pvm_get_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_PVM_RETS_RIP:
 		msr_info->data = pvm->msr_rets_rip_plus2 - 2;
 		break;
-	case MSR_PVM_SWITCH_CR3:
-		msr_info->data = pvm->msr_switch_cr3;
-		break;
 	case MSR_PVM_LINEAR_ADDRESS_RANGE:
 		msr_info->data = pvm->msr_linear_address_range;
 		break;
@@ -1239,9 +1237,6 @@ static int pvm_set_msr(struct kvm_vcpu *vcpu, struct msr_data *msr_info)
 	case MSR_PVM_RETS_RIP:
 		pvm->msr_rets_rip_plus2 = msr_info->data + 2;
 		break;
-	case MSR_PVM_SWITCH_CR3:
-		pvm->msr_switch_cr3 = msr_info->data;
-		break;
 	case MSR_PVM_LINEAR_ADDRESS_RANGE:
 		if (!pvm_check_and_set_msr_linear_address_range(pvm, msr_info->data))
 			return 1;
@@ -1848,29 +1843,26 @@ static int handle_hc_irq_halt(struct kvm_vcpu *vcpu)
 	return kvm_emulate_halt_noskip(vcpu);
 }
 
-static void pvm_flush_tlb_guest_current_kernel_user(struct kvm_vcpu *vcpu)
+static void pvm_flush_tlb_guest_current(struct kvm_vcpu *vcpu)
 {
 	/*
-	 * sync the current pgd and user_pgd (pvm->msr_switch_cr3)
-	 * which is a subset work of KVM_REQ_TLB_FLUSH_GUEST.
+	 * sync the current pgd which is a subset work of KVM_REQ_TLB_FLUSH_GUEST.
 	 */
 	kvm_make_request(KVM_REQ_TLB_FLUSH_GUEST, vcpu);
 }
 
 /*
  * Hypercall: PVM_HC_LOAD_PGTBL
- *	Load two PGDs into the current CR3 and MSR_PVM_SWITCH_CR3.
+ *	Load two PGDs into the current CR3.
  *
  * Arguments:
  *	flags:	bit0: flush the TLBs tagged with @pgd and @user_pgd.
  *		bit1: 4 (bit1=0) or 5 (bit1=1 && cpuid_has(LA57)) level paging.
  *	pgd: to be loaded into CR3.
- *	user_pgd: to be loaded into MSR_PVM_SWITCH_CR3.
  */
 static int handle_hc_load_pagetables(struct kvm_vcpu *vcpu, unsigned long flags,
-				     unsigned long pgd, unsigned long user_pgd)
+				     unsigned long pgd)
 {
-	struct vcpu_pvm *pvm = to_pvm(vcpu);
 	unsigned long cr4 = vcpu->arch.cr4;
 
 	if (!(flags & PVM_LOAD_PGTBL_FLAGS_LA57))
@@ -1885,10 +1877,9 @@ static int handle_hc_load_pagetables(struct kvm_vcpu *vcpu, unsigned long flags,
 
 	kvm_mmu_new_pgd(vcpu, pgd);
 	vcpu->arch.cr3 = pgd;
-	pvm->msr_switch_cr3 = user_pgd;
 
 	if (flags & PVM_LOAD_PGTBL_FLAGS_TLB)
-		pvm_flush_tlb_guest_current_kernel_user(vcpu);
+		pvm_flush_tlb_guest_current(vcpu);
 
 	return 1;
 }
@@ -1906,11 +1897,11 @@ static int handle_hc_flush_tlb_all(struct kvm_vcpu *vcpu)
 
 /*
  * Hypercall: PVM_HC_TLB_FLUSH_CURRENT
- *	Flush all TLBs tagged with the current CR3 and MSR_PVM_SWITCH_CR3.
+ *	Flush all TLBs tagged with the current CR3.
  */
-static int handle_hc_flush_tlb_current_kernel_user(struct kvm_vcpu *vcpu)
+static int handle_hc_flush_tlb_current(struct kvm_vcpu *vcpu)
 {
-	pvm_flush_tlb_guest_current_kernel_user(vcpu);
+	pvm_flush_tlb_guest_current(vcpu);
 
 	return 1;
 }
@@ -2118,11 +2109,11 @@ static int handle_exit_syscall(struct kvm_vcpu *vcpu)
 	case PVM_HC_IRQ_HALT:
 		return handle_hc_irq_halt(vcpu);
 	case PVM_HC_LOAD_PGTBL:
-		return handle_hc_load_pagetables(vcpu, a0, a1, a2);
+		return handle_hc_load_pagetables(vcpu, a0, a1);
 	case PVM_HC_TLB_FLUSH:
 		return handle_hc_flush_tlb_all(vcpu);
 	case PVM_HC_TLB_FLUSH_CURRENT:
-		return handle_hc_flush_tlb_current_kernel_user(vcpu);
+		return handle_hc_flush_tlb_current(vcpu);
 	case PVM_HC_TLB_INVLPG:
 		return handle_hc_invlpg(vcpu, a0);
 	case PVM_HC_LOAD_GS:
@@ -2740,8 +2731,8 @@ static fastpath_t pvm_vcpu_run(struct kvm_vcpu *vcpu)
 	pvm_vcpu_run_noinstr(vcpu);
 
 	if (is_smod_befor_run != is_smod(pvm)) {
+		vcpu->arch.mmu->root_role.access ^= ACC_USER_MASK;
 		swap(pvm->vcpu.arch.mmu->root, pvm->vcpu.arch.mmu->prev_roots[0]);
-		swap(pvm->msr_switch_cr3, pvm->vcpu.arch.cr3);
 	}
 
 	/* MSR_IA32_DEBUGCTLMSR is zeroed before vmenter. Restore it if needed */
@@ -2863,7 +2854,6 @@ static void pvm_vcpu_reset(struct kvm_vcpu *vcpu, bool init_event)
 	pvm->msr_event_entry = 0;
 	pvm->msr_retu_rip_plus2 = 0;
 	pvm->msr_rets_rip_plus2 = 0;
-	pvm->msr_switch_cr3 = 0;
 	pvm_set_default_msr_linear_address_range(pvm);
 }
 
@@ -2991,7 +2981,8 @@ static __init void pvm_set_cpu_caps(void)
 	 * PVM doesn't support SMEP.  When NX is supported and the guest can
 	 * use NX on the user pagetable to emulate the same protection as SMEP.
 	 */
-	kvm_cpu_cap_clear(X86_FEATURE_SMEP);
+	if (boot_cpu_has(X86_FEATURE_NX))
+		kvm_cpu_cap_set(X86_FEATURE_SMEP);
 
 	/*
 	 * Unlike VMX/SVM which can switches paging mode atomically, PVM

arch/x86/kvm/pvm/pvm.h

@@ -137,7 +137,6 @@ struct vcpu_pvm {
 	unsigned long msr_event_entry;
 	unsigned long msr_retu_rip_plus2;
 	unsigned long msr_rets_rip_plus2;
-	unsigned long msr_switch_cr3;
 	unsigned long msr_linear_address_range;
 
 	u64 l4_range_start;

arch/x86/kvm/x86.c

@@ -1528,7 +1528,6 @@ static const u32 emulated_msrs_all[] = {
 
 	MSR_PVM_LINEAR_ADDRESS_RANGE,
 	MSR_PVM_VCPU_STRUCT,
-	MSR_PVM_SWITCH_CR3,
 	MSR_PVM_EVENT_ENTRY,
 	MSR_PVM_RETU_RIP,
 	MSR_PVM_RETS_RIP,

arch/x86/mm/pti.c

@@ -85,9 +85,8 @@ void __init pti_check_boottime_disable(void)
 	}
 
 	if (boot_cpu_has(X86_FEATURE_KVM_PVM_GUEST)) {
-		pti_mode = PTI_FORCE_ON;
-		pti_print_if_insecure("force enabled on kvm pvm guest.");
-		setup_force_cpu_cap(X86_FEATURE_PTI);
+		pti_mode = PTI_FORCE_OFF;
+		pti_print_if_insecure("disabled on PVM guest.");
 		return;
 	}