Skip to content

Commit 36cc29e

Browse files
author
Lai Jiangshan
committed
vector controlled async exception
Signed-off-by: Lai Jiangshan <jiangshan.ljs@antgroup.com> Signed-off-by: Hou Wenlong <houwenlong.hwl@antgroup.com>
1 parent 8dc9377 commit 36cc29e

7 files changed

Lines changed: 212 additions & 194 deletions

File tree

‎Documentation/virt/kvm/x86/pvm-spec.rst‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -245,8 +245,10 @@ PVCS::event_flags
245245
due to the interrupt-enable flag being cleared in supervisor mode.
246246
| The guest is responsible for issuing a hypercall PVM_HC_IRQ_WIN when
247247
the guest sees this bit after setting the PVCS::event_flags.IF.
248-
The hypervisor clears this bit in handling
249-
PVM_HC_IRQ_WIN/IRQ_HLT/EVENT_RETURN_USER/EVENT_RETURN_HYPERVISOR.
248+
This bit might be left over without actual pending interrupt in some
249+
cases, which is harmless. The hypervisor clears this bit in handling
250+
PVM_HC_IRQ_WIN/IRQ_HLT/EVENT_RETURN_SUPERVISOR when interrupt is
251+
enabled.
250252
251253
Other bits are reserved (Software should set them to zero).
252254

‎arch/x86/entry/entry_64_pvm.S‎

Lines changed: 63 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
.macro PUSH_IRET_FRAME_FROM_PVCS user:req
1212
.if \user == 1
1313
movl PER_CPU_VAR(pvm_vcpu_struct + PVCS_user_ss), %ecx
14-
andq $0xff, %rcx
14+
andl $0xFFFF, %ecx
1515
pushq %rcx /* pt_regs->ss */
1616
.else
1717
pushq $__KERNEL_DS
@@ -23,7 +23,7 @@
2323

2424
.if \user == 1
2525
movl PER_CPU_VAR(pvm_vcpu_struct + PVCS_user_cs), %ecx
26-
andq $0xff, %rcx
26+
andl $0xFFFF, %ecx
2727
pushq %rcx /* pt_regs->cs */
2828
.else
2929
pushq $__KERNEL_CS
@@ -36,19 +36,41 @@
3636
movq PER_CPU_VAR(pvm_vcpu_struct + PVCS_r11), %r11
3737
.endm
3838

39-
.macro pvm_enable_events
40-
orq $PVM_EVENT_FLAGS_EF, PER_CPU_VAR(pvm_vcpu_struct + PVCS_event_flags)
41-
btq $PVM_EVENT_FLAGS_EP_BIT, PER_CPU_VAR(pvm_vcpu_struct + PVCS_event_flags)
42-
jnc .L_no_event_pending_\@
43-
pushq %rax
44-
movq $PVM_HC_EVENT_WIN, %rax
45-
call pvm_hypercall
46-
popq %rax
47-
.L_no_event_pending_\@:
39+
.macro PUSH_REGS_AND_HANDLE_EVENT handler:req
40+
/* Invalidate orig_ax so that syscall_get_nr() works correctly */
41+
pushq $-1
42+
PUSH_AND_CLEAR_REGS
43+
44+
movw PER_CPU_VAR(pvm_vcpu_struct + PVCS_event_errcode), %dx
45+
xorl %esi, %esi
46+
// get all async exceptions and put PVCS with non-contended xchg
47+
xchgw %si, PER_CPU_VAR(pvm_vcpu_struct + PVCS_event_vector)
48+
movq %rsp, %rdi /* %rdi -> pt_regs */
49+
call \handler
4850
.endm
4951

50-
.macro pvm_disable_events
51-
btrq $PVM_EVENT_FLAGS_EF_BIT, PER_CPU_VAR(pvm_vcpu_struct + PVCS_event_flags)
52+
.macro POP_REGS_AND_LOAD_IRET_FRAME_INTO_PVCS
53+
POP_REGS
54+
55+
// get PVCS before access to it
56+
btsw $PVM_PVCS_EVENT_VECTOR_STD_BIT, PER_CPU_VAR(pvm_vcpu_struct + PVCS_event_vector)
57+
58+
// Copy %rcx, %r11 to the PVM CPU structure
59+
movq %rcx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_rcx)
60+
movq %r11, PER_CPU_VAR(pvm_vcpu_struct + PVCS_r11)
61+
62+
// Copy the IRET frame to the PVM CPU structure
63+
// PVCS_user_cs/ss are ignored for ERETS, load them anyway
64+
movq 1*8(%rsp), %rcx /* RIP */
65+
movq %rcx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_rip)
66+
movq 2*8(%rsp), %rcx /* CS */
67+
movw %cx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_user_cs)
68+
movq 3*8(%rsp), %rcx /* RFLAGS */
69+
movl %ecx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_eflags)
70+
movq 4*8(%rsp), %rcx /* RSP */
71+
movq %rcx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_rsp)
72+
movq 5*8(%rsp), %rcx /* SS */
73+
movw %cx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_user_ss)
5274
.endm
5375

5476
.code64
@@ -60,8 +82,14 @@ SYM_CODE_START(entry_SYSCALL_64_pvm)
6082

6183
PUSH_IRET_FRAME_FROM_PVCS user=1
6284

63-
pvm_enable_events
85+
btrw $PVM_PVCS_EVENT_VECTOR_STD_BIT, PER_CPU_VAR(pvm_vcpu_struct + PVCS_event_vector)
86+
cmpw $0, PER_CPU_VAR(pvm_vcpu_struct + PVCS_event_vector)
87+
jz entry_SYSCALL_64_after_hwframe
6488

89+
.L_async_exception:
90+
PUSH_REGS_AND_HANDLE_EVENT handler=pvm_event
91+
POP_REGS
92+
addq $8, %rsp
6593
jmp entry_SYSCALL_64_after_hwframe
6694
SYM_CODE_END(entry_SYSCALL_64_pvm)
6795

@@ -97,7 +125,7 @@ SYM_FUNC_START(pvm_irq_enable)
97125
RET
98126
.L_maybe_interrupt_pending:
99127
/* handle pending IRQ */
100-
movq $PVM_HC_EVENT_WIN, %rax
128+
movq $PVM_HC_IRQ_WIN, %rax
101129
jmp pvm_hypercall
102130
SYM_FUNC_END(pvm_irq_enable)
103131
.popsection
@@ -112,35 +140,10 @@ SYM_CODE_START(pvm_user_event_entry)
112140
ENDBR
113141

114142
PUSH_IRET_FRAME_FROM_PVCS user=1
115-
/* pt_regs->orig_ax: errcode and vector */
116-
pushq PER_CPU_VAR(pvm_vcpu_struct + PVCS_event_errcode)
117-
118-
pvm_enable_events
119-
120-
PUSH_AND_CLEAR_REGS
121-
movq %rsp, %rdi /* %rdi -> pt_regs */
122-
call pvm_event
143+
PUSH_REGS_AND_HANDLE_EVENT handler=pvm_event
123144

124145
SYM_INNER_LABEL(pvm_restore_regs_and_return_to_usermode, SYM_L_GLOBAL)
125-
POP_REGS
126-
127-
pvm_disable_events
128-
129-
/* Copy %rcx, %r11 to the PVM CPU structure. */
130-
movq %rcx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_rcx)
131-
movq %r11, PER_CPU_VAR(pvm_vcpu_struct + PVCS_r11)
132-
133-
/* Copy the IRET frame to the PVM CPU structure. */
134-
movq 1*8(%rsp), %rcx /* RIP */
135-
movq %rcx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_rip)
136-
movq 2*8(%rsp), %rcx /* CS */
137-
movw %cx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_user_cs)
138-
movq 3*8(%rsp), %rcx /* RFLAGS */
139-
movl %ecx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_eflags)
140-
movq 4*8(%rsp), %rcx /* RSP */
141-
movq %rcx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_rsp)
142-
movq 5*8(%rsp), %rcx /* SS */
143-
movw %cx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_user_ss)
146+
POP_REGS_AND_LOAD_IRET_FRAME_INTO_PVCS
144147

145148
/*
146149
* We are on the trampoline stack. All regs are live.
@@ -172,34 +175,26 @@ SYM_CODE_START(pvm_kernel_event_entry)
172175
/* TODO: check stack overflow */
173176

174177
PUSH_IRET_FRAME_FROM_PVCS user=0
175-
/* pt_regs->orig_ax: errcode and vector */
176-
pushq PER_CPU_VAR(pvm_vcpu_struct + PVCS_event_errcode)
177-
178-
pvm_enable_events
179-
180-
PUSH_AND_CLEAR_REGS
181-
movq %rsp, %rdi /* %rdi -> pt_regs */
182-
call pvm_event
183-
184-
SYM_INNER_LABEL(pvm_restore_regs_and_return_to_kernel, SYM_L_GLOBAL)
185-
POP_REGS
186-
187-
pvm_disable_events
188-
189-
/* Copy %rcx, %r11 to the PVM CPU structure. */
190-
movq %rcx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_rcx)
191-
movq %r11, PER_CPU_VAR(pvm_vcpu_struct + PVCS_r11)
192-
193-
/* Copy the IRET frame to the PVM CPU structure. */
194-
movq 1*8(%rsp), %rcx /* RIP */
195-
movq %rcx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_rip)
196-
movq 3*8(%rsp), %rcx /* RFLAGS */
197-
movl %ecx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_eflags)
198-
movq 4*8(%rsp), %rcx /* RSP */
199-
movq %rcx, PER_CPU_VAR(pvm_vcpu_struct + PVCS_rsp)
178+
PUSH_REGS_AND_HANDLE_EVENT handler=pvm_event
179+
POP_REGS_AND_LOAD_IRET_FRAME_INTO_PVCS
200180

201-
addq $6*8, %rsp
181+
addq $(6*8+16), %rsp
202182
SYM_INNER_LABEL(pvm_rets_rip, SYM_L_GLOBAL)
203183
ANNOTATE_NOENDBR
204184
syscall
205185
SYM_CODE_END(pvm_kernel_event_entry)
186+
187+
.pushsection .head.text, "ax"
188+
SYM_CODE_START_NOALIGN(pvm_early_kernel_event_entry)
189+
UNWIND_HINT_ENTRY
190+
ENDBR
191+
192+
incl early_recursion_flag(%rip)
193+
PUSH_IRET_FRAME_FROM_PVCS user=0
194+
PUSH_REGS_AND_HANDLE_EVENT handler=pvm_early_event
195+
POP_REGS_AND_LOAD_IRET_FRAME_INTO_PVCS
196+
decl early_recursion_flag(%rip)
197+
addq $6*8, %rsp
198+
jmp pvm_rets_rip
199+
SYM_CODE_END(pvm_early_kernel_event_entry)
200+
.popsection

‎arch/x86/include/uapi/asm/pvm_para.h‎

Lines changed: 44 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@
4747
#define PVM_HC_SPECIAL_MAX (PVM_HC_SPECIAL_BASE+PVM_HC_SPECIAL_MAX_NR)
4848

4949
#define PVM_HC_LOAD_PGTBL (PVM_HC_SPECIAL_BASE+0)
50-
#define PVM_HC_EVENT_WIN (PVM_HC_SPECIAL_BASE+1)
50+
#define PVM_HC_IRQ_WIN (PVM_HC_SPECIAL_BASE+1)
5151
#define PVM_HC_IRQ_HALT (PVM_HC_SPECIAL_BASE+2)
5252
#define PVM_HC_TLB_FLUSH (PVM_HC_SPECIAL_BASE+3)
5353
#define PVM_HC_TLB_FLUSH_CURRENT (PVM_HC_SPECIAL_BASE+4)
@@ -58,17 +58,6 @@
5858
#define PVM_HC_LOAD_TLS (PVM_HC_SPECIAL_BASE+9)
5959

6060
/*
61-
* PVM_EVENT_FLAGS_EF
62-
* - Event enable flag. The flag is set to respond to events;
63-
* and cleared to inhibit events. When the hypervisor try to inject
64-
* an event except for NMI with PVM_EVENT_FLAGS_EF cleared, it will
65-
* morph it to triple-fault.
66-
*
67-
* PVM_EVENT_FLAGS_EP
68-
* - Event pending flag. The hypervisor sets it if it fails to inject
69-
* an event (NMI) to the VCPU due to the event-enable flag being
70-
* cleared in supervisor mode.
71-
*
7261
* PVM_EVENT_FLAGS_IF
7362
* - Interrupt enable flag. The flag is set to respond to maskable
7463
* external interrupts; and cleared to inhibit maskable external
@@ -79,15 +68,49 @@
7968
* a maskable event to the VCPU due to the interrupt-enable flag being
8069
* cleared in supervisor mode.
8170
*/
82-
#define PVM_EVENT_FLAGS_EF_BIT 0
83-
#define PVM_EVENT_FLAGS_EF _BITUL(PVM_EVENT_FLAGS_EF_BIT)
84-
#define PVM_EVENT_FLAGS_EP_BIT 1
85-
#define PVM_EVENT_FLAGS_EP _BITUL(PVM_EVENT_FLAGS_EP_BIT)
8671
#define PVM_EVENT_FLAGS_IP_BIT 8
8772
#define PVM_EVENT_FLAGS_IP _BITUL(PVM_EVENT_FLAGS_IP_BIT)
8873
#define PVM_EVENT_FLAGS_IF_BIT 9
8974
#define PVM_EVENT_FLAGS_IF _BITUL(PVM_EVENT_FLAGS_IF_BIT)
9075

76+
/*
77+
* Bits for event_vector.
78+
*
79+
* The lowest 8-bit is the vector number for non async-exception vector events.
80+
*
81+
* The highest 8-bit is defined as following. When the highest 8-bit is
82+
* non-zero in supervisor mode, only async-exception can be delivered without
83+
* jumping to the entry point. The guest supervisor should check any pending
84+
* NMI/MCE when handling any events. The hypervisor will also check it
85+
* for pending NMI/MCE in handling ERETU/ERETS. Trying to deliver a non
86+
* async-exception when the highest 8-bit is non-zero in supervisor mode
87+
* will make it morphed into a triple-fault.
88+
*
89+
* The supervisor code should clear the highest 8-bit and get all the
90+
* delivered events from event_vector in a appropriate way.
91+
*
92+
* PVM_PVCS_EVENT_VECTOR_STD:
93+
* - The supervisor software should set this bit before access to PVCS
94+
* for ERETU/ERETS. The event_vector remains to be this value during
95+
* user mode and upon delivering a SYSCALL event from user mode.
96+
* Since async-exception can be dilivered and access to PVCS any time,
97+
* so it is required for protecting the fields related to supervisor
98+
* event delivering.
99+
* - When a non async-exception is delivered, this bit is set and the
100+
* lowest 8-bit is the vector number.
101+
*
102+
* PVM_PVCS_EVENT_VECTOR_NMI
103+
* PVM_PVCS_EVENT_VECTOR_MCE
104+
* - An NMI or MCE is being delivered (or along with other events) or
105+
* pending (during the period the supervisor is about to ERETU/ERETS)
106+
*/
107+
#define PVM_PVCS_EVENT_VECTOR_STD_BIT 8
108+
#define PVM_PVCS_EVENT_VECTOR_STD _BITUL(PVM_PVCS_EVENT_VECTOR_STD_BIT)
109+
#define PVM_PVCS_EVENT_VECTOR_NMI_BIT 9
110+
#define PVM_PVCS_EVENT_VECTOR_NMI _BITUL(PVM_PVCS_EVENT_VECTOR_NMI_BIT)
111+
#define PVM_PVCS_EVENT_VECTOR_MCE_BIT 10
112+
#define PVM_PVCS_EVENT_VECTOR_MCE _BITUL(PVM_PVCS_EVENT_VECTOR_MCE_BIT)
113+
91114
#define PVM_LOAD_PGTBL_FLAGS_TLB _BITUL(0)
92115
#define PVM_LOAD_PGTBL_FLAGS_LA57 _BITUL(1)
93116

@@ -105,17 +128,14 @@ struct pvm_vcpu_struct {
105128
* bit 9 being valid. The other bits are reserved.
106129
*/
107130
u64 event_flags;
108-
u32 event_errcode;
109-
u32 event_vector;
110131
u64 cr2;
111-
u64 reserved0[5];
132+
u64 reserved0[6];
112133

113-
/*
114-
* For the event from supervisor mode with vector >=32, only eflags,
115-
* rip, rsp, rcx and r11 are saved, and others keep untouched.
116-
*/
134+
// For ERETS and the event from supervisor mode, user_cs, user_ss
135+
// user_gsbase, and pkru are ignored and kept untouched.
117136
u16 user_cs, user_ss;
118-
u32 reserved1;
137+
u16 event_errcode;
138+
u16 event_vector;
119139
u64 reserved2;
120140
u64 user_gsbase;
121141
u32 eflags;

‎arch/x86/kernel/asm-offsets_64.c‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -80,6 +80,7 @@ int main(void)
8080
#define ENTRY(entry) OFFSET(PVCS_ ## entry, pvm_vcpu_struct, entry)
8181
ENTRY(event_flags);
8282
ENTRY(event_errcode);
83+
ENTRY(event_vector);
8384
ENTRY(user_cs);
8485
ENTRY(user_ss);
8586
ENTRY(user_gsbase);

‎arch/x86/kernel/head_64.S‎

Lines changed: 0 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -633,37 +633,6 @@ SYM_CODE_START_NOALIGN(vc_no_ghcb)
633633
SYM_CODE_END(vc_no_ghcb)
634634
#endif
635635

636-
#ifdef CONFIG_PVM_GUEST
637-
#include <asm/pvm_para.h>
638-
639-
.align 256
640-
SYM_CODE_START_NOALIGN(pvm_early_kernel_event_entry)
641-
UNWIND_HINT_ENTRY
642-
ENDBR
643-
644-
incl early_recursion_flag(%rip)
645-
646-
pushq $__KERNEL_DS
647-
pushq PER_CPU_VAR(pvm_vcpu_struct + PVCS_rsp) /* pt_regs->sp */
648-
movl PER_CPU_VAR(pvm_vcpu_struct + PVCS_eflags), %ecx
649-
pushq %rcx /* pt_regs->flags */
650-
pushq $__KERNEL_CS
651-
pushq PER_CPU_VAR(pvm_vcpu_struct + PVCS_rip) /* pt_regs->ip */
652-
movq PER_CPU_VAR(pvm_vcpu_struct + PVCS_rcx), %rcx
653-
movq PER_CPU_VAR(pvm_vcpu_struct + PVCS_r11), %r11
654-
pushq PER_CPU_VAR(pvm_vcpu_struct + PVCS_event_errcode)
655-
656-
orq $PVM_EVENT_FLAGS_EF, PER_CPU_VAR(pvm_vcpu_struct + PVCS_event_flags)
657-
658-
PUSH_AND_CLEAR_REGS
659-
movq %rsp, %rdi /* %rdi -> pt_regs */
660-
call pvm_early_event
661-
662-
decl early_recursion_flag(%rip)
663-
jmp pvm_restore_regs_and_return_to_kernel
664-
SYM_CODE_END(pvm_early_kernel_event_entry)
665-
#endif
666-
667636
#define SYM_DATA_START_PAGE_ALIGNED(name) \
668637
SYM_START(name, SYM_L_GLOBAL, .balign PAGE_SIZE)
669638

0 commit comments

Comments
 (0)