Skip to content

Commit 91dd7de

Browse files
author
Fox Snowpatch
committed
From patchwork series 498309
https://patchwork.ozlabs.org//project/linuxppc-dev/list/?series=498309
1 parent a2f7734 commit 91dd7de

4 files changed

Lines changed: 128 additions & 11 deletions

File tree

‎arch/powerpc/net/bpf_jit.h‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -212,7 +212,9 @@ void bpf_jit_build_epilogue(u32 *image, struct codegen_context *ctx);
212212
void bpf_jit_build_fentry_stubs(u32 *image, struct codegen_context *ctx);
213213
void bpf_jit_realloc_regs(struct codegen_context *ctx);
214214
int bpf_jit_emit_exit_insn(u32 *image, struct codegen_context *ctx, int tmp_reg, long exit_addr);
215-
215+
void prepare_for_fsession_fentry(u32 *image, struct codegen_context *ctx, int cookie_cnt,
216+
int cookie_off, int retval_off);
217+
void store_func_meta(u32 *image, struct codegen_context *ctx, u64 func_meta, int func_meta_off);
216218
int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass,
217219
struct codegen_context *ctx, int insn_idx,
218220
int jmp_off, int dst_reg, u32 code);

‎arch/powerpc/net/bpf_jit_comp.c‎

Lines changed: 65 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,17 @@ bool bpf_jit_supports_kfunc_call(void)
453453
return IS_ENABLED(CONFIG_PPC64);
454454
}
455455

456+
bool bpf_jit_supports_fsession(void)
457+
{
458+
/*
459+
* TODO: Remove after validating support
460+
* for fsession and trampoline on ppc32.
461+
*/
462+
if (IS_ENABLED(CONFIG_PPC32))
463+
return -EOPNOTSUPP;
464+
return true;
465+
}
466+
456467
bool bpf_jit_supports_arena(void)
457468
{
458469
return IS_ENABLED(CONFIG_PPC64);
@@ -725,12 +736,16 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
725736
struct bpf_tramp_links *tlinks,
726737
void *func_addr)
727738
{
728-
int regs_off, nregs_off, ip_off, run_ctx_off, retval_off, nvr_off, alt_lr_off, r4_off = 0;
739+
int regs_off, func_meta_off, ip_off, run_ctx_off, retval_off;
740+
int nvr_off, alt_lr_off, r4_off = 0;
729741
struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
730742
struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
731743
struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
732744
int i, ret, nr_regs, retaddr_off, bpf_frame_size = 0;
733745
struct codegen_context codegen_ctx, *ctx;
746+
int cookie_off, cookie_cnt, cookie_ctx_off;
747+
int fsession_cnt = bpf_fsession_cnt(tlinks);
748+
u64 func_meta;
734749
u32 *image = (u32 *)rw_image;
735750
ppc_inst_t branch_insn;
736751
u32 *branches = NULL;
@@ -766,9 +781,11 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
766781
* [ reg argN ]
767782
* [ ... ]
768783
* regs_off [ reg_arg1 ] prog_ctx
769-
* nregs_off [ args count ] ((u64 *)prog_ctx)[-1]
784+
* func_meta_off [ args count ] ((u64 *)prog_ctx)[-1]
770785
* ip_off [ traced function ] ((u64 *)prog_ctx)[-2]
786+
* [ stack cookieN ]
771787
* [ ... ]
788+
* cookie_off [ stack cookie1 ]
772789
* run_ctx_off [ bpf_tramp_run_ctx ]
773790
* [ reg argN ]
774791
* [ ... ]
@@ -800,16 +817,21 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
800817
run_ctx_off = bpf_frame_size;
801818
bpf_frame_size += round_up(sizeof(struct bpf_tramp_run_ctx), SZL);
802819

820+
/* room for session cookies */
821+
cookie_off = bpf_frame_size;
822+
cookie_cnt = bpf_fsession_cookie_cnt(tlinks);
823+
bpf_frame_size += cookie_cnt * 8;
824+
803825
/* Room for IP address argument */
804826
ip_off = bpf_frame_size;
805827
if (flags & BPF_TRAMP_F_IP_ARG)
806828
bpf_frame_size += SZL;
807829

808-
/* Room for args count */
809-
nregs_off = bpf_frame_size;
830+
/* Room for function metadata, arg regs count */
831+
func_meta_off = bpf_frame_size;
810832
bpf_frame_size += SZL;
811833

812-
/* Room for args */
834+
/* Room for arg regs */
813835
regs_off = bpf_frame_size;
814836
bpf_frame_size += nr_regs * SZL;
815837

@@ -908,9 +930,9 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
908930
EMIT(PPC_RAW_STL(_R3, _R1, retaddr_off));
909931
}
910932

911-
/* Save function arg count -- see bpf_get_func_arg_cnt() */
912-
EMIT(PPC_RAW_LI(_R3, nr_regs));
913-
EMIT(PPC_RAW_STL(_R3, _R1, nregs_off));
933+
/* Save function arg regs count -- see bpf_get_func_arg_cnt() */
934+
func_meta = nr_regs;
935+
store_func_meta(image, ctx, func_meta, func_meta_off);
914936

915937
/* Save nv regs */
916938
EMIT(PPC_RAW_STL(_R25, _R1, nvr_off));
@@ -924,10 +946,28 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
924946
return ret;
925947
}
926948

927-
for (i = 0; i < fentry->nr_links; i++)
949+
if (fsession_cnt) {
950+
/*
951+
* Clear all the session cookies' values
952+
* Clear the return value to make sure fentry always get 0
953+
*/
954+
prepare_for_fsession_fentry(image, ctx, cookie_cnt, cookie_off, retval_off);
955+
}
956+
957+
cookie_ctx_off = (regs_off - cookie_off) / 8;
958+
959+
for (i = 0; i < fentry->nr_links; i++) {
960+
if (bpf_prog_calls_session_cookie(fentry->links[i])) {
961+
u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT);
962+
963+
store_func_meta(image, ctx, meta, func_meta_off);
964+
cookie_ctx_off--;
965+
}
966+
928967
if (invoke_bpf_prog(image, ro_image, ctx, fentry->links[i], regs_off, retval_off,
929968
run_ctx_off, flags & BPF_TRAMP_F_RET_FENTRY_RET))
930969
return -EINVAL;
970+
}
931971

932972
if (fmod_ret->nr_links) {
933973
branches = kcalloc(fmod_ret->nr_links, sizeof(u32), GFP_KERNEL);
@@ -989,12 +1029,27 @@ static int __arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *rw_im
9891029
image[branches[i]] = ppc_inst_val(branch_insn);
9901030
}
9911031

992-
for (i = 0; i < fexit->nr_links; i++)
1032+
/* set the "is_return" flag for fsession */
1033+
func_meta |= (1ULL << BPF_TRAMP_IS_RETURN_SHIFT);
1034+
if (fsession_cnt)
1035+
store_func_meta(image, ctx, func_meta, func_meta_off);
1036+
1037+
cookie_ctx_off = (regs_off - cookie_off) / 8;
1038+
1039+
for (i = 0; i < fexit->nr_links; i++) {
1040+
if (bpf_prog_calls_session_cookie(fexit->links[i])) {
1041+
u64 meta = func_meta | (cookie_ctx_off << BPF_TRAMP_COOKIE_INDEX_SHIFT);
1042+
1043+
store_func_meta(image, ctx, meta, func_meta_off);
1044+
cookie_ctx_off--;
1045+
}
1046+
9931047
if (invoke_bpf_prog(image, ro_image, ctx, fexit->links[i], regs_off, retval_off,
9941048
run_ctx_off, false)) {
9951049
ret = -EINVAL;
9961050
goto cleanup;
9971051
}
1052+
}
9981053

9991054
if (flags & BPF_TRAMP_F_CALL_ORIG) {
10001055
if (ro_image) /* image is NULL for dummy pass */

‎arch/powerpc/net/bpf_jit_comp32.c‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,41 @@ void bpf_jit_realloc_regs(struct codegen_context *ctx)
123123
}
124124
}
125125

126+
void prepare_for_fsession_fentry(u32 *image, struct codegen_context *ctx, int cookie_cnt,
127+
int cookie_off, int retval_off)
128+
{
129+
/*
130+
* Set session cookies value
131+
* Clear cookies field on stack
132+
* Ensure retval to be cleared on fentry
133+
*/
134+
EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG), 0));
135+
136+
for (int i = 0; i < cookie_cnt; i++) {
137+
EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, cookie_off + 4 * i));
138+
EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, cookie_off + 4 * i + 4));
139+
}
140+
141+
EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, retval_off));
142+
EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, retval_off + 4));
143+
}
144+
145+
void store_func_meta(u32 *image, struct codegen_context *ctx,
146+
u64 func_meta, int func_meta_off)
147+
{
148+
/*
149+
* Store func_meta to stack: [R1 + func_meta_off] = func_meta
150+
* func_meta := argument count in first byte + cookie value
151+
*/
152+
/* Store lower word */
153+
EMIT(PPC_RAW_LI32(bpf_to_ppc(TMP_REG), (u32)func_meta));
154+
EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, func_meta_off));
155+
156+
/* Store upper word */
157+
EMIT(PPC_RAW_LI32(bpf_to_ppc(TMP_REG), (u32)(func_meta >> 32)));
158+
EMIT(PPC_RAW_STW(bpf_to_ppc(TMP_REG), _R1, func_meta_off + 4));
159+
}
160+
126161
void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx)
127162
{
128163
int i;

‎arch/powerpc/net/bpf_jit_comp64.c‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,31 @@ static int bpf_jit_stack_offsetof(struct codegen_context *ctx, int reg)
179179
BUG();
180180
}
181181

182+
void prepare_for_fsession_fentry(u32 *image, struct codegen_context *ctx, int cookie_cnt,
183+
int cookie_off, int retval_off)
184+
{
185+
EMIT(PPC_RAW_LI(bpf_to_ppc(TMP_REG_1), 0));
186+
187+
for (int i = 0; i < cookie_cnt; i++)
188+
EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, cookie_off + 8 * i));
189+
EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, retval_off));
190+
}
191+
192+
void store_func_meta(u32 *image, struct codegen_context *ctx,
193+
u64 func_meta, int func_meta_off)
194+
{
195+
/*
196+
* Store func_meta to stack at [R1 + func_meta_off] = func_meta
197+
*
198+
* func_meta :
199+
* bit[63]: is_return flag
200+
* byte[1]: cookie offset from ctx
201+
* byte[0]: args count
202+
*/
203+
PPC_LI64(bpf_to_ppc(TMP_REG_1), func_meta);
204+
EMIT(PPC_RAW_STD(bpf_to_ppc(TMP_REG_1), _R1, func_meta_off));
205+
}
206+
182207
void bpf_jit_realloc_regs(struct codegen_context *ctx)
183208
{
184209
}

0 commit comments

Comments
 (0)