nettrace: kfree_skb function call strack backtrace support

Add the new argument '--drop-stack' to print the call stack of
kfree_skb().

Signed-off-by: Menglong Dong <imagedong@tencent.com>

Author: menglongdong

README.md

@@ -153,6 +153,7 @@ Usage:
     --diag-keep      don't quit when abnormal packet found
     --hooks          print netfilter hooks if dropping by netfilter
     --drop           skb drop monitor mode, for replace of 'droptrace'
+    --drop-stack     print the kernel function call stack of kfree_skb
 
     -v               show log information
     --debug          show debug information
@@ -171,6 +172,7 @@ Usage:
 - `diag-keep`：持续跟踪。`diag`模式下，默认在跟踪到异常报文后会停止跟踪，使用该参数后，会持续跟踪下去。
 - `hooks`：结合netfilter做的适配，详见下文
 - `drop`：进行系统丢包监控，取代原先的`droptrace`
+- `drop-stack`: 打印kfree_skb内核函数的调用堆栈
 
 下面我们首先来看一下默认模式下的工具使用方法。
 

script/bash-completion.sh

@@ -1,2 +1,2 @@
-complete -W '-s --saddr --saddr6 -d --daddr --daddr6 --addr --addr6 -p -D --dport -S --sport -P --port --pid -t --trace -v --detail --debug --ret --basic --diag --diag-quiet --diag-keep --date --drop --hooks -h --help' nettrace
+complete -W '-s --saddr --saddr6 -d --daddr --daddr6 --addr --addr6 -p -D --dport -S --sport -P --port --pid -t --trace -v --detail --debug --ret --basic --diag --diag-quiet --diag-keep --date --drop --drop-stack --hooks -h --help' nettrace
 complete -W '-h -s -d --addr -p --dport --sport --port -t -v --detail --stack --stack-tracer --timeline -c --ret --skb-mode --force-stack --tcp-flags -o --output' nettrace-legacy

shared/bpf/skb_shared.h

@@ -79,4 +79,9 @@ typedef struct {
 
 #define CONFIG_MAP_SIZE	1024
 
+#ifndef PERF_MAX_STACK_DEPTH
+#define PERF_MAX_STACK_DEPTH 127
+#endif
+typedef __u64 stack_trace_t[PERF_MAX_STACK_DEPTH];
+
 #endif

src/Makefile

@@ -25,11 +25,15 @@ kprobe_gen_cmd = @make BPF_CFLAGS="$(BPF_CFLAGS) $(1)" \
 		mv progs/kprobe.skel.h progs/$(2).skel.h
 
 ifndef COMPAT
+	CFLAGS += -DSTACK_TRACE
 $(bpf_progs_ext):
 	rm -rf progs/kprobe*.skel.h
 	$(call kprobe_gen_cmd,-DCORE_FULL,kprobe_core)
 	make progs/kprobe.skel.h
 else
+ifneq ($(shell grep -c get_stackid $(HEADERS)/include/uapi/linux/bpf.h),0)
+	CFLAGS += -DSTACK_TRACE
+endif
 $(bpf_progs_ext): %: %.skel.h
 	@:
 endif

src/analysis.c

@@ -142,13 +142,14 @@ static void analy_entry_handle(analy_entry_t *entry)
 {
 	packet_t *pkt = &entry->event->pkt;
 	static char buf[1024], tinfo[128];
+	event_t *e = entry->event;
 	rule_t *rule;
 	trace_t *t;
 
 	t = get_trace_from_analy_entry(entry);
 	pr_debug("output entry(%llx)\n", PTR2X(entry));
 	if (trace_ctx.detail) {
-		detail_event_t *detail = (void *)entry->event;
+		detail_event_t *detail = (void *)e;
 		static char ifbuf[IF_NAMESIZE];
 		char *ifname = detail->ifname;
 
@@ -195,6 +196,8 @@ static void analy_entry_handle(analy_entry_t *entry)
 	}
 out:
 	pr_info("%s\n", buf);
+	if (trace_is_stack(t))
+		trace_ctx.ops->print_stack(e->stack_id);
 }
 
 static void analy_ctx_free(analy_ctx_t *ctx)

src/gen_trace.py

@@ -49,7 +49,7 @@ def gen_group_init(group, name):
 
 global_status = {}
 global_names = {}
-global_status['trace_index'] = 0
+global_status['trace_index'] = 1
 
 rule_levels = {
     'info': 'RULE_INFO',

src/nettrace.c

@@ -78,6 +78,13 @@ static void do_parse_args(int argc, char *argv[])
 			.type = OPTION_BOOL,
 			.desc = "skb drop monitor mode, for replace of 'droptrace'",
 		},
+#ifdef STACK_TRACE
+		{
+			.lname = "drop-stack", .dest = &trace_args->drop_stack,
+			.type = OPTION_BOOL,
+			.desc = "print the kernel function call stack of kfree_skb",
+		},
+#endif
 		{ .type = OPTION_BLANK },
 		{
 			.sname = 'v', .dest = &show_log,

src/progs/kprobe.c

@@ -19,6 +19,15 @@ struct {
 	__uint(max_entries, TRACE_MAX);
 } m_ret SEC(".maps");
 
+#ifdef STACK_TRACE
+struct {
+	__uint(type, BPF_MAP_TYPE_STACK_TRACE);
+	__uint(max_entries, 16384);
+	__uint(key_size, sizeof(__u32));
+	__uint(value_size, sizeof(stack_trace_t));
+} m_stack SEC(".maps");
+#endif
+
 #ifdef KERN_VER
 __u32 kern_ver SEC("version") = KERN_VER;
 #endif
@@ -64,8 +73,38 @@ static try_inline int put_ret(int func)
 	return 0;
 }
 
-static try_inline int handle_entry(void *regs, struct sk_buff *skb, event_t *e,
-			       int size, int func)
+#ifdef STACK_TRACE
+static try_inline void try_trace_stack(void *regs, bpf_args_t *bpf_args,
+				       event_t *e, int func)
+{
+	int i = 0, key;
+	u16 *funcs;
+
+	if (!ARGS_GET(stack))
+		return;
+
+	funcs = ARGS_GET(stack_funs);
+
+#pragma unroll
+	for (; i < MAX_FUNC_STACK; i++) {
+		if (!funcs[i])
+			break;
+		if (funcs[i] == func)
+			goto do_stack;
+	}
+	return;
+
+do_stack:
+	key = bpf_get_stackid(regs, &m_stack, 0);
+	e->stack_id = key;
+}
+#else
+static try_inline void try_trace_stack(void *regs, bpf_args_t *bpf_args,
+				       event_t *e, int func) { }
+#endif
+
+static try_inline int handle_entry(void *regs, struct sk_buff *skb,
+				   event_t *e, int size, int func)
 {
 	packet_t *pkt = &e->pkt;
 	bool *matched;
@@ -111,6 +150,7 @@ static try_inline int handle_entry(void *regs, struct sk_buff *skb, event_t *e,
 	}
 
 out:
+	try_trace_stack(regs, bpf_args, e, func);
 	pkt->ts = bpf_ktime_get_ns();
 	e->key = (u64)(void *)skb;
 
@@ -129,8 +169,8 @@ static try_inline int handle_destroy(struct sk_buff *skb)
 }
 
 static try_inline int default_handle_entry(struct pt_regs *ctx,
-				       struct sk_buff *skb,
-				       int func)
+					   struct sk_buff *skb,
+					   int func)
 {
 	if (ARGS_GET_CONFIG(detail)) {
 		detail_event_t e = { .func = func };

src/progs/shared.h

@@ -1,6 +1,8 @@
 #ifndef _H_PROGS_SHARED
 #define _H_PROGS_SHARED
 
+#define MAX_FUNC_STACK 16
+
 #define DEFINE_BPF_ARGS()	\
 	u32  trace_mode;	\
 	u32  pid;		\
@@ -10,14 +12,19 @@
 	bool detail;		\
 	bool hooks;		\
 	bool ready;		\
-	bool nft_high;
+	bool nft_high;		\
+	bool stack;		\
+	u16  stack_funs[MAX_FUNC_STACK];
 
 #include <skb_shared.h>
 
 typedef struct __attribute__((__packed__)) {
 	packet_t	pkt;
 	u64		key;
 	u32		func;
+#ifdef STACK_TRACE
+	u32		stack_id;
+#endif
 } event_t;
 
 typedef struct __attribute__((__packed__)) {

src/trace.c

@@ -198,6 +198,9 @@ static int trace_prepare_args()
 	if (args->drop) {
 		trace_ctx.mode = TRACE_MODE_DROP;
 		traces = "kfree_skb";
+	} else if (args->drop_stack) {
+		pr_err("--drop should be set!\n");
+		goto err;
 	}
 
 	if (!traces) {
@@ -244,11 +247,17 @@ static int trace_prepare_args()
 			trace_group_enable("life");
 		break;
 	case TRACE_MODE_BASIC:
-	case TRACE_MODE_DROP:
+	case TRACE_MODE_DROP: {
+		trace_t *drop_trace = search_trace_enabled(traces);
+
 		if (!trace_ctx.drop_reason)
 			pr_warn("skb drop reason is not support by your kernel"
 				", drop reason will not be printed\n");
+
+		if (args->drop_stack && trace_set_stack(drop_trace))
+			goto err;
 		break;
+	}
 	default:
 		goto err;
 	}