Add 6.8 kernel to CI build. Update patch versions of existing kernels (#1995)

Author: ddelnano

bazel/pl_qemu_kernels.bzl

@@ -16,16 +16,18 @@
 
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file")
 
-kernel_build_date = 20230620214456
+kernel_build_date = 20240826223635
 kernel_catalog = {
-    "4.14.254": "857030b52d0a48bc95ab544439cf52da07bce8cc2b9bef2e0382f02c182fab78",
-    "4.19.254": "5203a52b48a966ddfc8564ed2d0fd1e84f20ef417cbb5d41eb0b1c8413e4853d",
-    "5.10.173": "8c746c5ee94964323ca69533950370f595426efd671bfed85473e5a4b570ec26",
-    "5.15.101": "e18b8c5e91e3856dd6705a324ec7ebb2f1be5ff503c1322f5ddc6c44255272e4",
-    "5.4.235": "828f69979065800a4f214a1be369ab9756fdc2c586bd5bbfcc6d2a0f501601d5",
-    "6.1.18": "31f125451a70249e206a1eba555e432d5e08d173a4ab9d675f0b5d57f879f81b",
+    "4.14.254": "007c1d9d7385c7aa9df8434b3c2a2c38963c88391d9f4b88d5ac67bbccbbccef",
+    "4.19.254": "946d914990ae5b2f6dae6ac0ecbc81c1f63df84832ecd7563c18e7dc99986d2f",
+    "5.4.254": "fbae23c3da8af04c2a039bff7a5926f71041506cd150d02209f1b1b10cd38329",
+    "5.10.224": "d5b3cd7282f0533fe1e2574a73f9164bc90fae1098fe0284cf37a2995ce0e762",
+    "5.15.165": "0b70925e4497d74d99fd43544b259c5ed2dbc4c91c4cf702f872698a2bf7d80d",
+    "6.1.106": "fa03bd44bd6a5a7e1dcf1ee6a1790add4f2297d85cfffdc61b52f123eda27dc7",
+    "6.8.12": "06db1941423384c4b956d1e7a46a888a5647d15504ee31fe8ae8bdff264f1678",
 }
 
+
 def kernel_version_to_name(version):
     return "linux_build_{}_x86_64".format(version.replace(".", "_"))
 
@@ -35,7 +37,6 @@ def qemu_kernel_deps():
             name = kernel_version_to_name(version),
             urls = [
                 "https://github.com/pixie-io/dev-artifacts/releases/download/kernel-build%2F{}/linux-build-{}.tar.gz".format(kernel_build_date, version),
-                "https://storage.googleapis.com/pixie-dev-public/kernel-build/{}/linux-build-{}.tar.gz".format(kernel_build_date, version),
             ],
             sha256 = sha,
             downloaded_file_path = "linux-build.tar.gz",

ci/github/matrix.sh

@@ -31,10 +31,11 @@ nightly_regression_test_iterations="${NIGHTLY_REGRESSION_TEST_ITERATIONS:-5}"
 all_kernel_versions=(
   "4.14.254"
   "4.19.254"
-  "5.10.173"
-  "5.15.101"
-  "5.4.235"
-  "6.1.18"
+  "5.4.254"
+  "5.10.224"
+  "5.15.165"
+  "6.1.106"
+  "6.8.12"
 )
 default_kernel_versions=(
   "4.14.254"

src/stirling/bpf_tools/bcc_bpf/task_struct_utils.h

@@ -25,6 +25,14 @@
 
 #include <linux/sched.h>
 
+// Between linux v6.1 and v6.8, NSEC_PER_SEC is no longer defined by including
+// linux/sched.h. This ifndef covers newer kernels that won't have it defined.
+// libbpf provides a trace_helpers.h header that provides NSEC_PER_SEC, so it seems
+// tools are moving towards defining their own instead of using a linux header for it.
+#ifndef NSEC_PER_SEC
+#define NSEC_PER_SEC 1000000000ULL
+#endif
+
 #include "src/stirling/bpf_tools/bcc_bpf/utils.h"
 
 // This is how Linux converts nanoseconds to clock ticks.

src/stirling/source_connectors/dynamic_tracer/dynamic_tracing/code_gen.cc

@@ -896,6 +896,11 @@ std::vector<std::string> GenIncludes() {
   return {
       // For struct task_struct.
       "#include <linux/sched.h>",
+      // NSEC_PER_SEC is not defined within linux/sched.h for
+      // 6.x kernels, so we define it here.
+      "#ifndef NSEC_PER_SEC",
+      "#define NSEC_PER_SEC 1000000000ULL",
+      "#endif",
   };
 }
 

src/stirling/source_connectors/dynamic_tracer/dynamic_tracing/code_gen_test.cc

@@ -395,6 +395,9 @@ TEST(GenProgramTest, SpecsAndCode) {
 
   const std::vector<std::string> expected_code_lines = {
       "#include <linux/sched.h>",
+      "#ifndef NSEC_PER_SEC",
+      "#define NSEC_PER_SEC 1000000000ULL",
+      "#endif",
       "#define __inline inline __attribute__((__always_inline__))",
       "static __inline uint64_t pl_nsec_to_clock_t(uint64_t x) {",
       "return div_u64(x, NSEC_PER_SEC / USER_HZ);",

src/stirling/source_connectors/dynamic_tracer/dynamic_tracing/dynamic_tracer_test.cc

@@ -314,6 +314,9 @@ tracepoints {
 
 const std::vector<std::string> kExpectedBCC = {
     "#include <linux/sched.h>",
+    "#ifndef NSEC_PER_SEC",
+    "#define NSEC_PER_SEC 1000000000ULL",
+    "#endif",
     "#define __inline inline __attribute__((__always_inline__))",
     "static __inline uint64_t pl_nsec_to_clock_t(uint64_t x) {",
     "return div_u64(x, NSEC_PER_SEC / USER_HZ);",

tools/docker/Makefile

@@ -137,7 +137,6 @@ KERNEL_BUILD_VERSIONS := 4.14.254 \
 KERNEL_BUILD_TEMPLATE := linux-build-%.tar.gz
 KERNEL_BUILD_TARGETS = $(addprefix $(KERNEL_BUILD_DIR)/, $(patsubst %,$(KERNEL_BUILD_TEMPLATE), $(KERNEL_BUILD_VERSIONS)))
 KERNEL_BUILD_TS := $(shell date +%Y%m%d%H%M%S)
-KERNEL_BUILD_GS_PATH := gs://pixie-dev-public/kernel-build/$(KERNEL_BUILD_TS)
 
 ETCD_IMAGE_VERSION := 3.5.9
 ETCD_IMAGE_TAG := "gcr.io/pixie-oss/pixie-dev-public/etcd:$(ETCD_IMAGE_VERSION)"
@@ -287,7 +286,6 @@ $(KERNEL_BUILD_DIR)/linux-build-%.tar.gz: kernel_builder/Dockerfile kernel_build
 
 .PHONY: upload_kernel_build
 upload_kernel_build:  $(KERNEL_BUILD_TARGETS)
-	gsutil cp $^ $(KERNEL_BUILD_GS_PATH)
 	$(GH_RELEASE_UPLOAD) kernel-build $(KERNEL_BUILD_TS) $^
 	@echo "Add this to the qemu bzl file:"
 	@echo "============================="