feat: add Large File Support (LFS) subsystem (#134)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: kamir

.github/workflows/ci.yml

@@ -76,6 +76,7 @@ jobs:
       - name: Enforce coverage floor
         run: bash hack/check_coverage.sh 45
 
+
   helm-lint:
     name: Helm Lint
     runs-on: ubuntu-latest

.github/workflows/codeql.yml

@@ -51,6 +51,12 @@ jobs:
         with:
           languages: ${{ matrix.language }}
           queries: security-extended,security-and-quality
+          config: |
+            paths-ignore:
+              - '**/node_modules/**'
+              - '**/target/**'
+              - '**/*.egg-info/**'
+              - 'third_party/**'
 
       - name: Autobuild
         uses: github/codeql-action/autobuild@v4

.github/workflows/docker.yml

@@ -36,6 +36,10 @@ jobs:
             image: kafscale-broker
             context: .
             file: deploy/docker/broker.Dockerfile
+          - name: lfs-proxy
+            image: kafscale-lfs-proxy
+            context: .
+            file: deploy/docker/lfs-proxy.Dockerfile
           - name: operator
             image: kafscale-operator
             context: .

.gitignore

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+Makefile-MK # Ignore build and IDE files
+
 # Binaries
 /bin/
 /dist/
@@ -31,6 +33,9 @@
 *.key
 coverage*.out
 .build/
+target/
+**/target/
+spark-warehouse/
 
 # Local Go cache (use GOCACHE=.gocache for hermetic builds/tests)
 .gocache/
@@ -53,6 +58,7 @@ coverage*.out
 *.swp
 *.swo
 .DS_Store
+.claude/
 
 # Addon processor artifacts
 addons/processors/**/bin/
@@ -79,3 +85,27 @@ proto/**/*.swagger.json
 _site/
 Gemfile
 Gemfile.lock
+
+# Ignore demo node_modules
+examples/E50_JS-kafscale-demo/node_modules/
+
+# Go compiled binaries (top-level)
+/e2e-client
+/lfs-proxy
+/proxy
+
+# Java build artifacts
+target/
+dependency-reduced-pom.xml
+
+# JavaScript/Node.js
+node_modules/
+package-lock.json
+
+# Python
+__pycache__/
+*.pyc
+*.egg-info/
+
+# Test output
+records.txt

Makefile

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-.PHONY: proto build test tidy lint generate docker-build docker-build-e2e-client docker-build-etcd-tools docker-clean ensure-minio start-minio stop-containers release-broker-ports test-produce-consume test-produce-consume-debug test-consumer-group test-ops-api test-mcp test-multi-segment-durability test-full test-operator test-acl demo demo-platform demo-platform-bootstrap iceberg-demo kafsql-demo platform-demo help clean-kind-all
+.PHONY: proto build test tidy lint generate build-sdk docker-build docker-build-e2e-client docker-build-etcd-tools docker-clean ensure-minio start-minio stop-containers release-broker-ports test-produce-consume test-produce-consume-debug test-consumer-group test-ops-api test-mcp test-multi-segment-durability test-full test-operator test-acl demo demo-platform demo-platform-bootstrap iceberg-demo kafsql-demo platform-demo help clean-kind-all
 
 REGISTRY ?= ghcr.io/kafscale
 STAMP_DIR ?= .build
@@ -71,6 +71,10 @@ KAFSCALE_DEMO_ETCD_INMEM ?= 1
 KAFSCALE_DEMO_ETCD_REPLICAS ?= 3
 BROKER_PORT ?= 39092
 BROKER_PORTS ?= 39092 39093 39094
+SDK_JAVA_BUILD_CMD ?= mvn -DskipTests clean package
+SDK_JS_BUILD_CMD ?= npm install && npm run build
+SDK_PY_BUILD_CMD ?= python -m build
+SKIP_JS_SDK ?= 1
 
 proto: ## Generate protobuf + gRPC stubs
 	buf generate
@@ -80,6 +84,19 @@ generate: proto
 build: ## Build all binaries
 	go build ./...
 
+build-sdk: ## Build all LFS client SDKs
+	@echo "Building Java SDK..."
+	@cd lfs-client-sdk/java && $(SDK_JAVA_BUILD_CMD)
+	@test -d lfs-client-sdk/java/target || { echo "Java SDK target/ missing"; exit 1; }
+	@if [ "$(SKIP_JS_SDK)" = "1" ]; then \
+		echo "Skipping JS SDK build (SKIP_JS_SDK=1)"; \
+	else \
+		echo "Building JS SDK..."; \
+		cd lfs-client-sdk/js && $(SDK_JS_BUILD_CMD); \
+	fi
+	@echo "Building Python SDK..."
+	@cd lfs-client-sdk/python && $(SDK_PY_BUILD_CMD)
+
 test: ## Run unit tests + vet + race
 	go vet ./...
 	go test -race ./...

README.md

@@ -117,6 +117,13 @@ For the technical specification and data formats, see `kafscale-spec.md`.
 A detailed architecture overview and design rationale are available here:
 https://www.novatechflow.com/p/kafscale.html
 
+## Examples
+
+- Quickstart guide: `examples/101_kafscale-dev-guide/README.md`
+- Spring Boot app demo (E20): `examples/E20_spring-boot-kafscale-demo/README.md`
+- Flink demo (E30): `examples/E30_flink-kafscale-demo/README.md`
+- Spark demo (E40): `examples/E40_spark-kafscale-demo/README.md`
+
 ## Community
 
 - License: Apache 2.0 (`LICENSE`)

addons/processors/iceberg-processor/Dockerfile

@@ -16,12 +16,21 @@
 
 FROM golang:1.25-alpine AS build
 RUN apk add --no-cache git
+ARG REPO_ROOT=.
+ARG MODULE_DIR=.
 WORKDIR /src
-COPY go.mod go.sum ./
+COPY ${REPO_ROOT} /src
+WORKDIR /src/${MODULE_DIR}
+ARG USE_LOCAL_PLATFORM=0
+RUN if [ "${USE_LOCAL_PLATFORM}" != "1" ]; then \
+      go mod edit -dropreplace github.com/KafScale/platform || true; \
+    fi
+RUN if [ "${USE_LOCAL_PLATFORM}" != "1" ]; then \
+      go mod download github.com/KafScale/platform@v1.5.0 || true; \
+    fi
 RUN --mount=type=cache,target=/go/pkg/mod \
     --mount=type=cache,target=/root/.cache/go-build \
     go mod download
-COPY . ./
 ARG GO_BUILD_FLAGS=
 RUN --mount=type=cache,target=/go/pkg/mod \
     --mount=type=cache,target=/root/.cache/go-build \

addons/processors/iceberg-processor/Makefile

@@ -19,6 +19,30 @@ BINARY := iceberg-processor
 BUILD_DIR := bin
 IMAGE ?= $(BINARY):dev
 DOCKER_BUILD_ARGS ?=
+DOCKER_BUILD_ARGS_LOCAL = --build-arg USE_LOCAL_PLATFORM=1 --build-arg REPO_ROOT=. --build-arg MODULE_DIR=addons/processors/iceberg-processor
+RSYNC_EXCLUDES = \
+	--exclude ".dockerignore" \
+	--exclude ".git" \
+	--exclude ".build" \
+	--exclude ".gocache" \
+	--exclude ".idea" \
+	--exclude ".vscode" \
+	--exclude "_site" \
+	--exclude "bin" \
+	--exclude "coverage.out" \
+	--exclude "dist" \
+	--exclude "docs" \
+	--exclude "deploy/helm" \
+	--exclude "test" \
+	--exclude "tmp" \
+	--exclude "**/.DS_Store" \
+	--exclude "**/*.log" \
+	--exclude "**/*.swp" \
+	--exclude "**/*_test.go" \
+	--exclude "**/node_modules" \
+	--exclude "ui/.next" \
+	--exclude "ui/dist" \
+	--exclude "ui/build"
 
 DOCKER_BUILD_CMD := $(shell \
 	if command -v docker >/dev/null 2>&1 && docker buildx version >/dev/null 2>&1; then \
@@ -46,4 +70,7 @@ clean:
 	rm -rf $(BUILD_DIR)
 
 docker-build:
-	$(DOCKER_BUILD_CMD) $(DOCKER_BUILD_ARGS) -t $(IMAGE) .
+	@tmp=$$(mktemp -d); \
+	rsync -a --delete $(RSYNC_EXCLUDES) ../../.. "$$tmp/"; \
+	$(DOCKER_BUILD_CMD) $(DOCKER_BUILD_ARGS) $(DOCKER_BUILD_ARGS_LOCAL) -t $(IMAGE) -f Dockerfile "$$tmp"; \
+	rm -rf "$$tmp"

addons/processors/iceberg-processor/config/config.yaml

@@ -62,3 +62,9 @@ mappings:
           type: string
           required: false
       allow_type_widening: true
+    lfs:
+      mode: off
+      max_inline_size: 1048576
+      store_metadata: false
+      validate_checksum: true
+      resolve_concurrency: 4

addons/processors/iceberg-processor/deploy/helm/iceberg-processor/config/config.yaml

@@ -57,3 +57,9 @@ mappings:
           type: string
           required: false
       allow_type_widening: true
+    lfs:
+      mode: off
+      max_inline_size: 1048576
+      store_metadata: false
+      validate_checksum: true
+      resolve_concurrency: 4