diff --git a/Dockerfile.android b/Dockerfile.android new file mode 100644 index 0000000..dcdbcfa --- /dev/null +++ b/Dockerfile.android @@ -0,0 +1,168 @@ +# Dockerfile.android — cross-compile android-arm64 static libraries via Android NDK +# +# Usage: +# docker build -f Dockerfile.android -o ./out . +# +# Build + link test (ensures .a files cross-link correctly with Go CGO): +# docker build -f Dockerfile.android --target build-test . +# +# This extracts prebuilt .a files + headers for android-arm64 into ./out/. + +# ============================================================================ +# Stage: Download sources +# ============================================================================ +FROM golang:1.24-bookworm AS sources + +RUN apt-get update && apt-get install -y --no-install-recommends \ + wget && \ + rm -rf /var/lib/apt/lists/* + +WORKDIR /src + +# Copy version files so we can read versions from Go +COPY go.mod ./ +COPY version.go ./ +COPY cmd/versioncmd/ ./cmd/versioncmd/ + +# Download llama.cpp +RUN LLAMA_VERSION=$(go run ./cmd/versioncmd llama.cpp) && \ + echo "Downloading llama.cpp ${LLAMA_VERSION}..." && \ + wget -qO llama.cpp.tar.gz "https://github.com/ggerganov/llama.cpp/archive/refs/tags/${LLAMA_VERSION}.tar.gz" && \ + mkdir -p llama-src && \ + tar xzf llama.cpp.tar.gz --strip-components=1 -C llama-src && \ + rm llama.cpp.tar.gz + +# Download whisper.cpp +RUN WHISPER_VERSION=$(go run ./cmd/versioncmd whisper.cpp) && \ + echo "Downloading whisper.cpp ${WHISPER_VERSION}..." && \ + wget -qO whisper.cpp.tar.gz "https://github.com/ggerganov/whisper.cpp/archive/refs/tags/${WHISPER_VERSION}.tar.gz" && \ + mkdir -p whisper-src && \ + tar xzf whisper.cpp.tar.gz --strip-components=1 -C whisper-src && \ + rm whisper.cpp.tar.gz + +# ============================================================================ +# Builder: Android ARM64 via NDK +# ============================================================================ +FROM golang:1.24-bookworm AS builder + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential cmake wget unzip && \ + rm -rf /var/lib/apt/lists/* + +# Install Android NDK r27c +ARG NDK_VERSION=r27c +RUN wget -qO ndk.zip "https://dl.google.com/android/repository/android-ndk-${NDK_VERSION}-linux.zip" && \ + unzip -q ndk.zip -d /opt && \ + rm ndk.zip +ENV ANDROID_NDK_HOME=/opt/android-ndk-${NDK_VERSION} + +WORKDIR /src +COPY --from=sources /src/llama-src llama-src +COPY --from=sources /src/whisper-src whisper-src + +# NDK strip tool for removing debug symbols from .a files +ENV NDK_STRIP=${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/llvm-strip + +# Build llama.cpp for android-arm64 +RUN cd llama-src && \ + cmake -B build \ + -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_HOME}/build/cmake/android.toolchain.cmake \ + -DANDROID_ABI=arm64-v8a \ + -DANDROID_PLATFORM=android-28 \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF \ + -DGGML_OPENMP=OFF && \ + cmake --build build --config Release -j$(nproc) + +# Build whisper.cpp for android-arm64 +RUN cd whisper-src && \ + cmake -B build \ + -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_HOME}/build/cmake/android.toolchain.cmake \ + -DANDROID_ABI=arm64-v8a \ + -DANDROID_PLATFORM=android-28 \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_SHARED_LIBS=OFF \ + -DGGML_OPENMP=OFF && \ + cmake --build build --config Release -j$(nproc) + +# Collect llama.cpp artifacts and strip debug symbols +RUN mkdir -p /out/llama.cpp/android-arm64 /out/llama.cpp/include /out/llama.cpp/ggml/include /out/llama.cpp/common && \ + find llama-src/build -name "*.a" -exec cp {} /out/llama.cpp/android-arm64/ \; && \ + find /out/llama.cpp/android-arm64 -name "*.a" -exec ${NDK_STRIP} --strip-debug {} \; && \ + cp llama-src/include/*.h /out/llama.cpp/include/ && \ + cp llama-src/ggml/include/*.h /out/llama.cpp/ggml/include/ && \ + cp llama-src/common/common.h /out/llama.cpp/common/ && \ + cp llama-src/common/sampling.h /out/llama.cpp/common/ + +# Collect whisper.cpp artifacts and strip debug symbols +RUN mkdir -p /out/whisper.cpp/android-arm64 /out/whisper.cpp/include /out/whisper.cpp/ggml/include && \ + find whisper-src/build -name "*.a" -exec cp {} /out/whisper.cpp/android-arm64/ \; && \ + find /out/whisper.cpp/android-arm64 -name "*.a" -exec ${NDK_STRIP} --strip-debug {} \; && \ + cp whisper-src/include/*.h /out/whisper.cpp/include/ && \ + cp whisper-src/ggml/include/*.h /out/whisper.cpp/ggml/include/ + +# ============================================================================ +# Build test — verifies .a files cross-link correctly with Go CGO + NDK +# ============================================================================ +FROM golang:1.24-bookworm AS build-test + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential wget unzip && \ + rm -rf /var/lib/apt/lists/* + +# Install Android NDK (same version as builder) +ARG NDK_VERSION=r27c +RUN wget -qO ndk.zip "https://dl.google.com/android/repository/android-ndk-${NDK_VERSION}-linux.zip" && \ + unzip -q ndk.zip -d /opt && \ + rm ndk.zip +ENV ANDROID_NDK_HOME=/opt/android-ndk-${NDK_VERSION} + +# NDK toolchain paths +ENV NDK_TOOLCHAIN=${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/linux-x86_64 +ENV CC_android_arm64=${NDK_TOOLCHAIN}/bin/aarch64-linux-android28-clang +ENV CXX_android_arm64=${NDK_TOOLCHAIN}/bin/aarch64-linux-android28-clang++ + +WORKDIR /src +COPY . . + +# Copy freshly built .a files and headers into the source tree +COPY --from=builder /out/llama.cpp/android-arm64/ /src/ggml/llamacpp/third_party/prebuilt/android-arm64/ +COPY --from=builder /out/llama.cpp/include/ /src/ggml/llamacpp/third_party/include/ +COPY --from=builder /out/llama.cpp/ggml/include/ /src/ggml/llamacpp/third_party/ggml/include/ +COPY --from=builder /out/llama.cpp/common/ /src/ggml/llamacpp/third_party/common/ +COPY --from=builder /out/whisper.cpp/android-arm64/ /src/ggml/whispercpp/third_party/prebuilt/android-arm64/ +COPY --from=builder /out/whisper.cpp/include/ /src/ggml/whispercpp/third_party/include/ +COPY --from=builder /out/whisper.cpp/ggml/include/ /src/ggml/whispercpp/third_party/ggml/include/ + +# Verify stub builds (no tags, no CGO) +RUN CGO_ENABLED=0 go build ./ggml/llamacpp/... && \ + CGO_ENABLED=0 go build ./ggml/whispercpp/... && \ + echo "stub builds OK" + +# Verify CGO cross-compilation links against android-arm64 .a files +RUN CGO_ENABLED=1 \ + GOOS=android \ + GOARCH=arm64 \ + CC=${CC_android_arm64} \ + CXX=${CXX_android_arm64} \ + go build -tags llamacpp ./ggml/llamacpp/... && \ + echo "llamacpp android-arm64 CGO build OK" + +RUN CGO_ENABLED=1 \ + GOOS=android \ + GOARCH=arm64 \ + CC=${CC_android_arm64} \ + CXX=${CXX_android_arm64} \ + go build -tags whispercpp ./ggml/whispercpp/... && \ + echo "whispercpp android-arm64 CGO build OK" + +# Run stub tests +RUN CGO_ENABLED=0 go test ./ggml/llamacpp/... && \ + CGO_ENABLED=0 go test ./ggml/whispercpp/... && \ + echo "all tests passed" + +# ============================================================================ +# Output stage — docker build -o extracts from here +# ============================================================================ +FROM scratch +COPY --from=builder /out/ / diff --git a/Dockerfile.libs b/Dockerfile.libs index c2ca9b0..ac82e85 100644 --- a/Dockerfile.libs +++ b/Dockerfile.libs @@ -63,25 +63,27 @@ COPY --from=sources /src/whisper-src whisper-src # Build llama.cpp (CPU) RUN cd llama-src && \ - cmake -B build -DBUILD_SHARED_LIBS=OFF && \ + cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF && \ cmake --build build --config Release -j$(nproc) # Build whisper.cpp (CPU) RUN cd whisper-src && \ - cmake -B build -DBUILD_SHARED_LIBS=OFF && \ + cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF && \ cmake --build build --config Release -j$(nproc) -# Collect llama.cpp artifacts +# Collect llama.cpp artifacts and strip debug symbols RUN mkdir -p /out/llama.cpp/linux-amd64 /out/llama.cpp/include /out/llama.cpp/ggml/include /out/llama.cpp/common && \ find llama-src/build -name "*.a" -exec cp {} /out/llama.cpp/linux-amd64/ \; && \ + find /out/llama.cpp/linux-amd64 -name "*.a" -exec strip --strip-debug {} \; && \ cp llama-src/include/*.h /out/llama.cpp/include/ && \ cp llama-src/ggml/include/*.h /out/llama.cpp/ggml/include/ && \ cp llama-src/common/common.h /out/llama.cpp/common/ && \ cp llama-src/common/sampling.h /out/llama.cpp/common/ -# Collect whisper.cpp artifacts +# Collect whisper.cpp artifacts and strip debug symbols RUN mkdir -p /out/whisper.cpp/linux-amd64 /out/whisper.cpp/include /out/whisper.cpp/ggml/include && \ find whisper-src/build -name "*.a" -exec cp {} /out/whisper.cpp/linux-amd64/ \; && \ + find /out/whisper.cpp/linux-amd64 -name "*.a" -exec strip --strip-debug {} \; && \ cp whisper-src/include/*.h /out/whisper.cpp/include/ && \ cp whisper-src/ggml/include/*.h /out/whisper.cpp/ggml/include/ @@ -100,25 +102,27 @@ COPY --from=sources /src/whisper-src whisper-src # Build llama.cpp (CUDA) RUN cd llama-src && \ - cmake -B build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=ON && \ + cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=ON && \ cmake --build build --config Release -j$(nproc) # Build whisper.cpp (CUDA) RUN cd whisper-src && \ - cmake -B build -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=ON && \ + cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DGGML_CUDA=ON && \ cmake --build build --config Release -j$(nproc) -# Collect llama.cpp artifacts (CUDA variant) +# Collect llama.cpp artifacts (CUDA variant) and strip debug symbols RUN mkdir -p /out/llama.cpp/linux-amd64-cuda /out/llama.cpp/include /out/llama.cpp/ggml/include /out/llama.cpp/common && \ find llama-src/build -name "*.a" -exec cp {} /out/llama.cpp/linux-amd64-cuda/ \; && \ + find /out/llama.cpp/linux-amd64-cuda -name "*.a" -exec strip --strip-debug {} \; && \ cp llama-src/include/*.h /out/llama.cpp/include/ && \ cp llama-src/ggml/include/*.h /out/llama.cpp/ggml/include/ && \ cp llama-src/common/common.h /out/llama.cpp/common/ && \ cp llama-src/common/sampling.h /out/llama.cpp/common/ -# Collect whisper.cpp artifacts (CUDA variant) +# Collect whisper.cpp artifacts (CUDA variant) and strip debug symbols RUN mkdir -p /out/whisper.cpp/linux-amd64-cuda /out/whisper.cpp/include /out/whisper.cpp/ggml/include && \ find whisper-src/build -name "*.a" -exec cp {} /out/whisper.cpp/linux-amd64-cuda/ \; && \ + find /out/whisper.cpp/linux-amd64-cuda -name "*.a" -exec strip --strip-debug {} \; && \ cp whisper-src/include/*.h /out/whisper.cpp/include/ && \ cp whisper-src/ggml/include/*.h /out/whisper.cpp/ggml/include/ @@ -139,25 +143,27 @@ COPY --from=sources /src/whisper-src whisper-src # Build llama.cpp (Vulkan) RUN cd llama-src && \ - cmake -B build -DBUILD_SHARED_LIBS=OFF -DGGML_VULKAN=ON && \ + cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DGGML_VULKAN=ON && \ cmake --build build --config Release -j$(nproc) # Build whisper.cpp (Vulkan) RUN cd whisper-src && \ - cmake -B build -DBUILD_SHARED_LIBS=OFF -DGGML_VULKAN=ON && \ + cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF -DGGML_VULKAN=ON && \ cmake --build build --config Release -j$(nproc) -# Collect llama.cpp artifacts (Vulkan variant) +# Collect llama.cpp artifacts (Vulkan variant) and strip debug symbols RUN mkdir -p /out/llama.cpp/linux-amd64-vulkan /out/llama.cpp/include /out/llama.cpp/ggml/include /out/llama.cpp/common && \ find llama-src/build -name "*.a" -exec cp {} /out/llama.cpp/linux-amd64-vulkan/ \; && \ + find /out/llama.cpp/linux-amd64-vulkan -name "*.a" -exec strip --strip-debug {} \; && \ cp llama-src/include/*.h /out/llama.cpp/include/ && \ cp llama-src/ggml/include/*.h /out/llama.cpp/ggml/include/ && \ cp llama-src/common/common.h /out/llama.cpp/common/ && \ cp llama-src/common/sampling.h /out/llama.cpp/common/ -# Collect whisper.cpp artifacts (Vulkan variant) +# Collect whisper.cpp artifacts (Vulkan variant) and strip debug symbols RUN mkdir -p /out/whisper.cpp/linux-amd64-vulkan /out/whisper.cpp/include /out/whisper.cpp/ggml/include && \ find whisper-src/build -name "*.a" -exec cp {} /out/whisper.cpp/linux-amd64-vulkan/ \; && \ + find /out/whisper.cpp/linux-amd64-vulkan -name "*.a" -exec strip --strip-debug {} \; && \ cp whisper-src/include/*.h /out/whisper.cpp/include/ && \ cp whisper-src/ggml/include/*.h /out/whisper.cpp/ggml/include/ diff --git a/Makefile b/Makefile index 4ff5f7c..1bf8439 100644 --- a/Makefile +++ b/Makefile @@ -6,12 +6,16 @@ # go build -tags whispercpp ./... # whisper.cpp bindings # # For maintainers (rebuild .a files from source): -# make build-libs # Build all libraries for current platform -# make build-libs-llama # Build llama.cpp only -# make build-libs-whisper # Build whisper.cpp only -# make build-libs-linux # Build linux-amd64 .a files via Docker -# make build-libs-all # Build native + linux-amd64 -# make clean # Remove temp build dirs (keeps prebuilt .a + headers) +# make build-libs # Build all libraries for current platform +# make build-libs-llama # Build llama.cpp only +# make build-libs-whisper # Build whisper.cpp only +# make build-libs-linux # Build all linux-amd64 variants (cpu, cuda, vulkan) +# make build-libs-linux-cpu # Build linux-amd64 CPU only +# make build-libs-linux-cuda # Build linux-amd64 CUDA only +# make build-libs-linux-vulkan # Build linux-amd64 Vulkan only +# make build-libs-android # Build android-arm64 via NDK +# make build-libs-all # Build native + all linux + android +# make clean # Remove temp build dirs (keeps prebuilt .a + headers) # Version sync via Go toolchain (single source of truth: version.go) LLAMA_VERSION := $(shell go run ./cmd/versioncmd llama.cpp) @@ -32,12 +36,17 @@ WHISPER_THIRD_PARTY := ggml/whispercpp/third_party WHISPER_SRC := $(WHISPER_THIRD_PARTY)/src WHISPER_PREBUILT := $(WHISPER_THIRD_PARTY)/prebuilt/$(PLATFORM) -.PHONY: build-libs build-libs-llama build-libs-whisper build-libs-linux build-libs-all clean verify +.PHONY: build-libs build-libs-llama build-libs-whisper \ + build-libs-linux build-libs-linux-cpu build-libs-linux-cuda build-libs-linux-vulkan \ + build-libs-android build-libs-all clean verify build-libs: build-libs-llama build-libs-whisper -# Build both native platform and linux-amd64 (via Docker) -build-libs-all: build-libs build-libs-linux +# Build native + all linux variants + android +build-libs-all: build-libs build-libs-linux build-libs-android + +# Build all linux-amd64 variants (cpu, cuda, vulkan) +build-libs-linux: build-libs-linux-cpu build-libs-linux-cuda build-libs-linux-vulkan # ============================================================================ # llama.cpp @@ -46,10 +55,11 @@ build-libs-llama: $(LLAMA_PREBUILT) $(LLAMA_PREBUILT): $(LLAMA_SRC) @echo "==> Building llama.cpp $(LLAMA_VERSION) for $(PLATFORM)..." - cd $(LLAMA_SRC) && cmake -B build -DBUILD_SHARED_LIBS=OFF && \ + cd $(LLAMA_SRC) && cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF && \ cmake --build build --config Release -j$(NPROC) @mkdir -p $(LLAMA_PREBUILT) find $(LLAMA_SRC)/build -name "*.a" -exec cp {} $(LLAMA_PREBUILT)/ \; + find $(LLAMA_PREBUILT) -name "*.a" -exec strip -S {} \; 2>/dev/null || true @echo "==> Copying llama.cpp headers..." @mkdir -p $(LLAMA_THIRD_PARTY)/include cp $(LLAMA_SRC)/include/*.h $(LLAMA_THIRD_PARTY)/include/ @@ -74,10 +84,11 @@ build-libs-whisper: $(WHISPER_PREBUILT) $(WHISPER_PREBUILT): $(WHISPER_SRC) @echo "==> Building whisper.cpp $(WHISPER_VERSION) for $(PLATFORM)..." - cd $(WHISPER_SRC) && cmake -B build -DBUILD_SHARED_LIBS=OFF && \ + cd $(WHISPER_SRC) && cmake -B build -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=OFF && \ cmake --build build --config Release -j$(NPROC) @mkdir -p $(WHISPER_PREBUILT) find $(WHISPER_SRC)/build -name "*.a" -exec cp {} $(WHISPER_PREBUILT)/ \; + find $(WHISPER_PREBUILT) -name "*.a" -exec strip -S {} \; 2>/dev/null || true @echo "==> Copying whisper.cpp headers..." @mkdir -p $(WHISPER_THIRD_PARTY)/include cp $(WHISPER_SRC)/include/*.h $(WHISPER_THIRD_PARTY)/include/ @@ -93,27 +104,62 @@ $(WHISPER_SRC): rm whisper.cpp.tar.gz # ============================================================================ -# Docker build for linux-amd64 (cross-compile from macOS) +# Docker build for linux-amd64 variants (cross-compile from macOS) +# ============================================================================ + +# Helper: build a linux-amd64 variant via Dockerfile.libs +# $(1) = GPU_BACKEND (cpu, cuda, vulkan) +# $(2) = prebuilt directory suffix ("" for cpu, "-cuda", "-vulkan") +define build-linux-variant + @echo "==> Building linux-amd64$(2) static libraries via Docker ($(1))..." + docker build -f Dockerfile.libs --build-arg GPU_BACKEND=$(1) -o ./out . + @mkdir -p $(LLAMA_THIRD_PARTY)/prebuilt/linux-amd64$(2) + cp out/llama.cpp/linux-amd64$(2)/*.a $(LLAMA_THIRD_PARTY)/prebuilt/linux-amd64$(2)/ + @mkdir -p $(LLAMA_THIRD_PARTY)/include $(LLAMA_THIRD_PARTY)/ggml/include $(LLAMA_THIRD_PARTY)/common + cp out/llama.cpp/include/*.h $(LLAMA_THIRD_PARTY)/include/ + cp out/llama.cpp/ggml/include/*.h $(LLAMA_THIRD_PARTY)/ggml/include/ + cp out/llama.cpp/common/common.h $(LLAMA_THIRD_PARTY)/common/ + cp out/llama.cpp/common/sampling.h $(LLAMA_THIRD_PARTY)/common/ + @mkdir -p $(WHISPER_THIRD_PARTY)/prebuilt/linux-amd64$(2) + cp out/whisper.cpp/linux-amd64$(2)/*.a $(WHISPER_THIRD_PARTY)/prebuilt/linux-amd64$(2)/ + @mkdir -p $(WHISPER_THIRD_PARTY)/include $(WHISPER_THIRD_PARTY)/ggml/include + cp out/whisper.cpp/include/*.h $(WHISPER_THIRD_PARTY)/include/ + cp out/whisper.cpp/ggml/include/*.h $(WHISPER_THIRD_PARTY)/ggml/include/ + rm -rf out + @echo "==> linux-amd64$(2) libraries ready" +endef + +build-libs-linux-cpu: + $(call build-linux-variant,cpu,) + +build-libs-linux-cuda: + $(call build-linux-variant,cuda,-cuda) + +build-libs-linux-vulkan: + $(call build-linux-variant,vulkan,-vulkan) + +# ============================================================================ +# Docker build for android-arm64 (cross-compile via Android NDK) # ============================================================================ -build-libs-linux: - @echo "==> Building linux-amd64 static libraries via Docker..." - docker build -f Dockerfile.libs -o ./out . +build-libs-android: + @echo "==> Building android-arm64 static libraries via Docker (NDK)..." + docker build -f Dockerfile.android -o ./out . @# llama.cpp - @mkdir -p $(LLAMA_THIRD_PARTY)/prebuilt/linux-amd64 - cp out/llama.cpp/linux-amd64/*.a $(LLAMA_THIRD_PARTY)/prebuilt/linux-amd64/ + @mkdir -p $(LLAMA_THIRD_PARTY)/prebuilt/android-arm64 + cp out/llama.cpp/android-arm64/*.a $(LLAMA_THIRD_PARTY)/prebuilt/android-arm64/ @mkdir -p $(LLAMA_THIRD_PARTY)/include $(LLAMA_THIRD_PARTY)/ggml/include $(LLAMA_THIRD_PARTY)/common cp out/llama.cpp/include/*.h $(LLAMA_THIRD_PARTY)/include/ cp out/llama.cpp/ggml/include/*.h $(LLAMA_THIRD_PARTY)/ggml/include/ cp out/llama.cpp/common/common.h $(LLAMA_THIRD_PARTY)/common/ cp out/llama.cpp/common/sampling.h $(LLAMA_THIRD_PARTY)/common/ @# whisper.cpp - @mkdir -p $(WHISPER_THIRD_PARTY)/prebuilt/linux-amd64 - cp out/whisper.cpp/linux-amd64/*.a $(WHISPER_THIRD_PARTY)/prebuilt/linux-amd64/ + @mkdir -p $(WHISPER_THIRD_PARTY)/prebuilt/android-arm64 + cp out/whisper.cpp/android-arm64/*.a $(WHISPER_THIRD_PARTY)/prebuilt/android-arm64/ @mkdir -p $(WHISPER_THIRD_PARTY)/include $(WHISPER_THIRD_PARTY)/ggml/include cp out/whisper.cpp/include/*.h $(WHISPER_THIRD_PARTY)/include/ cp out/whisper.cpp/ggml/include/*.h $(WHISPER_THIRD_PARTY)/ggml/include/ rm -rf out - @echo "==> linux-amd64 libraries ready" + @echo "==> android-arm64 libraries ready" # ============================================================================ # Verification diff --git a/ggml/llamacpp/embed.go b/ggml/llamacpp/embed.go index b80a83e..78fe54e 100644 --- a/ggml/llamacpp/embed.go +++ b/ggml/llamacpp/embed.go @@ -16,4 +16,5 @@ import "embed" //go:embed third_party/prebuilt/linux-amd64-cuda/*.a //go:embed third_party/prebuilt/linux-amd64-vulkan/*.a //go:embed third_party/prebuilt/linux-arm64/*.a +//go:embed third_party/prebuilt/android-arm64/*.a var _ embed.FS diff --git a/ggml/llamacpp/ggml_type.go b/ggml/llamacpp/ggml_type.go new file mode 100644 index 0000000..57ba61c --- /dev/null +++ b/ggml/llamacpp/ggml_type.go @@ -0,0 +1,28 @@ +// Copyright 2025 FootprintAI +// SPDX-License-Identifier: Apache-2.0 + +package llamacpp + +// GGMLType represents ggml tensor data types, used for KV cache quantization. +// Values match the ggml_type enum in ggml.h. +type GGMLType int + +const ( + GGMLTypeF32 GGMLType = 0 + GGMLTypeF16 GGMLType = 1 + GGMLTypeQ4_0 GGMLType = 2 + GGMLTypeQ4_1 GGMLType = 3 + GGMLTypeQ5_0 GGMLType = 6 + GGMLTypeQ5_1 GGMLType = 7 + GGMLTypeQ8_0 GGMLType = 8 + GGMLTypeQ8_1 GGMLType = 9 + GGMLTypeQ2_K GGMLType = 10 + GGMLTypeQ3_K GGMLType = 11 + GGMLTypeQ4_K GGMLType = 12 + GGMLTypeQ5_K GGMLType = 13 + GGMLTypeQ6_K GGMLType = 14 + GGMLTypeQ8_K GGMLType = 15 + GGMLTypeBF16 GGMLType = 30 + GGMLTypeTQ1_0 GGMLType = 34 + GGMLTypeTQ2_0 GGMLType = 35 +) diff --git a/ggml/llamacpp/llamacpp.go b/ggml/llamacpp/llamacpp.go index 4ea44f2..96ce0ab 100644 --- a/ggml/llamacpp/llamacpp.go +++ b/ggml/llamacpp/llamacpp.go @@ -91,6 +91,8 @@ func (m *Model) NewContext(opts ...ContextOption) (*Context, error) { params.n_threads_batch = C.int32_t(cfg.threads) } params.embeddings = C.bool(cfg.embeddings) + params.type_k = C.enum_ggml_type(cfg.typeK) + params.type_v = C.enum_ggml_type(cfg.typeV) ctx := C.llama_init_from_model(m.c, params) if ctx == nil { diff --git a/ggml/llamacpp/llamacpp_android.go b/ggml/llamacpp/llamacpp_android.go new file mode 100644 index 0000000..1e2265f --- /dev/null +++ b/ggml/llamacpp/llamacpp_android.go @@ -0,0 +1,12 @@ +//go:build llamacpp && android && arm64 + +// Copyright 2025 FootprintAI +// SPDX-License-Identifier: Apache-2.0 + +package llamacpp + +/* +#cgo android,arm64 LDFLAGS: -L${SRCDIR}/third_party/prebuilt/android-arm64 +#cgo android LDFLAGS: -Wl,--start-group -lcommon -lllama -lggml-cpu -lggml-base -lggml -Wl,--end-group -lstdc++ -lm -ldl -llog +*/ +import "C" diff --git a/ggml/llamacpp/llamacpp_stub.go b/ggml/llamacpp/llamacpp_stub.go index f6c1018..d63a33b 100644 --- a/ggml/llamacpp/llamacpp_stub.go +++ b/ggml/llamacpp/llamacpp_stub.go @@ -74,10 +74,12 @@ func WithMLock(enabled bool) ModelOption { return func(*modelConfig) {} } type ContextOption func(*contextConfig) type contextConfig struct{} -func WithContextSize(n int) ContextOption { return func(*contextConfig) {} } -func WithBatchSize(n int) ContextOption { return func(*contextConfig) {} } -func WithThreads(n int) ContextOption { return func(*contextConfig) {} } -func WithEmbeddings() ContextOption { return func(*contextConfig) {} } +func WithContextSize(n int) ContextOption { return func(*contextConfig) {} } +func WithBatchSize(n int) ContextOption { return func(*contextConfig) {} } +func WithThreads(n int) ContextOption { return func(*contextConfig) {} } +func WithEmbeddings() ContextOption { return func(*contextConfig) {} } +func WithTypeK(t GGMLType) ContextOption { return func(*contextConfig) {} } +func WithTypeV(t GGMLType) ContextOption { return func(*contextConfig) {} } // GenerateOption configures text generation (stub). type GenerateOption func(*generateConfig) diff --git a/ggml/llamacpp/options.go b/ggml/llamacpp/options.go index 61a49d9..9964765 100644 --- a/ggml/llamacpp/options.go +++ b/ggml/llamacpp/options.go @@ -45,6 +45,8 @@ type contextConfig struct { batchSize int threads int embeddings bool + typeK GGMLType + typeV GGMLType } func defaultContextConfig() contextConfig { @@ -53,6 +55,8 @@ func defaultContextConfig() contextConfig { batchSize: 0, // 0 = use default threads: 0, // 0 = use default embeddings: false, + typeK: GGMLTypeF16, // llama.cpp default + typeV: GGMLTypeF16, // llama.cpp default } } @@ -76,6 +80,18 @@ func WithEmbeddings() ContextOption { return func(c *contextConfig) { c.embeddings = true } } +// WithTypeK sets the data type for the KV cache K values (EXPERIMENTAL). +// Use lower-precision types (e.g. GGMLTypeQ8_0, GGMLTypeQ4_0) to reduce memory usage. +func WithTypeK(t GGMLType) ContextOption { + return func(c *contextConfig) { c.typeK = t } +} + +// WithTypeV sets the data type for the KV cache V values (EXPERIMENTAL). +// Use lower-precision types (e.g. GGMLTypeQ8_0, GGMLTypeQ4_0) to reduce memory usage. +func WithTypeV(t GGMLType) ContextOption { + return func(c *contextConfig) { c.typeV = t } +} + // GenerateOption configures text generation. type GenerateOption func(*generateConfig) diff --git a/ggml/llamacpp/third_party/common/common.h b/ggml/llamacpp/third_party/common/common.h index 3e1b23f..020b6a7 100644 --- a/ggml/llamacpp/third_party/common/common.h +++ b/ggml/llamacpp/third_party/common/common.h @@ -3,12 +3,14 @@ #pragma once #include "ggml-opt.h" +#include "ggml.h" #include "llama-cpp.h" #include #include #include #include +#include #include #include @@ -104,6 +106,8 @@ enum llama_example { LLAMA_EXAMPLE_DIFFUSION, LLAMA_EXAMPLE_FINETUNE, LLAMA_EXAMPLE_FIT_PARAMS, + LLAMA_EXAMPLE_RESULTS, + LLAMA_EXAMPLE_EXPORT_GRAPH_OPS, LLAMA_EXAMPLE_COUNT, }; @@ -176,6 +180,43 @@ enum common_speculative_type { COMMON_SPECULATIVE_TYPE_COUNT // number of types, unknown type }; +// Grammar type enumeration +enum common_grammar_type { + COMMON_GRAMMAR_TYPE_NONE, // no grammar set + COMMON_GRAMMAR_TYPE_USER, // user-provided GBNF (--grammar / "grammar" API field) + COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT, // auto-generated from JSON schema (--json-schema / "json_schema" API field) + COMMON_GRAMMAR_TYPE_TOOL_CALLS, // auto-generated by chat template parser for function calling +}; + +// Grammar variant struct with type and grammar string +struct common_grammar { + common_grammar_type type = COMMON_GRAMMAR_TYPE_NONE; + std::string grammar; + + // Default constructor - no grammar + common_grammar() = default; + + // Constructor with type and grammar string + common_grammar(common_grammar_type t, std::string g) : type(t), grammar(std::move(g)) { + GGML_ASSERT(type != COMMON_GRAMMAR_TYPE_NONE || !grammar.empty()); + } + + // Check if a grammar is set + bool empty() const { return type == COMMON_GRAMMAR_TYPE_NONE || grammar.empty(); } +}; + +// Returns the raw grammar string, or empty string if no grammar is set. +inline const std::string & common_grammar_value(const common_grammar & g) { + return g.grammar; +} + +// Returns true when the generation_prompt should be prefilled into the grammar sampler. +// Only output-format and tool-call grammars need prefill; user-supplied grammars must not be prefilled. +inline bool common_grammar_needs_prefill(const common_grammar & g) { + return g.type == COMMON_GRAMMAR_TYPE_OUTPUT_FORMAT + || g.type == COMMON_GRAMMAR_TYPE_TOOL_CALLS; +} + // sampling parameters struct common_params_sampling { uint32_t seed = LLAMA_DEFAULT_SEED; // the seed used to initialize llama_sampler @@ -226,7 +267,7 @@ struct common_params_sampling { COMMON_SAMPLER_TYPE_TEMPERATURE, }; - std::string grammar; // optional BNF-like grammar to constrain sampling + common_grammar grammar; // optional grammar constraint (user / output-format / tool-calls) bool grammar_lazy = false; std::vector grammar_triggers; // optional triggers (for lazy grammars) std::set preserved_tokens; @@ -234,6 +275,19 @@ struct common_params_sampling { std::vector logit_bias; // logit biases to apply std::vector logit_bias_eog; // pre-calculated logit biases for EOG tokens + // The assistant generation prompt already prefilled into the prompt. + // Fed to the grammar sampler (to advance past pre-existing tokens) and used + // to determine the reasoning budget sampler's initial state. + // Only applied when the grammar is of output-format or tool-calls type. + std::string generation_prompt; + + // reasoning budget sampler parameters + // these are populated by the server/CLI based on chat template params + int32_t reasoning_budget_tokens = -1; // -1 = disabled, >= 0 = token budget + std::vector reasoning_budget_start; // start tag token sequence + std::vector reasoning_budget_end; // end tag token sequence + std::vector reasoning_budget_forced; // forced sequence (message + end tag) + bool backend_sampling = false; bool has_logit_bias() const { @@ -456,6 +510,8 @@ struct common_params { bool kl_divergence = false; // compute KL divergence + bool check = false; // check rather than generate results for llama-results + bool usage = false; // print usage bool completion = false; // print source-able completion script bool use_color = false; // use color to distinguish generations and inputs @@ -517,13 +573,15 @@ struct common_params { // server params int32_t port = 8080; // server listens on this network port + bool reuse_port = false; // allow multiple sockets to bind to the same port int32_t timeout_read = 600; // http read timeout in seconds int32_t timeout_write = timeout_read; // http write timeout in seconds int32_t n_threads_http = -1; // number of threads to process HTTP requests (TODO: support threadpool) int32_t n_cache_reuse = 0; // min chunk size to reuse from the cache via KV shifting bool cache_prompt = true; // whether to enable prompt caching - int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot - int32_t checkpoint_every_nt = 8192; // make a checkpoint every n tokens during prefill + bool clear_idle = true; // save and clear idle slots upon starting a new task + int32_t n_ctx_checkpoints = 32; // max number of context checkpoints per slot + int32_t checkpoint_every_nt = 8192; // make a checkpoint every n tokens during prefill int32_t cache_ram_mib = 8192; // -1 = no limit, 0 - disable, 1 = 1 MiB, etc. std::string hostname = "127.0.0.1"; @@ -532,8 +590,11 @@ struct common_params { std::string chat_template = ""; // NOLINT bool use_jinja = true; // NOLINT bool enable_chat_template = true; + bool force_pure_content_parser = false; common_reasoning_format reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK; + int enable_reasoning = -1; // -1 = auto, 0 = disable, 1 = enable int reasoning_budget = -1; + std::string reasoning_budget_message; // message injected before end tag when budget exhausted bool prefill_assistant = true; // if true, any trailing assistant message will be prefilled into the response int sleep_idle_seconds = -1; // if >0, server will sleep after this many seconds of idle time @@ -554,6 +615,9 @@ struct common_params { bool endpoint_props = false; // only control POST requests, not GET bool endpoint_metrics = false; + // enable built-in tools + std::vector server_tools; + // router server configs std::string models_dir = ""; // directory containing models for the router server std::string models_preset = ""; // directory containing model presets for the router server @@ -616,6 +680,7 @@ struct common_params { // return false from callback to abort model loading or true to continue llama_progress_callback load_progress_callback = NULL; void * load_progress_callback_user_data = NULL; + bool no_alloc = false; // Don't allocate model buffers }; // call once at the start of a program if it uses libcommon @@ -731,6 +796,8 @@ std::string string_from(const std::vector & values); std::string string_from(const struct llama_context * ctx, const std::vector & tokens); std::string string_from(const struct llama_context * ctx, const struct llama_batch & batch); +bool glob_match(const std::string & pattern, const std::string & str); + // // Filesystem utils // @@ -913,7 +980,7 @@ const char * const LLM_KV_SPLIT_TENSORS_COUNT = "split.tensors.count"; // MoE utils // -const char * const LLM_FFN_EXPS_REGEX = "\\.ffn_(up|down|gate)_(ch|)exps"; +const char * const LLM_FFN_EXPS_REGEX = "\\.ffn_(up|down|gate|gate_up)_(ch|)exps"; inline std::string llm_ffn_exps_block_regex(int idx) { return string_format("blk\\.%d%s", idx, LLM_FFN_EXPS_REGEX); diff --git a/ggml/llamacpp/third_party/ggml/include/ggml-backend.h b/ggml/llamacpp/third_party/ggml/include/ggml-backend.h index 9fd3f7f..3c06aea 100644 --- a/ggml/llamacpp/third_party/ggml/include/ggml-backend.h +++ b/ggml/llamacpp/third_party/ggml/include/ggml-backend.h @@ -68,7 +68,7 @@ extern "C" { GGML_API void ggml_backend_buffer_reset (ggml_backend_buffer_t buffer); // tensor copy between different backends - GGML_API void ggml_backend_tensor_copy(struct ggml_tensor * src, struct ggml_tensor * dst); + GGML_API void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor * dst); // // Backend (stream) @@ -83,13 +83,17 @@ extern "C" { GGML_API size_t ggml_backend_get_alignment(ggml_backend_t backend); GGML_API size_t ggml_backend_get_max_size(ggml_backend_t backend); - GGML_API void ggml_backend_tensor_set_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); - GGML_API void ggml_backend_tensor_get_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_async (ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get_async (ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d_async(ggml_backend_t backend, struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d_async(ggml_backend_t backend, const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); // "offset" refers to the offset in tensor->data for setting/getting data - GGML_API void ggml_backend_tensor_set( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); - GGML_API void ggml_backend_tensor_get(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); - GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set ( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_get (const struct ggml_tensor * tensor, void * data, size_t offset, size_t size); + GGML_API void ggml_backend_tensor_set_2d( struct ggml_tensor * tensor, const void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_get_2d(const struct ggml_tensor * tensor, void * data, size_t offset, size_t size, size_t n_copies, size_t stride_tensor, size_t stride_data); + GGML_API void ggml_backend_tensor_memset( struct ggml_tensor * tensor, uint8_t value, size_t offset, size_t size); GGML_API void ggml_backend_synchronize(ggml_backend_t backend); @@ -109,7 +113,7 @@ extern "C" { // the copy is performed after all the currently queued operations in backend_src // backend_dst will wait for the copy to complete before performing other operations // automatic fallback to sync copy if async is not supported - GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, struct ggml_tensor * src, struct ggml_tensor * dst); + GGML_API void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst); GGML_API ggml_backend_dev_t ggml_backend_get_device(ggml_backend_t backend); @@ -135,7 +139,9 @@ extern "C" { // integrated GPU device using host memory GGML_BACKEND_DEVICE_TYPE_IGPU, // accelerator devices intended to be used together with the CPU backend (e.g. BLAS or AMX) - GGML_BACKEND_DEVICE_TYPE_ACCEL + GGML_BACKEND_DEVICE_TYPE_ACCEL, + // "meta" device wrapping multiple other devices for tensor parallelism + GGML_BACKEND_DEVICE_TYPE_META, }; // functionality supported by the device @@ -196,7 +202,9 @@ extern "C" { // Common functions that may be obtained using ggml_backend_reg_get_proc_address - // Split buffer type for tensor parallelism + // AllReduce operation for tensor parallelism (meta backend) + typedef bool (*ggml_backend_allreduce_tensor_t)(ggml_backend_t * backends, struct ggml_tensor ** tensors, size_t n_backends); + // Split buffer type for tensor parallelism (old) typedef ggml_backend_buffer_type_t (*ggml_backend_split_buffer_type_t)(int main_device, const float * tensor_split); // Set the number of threads for the backend typedef void (*ggml_backend_set_n_threads_t)(ggml_backend_t backend, int n_threads); diff --git a/ggml/llamacpp/third_party/ggml/include/ggml-cuda.h b/ggml/llamacpp/third_party/ggml/include/ggml-cuda.h index 22ad2c0..5436c7e 100644 --- a/ggml/llamacpp/third_party/ggml/include/ggml-cuda.h +++ b/ggml/llamacpp/third_party/ggml/include/ggml-cuda.h @@ -27,6 +27,9 @@ GGML_BACKEND_API bool ggml_backend_is_cuda(ggml_backend_t backend); // device buffer GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_buffer_type(int device); +// conduct allreduce operation between devices +GGML_BACKEND_API bool ggml_backend_cuda_allreduce_tensor(ggml_backend_t * backends, struct ggml_tensor ** tensors, size_t n_backends); + // split tensor buffer that splits matrices by rows across multiple devices GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_cuda_split_buffer_type(int main_device, const float * tensor_split); diff --git a/ggml/llamacpp/third_party/ggml/include/ggml-openvino.h b/ggml/llamacpp/third_party/ggml/include/ggml-openvino.h new file mode 100644 index 0000000..c43beb0 --- /dev/null +++ b/ggml/llamacpp/third_party/ggml/include/ggml-openvino.h @@ -0,0 +1,37 @@ +#pragma once + +#include "ggml-backend.h" + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +#define GGML_OPENVINO_NAME "OPENVINO" + +// backend API +GGML_BACKEND_API ggml_backend_t ggml_backend_openvino_init(int device); + +GGML_BACKEND_API bool ggml_backend_is_openvino(ggml_backend_t backend); + +GGML_BACKEND_API bool ggml_backend_buffer_is_openvino(ggml_backend_buffer_t buffer); + +GGML_BACKEND_API bool ggml_backend_buft_is_openvino(ggml_backend_buffer_type_t buft); + +GGML_BACKEND_API bool ggml_backend_buft_is_openvino_host(ggml_backend_buffer_type_t buft); + +GGML_BACKEND_API size_t ggml_backend_openvino_buffer_get_ctx_id(ggml_backend_buffer_t buffer); + +// device buffer +GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_openvino_buffer_type(int device); + +GGML_BACKEND_API ggml_backend_buffer_type_t ggml_backend_openvino_host_buffer_type(int device); + +GGML_BACKEND_API int ggml_backend_openvino_get_device_count(void); + +GGML_BACKEND_API ggml_backend_reg_t ggml_backend_openvino_reg(void); + +#ifdef __cplusplus +} +#endif diff --git a/ggml/llamacpp/third_party/ggml/include/ggml-rpc.h b/ggml/llamacpp/third_party/ggml/include/ggml-rpc.h index df1ad2a..1c11495 100644 --- a/ggml/llamacpp/third_party/ggml/include/ggml-rpc.h +++ b/ggml/llamacpp/third_party/ggml/include/ggml-rpc.h @@ -8,7 +8,12 @@ extern "C" { #define RPC_PROTO_MAJOR_VERSION 3 #define RPC_PROTO_MINOR_VERSION 6 -#define RPC_PROTO_PATCH_VERSION 0 +#define RPC_PROTO_PATCH_VERSION 1 + +#ifdef __cplusplus +static_assert(GGML_OP_COUNT == 96, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION"); +#endif + #define GGML_RPC_MAX_SERVERS 16 // backend API diff --git a/ggml/llamacpp/third_party/ggml/include/ggml.h b/ggml/llamacpp/third_party/ggml/include/ggml.h index 784d692..11d3e8a 100644 --- a/ggml/llamacpp/third_party/ggml/include/ggml.h +++ b/ggml/llamacpp/third_party/ggml/include/ggml.h @@ -427,7 +427,9 @@ extern "C" { // GGML_TYPE_IQ4_NL_4_8 = 37, // GGML_TYPE_IQ4_NL_8_8 = 38, GGML_TYPE_MXFP4 = 39, // MXFP4 (1 block) - GGML_TYPE_COUNT = 40, + GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) + GGML_TYPE_Q1_0 = 41, + GGML_TYPE_COUNT = 42, }; // precision @@ -463,6 +465,8 @@ extern "C" { GGML_FTYPE_MOSTLY_IQ1_M = 23, // except 1d tensors GGML_FTYPE_MOSTLY_BF16 = 24, // except 1d tensors GGML_FTYPE_MOSTLY_MXFP4 = 25, // except 1d tensors + GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors + GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors }; // available tensor operations: @@ -556,6 +560,7 @@ extern "C" { GGML_OP_GATED_LINEAR_ATTN, GGML_OP_RWKV_WKV7, GGML_OP_SOLVE_TRI, + GGML_OP_GATED_DELTA_NET, GGML_OP_UNARY, @@ -730,6 +735,10 @@ extern "C" { GGML_API size_t ggml_type_size(enum ggml_type type); // size in bytes for all elements in a block GGML_API size_t ggml_row_size (enum ggml_type type, int64_t ne); // size in bytes for all elements in a row + GGML_DEPRECATED( + GGML_API double ggml_type_sizef(enum ggml_type type), // ggml_type_size()/ggml_blck_size() as float + "use ggml_row_size() instead"); + GGML_API const char * ggml_type_name(enum ggml_type type); GGML_API const char * ggml_op_name (enum ggml_op op); GGML_API const char * ggml_op_symbol(enum ggml_op op); @@ -893,15 +902,17 @@ extern "C" { struct ggml_tensor * b, struct ggml_tensor * ids); - GGML_API struct ggml_tensor * ggml_add1( + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1( struct ggml_context * ctx, struct ggml_tensor * a, - struct ggml_tensor * b); + struct ggml_tensor * b), + "use ggml_add instead"); - GGML_API struct ggml_tensor * ggml_add1_inplace( + GGML_DEPRECATED(GGML_API struct ggml_tensor * ggml_add1_inplace( struct ggml_context * ctx, struct ggml_tensor * a, - struct ggml_tensor * b); + struct ggml_tensor * b), + "use ggml_add_inplace instead"); // dst = a // view(dst, nb1, nb2, nb3, offset) += b @@ -2463,6 +2474,17 @@ extern "C" { bool lower, bool uni); + // TODO: add ggml_gated_delta_net_set_bcast() to be able to configure Q, K broadcast type: tiled vs interleaved [TAG_GGML_GDN_BCAST] + // ref: https://github.com/ggml-org/llama.cpp/pull/19468#discussion_r2786394306 + GGML_API struct ggml_tensor * ggml_gated_delta_net( + struct ggml_context * ctx, + struct ggml_tensor * q, + struct ggml_tensor * k, + struct ggml_tensor * v, + struct ggml_tensor * g, + struct ggml_tensor * beta, + struct ggml_tensor * state); + // custom operators typedef void (*ggml_custom1_op_t)(struct ggml_tensor * dst , const struct ggml_tensor * a, int ith, int nth, void * userdata); diff --git a/ggml/llamacpp/third_party/ggml/include/gguf.h b/ggml/llamacpp/third_party/ggml/include/gguf.h index 79ee202..02d5f22 100644 --- a/ggml/llamacpp/third_party/ggml/include/gguf.h +++ b/ggml/llamacpp/third_party/ggml/include/gguf.h @@ -77,6 +77,7 @@ extern "C" { }; GGML_API struct gguf_context * gguf_init_empty(void); + GGML_API struct gguf_context * gguf_init_from_file_ptr(FILE * file, struct gguf_init_params params); GGML_API struct gguf_context * gguf_init_from_file(const char * fname, struct gguf_init_params params); //GGML_API struct gguf_context * gguf_init_from_buffer(..); @@ -189,6 +190,7 @@ extern "C" { // // write the entire context to a binary file + GGML_API bool gguf_write_to_file_ptr(const struct gguf_context * ctx, FILE * file, bool only_meta); GGML_API bool gguf_write_to_file(const struct gguf_context * ctx, const char * fname, bool only_meta); // get the size in bytes of the meta data (header, kv pairs, tensor info) including padding diff --git a/ggml/llamacpp/third_party/include/llama-cpp.h b/ggml/llamacpp/third_party/include/llama-cpp.h index 807e77f..8f63681 100644 --- a/ggml/llamacpp/third_party/include/llama-cpp.h +++ b/ggml/llamacpp/third_party/include/llama-cpp.h @@ -21,9 +21,7 @@ struct llama_sampler_deleter { }; struct llama_adapter_lora_deleter { - void operator()(llama_adapter_lora *) { - // llama_adapter_lora_free is deprecated - } + void operator()(llama_adapter_lora * adapter) { llama_adapter_lora_free(adapter); } }; typedef std::unique_ptr llama_model_ptr; diff --git a/ggml/llamacpp/third_party/include/llama.h b/ggml/llamacpp/third_party/include/llama.h index a84d56a..ac267b5 100644 --- a/ggml/llamacpp/third_party/include/llama.h +++ b/ggml/llamacpp/third_party/include/llama.h @@ -5,6 +5,7 @@ #include "ggml-cpu.h" #include "ggml-backend.h" #include "ggml-opt.h" +#include "gguf.h" #include #include @@ -152,6 +153,8 @@ extern "C" { LLAMA_FTYPE_MOSTLY_TQ1_0 = 36, // except 1d tensors LLAMA_FTYPE_MOSTLY_TQ2_0 = 37, // except 1d tensors LLAMA_FTYPE_MOSTLY_MXFP4_MOE = 38, // except 1d tensors + LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors + LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file }; @@ -189,9 +192,10 @@ extern "C" { LLAMA_API const char * llama_flash_attn_type_name(enum llama_flash_attn_type flash_attn_type); enum llama_split_mode { - LLAMA_SPLIT_MODE_NONE = 0, // single GPU - LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs - LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported + LLAMA_SPLIT_MODE_NONE = 0, // single GPU + LLAMA_SPLIT_MODE_LAYER = 1, // split layers and KV across GPUs + LLAMA_SPLIT_MODE_ROW = 2, // split layers and KV across GPUs, use tensor parallelism if supported + LLAMA_SPLIT_MODE_TENSOR = 3, }; // TODO: simplify (https://github.com/ggml-org/llama.cpp/pull/9294#pullrequestreview-2286561979) @@ -378,22 +382,33 @@ extern "C" { size_t n_samplers; }; + struct llama_model_tensor_override { + const char * pattern; + enum ggml_type type; + }; + + struct llama_model_imatrix_data { + const char * name; + const float * data; + size_t size; + }; + // model quantization parameters typedef struct llama_model_quantize_params { - int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() - enum llama_ftype ftype; // quantize to this llama_ftype - enum ggml_type output_tensor_type; // output tensor type - enum ggml_type token_embedding_type; // token embeddings tensor type - bool allow_requantize; // allow quantizing non-f32/f16 tensors - bool quantize_output_tensor; // quantize output.weight - bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored - bool pure; // quantize all tensors to the default type - bool keep_split; // quantize to the same number of shards - bool dry_run; // calculate and show the final quantization size without performing quantization - void * imatrix; // pointer to importance matrix data - void * kv_overrides; // pointer to vector containing overrides - void * tensor_types; // pointer to vector containing tensor types - void * prune_layers; // pointer to vector containing layer indices to prune + int32_t nthread; // number of threads to use for quantizing, if <=0 will use std::thread::hardware_concurrency() + enum llama_ftype ftype; // quantize to this llama_ftype + enum ggml_type output_tensor_type; // output tensor type + enum ggml_type token_embedding_type; // token embeddings tensor type + bool allow_requantize; // allow quantizing non-f32/f16 tensors + bool quantize_output_tensor; // quantize output.weight + bool only_copy; // only copy tensors - ftype, allow_requantize and quantize_output_tensor are ignored + bool pure; // quantize all tensors to the default type + bool keep_split; // quantize to the same number of shards + bool dry_run; // calculate and show the final quantization size without performing quantization + const struct llama_model_imatrix_data * imatrix; // pointer to importance matrix data + const struct llama_model_kv_override * kv_overrides; // pointer to kv overrides + const struct llama_model_tensor_override * tt_overrides; // pointer to tensor overrides + const int32_t * prune_layers; // pointer to layer indices to prune } llama_model_quantize_params; typedef struct llama_logit_bias { @@ -440,19 +455,35 @@ extern "C" { LLAMA_API void llama_detach_threadpool(struct llama_context * ctx); + typedef void (*llama_model_set_tensor_data_t)(struct ggml_tensor * tensor, void * userdata); + + // Create a new model from GGUF metadata as well as a function to set the tensor data + // - tensors are created as GGML_TYPE_F32 by default, + // override by adding a tensor with the same name but a different name to the context + LLAMA_API struct llama_model * llama_model_init_from_user( + struct gguf_context * metadata, + llama_model_set_tensor_data_t set_tensor_data, // function to initialize tensor data with + void * set_tensor_data_ud, // userdata for function + struct llama_model_params params); + DEPRECATED(LLAMA_API struct llama_model * llama_load_model_from_file( const char * path_model, struct llama_model_params params), "use llama_model_load_from_file instead"); - // Load the model from a file + // Load a model from a file // If the file is split into multiple parts, the file name must follow this pattern: -%05d-of-%05d.gguf // If the split file name does not follow this pattern, use llama_model_load_from_splits LLAMA_API struct llama_model * llama_model_load_from_file( const char * path_model, struct llama_model_params params); - // Load the model from multiple splits (support custom naming scheme) + // Load a model from an open FILE pointer + LLAMA_API struct llama_model * llama_model_load_from_file_ptr( + FILE * file, + struct llama_model_params params); + + // Load a model from multiple splits (support custom naming scheme) // The paths must be in the correct order LLAMA_API struct llama_model * llama_model_load_from_splits( const char ** paths, @@ -623,7 +654,6 @@ extern "C" { // Load a LoRA adapter from file // The adapter is valid as long as the associated model is not freed - // All adapters must be loaded before context creation LLAMA_API struct llama_adapter_lora * llama_adapter_lora_init( struct llama_model * model, const char * path_lora); @@ -647,9 +677,8 @@ extern "C" { LLAMA_API int32_t llama_adapter_meta_val_str_by_index(const struct llama_adapter_lora * adapter, int32_t i, char * buf, size_t buf_size); // Manually free a LoRA adapter - // NOTE: loaded adapters will be free when the associated model is deleted - LLAMA_API DEPRECATED(void llama_adapter_lora_free(struct llama_adapter_lora * adapter), - "adapters are now freed together with the associated model"); + // NOTE: loaded adapters that are not manually freed will be freed when the associated model is deleted + LLAMA_API void llama_adapter_lora_free(struct llama_adapter_lora * adapter); // Get the invocation tokens if the current lora is an alora LLAMA_API uint64_t llama_adapter_get_alora_n_invocation_tokens(const struct llama_adapter_lora * adapter); diff --git a/ggml/llamacpp/third_party/prebuilt/android-arm64/.gitkeep b/ggml/llamacpp/third_party/prebuilt/android-arm64/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/ggml/llamacpp/third_party/prebuilt/android-arm64/libcommon.a b/ggml/llamacpp/third_party/prebuilt/android-arm64/libcommon.a new file mode 100644 index 0000000..6658711 Binary files /dev/null and b/ggml/llamacpp/third_party/prebuilt/android-arm64/libcommon.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/android-arm64/libcpp-httplib.a b/ggml/llamacpp/third_party/prebuilt/android-arm64/libcpp-httplib.a new file mode 100644 index 0000000..7170f50 Binary files /dev/null and b/ggml/llamacpp/third_party/prebuilt/android-arm64/libcpp-httplib.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/android-arm64/libggml-base.a b/ggml/llamacpp/third_party/prebuilt/android-arm64/libggml-base.a new file mode 100644 index 0000000..2e703f0 Binary files /dev/null and b/ggml/llamacpp/third_party/prebuilt/android-arm64/libggml-base.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/android-arm64/libggml-cpu.a b/ggml/llamacpp/third_party/prebuilt/android-arm64/libggml-cpu.a new file mode 100644 index 0000000..017b54f Binary files /dev/null and b/ggml/llamacpp/third_party/prebuilt/android-arm64/libggml-cpu.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/android-arm64/libggml.a b/ggml/llamacpp/third_party/prebuilt/android-arm64/libggml.a new file mode 100644 index 0000000..f9ad770 Binary files /dev/null and b/ggml/llamacpp/third_party/prebuilt/android-arm64/libggml.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/android-arm64/libllama.a b/ggml/llamacpp/third_party/prebuilt/android-arm64/libllama.a new file mode 100644 index 0000000..5ab86cb Binary files /dev/null and b/ggml/llamacpp/third_party/prebuilt/android-arm64/libllama.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/android-arm64/libmtmd.a b/ggml/llamacpp/third_party/prebuilt/android-arm64/libmtmd.a new file mode 100644 index 0000000..b97718a Binary files /dev/null and b/ggml/llamacpp/third_party/prebuilt/android-arm64/libmtmd.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/android-arm64/libserver-context.a b/ggml/llamacpp/third_party/prebuilt/android-arm64/libserver-context.a new file mode 100644 index 0000000..a8fb085 Binary files /dev/null and b/ggml/llamacpp/third_party/prebuilt/android-arm64/libserver-context.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libcommon.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libcommon.a index d57f032..f6f06ff 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libcommon.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libcommon.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libcpp-httplib.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libcpp-httplib.a index 9f6a288..e1fefa7 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libcpp-httplib.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libcpp-httplib.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml-base.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml-base.a index 911ee7b..94e4ce7 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml-base.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml-base.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml-cpu.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml-cpu.a index 16ea255..ff49e8f 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml-cpu.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml-cpu.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml-cuda.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml-cuda.a index e7dbc19..fffbd13 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml-cuda.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml-cuda.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml.a index c926018..249cdb8 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libggml.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libllama.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libllama.a index 7cc2691..3379693 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libllama.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libllama.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libmtmd.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libmtmd.a index 99d1f0b..6cd24ec 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libmtmd.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libmtmd.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libserver-context.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libserver-context.a index 883c322..dbeb59d 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libserver-context.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-cuda/libserver-context.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libcommon.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libcommon.a index 3523ef1..a8468fe 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libcommon.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libcommon.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libcpp-httplib.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libcpp-httplib.a index 85aba27..9c5a1ac 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libcpp-httplib.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libcpp-httplib.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml-base.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml-base.a index 20e299a..ba99cb7 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml-base.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml-base.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml-cpu.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml-cpu.a index 049c7cf..da2df04 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml-cpu.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml-cpu.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml-vulkan.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml-vulkan.a index deebee9..88a9ac4 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml-vulkan.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml-vulkan.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml.a index 91c4dc1..72fc96e 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libggml.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libllama.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libllama.a index 1fbd114..9da3eae 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libllama.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libllama.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libmtmd.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libmtmd.a index 0a8e58b..6ecf783 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libmtmd.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libmtmd.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libserver-context.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libserver-context.a index 837476f..213ce33 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libserver-context.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64-vulkan/libserver-context.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libcommon.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libcommon.a index 193e683..aeceb6b 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libcommon.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libcommon.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libcpp-httplib.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libcpp-httplib.a index 020cf4f..f8f5d13 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libcpp-httplib.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libcpp-httplib.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libggml-base.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libggml-base.a index 0e7587b..bd61e10 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libggml-base.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libggml-base.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libggml-cpu.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libggml-cpu.a index decb33f..e690af0 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libggml-cpu.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libggml-cpu.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libggml.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libggml.a index 0f9aa0e..ff7fa77 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libggml.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libggml.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libllama.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libllama.a index fccc571..1ed1847 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libllama.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libllama.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libmtmd.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libmtmd.a index 6197b1e..0792c2c 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libmtmd.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libmtmd.a differ diff --git a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libserver-context.a b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libserver-context.a index f8ad75b..8b56512 100644 Binary files a/ggml/llamacpp/third_party/prebuilt/linux-amd64/libserver-context.a and b/ggml/llamacpp/third_party/prebuilt/linux-amd64/libserver-context.a differ diff --git a/ggml/whispercpp/embed.go b/ggml/whispercpp/embed.go index 3350927..e034b3e 100644 --- a/ggml/whispercpp/embed.go +++ b/ggml/whispercpp/embed.go @@ -15,4 +15,5 @@ import "embed" //go:embed third_party/prebuilt/linux-amd64-cuda/*.a //go:embed third_party/prebuilt/linux-amd64-vulkan/*.a //go:embed third_party/prebuilt/linux-arm64/*.a +//go:embed third_party/prebuilt/android-arm64/*.a var _ embed.FS diff --git a/ggml/whispercpp/third_party/prebuilt/android-arm64/.gitkeep b/ggml/whispercpp/third_party/prebuilt/android-arm64/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/ggml/whispercpp/third_party/prebuilt/android-arm64/libcommon.a b/ggml/whispercpp/third_party/prebuilt/android-arm64/libcommon.a new file mode 100644 index 0000000..29769cb Binary files /dev/null and b/ggml/whispercpp/third_party/prebuilt/android-arm64/libcommon.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/android-arm64/libggml-base.a b/ggml/whispercpp/third_party/prebuilt/android-arm64/libggml-base.a new file mode 100644 index 0000000..c355d0f Binary files /dev/null and b/ggml/whispercpp/third_party/prebuilt/android-arm64/libggml-base.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/android-arm64/libggml-cpu.a b/ggml/whispercpp/third_party/prebuilt/android-arm64/libggml-cpu.a new file mode 100644 index 0000000..2321ccb Binary files /dev/null and b/ggml/whispercpp/third_party/prebuilt/android-arm64/libggml-cpu.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/android-arm64/libggml.a b/ggml/whispercpp/third_party/prebuilt/android-arm64/libggml.a new file mode 100644 index 0000000..e19d103 Binary files /dev/null and b/ggml/whispercpp/third_party/prebuilt/android-arm64/libggml.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/android-arm64/libwhisper.a b/ggml/whispercpp/third_party/prebuilt/android-arm64/libwhisper.a new file mode 100644 index 0000000..f19800f Binary files /dev/null and b/ggml/whispercpp/third_party/prebuilt/android-arm64/libwhisper.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libcommon.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libcommon.a index 08c9719..61aa619 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libcommon.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libcommon.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml-base.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml-base.a index 5936da8..4207df2 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml-base.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml-base.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml-cpu.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml-cpu.a index 1c1e04a..dbaf36f 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml-cpu.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml-cpu.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml-cuda.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml-cuda.a index 553e222..2cc3042 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml-cuda.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml-cuda.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml.a index 0dcb79c..8db5376 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libggml.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libwhisper.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libwhisper.a index 1df550b..a0faa0f 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libwhisper.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64-cuda/libwhisper.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libcommon.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libcommon.a index 9b08f50..79bf1cb 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libcommon.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libcommon.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml-base.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml-base.a index b251abe..6312e47 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml-base.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml-base.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml-cpu.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml-cpu.a index c208662..a6adb24 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml-cpu.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml-cpu.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml-vulkan.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml-vulkan.a index b2a82e6..c93f533 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml-vulkan.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml-vulkan.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml.a index 99b27d7..00e15d4 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libggml.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libwhisper.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libwhisper.a index 1d066ee..babf534 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libwhisper.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64-vulkan/libwhisper.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64/libcommon.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64/libcommon.a index 271b24f..f46b673 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64/libcommon.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64/libcommon.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64/libggml-base.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64/libggml-base.a index 8ccf34d..c35b2c3 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64/libggml-base.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64/libggml-base.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64/libggml-cpu.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64/libggml-cpu.a index 371a07c..d6395a7 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64/libggml-cpu.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64/libggml-cpu.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64/libggml.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64/libggml.a index 0ee2671..ec94612 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64/libggml.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64/libggml.a differ diff --git a/ggml/whispercpp/third_party/prebuilt/linux-amd64/libwhisper.a b/ggml/whispercpp/third_party/prebuilt/linux-amd64/libwhisper.a index 78ec7dd..b5347fa 100644 Binary files a/ggml/whispercpp/third_party/prebuilt/linux-amd64/libwhisper.a and b/ggml/whispercpp/third_party/prebuilt/linux-amd64/libwhisper.a differ diff --git a/ggml/whispercpp/whispercpp_android.go b/ggml/whispercpp/whispercpp_android.go new file mode 100644 index 0000000..87b91db --- /dev/null +++ b/ggml/whispercpp/whispercpp_android.go @@ -0,0 +1,12 @@ +//go:build whispercpp && android && arm64 + +// Copyright 2025 FootprintAI +// SPDX-License-Identifier: Apache-2.0 + +package whispercpp + +/* +#cgo android,arm64 LDFLAGS: -L${SRCDIR}/third_party/prebuilt/android-arm64 +#cgo android LDFLAGS: -Wl,--start-group -lwhisper -lcommon -lggml-cpu -lggml-base -lggml -Wl,--end-group -lstdc++ -lm -ldl -llog +*/ +import "C" diff --git a/version.go b/version.go index 7d553ab..aafe579 100644 --- a/version.go +++ b/version.go @@ -1,7 +1,7 @@ package gonativeml const ( - Version = "v0.1.5" - LlamaCppVersion = "b8220" + Version = "v0.1.7" + LlamaCppVersion = "b8772" WhisperCppVersion = "v1.8.3" )