diff --git a/.github/workflows/_release_docker.yml b/.github/workflows/_release_docker.yml index 54a11d7a6a..20cfefef04 100644 --- a/.github/workflows/_release_docker.yml +++ b/.github/workflows/_release_docker.yml @@ -15,13 +15,11 @@ on: type: boolean default: false python: - type: boolean - description: If selected will build the python version of the image - default: false + type: string + description: If set, will build the python image using this as the tag for the base python image, e.g. '3.14.2-slim-trixie' workflow_call: inputs: *inputs - permissions: contents: read @@ -52,10 +50,12 @@ jobs: - name: Set dockerfile path and cache image run: | - if [ "${{ inputs.python }}" = "true" ]; then + if [ -n "${{ inputs.python }}" ]; then + echo "Building Python image with base python:${{ inputs.python }}" echo "DOCKERFILE_PATH=python.Dockerfile" >> $GITHUB_ENV echo "CACHE_IMAGE=${{ env.REGISTRY_IMAGE }}:buildcache-python-${{ env.PLATFORM_PAIR }}" >> $GITHUB_ENV else + echo "Building Rust image" echo "DOCKERFILE_PATH=Dockerfile" >> $GITHUB_ENV echo "CACHE_IMAGE=${{ env.REGISTRY_IMAGE }}:buildcache-rust-${{ env.PLATFORM_PAIR }}" >> $GITHUB_ENV fi @@ -81,7 +81,9 @@ jobs: uses: docker/build-push-action@v6 with: context: . - build-args: RUST_VERSION=${{ env.RUST_VERSION }} + build-args: | + RUST_VERSION=${{ env.RUST_VERSION }} + BASE_PYTHON_IMAGE_TAG=${{ inputs.python }} file: ${{ env.DOCKERFILE_PATH }} platforms: ${{ matrix.platform }} labels: ${{ steps.meta.outputs.labels }} diff --git a/.github/workflows/release_auto.yml b/.github/workflows/release_auto.yml index dd612f82a2..030f34284c 100644 --- a/.github/workflows/release_auto.yml +++ b/.github/workflows/release_auto.yml @@ -1,4 +1,9 @@ name: 🚀🦀🐍🐳🐙 Publish all packages to crates.io, PyPi, Docker Hub & Github + +env: + PYTHON_VERSION: 3.14.2 + DEBIAN_VERSION: trixie + on: workflow_dispatch: inputs: @@ -15,7 +20,7 @@ jobs: read-raphtory-version: runs-on: ubuntu-latest outputs: - version: ${{ steps.version.outputs.version }} + version: ${{ steps.version.outputs.version }} steps: - uses: actions/checkout@v4 with: @@ -62,7 +67,8 @@ jobs: base: ${{ inputs.base }} dry_run: ${{ inputs.dry_run == true }} tag: | + ${{ needs.read-raphtory-version.outputs.version }}-python${{ env.PYTHON_VERSION }}-${{ env.DEBIAN_VERSION }} ${{ needs.read-raphtory-version.outputs.version }}-python latest-python - python: true + python: ${{ env.PYTHON_VERSION }}-slim-${{ env.DEBIAN_VERSION }} secrets: inherit diff --git a/python.Dockerfile b/python.Dockerfile index e77d0ed171..181a986db7 100644 --- a/python.Dockerfile +++ b/python.Dockerfile @@ -1,22 +1,22 @@ -ARG PYTHON_VERSION=3.13.5 ARG RUST_VERSION=1.86.0 +ARG BASE_PYTHON_IMAGE_TAG FROM rust:${RUST_VERSION} AS build -ARG PYTHON_VERSION +ARG BASE_PYTHON_IMAGE_TAG WORKDIR /app ENV HOME=/root ENV PYENV_ROOT=$HOME/.pyenv ENV PATH=$PYENV_ROOT/shims:$PYENV_ROOT/bin:$PATH RUN apt-get update && apt-get install -y protobuf-compiler RUN curl -fsSL https://pyenv.run | bash -RUN pyenv install ${PYTHON_VERSION} -RUN pyenv global ${PYTHON_VERSION} +RUN PYTHON_VERSION=$(echo ${BASE_PYTHON_IMAGE_TAG} | cut -d'-' -f1) && \ + pyenv install ${PYTHON_VERSION} && \ + pyenv global ${PYTHON_VERSION} RUN pip install maturin==1.8.3 patchelf==0.17.2.2 COPY . . RUN cd python && maturin build --release -FROM python:${PYTHON_VERSION}-slim -ARG PYTHON_VERSION +FROM python:${BASE_PYTHON_IMAGE_TAG} WORKDIR /var/lib/raphtory COPY --from=build /app/target/wheels/*.whl / RUN pip install --no-cache-dir /*.whl && rm /*.whl diff --git a/raphtory-graphql/src/data.rs b/raphtory-graphql/src/data.rs index dbcaa30c6a..4605eeb96e 100644 --- a/raphtory-graphql/src/data.rs +++ b/raphtory-graphql/src/data.rs @@ -55,7 +55,7 @@ pub(crate) fn get_relative_path( #[derive(Clone)] pub struct Data { pub(crate) work_dir: PathBuf, - cache: Cache, + pub(crate) cache: Cache, pub(crate) create_index: bool, pub(crate) embedding_conf: Option, } @@ -414,6 +414,14 @@ pub(crate) mod data_tests { sleep(Duration::from_secs(3)).await; assert!(!data.cache.contains_key(Path::new("test_g"))); assert!(!data.cache.contains_key(Path::new("test_g2"))); + // FIXME: this test is not doing anything because calling cache.contains_key() runs + // any pending evictions. To actually test it we need this assertion: + // assert_eq!(data.cache.entry_count(), 0); + // Which currently does not work because the server task to trigger evictions is not running + // in this context. The problem is if we do run it by creating a server and calling + // server.start() the server gets consumed and we loose access to the cache to be able to run + // the check. If rework the server implementation and this becomes feasible we should change + // this test } #[tokio::test] diff --git a/raphtory-graphql/src/server.rs b/raphtory-graphql/src/server.rs index 2f4c9b43a3..2e56df9e75 100644 --- a/raphtory-graphql/src/server.rs +++ b/raphtory-graphql/src/server.rs @@ -200,6 +200,16 @@ impl GraphServer { self.data.vectorise_all_graphs_that_are_not().await?; let work_dir = self.data.work_dir.clone(); + // Otherwise evictions are only triggered when the cache is actively touched + let cache_clone = self.data.cache.clone(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(1)); + loop { + interval.tick().await; + cache_clone.run_pending_tasks().await; + } + }); + // it is important that this runs after algorithms have been pushed to PLUGIN_ALGOS static variable let app = self .generate_endpoint(tp.clone().map(|tp| tp.tracer(tracer_name)))