From c88d11905cdbf53da71ce5263a284ab692932744 Mon Sep 17 00:00:00 2001 From: Enrique Molina Date: Tue, 28 Oct 2025 13:01:22 +0100 Subject: [PATCH 1/2] adding runtime_memory and runtime_cpu limits to docker local impl. --- lithops/localhost/v1/localhost.py | 9 +++++++++ lithops/localhost/v2/localhost.py | 12 ++++++++++++ runtime/kubernetes/Dockerfile | 2 +- 3 files changed, 22 insertions(+), 1 deletion(-) diff --git a/lithops/localhost/v1/localhost.py b/lithops/localhost/v1/localhost.py index 5d2cd9cfc..e2b5fd9c4 100644 --- a/lithops/localhost/v1/localhost.py +++ b/lithops/localhost/v1/localhost.py @@ -172,6 +172,7 @@ def get_runtime_info(self): return { 'runtime_name': self.config['runtime'], 'runtime_memory': self.config.get('runtime_memory'), + 'runtime_cpu': self.config.get('runtime_cpu'), 'runtime_timeout': self.config.get('runtime_timeout'), 'max_workers': self.config['max_workers'], } @@ -346,9 +347,13 @@ def get_metadata(self): cmd = f'{self.docker_path} run --name lithops_metadata ' cmd += f'--user {self.uid}:{self.gid} ' if self.is_unix_system and not self.is_podman else '' cmd += f'--env USER={os.getenv("USER", "root")} ' + cmd += f'--cpus "{self.config.get("runtime_cpu")}" ' if self.config.get('runtime_cpu') else '' + cmd += f'--memory "{self.config.get("runtime_memory")}m" ' if self.config.get('runtime_memory') else '' cmd += f'--rm -v {tmp_path}:/tmp --entrypoint "python3" ' cmd += f'{self.runtime_name} /tmp/{USER_TEMP_DIR}/localhost-runner.py get_metadata' + logger.info(cmd) + process = sp.run( shlex.split(cmd), check=True, stdout=sp.PIPE, universal_newlines=True, start_new_session=True @@ -370,9 +375,13 @@ def run_job(self, job_key, job_filename): cmd += '--gpus all ' if self.use_gpu else '' cmd += f'--user {self.uid}:{self.gid} ' if self.is_unix_system and not self.is_podman else '' cmd += f'--env USER={os.getenv("USER", "root")} ' + cmd += f'--cpus "{self.config.get("runtime_cpu")}" ' if self.config.get('runtime_cpu') else '' + cmd += f'--memory "{self.config.get("runtime_memory")}m" ' if self.config.get('runtime_memory') else '' cmd += f'--rm -v {tmp_path}:/tmp --entrypoint "python3" ' cmd += f'{self.runtime_name} /tmp/{USER_TEMP_DIR}/localhost-runner.py run_job {job_filename}' + logger.info(cmd) + process = sp.Popen(shlex.split(cmd), stdout=sp.PIPE, stderr=sp.PIPE, start_new_session=True) self.jobs[job_key] = process diff --git a/lithops/localhost/v2/localhost.py b/lithops/localhost/v2/localhost.py index a873104af..50543fd7a 100644 --- a/lithops/localhost/v2/localhost.py +++ b/lithops/localhost/v2/localhost.py @@ -384,6 +384,10 @@ def setup(self): check=True, stdout=sp.PIPE, universal_newlines=True ) + def _get_memory_limit(self): + # with os + return self.config.get('runtime_memory', '512') + def get_metadata(self): if not os.path.isfile(RUNNER_FILE): self.setup() @@ -395,9 +399,13 @@ def get_metadata(self): cmd = f'{self.docker_path} run --name lithops_metadata ' cmd += f'--user {self.uid}:{self.gid} ' if self.is_unix_system and not self.is_podman else '' cmd += f'--env USER={os.getenv("USER", "root")} ' + cmd += f'--cpus "{self.config.get("runtime_cpu")}" ' if self.config.get('runtime_cpu') else '' + cmd += f'--memory "{self.config.get("runtime_memory")}m" ' if self.config.get('runtime_memory') else '' cmd += f'--rm -v {tmp_path}:/tmp --entrypoint "python3" ' cmd += f'{self.runtime_name} /tmp/{USER_TEMP_DIR}/localhost-runner.py get_metadata' + logger.info(cmd) + process = sp.run( shlex.split(cmd), check=True, stdout=sp.PIPE, universal_newlines=True, start_new_session=True @@ -416,9 +424,13 @@ def start(self): cmd += '--gpus all ' if self.use_gpu else '' cmd += f'--user {self.uid}:{self.gid} ' if self.is_unix_system and not self.is_podman else '' cmd += f'--env USER={os.getenv("USER", "root")} ' + cmd += f'--cpus "{self.config.get("runtime_cpu")}" ' if self.config.get('runtime_cpu') else '' + cmd += f'--memory "{self.config.get("runtime_memory")}m" ' if self.config.get('runtime_memory') else '' cmd += f'--rm -v {tmp_path}:/tmp -it --detach ' cmd += f'--entrypoint=/bin/bash {self.runtime_name}' + logger.info(cmd) + self.container_process = sp.Popen(shlex.split(cmd), stdout=sp.DEVNULL, start_new_session=True) self.container_process.communicate() # blocks until the process finishes diff --git a/runtime/kubernetes/Dockerfile b/runtime/kubernetes/Dockerfile index f1d682d90..6883dacfe 100644 --- a/runtime/kubernetes/Dockerfile +++ b/runtime/kubernetes/Dockerfile @@ -11,7 +11,7 @@ #FROM python:3.9-slim-buster # Python 3.10 -FROM python:3.10-slim-buster +FROM python:3.10-slim-trixie RUN apt-get update && apt-get install -y \ zip redis-server curl \ From 8f1fc06f08a9c04d855557ec0782841f08915326 Mon Sep 17 00:00:00 2001 From: Enrique Molina Date: Thu, 30 Oct 2025 19:40:02 +0100 Subject: [PATCH 2/2] using --cpuset-cpus to just use a subset of cpus --- lithops/localhost/v1/localhost.py | 6 ++++-- lithops/localhost/v2/localhost.py | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lithops/localhost/v1/localhost.py b/lithops/localhost/v1/localhost.py index e2b5fd9c4..28e67c7cd 100644 --- a/lithops/localhost/v1/localhost.py +++ b/lithops/localhost/v1/localhost.py @@ -347,7 +347,8 @@ def get_metadata(self): cmd = f'{self.docker_path} run --name lithops_metadata ' cmd += f'--user {self.uid}:{self.gid} ' if self.is_unix_system and not self.is_podman else '' cmd += f'--env USER={os.getenv("USER", "root")} ' - cmd += f'--cpus "{self.config.get("runtime_cpu")}" ' if self.config.get('runtime_cpu') else '' + #cmd += f'--cpus "{self.config.get("runtime_cpu")}" ' if self.config.get('runtime_cpu') else '' + cmd += f'--cpuset-cpus="0-{self.config.get("runtime_cpu")-1}" ' if self.config.get('runtime_cpu') else '' cmd += f'--memory "{self.config.get("runtime_memory")}m" ' if self.config.get('runtime_memory') else '' cmd += f'--rm -v {tmp_path}:/tmp --entrypoint "python3" ' cmd += f'{self.runtime_name} /tmp/{USER_TEMP_DIR}/localhost-runner.py get_metadata' @@ -375,7 +376,8 @@ def run_job(self, job_key, job_filename): cmd += '--gpus all ' if self.use_gpu else '' cmd += f'--user {self.uid}:{self.gid} ' if self.is_unix_system and not self.is_podman else '' cmd += f'--env USER={os.getenv("USER", "root")} ' - cmd += f'--cpus "{self.config.get("runtime_cpu")}" ' if self.config.get('runtime_cpu') else '' + # cmd += f'--cpus "{self.config.get("runtime_cpu")}" ' if self.config.get('runtime_cpu') else '' + cmd += f'--cpuset-cpus="0-{self.config.get("runtime_cpu")-1}" ' if self.config.get('runtime_cpu') else '' cmd += f'--memory "{self.config.get("runtime_memory")}m" ' if self.config.get('runtime_memory') else '' cmd += f'--rm -v {tmp_path}:/tmp --entrypoint "python3" ' cmd += f'{self.runtime_name} /tmp/{USER_TEMP_DIR}/localhost-runner.py run_job {job_filename}' diff --git a/lithops/localhost/v2/localhost.py b/lithops/localhost/v2/localhost.py index 50543fd7a..922cd3899 100644 --- a/lithops/localhost/v2/localhost.py +++ b/lithops/localhost/v2/localhost.py @@ -399,7 +399,8 @@ def get_metadata(self): cmd = f'{self.docker_path} run --name lithops_metadata ' cmd += f'--user {self.uid}:{self.gid} ' if self.is_unix_system and not self.is_podman else '' cmd += f'--env USER={os.getenv("USER", "root")} ' - cmd += f'--cpus "{self.config.get("runtime_cpu")}" ' if self.config.get('runtime_cpu') else '' + #cmd += f'--cpus "{self.config.get("runtime_cpu")}" ' if self.config.get('runtime_cpu') else '' + cmd += f'--cpuset-cpus="0-{self.config.get("runtime_cpu")-1}" ' if self.config.get('runtime_cpu') else '' cmd += f'--memory "{self.config.get("runtime_memory")}m" ' if self.config.get('runtime_memory') else '' cmd += f'--rm -v {tmp_path}:/tmp --entrypoint "python3" ' cmd += f'{self.runtime_name} /tmp/{USER_TEMP_DIR}/localhost-runner.py get_metadata' @@ -424,7 +425,8 @@ def start(self): cmd += '--gpus all ' if self.use_gpu else '' cmd += f'--user {self.uid}:{self.gid} ' if self.is_unix_system and not self.is_podman else '' cmd += f'--env USER={os.getenv("USER", "root")} ' - cmd += f'--cpus "{self.config.get("runtime_cpu")}" ' if self.config.get('runtime_cpu') else '' + #cmd += f'--cpus "{self.config.get("runtime_cpu")}" ' if self.config.get('runtime_cpu') else '' + cmd += f'--cpuset-cpus="0-{self.config.get("runtime_cpu")-1}" ' if self.config.get('runtime_cpu') else '' cmd += f'--memory "{self.config.get("runtime_memory")}m" ' if self.config.get('runtime_memory') else '' cmd += f'--rm -v {tmp_path}:/tmp -it --detach ' cmd += f'--entrypoint=/bin/bash {self.runtime_name}'