From e2133848d154686de775b866e1c33a63a91d865b Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Fri, 10 Apr 2026 13:43:59 +0500 Subject: [PATCH 1/2] Init gateways in background --- src/dstack/_internal/server/app.py | 4 ++-- .../_internal/server/background/scheduled_tasks/__init__.py | 4 ++++ .../_internal/server/background/scheduled_tasks/gateways.py | 6 ++++++ src/dstack/_internal/server/services/gateways/__init__.py | 4 ++-- 4 files changed, 14 insertions(+), 4 deletions(-) diff --git a/src/dstack/_internal/server/app.py b/src/dstack/_internal/server/app.py index 8f7ff489a..394697b7f 100644 --- a/src/dstack/_internal/server/app.py +++ b/src/dstack/_internal/server/app.py @@ -52,7 +52,7 @@ volumes, ) from dstack._internal.server.services.config import ServerConfigManager -from dstack._internal.server.services.gateways import gateway_connections_pool, init_gateways +from dstack._internal.server.services.gateways import gateway_connections_pool from dstack._internal.server.services.locking import advisory_lock_ctx from dstack._internal.server.services.projects import get_or_create_default_project from dstack._internal.server.services.proxy.deps import ServerProxyDependencyInjector @@ -157,7 +157,7 @@ async def lifespan(app: FastAPI): {"show_path": False}, ) await server_config_manager.apply_config(session=session, owner=admin) - await init_gateways(session=session) + update_default_project( project_name=DEFAULT_PROJECT_NAME, url=SERVER_URL, diff --git a/src/dstack/_internal/server/background/scheduled_tasks/__init__.py b/src/dstack/_internal/server/background/scheduled_tasks/__init__.py index cd1925c33..53e4a46b0 100644 --- a/src/dstack/_internal/server/background/scheduled_tasks/__init__.py +++ b/src/dstack/_internal/server/background/scheduled_tasks/__init__.py @@ -1,9 +1,11 @@ from apscheduler.schedulers.asyncio import AsyncIOScheduler +from apscheduler.triggers.date import DateTrigger from apscheduler.triggers.interval import IntervalTrigger from dstack._internal.server import settings from dstack._internal.server.background.scheduled_tasks.events import delete_events from dstack._internal.server.background.scheduled_tasks.gateways import ( + init_gateways_in_background, process_gateways_connections, ) from dstack._internal.server.background.scheduled_tasks.idle_volumes import ( @@ -34,6 +36,8 @@ def start_scheduled_tasks() -> AsyncIOScheduler: Start periodic tasks triggered by `apscheduler` at specific times/intervals. Suitable for tasks that run infrequently and don't need to lock rows for a long time. """ + # DateTrigger() to init gateways immediately. + _scheduler.add_job(init_gateways_in_background, DateTrigger(), max_instances=1) _scheduler.add_job(process_probes, IntervalTrigger(seconds=3, jitter=1)) _scheduler.add_job(collect_metrics, IntervalTrigger(seconds=10), max_instances=1) _scheduler.add_job(delete_metrics, IntervalTrigger(minutes=5), max_instances=1) diff --git a/src/dstack/_internal/server/background/scheduled_tasks/gateways.py b/src/dstack/_internal/server/background/scheduled_tasks/gateways.py index 7f3a3d17c..f71ecacc0 100644 --- a/src/dstack/_internal/server/background/scheduled_tasks/gateways.py +++ b/src/dstack/_internal/server/background/scheduled_tasks/gateways.py @@ -10,6 +10,7 @@ from dstack._internal.server.services.gateways import ( GatewayConnection, gateway_connections_pool, + init_gateways, ) from dstack._internal.server.services.locking import advisory_lock_ctx from dstack._internal.utils.logging import get_logger @@ -17,6 +18,11 @@ logger = get_logger(__name__) +async def init_gateways_in_background(): + async with get_session_ctx() as session: + await init_gateways(session=session) + + async def process_gateways_connections(): await _remove_inactive_connections() await _process_active_connections() diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index 1298e452d..990287593 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -577,7 +577,7 @@ async def init_gateways(session: AsyncSession): return_exceptions=True, ): if isinstance(res, Exception): - logger.warning( + logger.error( "Failed to update gateway %s: %s", gateway_compute.ip_address, res ) elif isinstance(res, bool) and res: @@ -590,7 +590,7 @@ async def init_gateways(session: AsyncSession): return_exceptions=True, ): if isinstance(error, Exception): - logger.warning( + logger.error( "Failed to configure gateway %s: %r", gateway_compute.ip_address, error ) From fa928105daf2d8b4854dee7fc490a1cdad971125 Mon Sep 17 00:00:00 2001 From: Victor Skvortsov Date: Fri, 10 Apr 2026 13:54:55 +0500 Subject: [PATCH 2/2] Revert log level changes --- src/dstack/_internal/server/services/gateways/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/dstack/_internal/server/services/gateways/__init__.py b/src/dstack/_internal/server/services/gateways/__init__.py index 990287593..1298e452d 100644 --- a/src/dstack/_internal/server/services/gateways/__init__.py +++ b/src/dstack/_internal/server/services/gateways/__init__.py @@ -577,7 +577,7 @@ async def init_gateways(session: AsyncSession): return_exceptions=True, ): if isinstance(res, Exception): - logger.error( + logger.warning( "Failed to update gateway %s: %s", gateway_compute.ip_address, res ) elif isinstance(res, bool) and res: @@ -590,7 +590,7 @@ async def init_gateways(session: AsyncSession): return_exceptions=True, ): if isinstance(error, Exception): - logger.error( + logger.warning( "Failed to configure gateway %s: %r", gateway_compute.ip_address, error )