From 6d18976b3da63be1e60737784f4a0e04ae24a04e Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Thu, 19 Mar 2026 18:11:39 +0100 Subject: [PATCH 01/19] Remove ruff warnings in VScode --- python/pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/pyproject.toml b/python/pyproject.toml index 05a2b26e6..b61f27824 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -42,6 +42,9 @@ indent-width = 4 # Assume Python 3.9 target-version = "py39" +# _ is the i18n/gettext builtin injected at runtime +builtins = ["_"] + [tool.ruff.lint] # Enable preview mode, allow os.env changes before imports preview = true From 6c7d78f5eb6faf6c521d18d2401d74f1e8906f8f Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Thu, 19 Mar 2026 18:12:27 +0100 Subject: [PATCH 02/19] Fix error in comets, related to string coercion --- python/PiFinder/comets.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/python/PiFinder/comets.py b/python/PiFinder/comets.py index 9430094ff..e8734fc36 100644 --- a/python/PiFinder/comets.py +++ b/python/PiFinder/comets.py @@ -4,6 +4,7 @@ from skyfield.constants import GM_SUN_Pitjeva_2005_km3_s2 as GM_SUN from PiFinder.utils import Timer, comet_file from PiFinder.calc_utils import sf_utils +import pandas as pd import requests import os import logging @@ -198,6 +199,14 @@ def calc_comets( .set_index("designation", drop=False) ) + # groupby/last can coerce numeric columns to strings when NaN values + # are present; ensure perihelion date fields are numeric before use + for col in ("perihelion_year", "perihelion_month", "perihelion_day"): + comets_df[col] = pd.to_numeric(comets_df[col], errors="coerce") + comets_df = comets_df.dropna( + subset=["perihelion_year", "perihelion_month", "perihelion_day"] + ) + # Report progress after pandas processing (roughly 66% of setup time) if progress_callback: progress_callback(2) From c0aaa03477300ac78778952580589eb50fb9390f Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Thu, 19 Mar 2026 18:34:48 +0100 Subject: [PATCH 03/19] Use a dynamic path to the data_dir when searching for logs --- python/PiFinder/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/PiFinder/server.py b/python/PiFinder/server.py index ff9db6d14..8fd622bba 100644 --- a/python/PiFinder/server.py +++ b/python/PiFinder/server.py @@ -781,7 +781,7 @@ def logs_page(): def stream_logs(): try: position = int(request.query.get("position", 0)) - log_file = "/home/pifinder/PiFinder_data/pifinder.log" + log_file = str(utils.data_dir / "pifinder.log") try: file_size = os.path.getsize(log_file) From a9e21486f2a84e09ff55f2ae7215cca531b24d24 Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Thu, 19 Mar 2026 18:35:11 +0100 Subject: [PATCH 04/19] Silence warnings in VScode --- .vscode/settings.json | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..fb57207b6 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,9 @@ +{ + "python.analysis.diagnosticSeverityOverrides": { + "reportUndefinedVariable": "none" + }, + "files.associations": { + "*logconf*.json": "jsonc" + } +} + From 21cdc3062a1b5a640cb76092f6916d4c9d585b42 Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Thu, 19 Mar 2026 18:41:45 +0100 Subject: [PATCH 05/19] Patch early to avoid pickle errors in interprocess communication --- python/PiFinder/main.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/PiFinder/main.py b/python/PiFinder/main.py index d447bf66c..c8b00fcd0 100644 --- a/python/PiFinder/main.py +++ b/python/PiFinder/main.py @@ -35,7 +35,6 @@ from PiFinder import config from PiFinder import pos_server from PiFinder import utils -from PiFinder import server from PiFinder import keyboard_interface from PiFinder.multiproclogging import MultiprocLogging @@ -51,6 +50,8 @@ from PiFinder.displays import DisplayBase, get_display +import PiFinder.manager_patch as patch + from typing import Any, TYPE_CHECKING # Mypy i8n fix @@ -124,6 +125,9 @@ def setup_dirs(): os.chmod(Path(utils.data_dir), 0o777) +patch.apply() + + class StateManager(BaseManager): pass @@ -348,10 +352,6 @@ def main( ) langXX.install() - import PiFinder.manager_patch as patch - - patch.apply() - with StateManager() as manager: shared_state = manager.SharedState() # type: ignore[attr-defined] location = shared_state.location() From fb0dbfa3ce9359f0bcb4a842c47af9870ff5c4b0 Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Fri, 20 Mar 2026 08:03:44 +0100 Subject: [PATCH 06/19] Fix: Import server again --- python/PiFinder/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/PiFinder/main.py b/python/PiFinder/main.py index c8b00fcd0..83577e524 100644 --- a/python/PiFinder/main.py +++ b/python/PiFinder/main.py @@ -35,6 +35,7 @@ from PiFinder import config from PiFinder import pos_server from PiFinder import utils +from PiFinder import server from PiFinder import keyboard_interface from PiFinder.multiproclogging import MultiprocLogging From a6053266a7c282b0af4e1e6e4731dbc70ea46942 Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Fri, 20 Mar 2026 08:05:02 +0100 Subject: [PATCH 07/19] Fix: collect early logging also in log file --- python/PiFinder/multiproclogging.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/python/PiFinder/multiproclogging.py b/python/PiFinder/multiproclogging.py index 92d36dccd..83b7f4376 100644 --- a/python/PiFinder/multiproclogging.py +++ b/python/PiFinder/multiproclogging.py @@ -87,6 +87,10 @@ def start(self, initial_queue: Optional[Queue] = None): len(self._queues) >= 1 ), "No queues in use. You should have requested at least one queue." + # Create the main-process queue BEFORE starting the sink so the sink + # receives it in its queue list and monitors it. + queue = initial_queue if initial_queue is not None else self.get_queue() + self._proc = Process( target=self._run_sink, args=( @@ -97,7 +101,6 @@ def start(self, initial_queue: Optional[Queue] = None): # Start separate process that consumes from the queues. self._proc.start() # Now in this process we can divert logging to the newly created class - queue = self.get_queue() MultiprocLogging.configurer(queue) def join(self): From caa729208b492febf29506d4620819b38abec8c2 Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Tue, 31 Mar 2026 07:52:32 +0200 Subject: [PATCH 08/19] Server: Back off reading logs in the log tab, if the log file cannot be found. --- python/PiFinder/main.py | 46 +++++++++++++++++++---------- python/PiFinder/multiproclogging.py | 34 +++++++++++++++------ python/PiFinder/server.py | 4 +-- python/views/logs.tpl | 38 ++++++++++++++++++++---- 4 files changed, 90 insertions(+), 32 deletions(-) diff --git a/python/PiFinder/main.py b/python/PiFinder/main.py index 83577e524..119c9b77f 100644 --- a/python/PiFinder/main.py +++ b/python/PiFinder/main.py @@ -868,25 +868,36 @@ def main( if __name__ == "__main__": + import sys + + debug_no_file_logs = "--debug-no-file-logs" in sys.argv + if debug_no_file_logs: + os.environ["PIFINDER_DEBUG_NO_FILE_LOGS"] = "1" + print("Bootstrap logging configuration ...") logging.basicConfig(format="%(asctime)s BASIC %(name)s: %(levelname)s %(message)s") rlogger = logging.getLogger() - rlogger.setLevel(logging.INFO) - log_path = utils.data_dir / "pifinder.log" - try: - log_helper = MultiprocLogging( - Path("pifinder_logconf.json"), - log_path, - ) + rlogger.setLevel(logging.DEBUG if debug_no_file_logs else logging.INFO) + + if debug_no_file_logs: + log_helper = MultiprocLogging(Path("pifinder_logconf.json"), console_only=True) MultiprocLogging.configurer(log_helper.get_queue()) - except FileNotFoundError: - rlogger.warning( - "Cannot find log configuration file, proceeding with basic configuration." - ) - rlogger.warning("Logs will not be stored on disk, unless you use --log") - logging.getLogger("PIL.PngImagePlugin").setLevel(logging.WARNING) - logging.getLogger("tetra3.Tetra3").setLevel(logging.WARNING) - logging.getLogger("picamera2.picamera2").setLevel(logging.WARNING) + else: + log_path = utils.data_dir / "pifinder.log" + try: + log_helper = MultiprocLogging( + Path("pifinder_logconf.json"), + log_path, + ) + MultiprocLogging.configurer(log_helper.get_queue()) + except FileNotFoundError: + rlogger.warning( + "Cannot find log configuration file, proceeding with basic configuration." + ) + rlogger.warning("Logs will not be stored on disk, unless you use --log") + logging.getLogger("PIL.PngImagePlugin").setLevel(logging.WARNING) + logging.getLogger("tetra3.Tetra3").setLevel(logging.WARNING) + logging.getLogger("picamera2.picamera2").setLevel(logging.WARNING) rlogger.info("Starting PiFinder ...") parser = argparse.ArgumentParser(description="eFinder") @@ -948,6 +959,11 @@ def main( "-x", "--verbose", help="Set logging to debug mode", action="store_true" ) parser.add_argument("-l", "--log", help="Log to file", action="store_true") + parser.add_argument( + "--debug-no-file-logs", + help="Debug: log everything at DEBUG level to console only, bypassing log configuration and file output", + action="store_true", + ) parser.add_argument( "--lang", help="Force user interface language (iso2 code). Changes configuration", diff --git a/python/PiFinder/multiproclogging.py b/python/PiFinder/multiproclogging.py index 83b7f4376..6284e9697 100644 --- a/python/PiFinder/multiproclogging.py +++ b/python/PiFinder/multiproclogging.py @@ -67,14 +67,18 @@ def __init__( log_conf: Optional[Path] = None, out_file: Optional[Path] = None, formatter: str = "%(asctime)s %(processName)s-%(name)s:%(levelname)s:%(message)s", + console_only: bool = False, ): self._queues: List[Queue] = [] self._log_conf_file = log_conf self._log_output_file = out_file self._formatter = formatter + self._console_only = console_only self._proc: Optional[Process] = None self.apply_config() + if console_only: + logging.getLogger().setLevel(logging.DEBUG) def apply_config(self): if self._log_conf_file is not None: @@ -94,7 +98,7 @@ def start(self, initial_queue: Optional[Queue] = None): self._proc = Process( target=self._run_sink, args=( - self._log_output_file, + None if self._console_only else self._log_output_file, self._queues, ), ) @@ -109,7 +113,7 @@ def join(self): self._queues[0].put(None) self._proc.join() - def _run_sink(self, output: Path, queues: List[Queue]): + def _run_sink(self, output: Optional[Path], queues: List[Queue]): """ This is the process that consumes every log message (sink) @@ -124,15 +128,22 @@ def _run_sink(self, output: Path, queues: List[Queue]): for hdlr in hdlrs: rLogger.removeHandler(hdlr) - # Set maxBytes to 50MB (50 * 1024 * 1024 bytes) and keep 5 backup files - h = logging.handlers.RotatingFileHandler( - output, maxBytes=50 * 1024 * 1024, backupCount=5, encoding="utf-8" - ) + if output is None: + import sys + + h = logging.StreamHandler(sys.stderr) + rLogger.setLevel(logging.DEBUG) + rLogger.warning("Starting logging process (console only, no file output)") + else: + # Set maxBytes to 50MB (50 * 1024 * 1024 bytes) and keep 5 backup files + h = logging.handlers.RotatingFileHandler( + output, maxBytes=50 * 1024 * 1024, backupCount=5, encoding="utf-8" + ) + rLogger.warning("Starting logging process") + rLogger.warning("Logging to %s", output) f = logging.Formatter(self._formatter) h.setFormatter(f) rLogger.addHandler(h) - rLogger.warning("Starting logging process") - rLogger.warning("Logging to %s", output) # import logging_tree # logging_tree.printout() @@ -172,6 +183,8 @@ def configurer(queue: Queue): This method needs to be called once in each process, so that log records get forwarded to the single process writing log messages. """ + import os + assert queue is not None, "You passed a None to configurer! You cannot do that" assert isinstance( queue, multiprocessing.queues.Queue @@ -182,8 +195,11 @@ def configurer(queue: Queue): config = json5.load(logconf) logging.config.dictConfig(config) - h = logging.handlers.QueueHandler(queue) root = logging.getLogger() + if os.environ.get("PIFINDER_DEBUG_NO_FILE_LOGS"): + root.setLevel(logging.DEBUG) + + h = logging.handlers.QueueHandler(queue) root.addHandler(h) def set_log_conf_file(self, config: Path) -> None: diff --git a/python/PiFinder/server.py b/python/PiFinder/server.py index 8fd622bba..7816f4e11 100644 --- a/python/PiFinder/server.py +++ b/python/PiFinder/server.py @@ -802,8 +802,8 @@ def stream_logs(): else: return {"logs": [], "position": position} except FileNotFoundError: - logger.error(f"Log file not found: {log_file}") - return {"logs": [], "position": 0} + logger.warning(f"Log file not found: {log_file}") + return {"logs": [], "position": 0, "file_not_found": True} except Exception as e: logger.error(f"Error streaming logs: {e}") diff --git a/python/views/logs.tpl b/python/views/logs.tpl index c8c0cf957..d52b0b648 100644 --- a/python/views/logs.tpl +++ b/python/views/logs.tpl @@ -164,17 +164,41 @@ const LINE_HEIGHT = 20; let updateInterval; let lastLine = ''; +// Backoff state for when the log file is not yet available. +// This page intentionally does NOT fetch the home route, so iwgetid is never triggered here. +const MIN_POLL_INTERVAL = 1000; +const MAX_POLL_INTERVAL = 10000; +let currentPollInterval = MIN_POLL_INTERVAL; + +function scheduleFetch() { + clearInterval(updateInterval); + updateInterval = setInterval(fetchLogs, currentPollInterval); +} + function fetchLogs() { if (isPaused) return; - + fetch(`/logs/stream?position=${currentPosition}`) .then(response => response.json()) .then(data => { + if (data.file_not_found) { + // Back off exponentially up to MAX_POLL_INTERVAL + currentPollInterval = Math.min(currentPollInterval * 2, MAX_POLL_INTERVAL); + scheduleFetch(); + return; + } + + // Reset backoff on a successful response + if (currentPollInterval !== MIN_POLL_INTERVAL) { + currentPollInterval = MIN_POLL_INTERVAL; + scheduleFetch(); + } + if (!data.logs || data.logs.length === 0) return; - + currentPosition = data.position; const logContent = document.getElementById('logContent'); - + // Add new logs to buffer, skipping duplicates data.logs.forEach(line => { if (line !== lastLine) { @@ -182,12 +206,12 @@ function fetchLogs() { lastLine = line; } }); - + // Trim buffer if it exceeds size if (logBuffer.length > BUFFER_SIZE) { logBuffer = logBuffer.slice(-BUFFER_SIZE); } - + // Update display updateLogDisplay(); }) @@ -239,8 +263,10 @@ function restartFromEnd() { currentPosition = 0; logBuffer = []; isPaused = false; + currentPollInterval = MIN_POLL_INTERVAL; document.getElementById('pauseButton').textContent = 'Pause'; fetchLogs(); + scheduleFetch(); } // Start fetching logs when page loads @@ -251,7 +277,7 @@ document.addEventListener('DOMContentLoaded', () => { // Start log fetching fetchLogs(); - updateInterval = setInterval(fetchLogs, 1000); + scheduleFetch(); // Hide loading message after first logs appear const observer = new MutationObserver((mutations) => { From 35dea3aea03d660c73eb8467e58b5e030e2db813 Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Tue, 31 Mar 2026 08:37:20 +0200 Subject: [PATCH 09/19] webserver: Upload and choose log configuration --- .gitignore | 4 + python/PiFinder/main.py | 5 ++ python/PiFinder/server.py | 58 ++++++++++++++ python/logconf_debug.json | 151 +++++++++++++++++++++++++++++++++++ python/pifinder_logconf.json | 1 - python/views/logs.tpl | 143 +++++++++++++-------------------- 6 files changed, 273 insertions(+), 89 deletions(-) create mode 100644 python/logconf_debug.json delete mode 120000 python/pifinder_logconf.json diff --git a/.gitignore b/.gitignore index 8c26256e2..72089c9df 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# Active log configuration symlink (auto-created at startup) +python/pifinder_logconf.json +python/logconf_*.json + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/python/PiFinder/main.py b/python/PiFinder/main.py index 119c9b77f..073a894f4 100644 --- a/python/PiFinder/main.py +++ b/python/PiFinder/main.py @@ -870,6 +870,11 @@ def main( if __name__ == "__main__": import sys + # Ensure the active log config symlink exists, defaulting to logconf_default.json + _logconf_link = Path("pifinder_logconf.json") + if not _logconf_link.exists(): + _logconf_link.symlink_to("logconf_default.json") + debug_no_file_logs = "--debug-no-file-logs" in sys.argv if debug_no_file_logs: os.environ["PIFINDER_DEBUG_NO_FILE_LOGS"] = "1" diff --git a/python/PiFinder/server.py b/python/PiFinder/server.py index 7816f4e11..d3ea7cfab 100644 --- a/python/PiFinder/server.py +++ b/python/PiFinder/server.py @@ -844,6 +844,64 @@ def get_component_levels(): logging.error(f"Error reading log configuration: {e}") return {"status": "error", "message": str(e)} + @app.route("/logs/configs") + @auth_required + def list_log_configs(): + """Return all available logconf_*.json files with display names.""" + import glob + + configs = [] + active = os.path.realpath("pifinder_logconf.json") if os.path.exists("pifinder_logconf.json") else None + for path in sorted(glob.glob("logconf_*.json")): + stem = path[len("logconf_"):-len(".json")] + display = stem.replace("_", " ").title() + configs.append({ + "file": path, + "name": display, + "active": os.path.realpath(path) == active, + }) + return {"configs": configs} + + @app.route("/logs/switch_config", method="post") + @auth_required + def switch_log_config(): + """Atomically repoint pifinder_logconf.json to the chosen config, then restart.""" + logconf_file = request.forms.get("logconf_file", "").strip() + if not logconf_file or not logconf_file.startswith("logconf_") or not logconf_file.endswith(".json"): + return {"status": "error", "message": "Invalid log config file name"} + if not os.path.exists(logconf_file): + return {"status": "error", "message": f"Log config file not found: {logconf_file}"} + try: + link = "pifinder_logconf.json" + tmp = link + ".tmp" + os.symlink(logconf_file, tmp) + os.replace(tmp, link) + logger.info("Switched log config to %s", logconf_file) + except Exception as e: + logger.error("Failed to switch log config: %s", e) + return {"status": "error", "message": str(e)} + return template("restart_pifinder") + + @app.route("/logs/upload_config", method="post") + @auth_required + def upload_log_config(): + """Upload a new logconf_*.json file.""" + upload = request.files.get("config_file") + if not upload: + return {"status": "error", "message": "No file provided"} + filename = upload.filename + if not filename.startswith("logconf_") or not filename.endswith(".json"): + return {"status": "error", "message": "File must be named logconf_.json"} + if os.path.exists(filename): + return {"status": "error", "message": f"File already exists: {filename}"} + try: + upload.save(filename, overwrite=False) + logger.info("Uploaded log config: %s", filename) + return {"status": "ok", "file": filename} + except Exception as e: + logger.error("Failed to save uploaded log config: %s", e) + return {"status": "error", "message": str(e)} + @app.route("/logs/download") @auth_required def download_logs(): diff --git a/python/logconf_debug.json b/python/logconf_debug.json new file mode 100644 index 000000000..bb47960c2 --- /dev/null +++ b/python/logconf_debug.json @@ -0,0 +1,151 @@ +{ + // Note that the JSON5 library that we use for configuration supports comments in JSON files. + "version": 1, + "disable_existing_loggers": false, // THIS MUST BE FALSE for logging to work + "formatters": { + "default": { + "format": "%(asctime)s %(processName)s-%(name)s:%(levelname)s:%(message)s" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + "level": "DEBUG", + "formatter": "default", + "stream": "ext://sys.stdout" + }, + // Used for GRPC, avoids a "No handlers can be found" warning. + "null": { + "class": "logging.NullHandler", + "level": "ERROR" + } + }, + "loggers": { + + ///////////////////////////////////////////////////////////////// + ////// root logger + // + "": { + "level": "DEBUG", + "handlers": ["console"] + // The file handler is added automatically by code + }, + + ///////////////////////////////////////////////////////////////// + ////// State shared between Subsystems + // + "SharedState": { + "level": "DEBUG" + }, + + ///////////////////////////////////////////////////////////////// + ////// User Interface + // + "UI": { + "level": "DEBUG" + }, + + ///////////////////////////////////////////////////////////////// + ////// Camera Subsystem + // + "Camera": { + "level": "DEBUG" + }, + + ///////////////////////////////////////////////////////////////// + ////// Platesolver Subsystem + // + "Solver": { + "level": "DEBUG" + }, + + ///////////////////////////////////////////////////////////////// + ////// GPS Subsystem + // + "GPS": { + "level": "DEBUG" + }, + "GPS.parser": { + "level": "DEBUG" + }, + + ///////////////////////////////////////////////////////////////// + ////// Catalog Subsystem + // + "Catalog": { + "level": "DEBUG" + }, + + ///////////////////////////////////////////////////////////////// + ////// Database + // + "Database": { + "level": "DEBUG" + }, + + ///////////////////////////////////////////////////////////////// + ////// IMU Subsystem + // + "IMU": { + "level": "DEBUG" + }, + + ///////////////////////////////////////////////////////////////// + ////// Keyboard Subsystem + // + "Keyboard": { + "level": "DEBUG" + }, + + ///////////////////////////////////////////////////////////////// + ////// Observations Subsystem + // + "Observation": { + "level": "DEBUG" + }, + + ///////////////////////////////////////////////////////////////// + ////// Web Server Subsystem + // + "Server": { + "level": "DEBUG" + }, + + ///////////////////////////////////////////////////////////////// + ////// Pos Server Subsystem (SkySafari LX200 Interface) + // + "PosServer": { + "level": "DEBUG" + }, + + ///////////////////////////////////////////////////////////////// + ////// Utils Libraries + // + "SysUtils": { + "level": "DEBUG" + }, + "Utils": { + "level": "DEBUG" + }, + + ///////////////////////////////////////////////////////////////// + ////// Third party Libraries + // + // Keep these suppressed to avoid noise. + // + "PIL.PngImagePlugin": { + "level": "WARNING" + }, + "tetra3.Tetra3": { + "level": "WARNING" + }, + "picamera2.picamera2": { + "level": "WARNING" + }, + "grpc": { + "level": "ERROR", + "propagate": false, + "handlers": ["null"] + }, + } +} diff --git a/python/pifinder_logconf.json b/python/pifinder_logconf.json deleted file mode 120000 index cb17ea4b3..000000000 --- a/python/pifinder_logconf.json +++ /dev/null @@ -1 +0,0 @@ -logconf_default.json \ No newline at end of file diff --git a/python/views/logs.tpl b/python/views/logs.tpl index d52b0b648..64a997752 100644 --- a/python/views/logs.tpl +++ b/python/views/logs.tpl @@ -122,20 +122,12 @@ - - - +
@@ -325,94 +317,69 @@ document.getElementById('copyButton').addEventListener('click', function() { }); }); -// Log level management -function updateComponentLevels() { - fetch('/logs/components') +// Log configuration management +function loadLogConfigs() { + fetch('/logs/configs') .then(response => response.json()) .then(data => { - const componentSelect = document.getElementById('componentSelect'); - componentSelect.innerHTML = ''; - - // Sort components alphabetically - const sortedComponents = Object.entries(data.components).sort(([a], [b]) => a.localeCompare(b)); - - sortedComponents.forEach(([component, levels]) => { + const configSelect = document.getElementById('configSelect'); + configSelect.innerHTML = ''; + data.configs.forEach(cfg => { const option = document.createElement('option'); - option.value = component; - option.textContent = component; - componentSelect.appendChild(option); + option.value = cfg.file; + option.textContent = cfg.name; + if (cfg.active) option.selected = true; + configSelect.appendChild(option); }); }) - .catch(error => console.error('Error fetching component levels:', error)); + .catch(error => console.error('Error fetching log configs:', error)); } -// Handle global level change -document.getElementById('globalLevel').addEventListener('change', function(e) { - const newLevel = e.target.value; - fetch('/logs/level', { - method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - }, - body: `level=${encodeURIComponent(newLevel)}` - }) - .then(response => response.json()) - .then(result => { - if (result.status === 'success') { - console.log(`Changed global log level to ${newLevel}`); - } else { - console.error('Failed to update global log level:', result.message); - } - }) - .catch(error => console.error('Error updating global log level:', error)); +document.getElementById('configSelect').addEventListener('change', function(e) { + const configFile = e.target.value; + if (!configFile) return; + if (!confirm(`Switch log configuration to "${e.target.options[e.target.selectedIndex].text}" and restart PiFinder?`)) { + loadLogConfigs(); // reset selection + return; + } + // Use a real form POST so the browser navigates to the restart page HTML + const form = document.createElement('form'); + form.method = 'POST'; + form.action = '/logs/switch_config'; + const input = document.createElement('input'); + input.type = 'hidden'; + input.name = 'logconf_file'; + input.value = configFile; + form.appendChild(input); + document.body.appendChild(form); + form.submit(); }); -// Handle component selection -document.getElementById('componentSelect').addEventListener('change', function(e) { - const component = e.target.value; - if (!component) { - document.getElementById('componentLevel').style.display = 'none'; +document.getElementById('uploadLogConfInput').addEventListener('change', function(e) { + const file = e.target.files[0]; + if (!file) return; + if (!file.name.startsWith('logconf_') || !file.name.endsWith('.json')) { + alert('File must be named logconf_.json'); + e.target.value = ''; return; } - - // Show level select and set current level - const levelSelect = document.getElementById('componentLevel'); - levelSelect.style.display = 'block'; - - // Get current level for selected component - fetch('/logs/components') + const formData = new FormData(); + formData.append('config_file', file); + fetch('/logs/upload_config', { method: 'POST', body: formData }) .then(response => response.json()) - .then(data => { - const currentLevel = data.components[component].current_level; - levelSelect.value = currentLevel; - }); -}); - -// Handle component level change -document.getElementById('componentLevel').addEventListener('change', function(e) { - const component = document.getElementById('componentSelect').value; - const newLevel = e.target.value; - - fetch('/logs/component_level', { - method: 'POST', - headers: { - 'Content-Type': 'application/x-www-form-urlencoded', - }, - body: `component=${encodeURIComponent(component)}&level=${encodeURIComponent(newLevel)}` - }) - .then(response => response.json()) - .then(result => { - if (result.status === 'success') { - console.log(`Changed ${component} log level to ${newLevel}`); - } else { - console.error('Failed to update log level:', result.message); - } - }) - .catch(error => console.error('Error updating log level:', error)); + .then(result => { + if (result.status === 'ok') { + loadLogConfigs(); + } else { + alert('Upload failed: ' + result.message); + } + }) + .catch(error => console.error('Error uploading log config:', error)); + e.target.value = ''; }); -// Initial load of components -updateComponentLevels(); +// Initial load of configs +loadLogConfigs(); // Set up button event listeners document.getElementById('pauseButton').addEventListener('click', function() { From 7f830e2279c6b0a2b54b014ab1cd570b1583dde8 Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Tue, 7 Apr 2026 07:35:38 +0200 Subject: [PATCH 10/19] Add missing bracket. --- python/PiFinder/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/PiFinder/server.py b/python/PiFinder/server.py index 8fd622bba..f6bb2bcc8 100644 --- a/python/PiFinder/server.py +++ b/python/PiFinder/server.py @@ -781,7 +781,7 @@ def logs_page(): def stream_logs(): try: position = int(request.query.get("position", 0)) - log_file = str(utils.data_dir / "pifinder.log") + log_file = str(utils.data_dir / "pifinder.log") try: file_size = os.path.getsize(log_file) From 732bdd671442b0d4d3856e9d0b984577ef26ae83 Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Tue, 7 Apr 2026 07:38:00 +0200 Subject: [PATCH 11/19] multiproclogging: Add type annotation for mypy error --- python/PiFinder/multiproclogging.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/PiFinder/multiproclogging.py b/python/PiFinder/multiproclogging.py index 6284e9697..46c780a12 100644 --- a/python/PiFinder/multiproclogging.py +++ b/python/PiFinder/multiproclogging.py @@ -131,7 +131,7 @@ def _run_sink(self, output: Optional[Path], queues: List[Queue]): if output is None: import sys - h = logging.StreamHandler(sys.stderr) + h: logging.Handler = logging.StreamHandler(sys.stderr) rLogger.setLevel(logging.DEBUG) rLogger.warning("Starting logging process (console only, no file output)") else: From 96a2d96545fec683c9f4b1893553a2b41b5b13a8 Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Tue, 7 Apr 2026 10:29:32 +0200 Subject: [PATCH 12/19] Server: Back off reading logs in the log tab on error (#403) * Server: Back off reading logs in the log tab, if the log file cannot be found. * multiproclogging: Add type annotation for mypy error --- python/PiFinder/main.py | 46 +++++++++++++++++++---------- python/PiFinder/multiproclogging.py | 34 +++++++++++++++------ python/PiFinder/server.py | 4 +-- python/views/logs.tpl | 38 ++++++++++++++++++++---- 4 files changed, 90 insertions(+), 32 deletions(-) diff --git a/python/PiFinder/main.py b/python/PiFinder/main.py index 83577e524..119c9b77f 100644 --- a/python/PiFinder/main.py +++ b/python/PiFinder/main.py @@ -868,25 +868,36 @@ def main( if __name__ == "__main__": + import sys + + debug_no_file_logs = "--debug-no-file-logs" in sys.argv + if debug_no_file_logs: + os.environ["PIFINDER_DEBUG_NO_FILE_LOGS"] = "1" + print("Bootstrap logging configuration ...") logging.basicConfig(format="%(asctime)s BASIC %(name)s: %(levelname)s %(message)s") rlogger = logging.getLogger() - rlogger.setLevel(logging.INFO) - log_path = utils.data_dir / "pifinder.log" - try: - log_helper = MultiprocLogging( - Path("pifinder_logconf.json"), - log_path, - ) + rlogger.setLevel(logging.DEBUG if debug_no_file_logs else logging.INFO) + + if debug_no_file_logs: + log_helper = MultiprocLogging(Path("pifinder_logconf.json"), console_only=True) MultiprocLogging.configurer(log_helper.get_queue()) - except FileNotFoundError: - rlogger.warning( - "Cannot find log configuration file, proceeding with basic configuration." - ) - rlogger.warning("Logs will not be stored on disk, unless you use --log") - logging.getLogger("PIL.PngImagePlugin").setLevel(logging.WARNING) - logging.getLogger("tetra3.Tetra3").setLevel(logging.WARNING) - logging.getLogger("picamera2.picamera2").setLevel(logging.WARNING) + else: + log_path = utils.data_dir / "pifinder.log" + try: + log_helper = MultiprocLogging( + Path("pifinder_logconf.json"), + log_path, + ) + MultiprocLogging.configurer(log_helper.get_queue()) + except FileNotFoundError: + rlogger.warning( + "Cannot find log configuration file, proceeding with basic configuration." + ) + rlogger.warning("Logs will not be stored on disk, unless you use --log") + logging.getLogger("PIL.PngImagePlugin").setLevel(logging.WARNING) + logging.getLogger("tetra3.Tetra3").setLevel(logging.WARNING) + logging.getLogger("picamera2.picamera2").setLevel(logging.WARNING) rlogger.info("Starting PiFinder ...") parser = argparse.ArgumentParser(description="eFinder") @@ -948,6 +959,11 @@ def main( "-x", "--verbose", help="Set logging to debug mode", action="store_true" ) parser.add_argument("-l", "--log", help="Log to file", action="store_true") + parser.add_argument( + "--debug-no-file-logs", + help="Debug: log everything at DEBUG level to console only, bypassing log configuration and file output", + action="store_true", + ) parser.add_argument( "--lang", help="Force user interface language (iso2 code). Changes configuration", diff --git a/python/PiFinder/multiproclogging.py b/python/PiFinder/multiproclogging.py index 83b7f4376..46c780a12 100644 --- a/python/PiFinder/multiproclogging.py +++ b/python/PiFinder/multiproclogging.py @@ -67,14 +67,18 @@ def __init__( log_conf: Optional[Path] = None, out_file: Optional[Path] = None, formatter: str = "%(asctime)s %(processName)s-%(name)s:%(levelname)s:%(message)s", + console_only: bool = False, ): self._queues: List[Queue] = [] self._log_conf_file = log_conf self._log_output_file = out_file self._formatter = formatter + self._console_only = console_only self._proc: Optional[Process] = None self.apply_config() + if console_only: + logging.getLogger().setLevel(logging.DEBUG) def apply_config(self): if self._log_conf_file is not None: @@ -94,7 +98,7 @@ def start(self, initial_queue: Optional[Queue] = None): self._proc = Process( target=self._run_sink, args=( - self._log_output_file, + None if self._console_only else self._log_output_file, self._queues, ), ) @@ -109,7 +113,7 @@ def join(self): self._queues[0].put(None) self._proc.join() - def _run_sink(self, output: Path, queues: List[Queue]): + def _run_sink(self, output: Optional[Path], queues: List[Queue]): """ This is the process that consumes every log message (sink) @@ -124,15 +128,22 @@ def _run_sink(self, output: Path, queues: List[Queue]): for hdlr in hdlrs: rLogger.removeHandler(hdlr) - # Set maxBytes to 50MB (50 * 1024 * 1024 bytes) and keep 5 backup files - h = logging.handlers.RotatingFileHandler( - output, maxBytes=50 * 1024 * 1024, backupCount=5, encoding="utf-8" - ) + if output is None: + import sys + + h: logging.Handler = logging.StreamHandler(sys.stderr) + rLogger.setLevel(logging.DEBUG) + rLogger.warning("Starting logging process (console only, no file output)") + else: + # Set maxBytes to 50MB (50 * 1024 * 1024 bytes) and keep 5 backup files + h = logging.handlers.RotatingFileHandler( + output, maxBytes=50 * 1024 * 1024, backupCount=5, encoding="utf-8" + ) + rLogger.warning("Starting logging process") + rLogger.warning("Logging to %s", output) f = logging.Formatter(self._formatter) h.setFormatter(f) rLogger.addHandler(h) - rLogger.warning("Starting logging process") - rLogger.warning("Logging to %s", output) # import logging_tree # logging_tree.printout() @@ -172,6 +183,8 @@ def configurer(queue: Queue): This method needs to be called once in each process, so that log records get forwarded to the single process writing log messages. """ + import os + assert queue is not None, "You passed a None to configurer! You cannot do that" assert isinstance( queue, multiprocessing.queues.Queue @@ -182,8 +195,11 @@ def configurer(queue: Queue): config = json5.load(logconf) logging.config.dictConfig(config) - h = logging.handlers.QueueHandler(queue) root = logging.getLogger() + if os.environ.get("PIFINDER_DEBUG_NO_FILE_LOGS"): + root.setLevel(logging.DEBUG) + + h = logging.handlers.QueueHandler(queue) root.addHandler(h) def set_log_conf_file(self, config: Path) -> None: diff --git a/python/PiFinder/server.py b/python/PiFinder/server.py index f6bb2bcc8..a7284cb89 100644 --- a/python/PiFinder/server.py +++ b/python/PiFinder/server.py @@ -802,8 +802,8 @@ def stream_logs(): else: return {"logs": [], "position": position} except FileNotFoundError: - logger.error(f"Log file not found: {log_file}") - return {"logs": [], "position": 0} + logger.warning(f"Log file not found: {log_file}") + return {"logs": [], "position": 0, "file_not_found": True} except Exception as e: logger.error(f"Error streaming logs: {e}") diff --git a/python/views/logs.tpl b/python/views/logs.tpl index c8c0cf957..d52b0b648 100644 --- a/python/views/logs.tpl +++ b/python/views/logs.tpl @@ -164,17 +164,41 @@ const LINE_HEIGHT = 20; let updateInterval; let lastLine = ''; +// Backoff state for when the log file is not yet available. +// This page intentionally does NOT fetch the home route, so iwgetid is never triggered here. +const MIN_POLL_INTERVAL = 1000; +const MAX_POLL_INTERVAL = 10000; +let currentPollInterval = MIN_POLL_INTERVAL; + +function scheduleFetch() { + clearInterval(updateInterval); + updateInterval = setInterval(fetchLogs, currentPollInterval); +} + function fetchLogs() { if (isPaused) return; - + fetch(`/logs/stream?position=${currentPosition}`) .then(response => response.json()) .then(data => { + if (data.file_not_found) { + // Back off exponentially up to MAX_POLL_INTERVAL + currentPollInterval = Math.min(currentPollInterval * 2, MAX_POLL_INTERVAL); + scheduleFetch(); + return; + } + + // Reset backoff on a successful response + if (currentPollInterval !== MIN_POLL_INTERVAL) { + currentPollInterval = MIN_POLL_INTERVAL; + scheduleFetch(); + } + if (!data.logs || data.logs.length === 0) return; - + currentPosition = data.position; const logContent = document.getElementById('logContent'); - + // Add new logs to buffer, skipping duplicates data.logs.forEach(line => { if (line !== lastLine) { @@ -182,12 +206,12 @@ function fetchLogs() { lastLine = line; } }); - + // Trim buffer if it exceeds size if (logBuffer.length > BUFFER_SIZE) { logBuffer = logBuffer.slice(-BUFFER_SIZE); } - + // Update display updateLogDisplay(); }) @@ -239,8 +263,10 @@ function restartFromEnd() { currentPosition = 0; logBuffer = []; isPaused = false; + currentPollInterval = MIN_POLL_INTERVAL; document.getElementById('pauseButton').textContent = 'Pause'; fetchLogs(); + scheduleFetch(); } // Start fetching logs when page loads @@ -251,7 +277,7 @@ document.addEventListener('DOMContentLoaded', () => { // Start log fetching fetchLogs(); - updateInterval = setInterval(fetchLogs, 1000); + scheduleFetch(); // Hide loading message after first logs appear const observer = new MutationObserver((mutations) => { From 908560a2c9baff0bbce8acdcf459a81aba90cc29 Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Tue, 7 Apr 2026 18:53:46 +0200 Subject: [PATCH 13/19] ruff formatting --- python/PiFinder/server.py | 41 ++++++++++++++++------ python/PiFinder/sqm/save_sweep_metadata.py | 3 +- python/PiFinder/sqm/sqm.py | 17 +++++---- python/PiFinder/ui/sqm.py | 4 ++- python/PiFinder/ui/sqm_calibration.py | 4 ++- python/PiFinder/ui/sqm_sweep.py | 16 ++++++--- python/tests/test_sqm.py | 4 ++- 7 files changed, 62 insertions(+), 27 deletions(-) diff --git a/python/PiFinder/server.py b/python/PiFinder/server.py index a13b2151a..bae7bc6b3 100644 --- a/python/PiFinder/server.py +++ b/python/PiFinder/server.py @@ -851,15 +851,21 @@ def list_log_configs(): import glob configs = [] - active = os.path.realpath("pifinder_logconf.json") if os.path.exists("pifinder_logconf.json") else None + active = ( + os.path.realpath("pifinder_logconf.json") + if os.path.exists("pifinder_logconf.json") + else None + ) for path in sorted(glob.glob("logconf_*.json")): - stem = path[len("logconf_"):-len(".json")] + stem = path[len("logconf_") : -len(".json")] display = stem.replace("_", " ").title() - configs.append({ - "file": path, - "name": display, - "active": os.path.realpath(path) == active, - }) + configs.append( + { + "file": path, + "name": display, + "active": os.path.realpath(path) == active, + } + ) return {"configs": configs} @app.route("/logs/switch_config", method="post") @@ -867,10 +873,17 @@ def list_log_configs(): def switch_log_config(): """Atomically repoint pifinder_logconf.json to the chosen config, then restart.""" logconf_file = request.forms.get("logconf_file", "").strip() - if not logconf_file or not logconf_file.startswith("logconf_") or not logconf_file.endswith(".json"): + if ( + not logconf_file + or not logconf_file.startswith("logconf_") + or not logconf_file.endswith(".json") + ): return {"status": "error", "message": "Invalid log config file name"} if not os.path.exists(logconf_file): - return {"status": "error", "message": f"Log config file not found: {logconf_file}"} + return { + "status": "error", + "message": f"Log config file not found: {logconf_file}", + } try: link = "pifinder_logconf.json" tmp = link + ".tmp" @@ -891,9 +904,15 @@ def upload_log_config(): return {"status": "error", "message": "No file provided"} filename = upload.filename if not filename.startswith("logconf_") or not filename.endswith(".json"): - return {"status": "error", "message": "File must be named logconf_.json"} + return { + "status": "error", + "message": "File must be named logconf_.json", + } if os.path.exists(filename): - return {"status": "error", "message": f"File already exists: {filename}"} + return { + "status": "error", + "message": f"File already exists: {filename}", + } try: upload.save(filename, overwrite=False) logger.info("Uploaded log config: %s", filename) diff --git a/python/PiFinder/sqm/save_sweep_metadata.py b/python/PiFinder/sqm/save_sweep_metadata.py index 518434a97..293131716 100644 --- a/python/PiFinder/sqm/save_sweep_metadata.py +++ b/python/PiFinder/sqm/save_sweep_metadata.py @@ -80,7 +80,8 @@ def save_sweep_metadata( # Noise floor estimation details (from NoiseFloorEstimator) if noise_floor_details is not None: metadata["noise_floor_estimator"] = { - k: v for k, v in noise_floor_details.items() + k: v + for k, v in noise_floor_details.items() if k != "request_zero_sec_sample" # Exclude internal flags } if camera_type is not None: diff --git a/python/PiFinder/sqm/sqm.py b/python/PiFinder/sqm/sqm.py index db6186ab0..d5426a020 100644 --- a/python/PiFinder/sqm/sqm.py +++ b/python/PiFinder/sqm/sqm.py @@ -65,7 +65,9 @@ def _load_calibration(self) -> bool: ) if not calibration_file.exists(): - logger.debug(f"No calibration file found at {calibration_file}, using defaults") + logger.debug( + f"No calibration file found at {calibration_file}, using defaults" + ) return False try: @@ -193,7 +195,9 @@ def _measure_star_flux_with_local_background( # Check for saturation in aperture aperture_pixels = image_patch[aperture_mask] - max_aperture_pixel = np.max(aperture_pixels) if len(aperture_pixels) > 0 else 0 + max_aperture_pixel = ( + np.max(aperture_pixels) if len(aperture_pixels) > 0 else 0 + ) if max_aperture_pixel >= saturation_threshold: # Mark saturated star with flux=-1 to be excluded from mzero calculation @@ -202,7 +206,6 @@ def _measure_star_flux_with_local_background( n_saturated += 1 continue - # Total flux in aperture (includes background) total_flux = np.sum(aperture_pixels) @@ -263,9 +266,7 @@ def _calculate_mzero( # Flux-weighted mean: brighter stars contribute more valid_mzeros_arr = np.array(valid_mzeros) valid_fluxes_arr = np.array(valid_fluxes) - weighted_mzero = float( - np.average(valid_mzeros_arr, weights=valid_fluxes_arr) - ) + weighted_mzero = float(np.average(valid_mzeros_arr, weights=valid_fluxes_arr)) return weighted_mzero, mzeros @@ -540,7 +541,9 @@ def calculate( # Following ASTAP: zenith is reference point where extinction = 0 # Only ADDITIONAL extinction below zenith is added: 0.28 * (airmass - 1) # This allows comparing measurements at different altitudes - extinction_for_altitude = self._atmospheric_extinction(altitude_deg) # 0.28*(airmass-1) + extinction_for_altitude = self._atmospheric_extinction( + altitude_deg + ) # 0.28*(airmass-1) # Main SQM value: no extinction correction (raw measurement) sqm_final = sqm_uncorrected diff --git a/python/PiFinder/ui/sqm.py b/python/PiFinder/ui/sqm.py index 592e755fd..235b7a6c4 100644 --- a/python/PiFinder/ui/sqm.py +++ b/python/PiFinder/ui/sqm.py @@ -262,7 +262,9 @@ def _is_calibrated(self) -> bool: camera_type = self.shared_state.camera_type() camera_type_processed = f"{camera_type}_processed" calibration_file = ( - Path.home() / "PiFinder_data" / f"sqm_calibration_{camera_type_processed}.json" + Path.home() + / "PiFinder_data" + / f"sqm_calibration_{camera_type_processed}.json" ) return calibration_file.exists() diff --git a/python/PiFinder/ui/sqm_calibration.py b/python/PiFinder/ui/sqm_calibration.py index 450b54d98..b0eb61803 100644 --- a/python/PiFinder/ui/sqm_calibration.py +++ b/python/PiFinder/ui/sqm_calibration.py @@ -701,7 +701,9 @@ def _analyze_calibration(self): # 2. Compute read noise using temporal variance (not spatial) # Spatial std includes fixed pattern noise (PRNU), which is wrong. # Temporal variance at each pixel measures true read noise. - temporal_variance = np.var(bias_stack, axis=0) # variance across frames per pixel + temporal_variance = np.var( + bias_stack, axis=0 + ) # variance across frames per pixel self.read_noise = float(np.sqrt(np.mean(temporal_variance))) # 3. Compute dark current rate diff --git a/python/PiFinder/ui/sqm_sweep.py b/python/PiFinder/ui/sqm_sweep.py index 56e8310b2..277e47b7b 100644 --- a/python/PiFinder/ui/sqm_sweep.py +++ b/python/PiFinder/ui/sqm_sweep.py @@ -288,7 +288,8 @@ def _add_detailed_metadata(self): # Find the sweep directory captures_dir = Path(utils.data_dir) / "captures" sweep_dirs = [ - d for d in captures_dir.glob("sweep_*") + d + for d in captures_dir.glob("sweep_*") if d.stat().st_ctime >= (self.start_time - 1) ] if not sweep_dirs: @@ -313,7 +314,8 @@ def _add_detailed_metadata(self): "pifinder_value": sqm_state.value, "reference_value": self.reference_sqm, "difference": (self.reference_sqm - sqm_state.value) - if self.reference_sqm and sqm_state.value else None, + if self.reference_sqm and sqm_state.value + else None, "source": sqm_state.source, } @@ -327,7 +329,8 @@ def _add_detailed_metadata(self): if image_metadata: metadata["image"] = { "exposure_us": image_metadata.get("exposure_time"), - "exposure_sec": image_metadata.get("exposure_time", 0) / 1_000_000.0, + "exposure_sec": image_metadata.get("exposure_time", 0) + / 1_000_000.0, "gain": image_metadata.get("gain"), "imu_delta": image_metadata.get("imu_delta"), } @@ -348,8 +351,11 @@ def _add_detailed_metadata(self): # Add NoiseFloorEstimator output camera_type = self.shared_state.camera_type() camera_type_processed = f"{camera_type}_processed" - exposure_sec = (image_metadata.get("exposure_time", 500000) / 1_000_000.0 - if image_metadata else 0.5) + exposure_sec = ( + image_metadata.get("exposure_time", 500000) / 1_000_000.0 + if image_metadata + else 0.5 + ) if self.camera_image is not None: image_array = np.array(self.camera_image.convert("L")) diff --git a/python/tests/test_sqm.py b/python/tests/test_sqm.py index a8a9ac58b..7b9f610e9 100644 --- a/python/tests/test_sqm.py +++ b/python/tests/test_sqm.py @@ -256,7 +256,9 @@ def test_calculate_extinction_applied(self): # Check extinction values (ASTAP convention: 0 at zenith) # Pickering airmass at 30° ≈ 1.995, so extinction ≈ 0.28 * 0.995 ≈ 0.279 - assert details_zenith["extinction_for_altitude"] == pytest.approx(0.0, abs=0.001) + assert details_zenith["extinction_for_altitude"] == pytest.approx( + 0.0, abs=0.001 + ) expected_ext_30 = 0.28 * (sqm._pickering_airmass(30.0) - 1) assert details_30deg["extinction_for_altitude"] == pytest.approx( expected_ext_30, abs=0.001 From 5dce9498d833f4147f0409de38e9ed820f018ff4 Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Mon, 13 Apr 2026 08:05:02 +0200 Subject: [PATCH 14/19] ruff formatting --- python/PiFinder/calc_utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/PiFinder/calc_utils.py b/python/PiFinder/calc_utils.py index 112271a3d..d834f38d0 100644 --- a/python/PiFinder/calc_utils.py +++ b/python/PiFinder/calc_utils.py @@ -140,7 +140,6 @@ def b1950_to_j2000(ra_hours, dec_deg): """ return epoch_to_epoch(B1950, J2000, ra_hours, dec_deg) - def aim_degrees(shared_state, mount_type, screen_direction, target): """ Returns degrees in either From f18de0e5481b1e612fa47811ec96f9f7e3a299b8 Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Mon, 13 Apr 2026 08:12:18 +0200 Subject: [PATCH 15/19] Avoid mypy notes that bodies of untyped functions are not checked. --- python/PiFinder/calc_utils.py | 1 + python/PiFinder/catalog_base.py | 2 +- python/PiFinder/catalog_imports/harris_loader.py | 2 +- python/PiFinder/state.py | 2 +- python/PiFinder/ui/radec_entry.py | 2 +- python/PiFinder/ui/sqm_calibration.py | 2 +- python/PiFinder/ui/textentry.py | 2 +- 7 files changed, 7 insertions(+), 6 deletions(-) diff --git a/python/PiFinder/calc_utils.py b/python/PiFinder/calc_utils.py index d834f38d0..112271a3d 100644 --- a/python/PiFinder/calc_utils.py +++ b/python/PiFinder/calc_utils.py @@ -140,6 +140,7 @@ def b1950_to_j2000(ra_hours, dec_deg): """ return epoch_to_epoch(B1950, J2000, ra_hours, dec_deg) + def aim_degrees(shared_state, mount_type, screen_direction, target): """ Returns degrees in either diff --git a/python/PiFinder/catalog_base.py b/python/PiFinder/catalog_base.py index 6f07f3f47..12ad7d2ea 100644 --- a/python/PiFinder/catalog_base.py +++ b/python/PiFinder/catalog_base.py @@ -171,7 +171,7 @@ def assign_virtual_object_ids(catalog, low_id: int) -> int: class TimerMixin: """Provides timer functionality via composition""" - def __init__(self): + def __init__(self) -> None: self.timer: Optional[threading.Timer] = None self.is_running: bool = False self.time_delay_seconds: Union[int, Callable[[], int]] = ( diff --git a/python/PiFinder/catalog_imports/harris_loader.py b/python/PiFinder/catalog_imports/harris_loader.py index e3635a3cd..e9e7cd69a 100644 --- a/python/PiFinder/catalog_imports/harris_loader.py +++ b/python/PiFinder/catalog_imports/harris_loader.py @@ -381,7 +381,7 @@ def create_cluster_object(entry: npt.NDArray, seq: int) -> Dict[str, Any]: return result -def load_harris(): +def load_harris() -> None: logging.info("Loading Harris Globular Cluster catalog") catalog: str = "Har" obj_type: str = "Gb" # Globular Cluster diff --git a/python/PiFinder/state.py b/python/PiFinder/state.py index e96b1825b..8f886b31e 100644 --- a/python/PiFinder/state.py +++ b/python/PiFinder/state.py @@ -238,7 +238,7 @@ def from_json(cls, json_str): class SharedStateObj: - def __init__(self): + def __init__(self) -> None: self.__power_state = 1 # 0 = sleep state, 1 = awake state # self.__solve_state # None = No solve attempted yet diff --git a/python/PiFinder/ui/radec_entry.py b/python/PiFinder/ui/radec_entry.py index 34b8db27c..5f77be72d 100644 --- a/python/PiFinder/ui/radec_entry.py +++ b/python/PiFinder/ui/radec_entry.py @@ -586,7 +586,7 @@ class LayoutConfig: class UIRADecEntry(UIModule): __title__ = _("RA/DEC Entry") - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) self.callback = self.item_definition.get("callback") diff --git a/python/PiFinder/ui/sqm_calibration.py b/python/PiFinder/ui/sqm_calibration.py index b0eb61803..02bc8a513 100644 --- a/python/PiFinder/ui/sqm_calibration.py +++ b/python/PiFinder/ui/sqm_calibration.py @@ -52,7 +52,7 @@ class UISQMCalibration(UIModule): __title__ = "SQM CAL" __help_name__ = "" - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) # Wizard state machine diff --git a/python/PiFinder/ui/textentry.py b/python/PiFinder/ui/textentry.py index 0e41d81b2..7faae1f85 100644 --- a/python/PiFinder/ui/textentry.py +++ b/python/PiFinder/ui/textentry.py @@ -80,7 +80,7 @@ def __iter__(self): class UITextEntry(UIModule): - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs) -> None: super().__init__(*args, **kwargs) # Get mode from item_definition From 1e90d927efd317bda5aeae2d09f022751a284b75 Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Mon, 13 Apr 2026 08:21:40 +0200 Subject: [PATCH 16/19] type annotate objects_db and observations_db --- python/PiFinder/catalog_imports/database.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/python/PiFinder/catalog_imports/database.py b/python/PiFinder/catalog_imports/database.py index fd8419ce3..91b88cf2c 100644 --- a/python/PiFinder/catalog_imports/database.py +++ b/python/PiFinder/catalog_imports/database.py @@ -4,11 +4,16 @@ This module provides centralized access to database objects for all catalog loaders. """ +from typing import Optional + +from PiFinder.db.objects_db import ObjectsDatabase +from PiFinder.db.observations_db import ObservationsDatabase + from .catalog_import_utils import init_databases # Global database objects shared across all catalog loaders -objects_db = None -observations_db = None +objects_db: Optional[ObjectsDatabase] = None +observations_db: Optional[ObservationsDatabase] = None def init_shared_database(): From 08b7615ac6859974fb5becb74d95fb013225f5d4 Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Mon, 13 Apr 2026 08:22:07 +0200 Subject: [PATCH 17/19] Fix type errors --- python/PiFinder/catalog_imports/harris_loader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/python/PiFinder/catalog_imports/harris_loader.py b/python/PiFinder/catalog_imports/harris_loader.py index e9e7cd69a..3edd7fe3b 100644 --- a/python/PiFinder/catalog_imports/harris_loader.py +++ b/python/PiFinder/catalog_imports/harris_loader.py @@ -385,6 +385,9 @@ def load_harris() -> None: logging.info("Loading Harris Globular Cluster catalog") catalog: str = "Har" obj_type: str = "Gb" # Globular Cluster + + if objects_db is None: + raise RuntimeError("Database not initialized. Call init_shared_database() first.") conn, _ = objects_db.get_conn_cursor() # Enable bulk mode to prevent commits during insert operations From f8801318e7d928d5e9229b0453e435984e71cd2b Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Mon, 13 Apr 2026 08:26:48 +0200 Subject: [PATCH 18/19] Fix mypy errors --- python/PiFinder/ui/textentry.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/python/PiFinder/ui/textentry.py b/python/PiFinder/ui/textentry.py index 7faae1f85..e65738747 100644 --- a/python/PiFinder/ui/textentry.py +++ b/python/PiFinder/ui/textentry.py @@ -1,4 +1,5 @@ from PIL import Image, ImageDraw +from PiFinder.composite_object import CompositeObject from PiFinder.ui.base import UIModule from PiFinder.db.objects_db import ObjectsDatabase from PiFinder.ui.object_list import UIObjectList @@ -6,6 +7,8 @@ import time import threading from typing import Any, TYPE_CHECKING +import logging + if TYPE_CHECKING: @@ -105,7 +108,7 @@ def __init__(self, *args, **kwargs) -> None: self.KEYPRESS_TIMEOUT = 1 self.last_key_press_time = 0 self.char_index = 0 - self.search_results = [] + self.search_results: list[CompositeObject] = [] self.search_results_len_str = "0" self.show_keypad = True self.keys = KeyPad() @@ -231,8 +234,6 @@ def update_search_results(self): Debounced async search - waits 250ms after last keystroke before searching. Only updates search results in search mode. """ - import logging - logger = logging.getLogger("TextEntry") if self.text_entry_mode: @@ -271,8 +272,6 @@ def _perform_search(self, search_text, search_version): Perform the actual search in background thread. Only updates results if this search version is still current. """ - import logging - logger = logging.getLogger("TextEntry") try: From dd2d884cb8908989d522da7c1640c009cb90f209 Mon Sep 17 00:00:00 2001 From: Jens Scheidtmann Date: Sat, 18 Apr 2026 18:50:52 +0200 Subject: [PATCH 19/19] Clean up nox messages, errors and warnings. --- python/PiFinder/catalog_imports/harris_loader.py | 6 ++++-- python/PiFinder/catalog_imports/lynga_loader.py | 3 ++- python/PiFinder/ui/menu_structure.py | 5 ++++- python/PiFinder/ui/object_list.py | 13 ++++++------- python/PiFinder/ui/textentry.py | 1 - 5 files changed, 16 insertions(+), 12 deletions(-) diff --git a/python/PiFinder/catalog_imports/harris_loader.py b/python/PiFinder/catalog_imports/harris_loader.py index 2bd7d6b72..72649f02d 100644 --- a/python/PiFinder/catalog_imports/harris_loader.py +++ b/python/PiFinder/catalog_imports/harris_loader.py @@ -386,9 +386,11 @@ def load_harris() -> None: logging.info("Loading Harris Globular Cluster catalog") catalog: str = "Har" obj_type: str = "Gb" # Globular Cluster - + if objects_db is None: - raise RuntimeError("Database not initialized. Call init_shared_database() first.") + raise RuntimeError( + "Database not initialized. Call init_shared_database() first." + ) conn, _ = objects_db.get_conn_cursor() # Enable bulk mode to prevent commits during insert operations diff --git a/python/PiFinder/catalog_imports/lynga_loader.py b/python/PiFinder/catalog_imports/lynga_loader.py index 7c7af7b02..09f7df853 100644 --- a/python/PiFinder/catalog_imports/lynga_loader.py +++ b/python/PiFinder/catalog_imports/lynga_loader.py @@ -528,7 +528,8 @@ def create_cluster_object(entry: npt.NDArray, seq: int) -> Dict[str, Any]: return result -def load_lynga(): +def load_lynga() -> None: + assert objects_db is not None, "Database not initialized before load_lynga()" logging.info("Loading Lynga Open Cluster catalog") catalog: str = "Lyn" obj_type: str = "OC" # Open Cluster diff --git a/python/PiFinder/ui/menu_structure.py b/python/PiFinder/ui/menu_structure.py index d1a985d36..52c3bad7a 100644 --- a/python/PiFinder/ui/menu_structure.py +++ b/python/PiFinder/ui/menu_structure.py @@ -1122,7 +1122,10 @@ def _(key: str) -> Any: "custom_callback": callbacks.set_time, }, {"name": _("Reset Location"), "callback": callbacks.gps_reset}, - {"name": _("Reset Time/Date"), "callback": callbacks.datetime_reset}, + { + "name": _("Reset Time/Date"), + "callback": callbacks.datetime_reset, + }, ], }, {"name": _("Console"), "class": UIConsole}, diff --git a/python/PiFinder/ui/object_list.py b/python/PiFinder/ui/object_list.py index 6ce73240b..fe797b23b 100644 --- a/python/PiFinder/ui/object_list.py +++ b/python/PiFinder/ui/object_list.py @@ -364,19 +364,18 @@ def create_shortname_text(self, obj: CompositeObject) -> str: if obj.catalog_code == "PL" and obj.names: planet_abbrevs = { "Mercury": "MER", - "Venus": "VEN", - "Moon": "MON", - "Mars": "MAR", + "Venus": "VEN", + "Moon": "MON", + "Mars": "MAR", "Jupiter": "JUP", - "Saturn": "SAT", - "Uranus": "URA", + "Saturn": "SAT", + "Uranus": "URA", "Neptune": "NEP", - "Pluto": "PLU", + "Pluto": "PLU", } return planet_abbrevs.get(obj.names[0], obj.names[0]) return f"{obj.catalog_code}{obj.sequence}" - def create_locate_text(self, obj: CompositeObject) -> str: az, alt = aim_degrees( self.shared_state, self.mount_type, self.screen_direction, obj diff --git a/python/PiFinder/ui/textentry.py b/python/PiFinder/ui/textentry.py index 1da05163c..75b9b7a8e 100644 --- a/python/PiFinder/ui/textentry.py +++ b/python/PiFinder/ui/textentry.py @@ -8,7 +8,6 @@ import threading from typing import Any, List, TYPE_CHECKING import logging -from PiFinder.composite_object import CompositeObject if TYPE_CHECKING: