webserver: Upload and choose log configuration

.gitignore

@@ -1,3 +1,7 @@
+# Active log configuration symlink (auto-created at startup)
+python/pifinder_logconf.json
+python/logconf_*.json
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]

python/PiFinder/catalog_imports/database.py

@@ -5,7 +5,11 @@
 """
 
 from typing import Optional
-from .catalog_import_utils import init_databases, ObjectsDatabase, ObservationsDatabase
+
+from PiFinder.db.objects_db import ObjectsDatabase
+from PiFinder.db.observations_db import ObservationsDatabase
+
+from .catalog_import_utils import init_databases
 
 # Global database objects shared across all catalog loaders
 objects_db: Optional[ObjectsDatabase] = None

python/PiFinder/catalog_imports/harris_loader.py

@@ -386,6 +386,11 @@ def load_harris() -> None:
     logging.info("Loading Harris Globular Cluster catalog")
     catalog: str = "Har"
     obj_type: str = "Gb"  # Globular Cluster
+
+    if objects_db is None:
+        raise RuntimeError(
+            "Database not initialized. Call init_shared_database() first."
+        )
     conn, _ = objects_db.get_conn_cursor()
 
     # Enable bulk mode to prevent commits during insert operations

python/PiFinder/catalog_imports/lynga_loader.py

@@ -528,7 +528,8 @@ def create_cluster_object(entry: npt.NDArray, seq: int) -> Dict[str, Any]:
     return result
 
 
-def load_lynga():
+def load_lynga() -> None:
+    assert objects_db is not None, "Database not initialized before load_lynga()"
     logging.info("Loading Lynga Open Cluster catalog")
     catalog: str = "Lyn"
     obj_type: str = "OC"  # Open Cluster

python/PiFinder/main.py

@@ -873,25 +873,41 @@ def main(
 
 
 if __name__ == "__main__":
+    import sys
+
+    # Ensure the active log config symlink exists, defaulting to logconf_default.json
+    _logconf_link = Path("pifinder_logconf.json")
+    if not _logconf_link.exists():
+        _logconf_link.symlink_to("logconf_default.json")
+
+    debug_no_file_logs = "--debug-no-file-logs" in sys.argv
+    if debug_no_file_logs:
+        os.environ["PIFINDER_DEBUG_NO_FILE_LOGS"] = "1"
+
     print("Bootstrap logging configuration ...")
     logging.basicConfig(format="%(asctime)s BASIC %(name)s: %(levelname)s %(message)s")
     rlogger = logging.getLogger()
-    rlogger.setLevel(logging.INFO)
-    log_path = utils.data_dir / "pifinder.log"
-    try:
-        log_helper = MultiprocLogging(
-            Path("pifinder_logconf.json"),
-            log_path,
-        )
+    rlogger.setLevel(logging.DEBUG if debug_no_file_logs else logging.INFO)
+
+    if debug_no_file_logs:
+        log_helper = MultiprocLogging(Path("pifinder_logconf.json"), console_only=True)
         MultiprocLogging.configurer(log_helper.get_queue())
-    except FileNotFoundError:
-        rlogger.warning(
-            "Cannot find log configuration file, proceeding with basic configuration."
-        )
-        rlogger.warning("Logs will not be stored on disk, unless you use --log")
-        logging.getLogger("PIL.PngImagePlugin").setLevel(logging.WARNING)
-        logging.getLogger("tetra3.Tetra3").setLevel(logging.WARNING)
-        logging.getLogger("picamera2.picamera2").setLevel(logging.WARNING)
+    else:
+        log_path = utils.data_dir / "pifinder.log"
+        try:
+            log_helper = MultiprocLogging(
+                Path("pifinder_logconf.json"),
+                log_path,
+            )
+            MultiprocLogging.configurer(log_helper.get_queue())
+        except FileNotFoundError:
+            rlogger.warning(
+                "Cannot find log configuration file, proceeding with basic configuration."
+            )
+            rlogger.warning("Logs will not be stored on disk, unless you use --log")
+            logging.getLogger("PIL.PngImagePlugin").setLevel(logging.WARNING)
+            logging.getLogger("tetra3.Tetra3").setLevel(logging.WARNING)
+            logging.getLogger("picamera2.picamera2").setLevel(logging.WARNING)
 
     rlogger.info("Starting PiFinder ...")
     parser = argparse.ArgumentParser(description="eFinder")
@@ -953,6 +969,11 @@ def main(
         "-x", "--verbose", help="Set logging to debug mode", action="store_true"
     )
     parser.add_argument("-l", "--log", help="Log to file", action="store_true")
+    parser.add_argument(
+        "--debug-no-file-logs",
+        help="Debug: log everything at DEBUG level to console only, bypassing log configuration and file output",
+        action="store_true",
+    )
     parser.add_argument(
         "--lang",
         help="Force user interface language (iso2 code). Changes configuration",

python/PiFinder/multiproclogging.py

@@ -67,14 +67,18 @@ def __init__(
         log_conf: Optional[Path] = None,
         out_file: Optional[Path] = None,
         formatter: str = "%(asctime)s %(processName)s-%(name)s:%(levelname)s:%(message)s",
+        console_only: bool = False,
     ):
         self._queues: List[Queue] = []
         self._log_conf_file = log_conf
         self._log_output_file = out_file
         self._formatter = formatter
+        self._console_only = console_only
         self._proc: Optional[Process] = None
 
         self.apply_config()
+        if console_only:
+            logging.getLogger().setLevel(logging.DEBUG)
 
     def apply_config(self):
         if self._log_conf_file is not None:
@@ -94,7 +98,7 @@ def start(self, initial_queue: Optional[Queue] = None):
         self._proc = Process(
             target=self._run_sink,
             args=(
-                self._log_output_file,
+                None if self._console_only else self._log_output_file,
                 self._queues,
             ),
         )
@@ -109,7 +113,7 @@ def join(self):
         self._queues[0].put(None)
         self._proc.join()
 
-    def _run_sink(self, output: Path, queues: List[Queue]):
+    def _run_sink(self, output: Optional[Path], queues: List[Queue]):
         """
         This is the process that consumes every log message (sink)
 
@@ -124,15 +128,22 @@ def _run_sink(self, output: Path, queues: List[Queue]):
         for hdlr in hdlrs:
             rLogger.removeHandler(hdlr)
 
-        # Set maxBytes to 50MB (50 * 1024 * 1024 bytes) and keep 5 backup files
-        h = logging.handlers.RotatingFileHandler(
-            output, maxBytes=50 * 1024 * 1024, backupCount=5, encoding="utf-8"
-        )
+        if output is None:
+            import sys
+
+            h: logging.Handler = logging.StreamHandler(sys.stderr)
+            rLogger.setLevel(logging.DEBUG)
+            rLogger.warning("Starting logging process (console only, no file output)")
+        else:
+            # Set maxBytes to 50MB (50 * 1024 * 1024 bytes) and keep 5 backup files
+            h = logging.handlers.RotatingFileHandler(
+                output, maxBytes=50 * 1024 * 1024, backupCount=5, encoding="utf-8"
+            )
+            rLogger.warning("Starting logging process")
+            rLogger.warning("Logging to %s", output)
         f = logging.Formatter(self._formatter)
         h.setFormatter(f)
         rLogger.addHandler(h)
-        rLogger.warning("Starting logging process")
-        rLogger.warning("Logging to %s", output)
 
         # import logging_tree
         # logging_tree.printout()
@@ -172,6 +183,8 @@ def configurer(queue: Queue):
         This method needs to be called once in each process, so that log records get forwarded to the single process writing
         log messages.
         """
+        import os
+
         assert queue is not None, "You passed a None to configurer! You cannot do that"
         assert isinstance(
             queue, multiprocessing.queues.Queue
@@ -182,8 +195,11 @@ def configurer(queue: Queue):
             config = json5.load(logconf)
             logging.config.dictConfig(config)
 
-        h = logging.handlers.QueueHandler(queue)
         root = logging.getLogger()
+        if os.environ.get("PIFINDER_DEBUG_NO_FILE_LOGS"):
+            root.setLevel(logging.DEBUG)
+
+        h = logging.handlers.QueueHandler(queue)
         root.addHandler(h)
 
     def set_log_conf_file(self, config: Path) -> None:

python/PiFinder/server.py

@@ -803,8 +803,8 @@ def stream_logs():
                     else:
                         return {"logs": [], "position": position}
                 except FileNotFoundError:
-                    logger.error(f"Log file not found: {log_file}")
-                    return {"logs": [], "position": 0}
+                    logger.warning(f"Log file not found: {log_file}")
+                    return {"logs": [], "position": 0, "file_not_found": True}
 
             except Exception as e:
                 logger.error(f"Error streaming logs: {e}")
@@ -845,6 +845,83 @@ def get_component_levels():
                 logging.error(f"Error reading log configuration: {e}")
                 return {"status": "error", "message": str(e)}
 
+        @app.route("/logs/configs")
+        @auth_required
+        def list_log_configs():
+            """Return all available logconf_*.json files with display names."""
+            import glob
+
+            configs = []
+            active = (
+                os.path.realpath("pifinder_logconf.json")
+                if os.path.exists("pifinder_logconf.json")
+                else None
+            )
+            for path in sorted(glob.glob("logconf_*.json")):
+                stem = path[len("logconf_") : -len(".json")]
+                display = stem.replace("_", " ").title()
+                configs.append(
+                    {
+                        "file": path,
+                        "name": display,
+                        "active": os.path.realpath(path) == active,
+                    }
+                )
+            return {"configs": configs}
+
+        @app.route("/logs/switch_config", method="post")
+        @auth_required
+        def switch_log_config():
+            """Atomically repoint pifinder_logconf.json to the chosen config, then restart."""
+            logconf_file = request.forms.get("logconf_file", "").strip()
+            if (
+                not logconf_file
+                or not logconf_file.startswith("logconf_")
+                or not logconf_file.endswith(".json")
+            ):
+                return {"status": "error", "message": "Invalid log config file name"}
+            if not os.path.exists(logconf_file):
+                return {
+                    "status": "error",
+                    "message": f"Log config file not found: {logconf_file}",
+                }
+            try:
+                link = "pifinder_logconf.json"
+                tmp = link + ".tmp"
+                os.symlink(logconf_file, tmp)
+                os.replace(tmp, link)
+                logger.info("Switched log config to %s", logconf_file)
+            except Exception as e:
+                logger.error("Failed to switch log config: %s", e)
+                return {"status": "error", "message": str(e)}
+            return template("restart_pifinder")
+
+        @app.route("/logs/upload_config", method="post")
+        @auth_required
+        def upload_log_config():
+            """Upload a new logconf_*.json file."""
+            upload = request.files.get("config_file")
+            if not upload:
+                return {"status": "error", "message": "No file provided"}
+            filename = upload.filename
+            if not filename.startswith("logconf_") or not filename.endswith(".json"):
+                return {
+                    "status": "error",
+                    "message": "File must be named logconf_<name>.json",
+                }
+            if os.path.exists(filename):
+                return {
+                    "status": "error",
+                    "message": f"File already exists: {filename}",
+                }
+            try:
+                upload.save(filename, overwrite=False)
+                logger.info("Uploaded log config: %s", filename)
+                return {"status": "ok", "file": filename}
+            except Exception as e:
+                logger.error("Failed to save uploaded log config: %s", e)
+                return {"status": "error", "message": str(e)}
+
         @app.route("/logs/download")
         @auth_required
         def download_logs():

python/PiFinder/sqm/save_sweep_metadata.py

@@ -80,7 +80,8 @@ def save_sweep_metadata(
     # Noise floor estimation details (from NoiseFloorEstimator)
     if noise_floor_details is not None:
         metadata["noise_floor_estimator"] = {
-            k: v for k, v in noise_floor_details.items()
+            k: v
+            for k, v in noise_floor_details.items()
             if k != "request_zero_sec_sample"  # Exclude internal flags
         }
         if camera_type is not None:

python/PiFinder/sqm/sqm.py

@@ -65,7 +65,9 @@ def _load_calibration(self) -> bool:
         )
 
         if not calibration_file.exists():
-            logger.debug(f"No calibration file found at {calibration_file}, using defaults")
+            logger.debug(
+                f"No calibration file found at {calibration_file}, using defaults"
+            )
             return False
 
         try:
@@ -204,7 +206,6 @@ def _measure_star_flux_with_local_background(
                 n_saturated += 1
                 continue
 
-
             # Total flux in aperture (includes background)
             total_flux = np.sum(aperture_pixels)
 

python/PiFinder/ui/menu_structure.py

@@ -1122,7 +1122,10 @@ def _(key: str) -> Any:
                             "custom_callback": callbacks.set_time,
                         },
                         {"name": _("Reset Location"), "callback": callbacks.gps_reset},
-                        {"name": _("Reset Time/Date"), "callback": callbacks.datetime_reset},
+                        {
+                            "name": _("Reset Time/Date"),
+                            "callback": callbacks.datetime_reset,
+                        },
                     ],
                 },
                 {"name": _("Console"), "class": UIConsole},

python/PiFinder/ui/object_list.py

@@ -364,19 +364,18 @@ def create_shortname_text(self, obj: CompositeObject) -> str:
         if obj.catalog_code == "PL" and obj.names:
             planet_abbrevs = {
                 "Mercury": "MER",
-                "Venus":   "VEN",
-                "Moon":    "MON",
-                "Mars":    "MAR",
+                "Venus": "VEN",
+                "Moon": "MON",
+                "Mars": "MAR",
                 "Jupiter": "JUP",
-                "Saturn":  "SAT",
-                "Uranus":  "URA",
+                "Saturn": "SAT",
+                "Uranus": "URA",
                 "Neptune": "NEP",
-                "Pluto":   "PLU",
+                "Pluto": "PLU",
             }
             return planet_abbrevs.get(obj.names[0], obj.names[0])
         return f"{obj.catalog_code}{obj.sequence}"
 
-
     def create_locate_text(self, obj: CompositeObject) -> str:
         az, alt = aim_degrees(
             self.shared_state, self.mount_type, self.screen_direction, obj