Rescue: logger→ledger + tree-based checkpoint management (was orphaned branch 71)

agent_config.yaml

@@ -1,14 +1,14 @@
 # Agent Configuration
 agent:
-  # Select the model provider. 
+  # Select the model provider.
   # Local: 'ollama'
   # Remote: 'openrouter', 'google', 'openai' (requires API tokens in .env)
   provider: ollama
-  
+
   # Local Settings
   ollama_model: qwen2.5:3b-instruct
   fallback_to_local: true
-  
+
   # Remote Model Selection
   openrouter_model: mistralai/mistral-small-3.1-24b-instruct:free
   google_model: gemini-1.5-flash

requirements.txt

@@ -26,7 +26,11 @@ onnxruntime==1.23.2
 xxhash==3.6.0
 tables==3.10.2
 httpx==0.28.1
+<<<<<<< Updated upstream
 pydantic==2.11.10
+=======
+pydantic>=2.6
+>>>>>>> Stashed changes
 dotenv==0.9.9
 h5py==3.15.1
 

weightslab/__init__.py

@@ -41,7 +41,6 @@
 __maintainer__ = 'Guillaume PELLUET'
 __credits__ = 'GrayBox'
 __license__ = 'BSD 2-clause'
-
 __all__ = [
     "watch_or_edit",
     "serve",

weightslab/backend/cli.py

@@ -100,17 +100,29 @@ def _handle_command(cmd: str) -> Any:
                     'status': 'Show basic status: registered models and optimizers',
                     'list_models': 'List registered model names in the ledger',
                     'list_optimizers': 'List registered optimizer names in the ledger',
-                                    'dump': 'Return a sanitized dump of the ledger contents',
+                    'list_loaders': 'List registered dataloader names in the ledger',
+                    'list_uids': 'List data sample UIDs. Syntax: list_uids [loader_name] [--discarded] [--limit N]',
+                    'dump': 'Return a sanitized dump of the ledger contents',
                     'operate': 'Edit model architecture. Syntax: operate [<model_name>] <op_type:int> <layer_id:int> <nb|[list]>',
-                            'plot_model': 'Show ASCII tree of model architecture. Syntax: plot_model [<model_name>]',
+                    'plot_model': 'Show ASCII tree of model architecture. Syntax: plot_model [<model_name>]',
+                    'discard': 'Discard data samples. Syntax: discard <uid> [uid2 ...] [--loader loader_name]',
+                    'undiscard': 'Un-discard data samples. Syntax: undiscard <uid> [uid2 ...] [--loader loader_name]',
+                    'add_tag': 'Add tag to data sample. Syntax: add_tag <uid> <tag> [--loader loader_name]',
                     'hp / hyperparams': 'List or show hyperparameters. Syntax: hp -> list, hp <name> -> show',
-                    # editing hyperparameters is disabled in the CLI (read-only)
                     'quit / exit': 'Close the client connection'
                 },
                 'hyperparams_examples': {
                     'list': 'hp',
                     'show': 'hp fashion_mnist',
                     'set': "set_hp <hp_name?> <key.path> <value>  # e.g. set_hp fashion_mnist data.train_loader.batch_size 32",
+                },
+                'data_examples': {
+                    'list all UIDs': 'list_uids',
+                    'list specific loader': 'list_uids train_loader',
+                    'list discarded only': 'list_uids --discarded',
+                    'discard samples': 'discard sample_001 sample_002',
+                    'undiscard samples': 'undiscard sample_001',
+                    'add tag': 'add_tag sample_001 difficult',
                 }
             }
 
@@ -180,9 +192,120 @@ def _handle_command(cmd: str) -> Any:
         if verb == 'list_dataloaders':
             return {'ok': True, 'dataloaders': GLOBAL_LEDGER.list_dataloaders()}
 
+        if verb in ('list_loaders', 'loaders'):
+            return {'ok': True, 'loaders': GLOBAL_LEDGER.list_dataloaders()}
+
         if verb == 'list_optimizers':
             return {'ok': True, 'optimizers': GLOBAL_LEDGER.list_optimizers()}
 
+        if verb in ('list_uids', 'uids', 'samples'):
+            # Syntax: list_uids [loader_name] [--discarded] [--limit N]
+            loader_name = None
+            show_discarded_only = False
+            limit = None
+
+            # Parse arguments
+            i = 1
+            while i < len(parts):
+                if parts[i] == '--discarded':
+                    show_discarded_only = True
+                elif parts[i] == '--limit' and i + 1 < len(parts):
+                    try:
+                        limit = int(parts[i + 1])
+                        i += 1
+                    except ValueError:
+                        return {'ok': False, 'error': 'Invalid limit value'}
+                elif not parts[i].startswith('--'):
+                    loader_name = parts[i]
+                i += 1
+
+            try:
+                loaders_to_check = [loader_name] if loader_name else GLOBAL_LEDGER.list_dataloaders()
+
+                result = {'ok': True, 'uids': {}}
+
+                for lname in loaders_to_check:
+                    try:
+                        loader = GLOBAL_LEDGER.get_dataloader(lname)
+                        # Unwrap proxy
+                        if hasattr(loader, 'get') and callable(loader.get):
+                            loader = loader.get()
+
+                        if loader is None:
+                            continue
+
+                        # Try to get dataset from loader
+                        dataset = None
+                        if hasattr(loader, 'dataset'):
+                            dataset = loader.dataset
+
+                        if dataset is None:
+                            continue
+
+                        # Get UIDs and discard status
+                        uids_list = []
+
+                        # Try different methods to get UIDs
+                        if hasattr(dataset, 'get_sample_uids'):
+                            # Method to get all UIDs
+                            all_uids = dataset.get_sample_uids()
+                        elif hasattr(dataset, 'sample_ids'):
+                            all_uids = dataset.sample_ids
+                        elif hasattr(dataset, 'uids'):
+                            all_uids = dataset.uids
+                        elif hasattr(dataset, '__len__'):
+                            # Fallback: generate UIDs from indices
+                            all_uids = [f"sample_{i:06d}" for i in range(len(dataset))]
+                        else:
+                            all_uids = []
+
+                        # Check discard status for each UID
+                        for uid in all_uids:
+                            is_discarded = False
+
+                            # Try to get discard status
+                            if hasattr(dataset, 'is_discarded'):
+                                try:
+                                    is_discarded = dataset.is_discarded(uid)
+                                except Exception:
+                                    pass
+                            elif hasattr(dataset, 'discarded_samples'):
+                                is_discarded = uid in dataset.discarded_samples
+
+                            # Filter based on --discarded flag
+                            if show_discarded_only and not is_discarded:
+                                continue
+
+                            # Get tags if available
+                            tags = []
+                            if hasattr(dataset, 'get_tags'):
+                                try:
+                                    tags = dataset.get_tags(uid)
+                                except Exception:
+                                    pass
+                            elif hasattr(dataset, 'sample_tags') and hasattr(dataset.sample_tags, 'get'):
+                                tags = dataset.sample_tags.get(uid, [])
+
+                            uids_list.append({
+                                'uid': uid,
+                                'discarded': is_discarded,
+                                'tags': tags
+                            })
+
+                            # Apply limit
+                            if limit and len(uids_list) >= limit:
+                                break
+
+                        result['uids'][lname] = uids_list
+
+                    except Exception as e:
+                        result['uids'][lname] = {'error': str(e)}
+
+                return result
+
+            except Exception as e:
+                return {'ok': False, 'error': str(e)}
+
         # Return a lightweight snapshot of all ledger registries
         if verb in ('ledgers', 'ledger', 'snapshot'):
             snap = GLOBAL_LEDGER.snapshot()
@@ -314,6 +437,156 @@ def _handle_command(cmd: str) -> Any:
 
                     return {'ok': True, 'operated': True, 'op': (op_type, layer_id, nb), 'model': model_name}
 
+        if verb in ('discard', 'undiscard'):
+            # Syntax: discard <uid> [uid2 ...] [--loader loader_name]
+            # Syntax: undiscard <uid> [uid2 ...] [--loader loader_name]
+
+            if len(parts) < 2:
+                return {'ok': False, 'error': f'usage: {verb} <uid> [uid2 ...] [--loader loader_name]'}
+
+            loader_name = None
+            uids = []
+
+            # Parse arguments
+            i = 1
+            while i < len(parts):
+                if parts[i] == '--loader' and i + 1 < len(parts):
+                    loader_name = parts[i + 1]
+                    i += 2
+                else:
+                    uids.append(parts[i])
+                    i += 1
+
+            if not uids:
+                return {'ok': False, 'error': 'No UIDs specified'}
+
+            discard_status = 1 if verb == 'discard' else 0
+
+            try:
+                # Get loader(s)
+                if loader_name:
+                    loaders_to_update = {loader_name: GLOBAL_LEDGER.get_dataloader(loader_name)}
+                else:
+                    # Try all loaders
+                    loader_names = GLOBAL_LEDGER.list_dataloaders()
+                    loaders_to_update = {name: GLOBAL_LEDGER.get_dataloader(name) for name in loader_names}
+
+                results = {'ok': True, 'updated': {}, 'errors': {}}
+
+                for lname, loader in loaders_to_update.items():
+                    # Unwrap proxy
+                    if hasattr(loader, 'get') and callable(loader.get):
+                        loader = loader.get()
+
+                    if loader is None:
+                        continue
+
+                    # Get dataset
+                    dataset = getattr(loader, 'dataset', None) if loader else None
+
+                    if dataset is None:
+                        continue
+
+                    updated_uids = []
+
+                    for uid in uids:
+                        try:
+                            # Try different methods to set discard status
+                            if hasattr(dataset, 'set_discard'):
+                                dataset.set_discard(uid, discard_status)
+                                updated_uids.append(uid)
+                            elif hasattr(dataset, 'discard_sample'):
+                                if discard_status == 1:
+                                    dataset.discard_sample(uid)
+                                else:
+                                    # undiscard
+                                    if hasattr(dataset, 'undiscard_sample'):
+                                        dataset.undiscard_sample(uid)
+                                updated_uids.append(uid)
+                            elif hasattr(dataset, 'discarded_samples'):
+                                # Direct set manipulation
+                                if discard_status == 1:
+                                    dataset.discarded_samples.add(uid)
+                                else:
+                                    dataset.discarded_samples.discard(uid)
+                                updated_uids.append(uid)
+                            else:
+                                results['errors'][uid] = f'No discard method available in {lname}'
+                        except Exception as e:
+                            results['errors'][uid] = str(e)
+
+                    if updated_uids:
+                        results['updated'][lname] = updated_uids
+
+                return results
+
+            except Exception as e:
+                return {'ok': False, 'error': str(e)}
+
+        if verb in ('add_tag', 'tag'):
+            # Syntax: add_tag <uid> <tag> [--loader loader_name]
+
+            if len(parts) < 3:
+                return {'ok': False, 'error': 'usage: add_tag <uid> <tag> [--loader loader_name]'}
+
+            uid = parts[1]
+            tag = parts[2]
+            loader_name = None
+
+            # Parse optional --loader argument
+            if len(parts) >= 5 and parts[3] == '--loader':
+                loader_name = parts[4]
+
+            try:
+                # Get loader(s)
+                if loader_name:
+                    loaders_to_update = {loader_name: GLOBAL_LEDGER.get_dataloader(loader_name)}
+                else:
+                    # Try all loaders
+                    loader_names = GLOBAL_LEDGER.list_dataloaders()
+                    loaders_to_update = {name: GLOBAL_LEDGER.get_dataloader(name) for name in loader_names}
+
+                results = {'ok': True, 'updated': {}, 'errors': {}}
+
+                for lname, loader in loaders_to_update.items():
+                    # Unwrap proxy
+                    if hasattr(loader, 'get') and callable(loader.get):
+                        loader = loader.get()
+
+                    if loader is None:
+                        continue
+
+                    # Get dataset
+                    dataset = getattr(loader, 'dataset', None) if loader else None
+
+                    if dataset is None:
+                        continue
+
+                    try:
+                        # Try different methods to add tags
+                        if hasattr(dataset, 'add_tag'):
+                            dataset.add_tag(uid, tag)
+                            results['updated'][lname] = f'Added tag "{tag}" to {uid}'
+                        elif hasattr(dataset, 'sample_tags'):
+                            # Direct dict manipulation
+                            if uid not in dataset.sample_tags:
+                                dataset.sample_tags[uid] = []
+                            if tag not in dataset.sample_tags[uid]:
+                                dataset.sample_tags[uid].append(tag)
+                            results['updated'][lname] = f'Added tag "{tag}" to {uid}'
+                        else:
+                            results['errors'][lname] = 'No tag method available'
+                    except Exception as e:
+                        results['errors'][lname] = str(e)
+
+                if not results['updated']:
+                    return {'ok': False, 'error': 'Could not add tag to any loader'}
+
+                return results
+
+            except Exception as e:
+                return {'ok': False, 'error': str(e)}
+
         # Hyperparameters: list / show details and set
         if verb in ('hp', 'hyperparams'):
             # hp -> list