From 93eaf6c69fd2387ba89257b7e66cc570add807d5 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 15 Jan 2026 15:03:08 -0500 Subject: [PATCH 01/49] Add support for folder store / retrieve. This is a WIP. --- pyproject.toml | 2 +- src/hashstore/filehashstore.py | 168 ++++++++++++++++++++++++++++++--- src/hashstore/hashstore.py | 66 ++++++++++++- 3 files changed, 220 insertions(+), 16 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bddee275..a5d15884 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "hashstore" -version = "1.1.1" +version = "1.2.0" description = "HashStore, an object storage system using content identifiers." authors = [ { name = "Dou Mok", email = "douming.mok@gmail.com" }, diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 74b9c600..88e42a23 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1,38 +1,40 @@ """Core module for FileHashStore""" import atexit +import fcntl +import hashlib +import inspect import io +import logging import multiprocessing +import os import shutil import threading -import hashlib -import os -import logging -import inspect -import fcntl -import yaml -from typing import List, Dict, Union, Optional, IO, Tuple, Set, Any +from contextlib import closing from dataclasses import dataclass from pathlib import Path -from contextlib import closing from tempfile import NamedTemporaryFile +from typing import IO, Any, Dict, List, Optional, Set, Tuple, Union + +import yaml + from hashstore import HashStore from hashstore.filehashstore_exceptions import ( CidRefsContentError, - OrphanPidRefsFileFound, CidRefsFileNotFound, HashStoreRefsAlreadyExists, + IdentifierNotLocked, NonMatchingChecksum, NonMatchingObjSize, - PidRefsAlreadyExistsError, + OrphanPidRefsFileFound, PidNotFoundInCidRefsFile, + PidRefsAlreadyExistsError, PidRefsContentError, PidRefsDoesNotExist, PidRefsFileNotFound, RefsFileExistsButCidObjMissing, - UnsupportedAlgorithm, StoreObjectForPidAlreadyInProgress, - IdentifierNotLocked, + UnsupportedAlgorithm, ) @@ -246,7 +248,13 @@ def _write_properties(self, properties: Dict[str, Union[str, int]]) -> None: checked_properties = self._validate_properties(properties) # Collect configuration properties from validated & supplied dictionary - (_, store_depth, store_width, store_algorithm, store_metadata_namespace,) = [ + ( + _, + store_depth, + store_width, + store_algorithm, + store_metadata_namespace, + ) = [ checked_properties[property_name] for property_name in self.property_required_keys ] @@ -1022,8 +1030,140 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: logging.info(info_string) return hex_digest + def store_folder( + self, + pid: str, + root_path: str | Path, + child_path: Optional[str | Path] = None, + additional_algorithm: Optional[str] = None, + checksum: Optional[str] = None, + checksum_algorithm: Optional[str] = None, + expected_object_size: Optional[int] = None, + ) -> "ObjectMetadata": + """Store a folder (and subfolders) as container objects. + + Args: + pid (str): The context within which this folder is being stored + root_path (str): Path to the root of the folder. + child_path (str): Path to folder being stored relative to the root_path. If None, assumes root_path. + + Returns: + str: CID for the container + """ + # Get the relative path for the object / folder + root_path = Path(root_path) + if child_path is None: + child_path = root_path + else: + child_path = Path(child_path) + relative_path = child_path.relative_to(root_path) + path_pid = pid + container_name = "root" + if str(relative_path) != ".": + path_pid = f"{pid} {relative_path}" + container_name = str(relative_path) + + # Check if this container already exists + try: + # resolve pid, path to CID. This raises if not found + _entry = self._find_object(path_pid) + size = os.path.getsize( + self._build_hashstore_data_object_path(_entry["cid"]) + ) + return ObjectMetadata( + pid=path_pid, cid=_entry["cid"], obj_size=size, hex_digests={} + ) + except PidNotFoundInCidRefsFile: + pass + except PidRefsDoesNotExist: + pass + + manifest = [] + for item in child_path.iterdir(): + if item.is_dir(): + meta = self.store_folder( + pid, + root_path, + child_path=item.absolute(), + additional_algorithm=additional_algorithm, + checksum=checksum, + checksum_algorithm=checksum_algorithm, + expected_object_size=expected_object_size, + ) + manifest.append((0, meta.cid, item.name)) + elif item.is_file(): + item_pid = f"{pid} {item.relative_to(root_path)}" + meta = self.store_object(item_pid, str(item.absolute())) + manifest.append((1, meta.cid, item.name)) + manifest.sort(key=lambda x: (x[0], x[1])) + dest_stream = io.BytesIO() + dest_stream.name = container_name + for row in manifest: + dest_stream.write(f"{row[0]} {row[1]} {row[2]}\n".encode("utf-8")) + dest_stream.seek(0) + # TODO: error handling + return self.store_object( + path_pid, + data=dest_stream, + additional_algorithm=additional_algorithm, + checksum=checksum, + checksum_algorithm=checksum_algorithm, + expected_object_size=expected_object_size, + ) + + def retrieve_folder(self, pid:str, destination_path:str|Path, child_path:Optional[str|Path]=None): + """Retrieve a folder (and subfolders) stored as container objects. + + Args: + pid (str): The context within which this folder is being retrieved + destination_path (str|Path): Path to the root of the folder to create. + child_path (str|Path): Path to folder being retrieved relative to the destination_path. If None, assumes destination_path. + Returns: + None + """ + # TODO: Error handling + # TODO: read access control considerations + destination_path = Path(destination_path) + if child_path is None: + child_path = Path("") + else: + child_path = Path(child_path) + path_pid = pid + if str(child_path) != ".": + path_pid = f"{pid} {child_path}" + + # Retrieve the container object + obj_stream = self.retrieve_object(path_pid) + with closing(obj_stream): + for line in obj_stream: + line = line.decode("utf-8").strip() + type_flag, cid, name = line.split(" ", 2) + if type_flag == "0": + # Directory + (destination_path / child_path / name).mkdir(parents=True, exist_ok=True) + self.retrieve_folder( + pid, + destination_path, + child_path=child_path / name, + ) + elif type_flag == "1": + # File + item_pid = f"{pid} {child_path / name}" + file_stream = self.retrieve_object(item_pid) + with closing(file_stream): + dest_file_path = destination_path / child_path / name + dest_file_path.parent.mkdir(parents=True, exist_ok=True) + with open(dest_file_path, "wb") as dest_file: + shutil.copyfileobj(file_stream, dest_file) + # FileHashStore Core Methods + def _deserialize_container(self, cid) -> Dict[str, Any]: + pass + + def _serialize_container(self, container: Dict[str, Any]) -> str: + pass + def _find_object(self, pid: str) -> Dict[str, str]: """Check if an object referenced by a pid exists and retrieve its content identifier. The `find_object` method validates the existence of an object based on the provided @@ -2768,7 +2908,7 @@ def _check_string(string: str, arg: str) -> None: :param str string: Value to check. :param str arg: Name of the argument to check. """ - if string is None or string.strip() == "" or any(ch.isspace() for ch in string): + if string is None or string.strip() == "": method = inspect.stack()[1].function err_msg = ( f"FileHashStore - {method}: {arg} cannot be None" diff --git a/src/hashstore/hashstore.py b/src/hashstore/hashstore.py index 20a93fd8..c92d5257 100644 --- a/src/hashstore/hashstore.py +++ b/src/hashstore/hashstore.py @@ -1,8 +1,10 @@ """Hashstore Interface""" -from abc import ABC, abstractmethod import importlib.metadata import importlib.util +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Optional class HashStore(ABC): @@ -64,6 +66,68 @@ def store_object( """ raise NotImplementedError() + @abstractmethod + def store_folder( + self, + pid:str, + root_path:str|Path, + child_path:Optional[str|Path]=None, + additional_algorithm:Optional[str]=None, + checksum:Optional[str]=None, + checksum_algorithm:Optional[str]=None, + expected_object_size:Optional[int]=None, + ): + """Stores a folder and subfolders. + + The `store_folder` method stores a folder and its subfolders to HashStore. Each file within the folder + is processed and stored individually, following the same procedures as the `store_object` method. The + folder structure is preserved within HashStore, allowing for easy retrieval of the entire folder or + individual files as needed. + + The root of a folder is specified by the `root_path` argument and must be identified by a PID. + + This method performs a recursive, depth firth traversal of the folder structure, storing each file it encounters and + storing folders as a container object that lists the files and subfolders contained within it. + + Args: + pid (str): Identifier for the context of this folder hierarchy. + root_path (str | Path): The physical path to the root folder being stored. + child_path (Optional[str | Path], optional): Path to a subfolder of root_path. This is + normally None for the initial invocation of this method, and recursive calls will set the + child_path as needed. Defaults to None. + additional_algorithm (Optional[str], optional): See `store_object`. Defaults to None. + checksum (Optional[str], optional): See `store_object`. Defaults to None. + checksum_algorithm (Optional[str], optional): See `store_object`. Defaults to None. + expected_object_size (Optional[int], optional): See `store_object`. Defaults to None. + + Raises: + NotImplementedError: Must be implemented in subclass. + """ + raise NotImplementedError() + + @abstractmethod + def retrieve_folder(self, pid:str, destination_path:str|Path, child_path:Optional[str|Path]=None): + """Retrieves a folder and its subfolders from HashStore. + + The `retrieve_folder` method retrieves a folder and its subfolders from HashStore, reconstructing + the original folder structure at the specified target path. Each file within the folder is retrieved + individually, following the same procedures as the `retrieve_object` method. The folder structure + is preserved during retrieval, allowing for easy access to the entire folder or individual files as needed. + + The root of a folder is specified by the `pid` argument, which identifies the context of the folder hierarchy. The + optional child_path argument can be used to specify a subfolder within the root folder for retrieval. + + Output files and folders will be created under the `destination_path`. + + Args: + pid (str): Identifier for the context of this folder hierarchy. + destination_path (str | Path): The physical path where the retrieved folder will be reconstructed. + child_path (Optional[str|Path|], optional): Path to a subfolder of the root folder. This is + normally None for the initial invocation of this method, and recursive calls will set the + child_path as needed. Defaults to None. + """ + raise NotImplementedError() + @abstractmethod def tag_object(self, pid, cid): """Creates references that allow objects stored in HashStore to be discoverable. From 154a1cb29727fdef4320d4f0ea1c8a173cd61bfc Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 15 Jan 2026 15:29:43 -0500 Subject: [PATCH 02/49] Added notes about folders in hashstore --- folder_operations.md | 114 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 folder_operations.md diff --git a/folder_operations.md b/folder_operations.md new file mode 100644 index 00000000..91eb1fdd --- /dev/null +++ b/folder_operations.md @@ -0,0 +1,114 @@ +# hashtree + +Describes storing directory trees in hashstore. + +## Assumptions + +- The root of a folder hierarchy is identified by a PID +- A folder hierarchy (including content) identified by a PID is immutable +- A mutation to a folder hierarchy results in a new folder hierarchy identified by a new PID +- Any subfolder may optionally be identified by a PID +- Any file contained within a folder hierarchy may be identified by a PID +- Permissions are associated with a PID and so apply to content of PID identified containers or files. +- A folder hierarchy may reference all or part of another identified folder hierarchy +- A folder is represented by a `container` in hashstore. + +## Containers + +Hashstore is augmented by adding an additional type of content that represents a `container`, the contents of which represent a single folder. A `container` has two types of entries: `file` that represents a single file and `folder` which represents a single subfolder. Each entry in a `container` has properties: `type`, `cid`, and `name`, where: + +`type` - Indicates if the entry is a folder (`0`) or file (`1`). + +`cid` - The content ID for the respective file or container. + +`name` - The name component of the path to the entry. i.e. The last path segment for a subfolder or the file name (without path) for a file. + +The CID for a container is computed from the serialized content on the container which includes the CID values for any subfolders. Hence, computing the CID for folders in a hierarchy requires a depth-first approach where the CIDs for leaves of a branch are computed before the branch. + +A container is serialized space delimited rows in a text file. Each row represents an entry in the container, with values `type`, `cid`, and `name` in that order. Since folder or file names *may* contain whitespace, the `name` entry consumes the remainder of the row. + +Since the CID for a container is dependent on its content, the content order is sorted by the `type` and `cid` values so hashing is consistent. Hence rows referencing subfolder containers will always appear before rows referencing files. + +For example, given the folder hierarchy: + +``` +PID_1 +├── A +│ ├── a1.txt +│ └── a2.txt +└── B + └── b1.csv +``` + +The following `container` entries are created (`cid` values are truncated): + +Container `ad5eb`: +``` +1 10fbd a1.txt +1 c880c a2.txt +``` + +Container `cc08d`: +``` +1 00e99 b1.csv +``` + +Container `dbc15`: +``` +0 ad5eb A +0 cc08d B +``` + +The hashstore entry for `PID_1` might be: +``` +$ cat refs/pids/53/b2/f2/58a2f3061a7bee4ba8b157aab217795c4692e2a2d8856e2fd97eb7fa3f +dbc1516e49e7437ea441f279570d32b1e2f149c44ab0a77682629215f4a5970b + +$ cat refs/cids/db/c1/51/6e49e7437ea441f279570d32b1e2f149c44ab0a77682629215f4a5970b +PID_1 +``` + +Each container is resolveable by the combination of PID and path. So for example, +the folder `B` within the context of `PID_1` can be resolved using the identifier `PID_1 B`. +Similarly, the file `A/a2.txt` can be resolved with the identifier `PID_1 A/a2.txt`. +Corresponding entries in hashstore `refs/pids` and `refs/cids` are created. + +## Operations + +### Get an object by path + +Given a PID and a path, retrieve the corresponding object (file or folder) from hashstore. + +Persistent identifiers for objects within a folder hierarchy are constructed by concatenating the PID with the path using a space as a delimiter. For example, to retrieve the object at path `data/file1.txt` within the folder hierarchy identified by PID `abc123`, the identifier would be `abc123 data/file1.txt`. + +``` +hashstore = HashStore(...) +path_pid = "" + " " + "" +object_stream = hashstore.get_object(path_pid) +``` + +### Store a new folder hierarchy + +To store a new folder hierarchy, recursively create `container` entries for each folder in the hierarchy, starting from the leaves and working up to the root. For each folder, create a `container` with entries for its subfolders and files, compute the CID for the container, and store it in hashstore. Finally, associate the root container's CID with the PID representing the entire folder hierarchy. + +This is achieved by the `hash_store.store_folder()` method. + +``` +hashstore = HashStore(...) +pid = "" +source_path = "" +hashstore.store_folder(pid, source_path) +``` + +### Retrieve folder hierarchy structure + +To retrieve the structure of a folder hierarchy identified by a PID, recursively resolve each `container` starting from the root PID. For each folder, read its `container` entries to identify subfolders and files, and continue resolving subfolders until the entire hierarchy is reconstructed. + +This is achieved by the `hash_store.retrieve_folder()` method. + +``` +hashstore = HashStore(...) +pid = "" +destination_path = "" +hashstore.retrieve_folder(pid, destination_path) +``` From e3056b7a8d4990c146b0184f2cacd08da6a85dd3 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 15 Jan 2026 15:36:59 -0500 Subject: [PATCH 03/49] Adjust typehints for 3.9 --- folder_operations.md | 6 +++--- src/hashstore/filehashstore.py | 6 +++--- src/hashstore/hashstore.py | 8 ++++---- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/folder_operations.md b/folder_operations.md index 91eb1fdd..9ad9420c 100644 --- a/folder_operations.md +++ b/folder_operations.md @@ -84,14 +84,14 @@ Persistent identifiers for objects within a folder hierarchy are constructed by ``` hashstore = HashStore(...) path_pid = "" + " " + "" -object_stream = hashstore.get_object(path_pid) +object_stream = hashstore.retrieve_object(path_pid) ``` ### Store a new folder hierarchy To store a new folder hierarchy, recursively create `container` entries for each folder in the hierarchy, starting from the leaves and working up to the root. For each folder, create a `container` with entries for its subfolders and files, compute the CID for the container, and store it in hashstore. Finally, associate the root container's CID with the PID representing the entire folder hierarchy. -This is achieved by the `hash_store.store_folder()` method. +This is achieved by the `hashstore.store_folder()` method. ``` hashstore = HashStore(...) @@ -104,7 +104,7 @@ hashstore.store_folder(pid, source_path) To retrieve the structure of a folder hierarchy identified by a PID, recursively resolve each `container` starting from the root PID. For each folder, read its `container` entries to identify subfolders and files, and continue resolving subfolders until the entire hierarchy is reconstructed. -This is achieved by the `hash_store.retrieve_folder()` method. +This is achieved by the `hashstore.retrieve_folder()` method. ``` hashstore = HashStore(...) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 88e42a23..31af63a6 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1033,8 +1033,8 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: def store_folder( self, pid: str, - root_path: str | Path, - child_path: Optional[str | Path] = None, + root_path: Union[str, Path], + child_path: Optional[Union[str, Path]] = None, additional_algorithm: Optional[str] = None, checksum: Optional[str] = None, checksum_algorithm: Optional[str] = None, @@ -1111,7 +1111,7 @@ def store_folder( expected_object_size=expected_object_size, ) - def retrieve_folder(self, pid:str, destination_path:str|Path, child_path:Optional[str|Path]=None): + def retrieve_folder(self, pid:str, destination_path:Union[str, Path], child_path:Optional[Union[str, Path]]=None): """Retrieve a folder (and subfolders) stored as container objects. Args: diff --git a/src/hashstore/hashstore.py b/src/hashstore/hashstore.py index c92d5257..05b85f7b 100644 --- a/src/hashstore/hashstore.py +++ b/src/hashstore/hashstore.py @@ -4,7 +4,7 @@ import importlib.util from abc import ABC, abstractmethod from pathlib import Path -from typing import Optional +from typing import Optional, Union class HashStore(ABC): @@ -70,8 +70,8 @@ def store_object( def store_folder( self, pid:str, - root_path:str|Path, - child_path:Optional[str|Path]=None, + root_path:Union[str,Path], + child_path:Optional[Union[str,Path]]=None, additional_algorithm:Optional[str]=None, checksum:Optional[str]=None, checksum_algorithm:Optional[str]=None, @@ -106,7 +106,7 @@ def store_folder( raise NotImplementedError() @abstractmethod - def retrieve_folder(self, pid:str, destination_path:str|Path, child_path:Optional[str|Path]=None): + def retrieve_folder(self, pid:str, destination_path:Union[str,Path], child_path:Optional[Union[str,Path]]=None): """Retrieves a folder and its subfolders from HashStore. The `retrieve_folder` method retrieves a folder and its subfolders from HashStore, reconstructing From c5a5e7281bcd2e0bf1e863a2fed295355a583581 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 15 Jan 2026 15:48:25 -0500 Subject: [PATCH 04/49] Adjust check_string to check for leading or trailing whitespace --- src/hashstore/filehashstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 31af63a6..470de60e 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -2908,7 +2908,7 @@ def _check_string(string: str, arg: str) -> None: :param str string: Value to check. :param str arg: Name of the argument to check. """ - if string is None or string.strip() == "": + if string is None or string.strip() == "" or string.strip() != string: method = inspect.stack()[1].function err_msg = ( f"FileHashStore - {method}: {arg} cannot be None" From b8ff3bf679c8e18729613629ff013830d6db46cc Mon Sep 17 00:00:00 2001 From: Dave Vieglais <605409+datadavev@users.noreply.github.com> Date: Tue, 20 Jan 2026 12:05:52 -0500 Subject: [PATCH 05/49] Rename header and update folder hierarchy example --- folder_operations.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/folder_operations.md b/folder_operations.md index 9ad9420c..3e286216 100644 --- a/folder_operations.md +++ b/folder_operations.md @@ -1,4 +1,4 @@ -# hashtree +# Folders in HashStore Describes storing directory trees in hashstore. @@ -32,11 +32,11 @@ Since the CID for a container is dependent on its content, the content order is For example, given the folder hierarchy: ``` -PID_1 -├── A +PID_1 <- dbc15 +├── A <- ad5eb │ ├── a1.txt │ └── a2.txt -└── B +└── B <- cc08d └── b1.csv ``` From 1b72b4dd5f074e66bcbdbb6ae4120d78ae7fef63 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Fri, 20 Feb 2026 10:44:52 -0500 Subject: [PATCH 06/49] KeyError instead of ValueError, add list_pids() --- src/hashstore/filehashstore.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 470de60e..e0fc6129 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -8,13 +8,14 @@ import logging import multiprocessing import os +import re import shutil import threading from contextlib import closing from dataclasses import dataclass from pathlib import Path from tempfile import NamedTemporaryFile -from typing import IO, Any, Dict, List, Optional, Set, Tuple, Union +from typing import IO, Any, Dict, Generator, List, Optional, Set, Tuple, Union import yaml @@ -755,8 +756,8 @@ def retrieve_metadata(self, pid: str, format_id: Optional[str] = None) -> IO[byt return metadata_stream else: err_msg = f"No metadata found for pid: {pid}" - self.fhs_logger.error(err_msg) - raise ValueError(err_msg) + self.fhs_logger.warning(err_msg) + raise KeyError(err_msg) def delete_object(self, pid: str) -> None: self.fhs_logger.debug("Request to delete object for id: %s", pid) @@ -1156,6 +1157,24 @@ def retrieve_folder(self, pid:str, destination_path:Union[str, Path], child_path with open(dest_file_path, "wb") as dest_file: shutil.copyfileobj(file_stream, dest_file) + def list_pids(self, pattern: Optional[str]=None) -> Generator: + rpattern = None + if pattern is not None: + rpattern = re.compile(pattern) + ignore_names = [".DS_Store", ] + for cid_entry in self.cids.rglob('*'): + if cid_entry.is_file() and cid_entry.name not in ignore_names: + self.fhs_logger.debug(str(cid_entry)) + for _, entry in enumerate(open(cid_entry, "r", encoding="utf-8")): + pid = entry.strip() + if len(pid) > 0: + if rpattern is not None: + if rpattern.fullmatch(pid): + yield pid + else: + yield pid + + # FileHashStore Core Methods def _deserialize_container(self, cid) -> Dict[str, Any]: @@ -2908,7 +2927,7 @@ def _check_string(string: str, arg: str) -> None: :param str string: Value to check. :param str arg: Name of the argument to check. """ - if string is None or string.strip() == "" or string.strip() != string: + if not string or string.strip() != string: method = inspect.stack()[1].function err_msg = ( f"FileHashStore - {method}: {arg} cannot be None" From 040868e30f8b7c1c1b83b34e25df90b32d86ee7e Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 30 Apr 2026 09:32:54 -0400 Subject: [PATCH 07/49] some refactoring for folder support --- .vscode/settings.json | 3 +- folder_operations.md | 52 ++- hashstore_layout.drawio | 145 +++++++ pyproject.toml | 6 + src/hashstore/__init__.py | 2 +- src/hashstore/__main__.py | 312 ++++++++++++++ .../{hashstore.py => basehashstore.py} | 19 +- src/hashstore/filehashstore.py | 143 ++++++- src/hashstore/localstore.py | 403 ++++++++++++++++++ tests/test_hashstore.py | 2 +- 10 files changed, 1059 insertions(+), 28 deletions(-) create mode 100644 hashstore_layout.drawio create mode 100644 src/hashstore/__main__.py rename src/hashstore/{hashstore.py => basehashstore.py} (95%) create mode 100644 src/hashstore/localstore.py diff --git a/.vscode/settings.json b/.vscode/settings.json index b15ffaa4..cdd22c0f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -8,5 +8,6 @@ "editor.formatOnSave": true, "[python]": { "editor.defaultFormatter": "ms-python.black-formatter" - } + }, + "python-envs.pythonProjects": [] } \ No newline at end of file diff --git a/folder_operations.md b/folder_operations.md index 9ad9420c..23412dae 100644 --- a/folder_operations.md +++ b/folder_operations.md @@ -1,6 +1,6 @@ # hashtree -Describes storing directory trees in hashstore. +Describes storing directory trees in hashstore (hs). ## Assumptions @@ -13,6 +13,16 @@ Describes storing directory trees in hashstore. - A folder hierarchy may reference all or part of another identified folder hierarchy - A folder is represented by a `container` in hashstore. +## Virtual hashstore + +When a folder is added to `hs`, it is necessary to calculate file and folder hashes and compare these with any existing content in the target `hs`. The efficiency of updating an existing folder entry in `hs` can be significantly improved by computing the hashes locally and determining what may need to be sent to the target `hs`. This is especially important for large folder structures that may have isolated changes. + +A virtual `hs` (`vhs`) is a local folder structure that is similar to a `hs` except that the content bytes are not stored (except for containers), only hashes of the content. Time stamps of the hash entries are compared with content time stamps to identify candidates for hash recalculation. If hash values have changed, then the files are tagged for upload to the target hs. + +A `vhs` is composed of CID and PID ref files, and container files for folder hashes. Even though content ids are calculated, the content files are not stored. + + + ## Containers Hashstore is augmented by adding an additional type of content that represents a `container`, the contents of which represent a single folder. A `container` has two types of entries: `file` that represents a single file and `folder` which represents a single subfolder. Each entry in a `container` has properties: `type`, `cid`, and `name`, where: @@ -112,3 +122,43 @@ pid = "" destination_path = "" hashstore.retrieve_folder(pid, destination_path) ``` + + +--- + +## `add` + +`add(PID:str, path:pathlib.Path)->None` + +Add an object or folder to `vhs`. + + +## `init` + +`init(path:pathlib.Path)->None` + +Initializes a `vhs` folder within the current folder. + + +## `status` + +`status()->VhsStatus` + +Reports the status of the entries in the `vhs` versus the current contents of +registered content. + + +## `update` + +`update(PID:str|None)` + +Recalculates CID values based on the current content of registered entries. + + +## `commit` + +`commit()` + +Makes entries in the `vhs` immutable preventing any further updates to existing +PIDs. Any further changes require new PID. + diff --git a/hashstore_layout.drawio b/hashstore_layout.drawio new file mode 100644 index 00000000..4b6c4c73 --- /dev/null +++ b/hashstore_layout.drawio @@ -0,0 +1,145 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/pyproject.toml b/pyproject.toml index a5d15884..8153e587 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,14 +33,20 @@ classifiers = [ "Topic :: System :: Filesystems", ] dependencies = [ + "click>=8.3.1", "pathlib>=1.0.1", "pyyaml>=6.0", + "xattr-compat>=1.0.0", ] [project.scripts] hashstore = "hashstore.hashstoreclient:main" +hs = "hashstore.__main__:main" [dependency-groups] +cli = [ + "rich>=14.3.3", +] dev = [ "pytest>=7.2.0", "exceptiongroup>=1.1.0", diff --git a/src/hashstore/__init__.py b/src/hashstore/__init__.py index be656f5e..6d470289 100644 --- a/src/hashstore/__init__.py +++ b/src/hashstore/__init__.py @@ -16,7 +16,7 @@ system. """ -from hashstore.hashstore import HashStore, HashStoreFactory +from hashstore.basehashstore import HashStore, HashStoreFactory __all__ = ("HashStore", "HashStoreFactory") __version__ = "1.1.0" diff --git a/src/hashstore/__main__.py b/src/hashstore/__main__.py new file mode 100644 index 00000000..d4b09654 --- /dev/null +++ b/src/hashstore/__main__.py @@ -0,0 +1,312 @@ +import dataclasses +import datetime +import json +import logging +import os +import pathlib +import sys +import click +import rich +import rich.tree +import yaml +try: + from yaml import CLoader as Loader +except ImportError: + from yaml import Loader + +import hashstore +import hashstore.filehashstore_exceptions + +HASHSTORE_FOLDER_NAME = ".hashstore" +DEFAULT_HASHSTORE = f"./{HASHSTORE_FOLDER_NAME}" + +def get_logger(): + return logging.getLogger("hs") + +def sizeof_fmt(num, suffix="B"): + for unit in ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"): + if abs(num) < 1024.0: + return f"{num:3.1f}{unit}{suffix}" + num /= 1024.0 + return f"{num:.1f}Yi{suffix}" + +def enumerate_dict(d): + for key, value in d.items(): + if isinstance(value, dict): + for subkey, subvalue in enumerate_dict(value): + yield f"{key}.{subkey}", subvalue + else: + yield key, value + + +def load_hashstore_properties(path:pathlib.Path)-> dict: + properties_file = path / "hashstore.yaml" + if not properties_file.exists(): + raise FileNotFoundError(f"Hashstore properties file not found: {properties_file}") + with open(properties_file, 'r') as f: + properties = yaml.load(f, Loader=Loader) + properties["store_path"] = str(path) + return properties + + +def locate_hashstore(path:pathlib.Path)->pathlib.Path: + # Iterate through the current directory and all its parents + for directory in [path] + list(path.parents): + # Use glob() to find files matching the pattern within the current directory + for file_path in directory.glob(HASHSTORE_FOLDER_NAME): + if file_path.is_dir(): + return file_path + return None + + +@click.group() +@click.option( + "--store", + type=click.Path(path_type=pathlib.Path, file_okay=False), + envvar="HASHSTORE_PATH", + default=None +) +@click.option("--config", default='~/.config/hashstore/defaults.yml', type=click.Path(), help="Path to configuration file") +@click.option("--log-level", default=None, help="Set the logging level (INFO)") +@click.pass_context +def main(ctx, store, config, log_level): + """Implements a high level file and folder operations CLI for hashstore. + """ + ctx.ensure_object(dict) + # Load defaults from config file + config = os.path.expanduser(config) + if os.path.exists(config): + with open(config, 'r') as f: + cfg = yaml.load(f, Loader=Loader) + ctx.default_map = cfg + if store is None: + store = locate_hashstore(pathlib.Path.cwd()) + if store is not None: + ctx.obj["hashstore_path"] = store + else: + store.expanduser() + ctx.obj["hashstore_path"] = store + ctx.obj["module_name"] = ctx.default_map.get("module_name", "hashstore.filehashstore") + ctx.obj["class_name"] = ctx.default_map.get("class_name", "FileHashStore") + # Set up logging + if log_level is None: + log_level = ctx.default_map.get('log_level', 'INFO') if 'cfg' in locals() else 'INFO' + logger = get_logger() + numeric_level = getattr(logging, log_level.upper(), None) + if not isinstance(numeric_level, int): + raise ValueError(f"Invalid log level: {log_level}") + logging.basicConfig(level=numeric_level) + logger.setLevel(numeric_level) + logger.debug(f"Logging initialized at level: {log_level}") + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"Configuration:") + for key, value in enumerate_dict(ctx.default_map): + logger.debug(f" {key}: {value}") + return 0 + + +@main.command() +@click.pass_context +def version(ctx): + """Show the version of Hashstore.""" + print(f"Hashstore version {hashstore.__version__}") + return 0 + + +@main.command() +@click.pass_context +@click.option("--depth", "-d", type=int, default=3, help="Hashstore hierarchy depth") +@click.option("--width", "-w", type=int, default=2, help="Hashstore hierarchy width") +@click.option("--algorithm", "-a", type=str, default="SHA-256", help="Hashstore algorithm") +@click.option("--metadata_namespace", type=str, default=None, help="Hashstore metadata namespace") +def create(ctx, depth, width, algorithm, metadata_namespace): + """Create a new hashstore at the specified root path.""" + logger = get_logger() + store = ctx.obj["hashstore_path"] + if not store.parent.exists(): + logger.error(f"Parent directory does not exist: {store.parent}") + return 1 + if store.exists(): + if (store / "hashstore.yaml").exists(): + logger.error(f"Hashstore already exists at: {store}") + return 1 + properties = { + "store_path": store, + "store_depth": depth, + "store_width": width, + "store_algorithm": algorithm, + "store_metadata_namespace": metadata_namespace, + } + hashstore_factory = hashstore.HashStoreFactory() + try: + hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + logger.info(f"Hashstore created at: {store}") + except Exception as e: + logger.error(f"Failed to create hashstore: {e}") + return 1 + return 0 + + +@main.command(name="add_folder") +@click.pass_context +@click.argument( + "root_path", + type=click.Path(path_type=pathlib.Path, file_okay=False, exists=True) +) +@click.argument("pid", type=str) +@click.option( + "-s", + "--sysmeta_path", + type=click.Path(path_type=pathlib.Path), + default=None, + help="Path to system metadata XML file" +) +@click.option( + "-p", + "--pattern", + type=str, + default=None, + help="Glob pattern for files to include (defaults to all)" +) +def add_object( + ctx, + root_path:pathlib.Path, + pid: str|None, + sysmeta_path:pathlib.Path|None, + pattern:str|None, +): + """Add a folder to the hashstore. + + PID is required and is used to reference the folder root. + + If object_path is a folder, a PID is generated for the folder contents added + recursively using the object relative paths as suffix to the PID. + + if the file doesn't exist: + add bytes + store pid + store sysmeta if provided + if the file exists: + if pid doesn't exist: + store pid by adding to list + if sysmeta provided: + store sysmeta + """ + logger = get_logger() + store = ctx.obj["hashstore_path"] + logger.info(f"Adding folder at path: {store} with PID: {pid} and sysmeta: {sysmeta_path}") + properties = load_hashstore_properties(store) + hashstore_factory = hashstore.HashStoreFactory() + try: + hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + logger.debug(f"Hashstore opened at: {store}") + except Exception as e: + logger.error(f"Failed to open hashstore: {e}") + return 1 + # Does the object already exist in the store? + try: + info = hash_store._find_object(pid=pid) + hash_store.tag_object(pid, info.get("cid")) + except hashstore.filehashstore_exceptions.PidRefsDoesNotExist: + info = None + # Object doesn't exist in store + root_path = root_path.absolute() + if info is None: + try: + info = hash_store.commit_folder(pid=pid, root_path=root_path, pattern=pattern) + print(json.dumps(dataclasses.asdict(info))) + except Exception as e: + logger.error(f"Failed to add object: {e}") + return 1 + if sysmeta_path is not None and sysmeta_path.is_file(): + hash_store.store_metadata(pid=pid, metadata=str(sysmeta_path), format_id=properties.get("store_metadata_namespace")) + return 1 + return 0 + + +@main.command("get") +@click.pass_context +@click.argument("pid", type=str) +@click.option("-s", "--stream", is_flag=True, help="Stream to stdout") +@click.option("-r", "--recursive", is_flag=True, help="Recurse into contents if PID is a folder.") +def get_object(ctx, pid, stream, recursive): + """Retrieve an object or folder from hashstore.""" + store = ctx.obj["hashstore_path"] + pass + + +@main.command("ls") +@click.pass_context +@click.option("-p", "--pattern", default=None, help="Optional regex pattern for PID matching.") +@click.option("-h", "--human-readable", is_flag=True, help="Display sizes in human readable format.") +@click.option("-r", "--reference", is_flag=True, help="Include path references.") +def list_pids(ctx, pattern, human_readable, reference): + """List PIDs in the hashstore.""" + logger = get_logger() + store = ctx.obj["hashstore_path"] + properties = load_hashstore_properties(store) + hashstore_factory = hashstore.HashStoreFactory() + try: + hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + logger.debug(f"Hashstore opened at: {store}") + except Exception as e: + logger.error(f"Failed to open hashstore: {e}") + return 1 + total_objects = 0 + # iterate over the refs/cids folder, getting PIDs from each file. + print(f"Hashstore: {str(store.relative_to(pathlib.Path.cwd(), walk_up=True))}") + for pid in hash_store.list_pids(pattern=pattern): + try: + pid_stat = hash_store.get_object_status(pid) + fsize = pid_stat.get("size", 0) + total_objects += 1 + t_modified = pid_stat.get("modtime", "-") + if t_modified != "-": + t_modified = datetime.datetime.fromtimestamp(t_modified).isoformat(timespec="seconds") + t_access = pid_stat.get("accesstime", "-") + if t_access != "-": + t_access = datetime.datetime.fromtimestamp(t_access).isoformat(timespec="seconds") + if human_readable: + fsize = sizeof_fmt(fsize) + if fsize>0 or reference: + print(f"{fsize}\t{t_modified}\t{t_access}\t{pid}") + try: + meta = hash_store.retrieve_metadata(pid) + print(json.dumps(meta, indent=2)) + except KeyError: + pass + except KeyError: + logger.warning(f"PID status not available in hashstore: {pid}") + print(f"Total {total_objects}") + + +@main.command("tree") +@click.pass_context +@click.argument("pid", type=str) +def get_container_tree(ctx, pid)->None: + logger = get_logger() + store = ctx.obj["hashstore_path"] + properties = load_hashstore_properties(store) + hashstore_factory = hashstore.HashStoreFactory() + try: + hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + logger.debug(f"Hashstore opened at: {store}") + except Exception as e: + logger.error(f"Failed to open hashstore: {e}") + return 1 + tree = { + pid:rich.tree.Tree(pid), + } + for entry in hash_store.folder_content(pid, depth_first=True): + print(f"{entry[0]} {entry[1]} {entry[3]} {entry[2]}") + parts = entry[3].rsplit("/", 1) + if entry[1] == 0: + tree[entry[3]] = tree[parts[0]].add(entry[3]) + else: + # file + tree[parts[0]].add(entry[3]) + rich.print(tree[pid]) + + +if __name__ == "__main__": + sys.exit(main(auto_envvar_prefix="HASHSTORE")) diff --git a/src/hashstore/hashstore.py b/src/hashstore/basehashstore.py similarity index 95% rename from src/hashstore/hashstore.py rename to src/hashstore/basehashstore.py index 05b85f7b..2e50d8cb 100644 --- a/src/hashstore/hashstore.py +++ b/src/hashstore/basehashstore.py @@ -4,7 +4,7 @@ import importlib.util from abc import ABC, abstractmethod from pathlib import Path -from typing import Optional, Union +from typing import Optional, Union, Generator class HashStore(ABC): @@ -234,6 +234,23 @@ def get_hex_digest(self, pid, algorithm): """ raise NotImplementedError() + @abstractmethod + def list_pids(self, pattern:Optional[str]=None) -> Generator: + """Yields PIDs from the hashstore. + + :param str pattern: Optional regexp pattern to match. + """ + raise NotImplementedError() + + @abstractmethod + def get_object_status(self, pid) -> dict: + """Returns a dictionary of the object size, modtime, accesstime for the given pid. + + :param str pid: Object identifier + + :return: dict - Dictionary containing information about the object. + """ + raise NotImplementedError() class HashStoreFactory: """A factory class for creating `HashStore`-like objects. diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index e0fc6129..51e212ae 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -606,6 +606,30 @@ def tag_object(self, pid: str, cid: str) -> None: self.fhs_logger.error(err_msg) raise PidRefsAlreadyExistsError(err_msg) + def get_object_status(self, pid: str) -> dict: + logging.debug("Request to get object status for pid: %s", pid) + self._check_string(pid, "pid") + + object_status_dict = {} + try: + object_info_dict = self._find_object(pid) + object_cid = object_info_dict.get("cid") + if object_cid: + obj_path = self._get_hashstore_data_object_path(object_cid) + object_status_dict["size"] = os.path.getsize(obj_path) + object_status_dict["modtime"] = os.path.getmtime(obj_path) + object_status_dict["accesstime"] = os.path.getatime(obj_path) + else: + err_msg = f"No object found for pid: {pid}" + self.fhs_logger.warning(err_msg) + raise KeyError(err_msg) + except KeyError as ke: + err_msg = f"No object found for pid: {pid}. Details: {ke}" + self.fhs_logger.warning(err_msg) + raise KeyError(err_msg) + + return object_status_dict + def delete_if_invalid_object( self, object_metadata: "ObjectMetadata", @@ -1031,6 +1055,50 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: logging.info(info_string) return hex_digest + def _store_container( + self, + pid:str, + name:str, + manifest:list[tuple[int, str, str]], + additional_algorithm: Optional[str] = None, + checksum: Optional[str] = None, + checksum_algorithm: Optional[str] = None, + ) -> "ObjectMetadata": + self.fhs_logger.debug(f"_store_container {pid=}") + manifest.sort(key=lambda x: (x[0], x[1])) + dest_stream = io.BytesIO() + dest_stream.name = name + dest_stream.write(f"container {len(manifest)}\n".encode("utf-8")) + for row in manifest: + dest_stream.write(f"{row[0]} {row[1]} {row[2]}\n".encode("utf-8")) + dest_stream.seek(0) + return self.store_object( + pid, + data=dest_stream, + additional_algorithm=additional_algorithm, + checksum=checksum, + checksum_algorithm=checksum_algorithm, + expected_object_size=None, + ) + + def _load_container(self, pid:str) -> list[tuple[int, str, str]]: + self.fhs_logger.debug(f"_load_container {pid=}") + obj_stream = self.retrieve_object(pid) + row = 0 + manifest = [] + with closing(obj_stream): + for line in obj_stream: + line = line.decode("utf-8").strip() + if row == 0: + if not line.startswith("container "): + msg = f"{pid} is not a container." + raise ValueError(msg) + else: + type_flag, cid, name = line.split(" ", 2) + manifest.append((int(type_flag), cid, name)) + row += 1 + return manifest + def store_folder( self, pid: str, @@ -1040,9 +1108,13 @@ def store_folder( checksum: Optional[str] = None, checksum_algorithm: Optional[str] = None, expected_object_size: Optional[int] = None, - ) -> "ObjectMetadata": + pattern: Optional[str] = None, + ) -> Optional["ObjectMetadata"]: """Store a folder (and subfolders) as container objects. + Traverses the folder hierarchy in a depth first manner so that + the leaf elements are computed for inclusion in parent hashes. + Args: pid (str): The context within which this folder is being stored root_path (str): Path to the root of the folder. @@ -1051,6 +1123,7 @@ def store_folder( Returns: str: CID for the container """ + raise NotImplementedError # Get the relative path for the object / folder root_path = Path(root_path) if child_path is None: @@ -1061,9 +1134,10 @@ def store_folder( path_pid = pid container_name = "root" if str(relative_path) != ".": - path_pid = f"{pid} {relative_path}" + path_pid = f"{pid}/{relative_path}" container_name = str(relative_path) + self.fhs_logger.debug("store_folder {path_pid=}") # Check if this container already exists try: # resolve pid, path to CID. This raises if not found @@ -1079,6 +1153,7 @@ def store_folder( except PidRefsDoesNotExist: pass + # Container doesn't exist manifest = [] for item in child_path.iterdir(): if item.is_dir(): @@ -1090,27 +1165,55 @@ def store_folder( checksum=checksum, checksum_algorithm=checksum_algorithm, expected_object_size=expected_object_size, + pattern=pattern, ) - manifest.append((0, meta.cid, item.name)) - elif item.is_file(): - item_pid = f"{pid} {item.relative_to(root_path)}" + if meta is not None: + manifest.append((0, meta.cid, item.name)) + elif pattern is None and item.is_file(): + # If no pattern then grab all files, otherwise defer to + # globbing match later. + item_pid = f"{pid}/{item.relative_to(root_path)}" + self.fhs_logger.debug("store_folder {item_pid=}") meta = self.store_object(item_pid, str(item.absolute())) manifest.append((1, meta.cid, item.name)) - manifest.sort(key=lambda x: (x[0], x[1])) - dest_stream = io.BytesIO() - dest_stream.name = container_name - for row in manifest: - dest_stream.write(f"{row[0]} {row[1]} {row[2]}\n".encode("utf-8")) - dest_stream.seek(0) - # TODO: error handling - return self.store_object( + if pattern is not None: + # globbing pattern was specified. Grab the matching files here. + for item in child_path.glob(pattern): + if item.is_file(): + item_pid = f"{pid}/{item.relative_to(root_path)}" + meta = self.store_object(item_pid, str(item.absolute())) + manifest.append((1, meta.cid, item.name)) + if len(manifest) == 0: + return None + return self._store_container( path_pid, - data=dest_stream, + container_name, + manifest, additional_algorithm=additional_algorithm, checksum=checksum, - checksum_algorithm=checksum_algorithm, - expected_object_size=expected_object_size, - ) + checksum_algorithm=checksum_algorithm + ) + + def folder_content(self, pid:str, depth:int=0, depth_first:bool=True) -> Generator: + """Yield the content of a folder, breadth first, recursively. + (depth, type, CID, name) + """ + # Retrieve the container object + manifest = self._load_container(pid) + if depth_first: + for entry in manifest: + if entry[0] == 0: + yield (depth, entry[0], entry[1], f"{pid}/{entry[2]}") + yield from self.folder_content(f"{pid}/{entry[2]}", depth=depth+1, depth_first=depth_first) + else: + yield (depth, entry[0], entry[1], f"{pid}/{entry[2]}") + else: + for entry in manifest: + yield (depth, entry[0], entry[1], f"{pid}/{entry[2]}") + for entry in manifest: + if entry[0] == 0: + # folder, recurse + yield from self.folder_content(f"{pid}/{entry[2]}", depth=depth+1, depth_first=depth_first) def retrieve_folder(self, pid:str, destination_path:Union[str, Path], child_path:Optional[Union[str, Path]]=None): """Retrieve a folder (and subfolders) stored as container objects. @@ -1177,12 +1280,6 @@ def list_pids(self, pattern: Optional[str]=None) -> Generator: # FileHashStore Core Methods - def _deserialize_container(self, cid) -> Dict[str, Any]: - pass - - def _serialize_container(self, container: Dict[str, Any]) -> str: - pass - def _find_object(self, pid: str) -> Dict[str, str]: """Check if an object referenced by a pid exists and retrieve its content identifier. The `find_object` method validates the existence of an object based on the provided diff --git a/src/hashstore/localstore.py b/src/hashstore/localstore.py new file mode 100644 index 00000000..3b8d3ec8 --- /dev/null +++ b/src/hashstore/localstore.py @@ -0,0 +1,403 @@ +"""Implements a virtual hashstore. + +A virtual hashstore is used for staging content that is to be added to a +hashstore. + +Staging associates PIDs with objects and folders, and computes their hashes. +When changes are made, a difference can be computed and used to efficiently +transmit changes to a hashtore. +""" +import collections.abc +import contextlib +import dataclasses +import hashlib +import io +import json +import os +import pathlib +import typing + +import xattr_compat as xattrs + +import hashstore.filehashstore +import hashstore.filehashstore_exceptions + +ORG_DATAONE_CID = "org.dataone.hashstore.cid" +ORG_DATAONE_CID_ALGORITHM = "sha265" +FOLDER_ENTRY_FOLDER = 0 +FOLDER_ENTRY_FILE = 1 + + +def depth_first_walk(obj_path:pathlib.Path) -> collections.abc.Generator[pathlib.Path]: + if obj_path.is_file(): + yield obj_path + return + if obj_path.is_dir(): + for item in obj_path.iterdir(): + if item.is_dir(): + yield from depth_first_walk(item) + elif item.is_file(): + yield item + yield obj_path + + +def compute_object_hash(obj_path: pathlib.Path, algorithm:str=ORG_DATAONE_CID_ALGORITHM) -> str: + """Computes and returns the hash of the specified object path. + + This has the side effect of setting the ORG_DATAONE_CID xattr to a JSON + struct that contains the current modification time, size, and hash value. + + The existing ORG_DATAONE_CID xattr is examined if present, and is returned + unless there is a mismatch of size or modification time. + + If obj_path is a folder, then the hash is computed as the hash of the list of + type, cid, and name values. Since it is necessary to verify the consistency + of the hashes for subfolders, a traversal to the leaves is necessary. Hence, + this method can be expensive with deeply nested subfolders since even traversing + """ + #Xattrs accepts os.PathLike + attrs = xattrs.Xattrs(obj_path) + obj_stat = obj_path.stat() + if obj_path.is_file(): + try: + # check for current existing cid value and return that if present + entry = json.loads(attrs.get(ORG_DATAONE_CID, {}).decode("utf-8")) + if entry["mtime"] == obj_stat.st_mtime and entry["size"] == obj_stat.st_size: + # nothing has changed, so use the existing value + return entry["cid"] + except KeyError: + # No or invalid xattr value, ignore and continue + pass + with obj_path.open('rb') as f: + # Use the builtin chunker, requires python >= 3.11 + digest = hashlib.file_digest(f, algorithm) + hexdigest = digest.hexdigest() + # Set the xattr + entry = json.dumps({ + "cid": hexdigest, + "mtime": obj_stat.st_mtime, + "size": obj_stat.st_size + }, ensure_ascii=False) + attrs[ORG_DATAONE_CID] = entry.encode("utf-8") + elif obj_path.is_dir(): + # Hash of a folder is the hash of the content. To check a folder, it + # is necessary to traverse to the leaves to verify / compute the + # hashes. + pass + else: + raise ValueError(f"Path is not a file or folder.") + return hexdigest + + +@dataclasses.dataclass +class FolderEntry: + """A single entry in a Folder instance. + + An entry represents a file or folder record within a folder object. + """ + kind: int # FOLDER_ENTRY_FOLDER | FOLDER_ENTRY_FILE + cid: str # The content id + name: str # name of folder or file + + def get_cid(self, algorithm:str=ORG_DATAONE_CID_ALGORITHM) -> str: + """Retrieves the contentId from the ORG_DATAONE_CID xattr or computes + the CID if the xattr value is out of date. + + A CID value is computed if the file size or modified timestamp do + not match the entries stored in the xattr structure or if the xattr + is not set or otherwise cannot be read. + """ + hasher = hashlib.new(self.algorithm) + pass + + +@dataclasses.dataclass +class Folder: + """An object that represents a single folder and its content. + """ + name: str # PID + "/" + path to the folder relative to root + entries: list[FolderEntry] = dataclasses.field(default_factory=list) + + def __iter__(self): + return self.entries.__iter__() + + def append(self, item: FolderEntry) -> None: + self.entries.append(item) + + def compute_cid(self, algorithm:str=ORG_DATAONE_CID_ALGORITHM) -> str: + """Computes the CID for this folder based on the content of the entries. + + The CID is computed as the hash of the list of type, cid, and name values. + The entries are sorted by type and name to ensure a consistent hash value. + """ + hasher = hashlib.new(algorithm) + self.entries.sort(key=lambda x: (x.kind, x.name)) + for entry in self.entries: + hasher.update(f"{entry.kind} {entry.cid} {entry.name}\n".encode("utf-8")) + return hasher.hexdigest() + + @classmethod + def deserialize(cls, stream: io.BytesIO, pid:str) -> "Folder": + manifest = Folder(name=pid) + with contextlib.closing(stream): + header = stream.readline().decode("utf-8").strip() + if not header.startswith("container "): + msg = f"{pid} is not a container." + raise ValueError(msg) + for line in stream: + line = line.decode("utf-8").strip() + type_flag, cid, name = line.split(" ", 2) + manifest.entries.append(FolderEntry(kind=int(type_flag), cid=cid, name=name)) + return manifest + + def serialize( + self, + ) -> io.BytesIO: + self.entries.sort(key=lambda x: (x.kind, x.name)) + dest_stream = io.BytesIO() + dest_stream.name = "tmp" + dest_stream.write(f"container {len(self.entries)}\n".encode("utf-8")) + for entry in self.entries: + dest_stream.write(f"{entry.kind} {entry.cid} {entry.name}\n".encode("utf-8")) + dest_stream.seek(0) + return dest_stream + + @property + def size(self) -> int: + return len(self.entries) + + +class VirtualHashStore(hashstore.filehashstore.FileHashStore): + + def __init__(self, properties): + super().__init__(properties) + + def _find_object(self, pid: str) -> dict[str, str]: + """Check if an object referenced by a pid exists and retrieve its content identifier. + The `find_object` method validates the existence of an object based on the provided + pid and returns the associated content identifier. + + :param str pid: Authority-based or persistent identifier of the object. + + :return: obj_info_dict: + - cid: content identifier + - cid_object_path: path to the object + - cid_refs_path: path to the cid refs file + - pid_refs_path: path to the pid refs file + - sysmeta_path: path to the sysmeta file + """ + self.fhs_logger.debug("Request to find object for for pid: %s", pid) + self._check_string(pid, "pid") + + pid_ref_abs_path = self._get_hashstore_pid_refs_path(pid) + if os.path.isfile(pid_ref_abs_path): + # Read the file to get the cid from the pid reference + pid_refs_cid = self._read_small_file_content(pid_ref_abs_path) + + # Confirm that the cid reference file exists + cid_ref_abs_path = self._get_hashstore_cid_refs_path(pid_refs_cid) + if os.path.isfile(cid_ref_abs_path): + # Check that the pid is actually found in the cid reference file + if self._is_string_in_refs_file(pid, cid_ref_abs_path): + # Object must also exist in order to return the cid retrieved + if self._exists("objects", pid_refs_cid): + cid_object_path = self._get_hashstore_data_object_path( + pid_refs_cid + ) + else: + cid_object_path = None + sysmeta_doc_name = self._computehash(pid + self.sysmeta_ns) + metadata_directory = self._computehash(pid) + metadata_rel_path = pathlib.Path(*self._shard(metadata_directory)) + sysmeta_full_path = ( + self._get_store_path("metadata") + / metadata_rel_path + / sysmeta_doc_name + ) + obj_info_dict = { + "cid": pid_refs_cid, + "cid_object_path": cid_object_path, + "cid_refs_path": cid_ref_abs_path, + "pid_refs_path": pid_ref_abs_path, + "sysmeta_path": ( + sysmeta_full_path + if os.path.isfile(sysmeta_full_path) + else "Does not exist." + ), + } + return obj_info_dict + else: + # If not, it is an orphan pid refs file + err_msg = ( + f"Pid reference file exists with cid: {pid_refs_cid} for pid: {pid} but " + f"is missing from cid refs file: {cid_ref_abs_path}" + ) + self.fhs_logger.error(err_msg) + raise hashstore.filehashstore_exceptions.PidNotFoundInCidRefsFile(err_msg) + else: + err_msg = ( + f"Pid reference file exists with cid: {pid_refs_cid} but cid reference file " + + f"not found: {cid_ref_abs_path} for pid: {pid}" + ) + self.fhs_logger.error(err_msg) + raise hashstore.filehashstore_exceptions.OrphanPidRefsFileFound(err_msg) + else: + err_msg = ( + f"Pid reference file not found for pid ({pid}): {pid_ref_abs_path}" + ) + self.fhs_logger.error(err_msg) + raise hashstore.filehashstore_exceptions.PidRefsDoesNotExist(err_msg) + + + def commit_folder( + self, + pid: str, + root_path: typing.Union[str, pathlib.Path], + child_path: typing.Optional[typing.Union[str, pathlib.Path]] = None, + additional_algorithm: typing.Optional[str] = None, + checksum: typing.Optional[str] = None, + checksum_algorithm: typing.Optional[str] = None, + expected_object_size: typing.Optional[int] = None, + pattern: typing.Optional[str] = None, + ) -> typing.Optional[hashstore.filehashstore.ObjectMetadata]: + """Store a folder (and subfolders) as container objects. + + Traverses the folder hierarchy in a depth first manner so that + the leaf elements are computed for inclusion in parent hashes. + + Args: + pid (str): The context within which this folder is being stored + root_path (str): Path to the root of the folder. + child_path (str): Path to folder being stored relative to the root_path. If None, assumes root_path. + + Returns: + str: CID for the container + """ + # Get the relative path for the object / folder + root_path = pathlib.Path(root_path) + if child_path is None: + child_path = root_path + else: + child_path = pathlib.Path(child_path) + relative_path = child_path.relative_to(root_path) + path_pid = pid + container_name = "root" + if str(relative_path) != ".": + path_pid = f"{pid}/{relative_path}" + container_name = str(relative_path) + + # Check if this container already exists + try: + # resolve pid, path to CID. This raises if not found + _entry = self._find_object(path_pid) + size = os.path.getsize( + self._build_hashstore_data_object_path(_entry["cid"]) + ) + return hashstore.filehashstore.ObjectMetadata( + pid=path_pid, cid=_entry["cid"], obj_size=size, hex_digests={} + ) + except hashstore.filehashstore.PidNotFoundInCidRefsFile: + pass + except hashstore.filehashstore.PidRefsDoesNotExist: + pass + + # Container doesn't exist + manifest = Folder(name=path_pid) + for item in child_path.iterdir(): + if item.is_dir(): + meta = self.commit_folder( + pid, + root_path, + child_path=item.absolute(), + additional_algorithm=additional_algorithm, + checksum=checksum, + checksum_algorithm=checksum_algorithm, + expected_object_size=expected_object_size, + pattern=pattern, + ) + if meta is not None: + manifest.append(FolderEntry(kind=0, cid=meta.cid, name=item.name)) + elif pattern is None and item.is_file(): + # If no pattern then grab all files, otherwise defer to + # globbing match later. + item_pid = f"{pid}/{item.relative_to(root_path)}" + self.fhs_logger.debug("store_folder {item_pid=}") + with item.open("r") as item_stream: + _cid = self._computehash(item_stream) + self._store_hashstore_refs_files(item_pid, _cid) + manifest.append(FolderEntry(kind=1, cid=_cid, name=item.name)) + if pattern is not None: + # globbing pattern was specified. Grab the matching files here. + for item in child_path.glob(pattern): + if item.is_file(): + item_pid = f"{pid}/{item.relative_to(root_path)}" + with item.open("r") as item_stream: + _cid = self._computehash(item_stream) + self._store_hashstore_refs_files(item_pid, _cid) + manifest.append(FolderEntry(kind=1, cid=_cid, name=item.name)) + if manifest.size == 0: + return None + + return self.store_object( + path_pid, + data=manifest.serialize(), + additional_algorithm=additional_algorithm, + checksum=checksum, + checksum_algorithm=checksum_algorithm, + expected_object_size=None, + ) + + + def folder_content(self, pid:str, depth:int=0, depth_first:bool=True) -> typing.Generator: + """Yield the content of a folder, breadth first, recursively. + (depth, type, CID, name) + """ + # Retrieve the container object + with contextlib.closing(self.retrieve_object(pid)) as obj_stream: + manifest = Folder.deserialize(obj_stream, pid) + if depth_first: + for entry in manifest: + if entry.kind == 0: + yield (depth, entry.kind, entry.cid, f"{pid}/{entry.name}") + yield from self.folder_content(f"{pid}/{entry.name}", depth=depth+1, depth_first=depth_first) + else: + yield (depth, entry.kind, entry.cid, f"{pid}/{entry.name}") + else: + for entry in manifest: + yield (depth, entry.kind, entry.cid, f"{pid}/{entry.name}") + for entry in manifest: + if entry.kind == 0: + # folder, recurse + yield from self.folder_content(f"{pid}/{entry.name}", depth=depth+1, depth_first=depth_first) + + def get_object_status(self, pid: str) -> dict: + self.fhs_logger.debug("Request to get object status for pid: %s", pid) + self._check_string(pid, "pid") + + object_status_dict = {} + try: + object_info_dict = self._find_object(pid) + object_cid = object_info_dict.get("cid") + if object_cid: + try: + obj_path = self._get_hashstore_data_object_path(object_cid) + object_status_dict["size"] = os.path.getsize(obj_path) + object_status_dict["modtime"] = os.path.getmtime(obj_path) + object_status_dict["accesstime"] = os.path.getatime(obj_path) + except FileNotFoundError as e: + msg = f"No object found for CID {object_cid}" + self.fhs_logger.warning(msg) + object_status_dict["size"] = 0 + object_status_dict["modtime"] = "-" + object_status_dict["accesstime"] = "-" + else: + err_msg = f"No object found for pid: {pid}" + self.fhs_logger.warning(err_msg) + raise KeyError(err_msg) + except KeyError as ke: + err_msg = f"No object found for pid: {pid}. Details: {ke}" + self.fhs_logger.warning(err_msg) + raise KeyError(err_msg) + + return object_status_dict + diff --git a/tests/test_hashstore.py b/tests/test_hashstore.py index 02d83e3b..dbc24a2d 100644 --- a/tests/test_hashstore.py +++ b/tests/test_hashstore.py @@ -2,7 +2,7 @@ import os import pytest -from hashstore.hashstore import HashStoreFactory +from hashstore.basehashstore import HashStoreFactory from hashstore.filehashstore import FileHashStore From 74f1870e09a7d67739ecb4958060ae5fe210d4ef Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:38:27 -0400 Subject: [PATCH 08/49] Dependency updates require 3.10 minimum python version --- pyproject.toml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 8153e587..00fc7ed7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ authors = [ { name = "Jeanette Clark" }, { name = "Ian M. Nesbitt" }, ] -requires-python = ">=3.9, <4.0" +requires-python = ">=3.10, <4.0" readme = "README.md" keywords = [ "filesystem", @@ -35,6 +35,7 @@ classifiers = [ dependencies = [ "click>=8.3.1", "pathlib>=1.0.1", + "pyarrow>=24.0.0", "pyyaml>=6.0", "xattr-compat>=1.0.0", ] From fda4af6702f15346b54ac054de851b618c31f6ea Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:39:56 -0400 Subject: [PATCH 09/49] Adjust folder related method signatures --- src/hashstore/basehashstore.py | 104 ++++++++++++++++++--------------- 1 file changed, 57 insertions(+), 47 deletions(-) diff --git a/src/hashstore/basehashstore.py b/src/hashstore/basehashstore.py index 2e50d8cb..6620c845 100644 --- a/src/hashstore/basehashstore.py +++ b/src/hashstore/basehashstore.py @@ -4,7 +4,9 @@ import importlib.util from abc import ABC, abstractmethod from pathlib import Path -from typing import Optional, Union, Generator +from typing import Generator, Optional, Union + +import hashstore.folderentry class HashStore(ABC): @@ -69,36 +71,41 @@ def store_object( @abstractmethod def store_folder( self, - pid:str, - root_path:Union[str,Path], - child_path:Optional[Union[str,Path]]=None, - additional_algorithm:Optional[str]=None, - checksum:Optional[str]=None, - checksum_algorithm:Optional[str]=None, - expected_object_size:Optional[int]=None, + pid: str, + path: str, + entries: hashstore.folderentry.FolderEntries, + additional_algorithm: Optional[str] = None, + checksum: Optional[str] = None, + checksum_algorithm: Optional[str] = None, + verify_entry_cids: bool = True, ): - """Stores a folder and subfolders. - - The `store_folder` method stores a folder and its subfolders to HashStore. Each file within the folder - is processed and stored individually, following the same procedures as the `store_object` method. The - folder structure is preserved within HashStore, allowing for easy retrieval of the entire folder or - individual files as needed. - - The root of a folder is specified by the `root_path` argument and must be identified by a PID. - - This method performs a recursive, depth firth traversal of the folder structure, storing each file it encounters and - storing folders as a container object that lists the files and subfolders contained within it. + """Store a folder object. + + A Folder is a list of entries that appear in a folder. Each entry + may be a file or a Folder. This method is used instead of store_object + because Folders have special requirements to ensure deterministic serialization. + + The Folder is tagged with an identifier that is "{PID} {path}", that is, the + PID followed by a single space, then the path. If the path portion is an empty + string, ".", or "/" then the Folder is the root Folder. + + Note that since the hash of a Folder is computed from hashes of its content, + a Folder hierarchy must be stored starting with the leaves. This method + will raise a ValueError if the hash of an entry does not already exist in + the hashstore. Hence the general pattern for storing a folder hierarchy is + to do a depth first traversal of the hierarchy, storing the files (ensuring + their hashes are available) and computing the hash for the containing folder + for use in the parent folder reference to the child. Args: - pid (str): Identifier for the context of this folder hierarchy. - root_path (str | Path): The physical path to the root folder being stored. - child_path (Optional[str | Path], optional): Path to a subfolder of root_path. This is - normally None for the initial invocation of this method, and recursive calls will set the - child_path as needed. Defaults to None. - additional_algorithm (Optional[str], optional): See `store_object`. Defaults to None. - checksum (Optional[str], optional): See `store_object`. Defaults to None. - checksum_algorithm (Optional[str], optional): See `store_object`. Defaults to None. - expected_object_size (Optional[int], optional): See `store_object`. Defaults to None. + pid (str): The context within which this folder is being stored + path (str): Path to this folder relative to the root. + entries (list[FolderEntry]): A list of FolderEntry objects. + verify_entry_cids: If True then FolderEntry CID values are + verified to to ensure they exist in the hashstore. + + Returns: + ObjectMetadata: The computed ObjectMetadata for this entry. Raises: NotImplementedError: Must be implemented in subclass. @@ -106,25 +113,27 @@ def store_folder( raise NotImplementedError() @abstractmethod - def retrieve_folder(self, pid:str, destination_path:Union[str,Path], child_path:Optional[Union[str,Path]]=None): - """Retrieves a folder and its subfolders from HashStore. - - The `retrieve_folder` method retrieves a folder and its subfolders from HashStore, reconstructing - the original folder structure at the specified target path. Each file within the folder is retrieved - individually, following the same procedures as the `retrieve_object` method. The folder structure - is preserved during retrieval, allowing for easy access to the entire folder or individual files as needed. - - The root of a folder is specified by the `pid` argument, which identifies the context of the folder hierarchy. The - optional child_path argument can be used to specify a subfolder within the root folder for retrieval. - - Output files and folders will be created under the `destination_path`. + def retrieve_folder( + self, + pid: str, + path: str, + ) -> hashstore.folderentry.FolderEntries: + """Retrieve a FolderEntries instance from the hashstore. + + We first check to see if a CID is available for the combination of + "{PID} {path}", and if so, return that entry. Otherwise, we iterate + over path segments to find the correspoding FolderEntry, if any. + This iterative approach is necesary if since entire trees are not + stored when a new version of a folder hierarchy is stored. Hence, it + may be necessary to jump back to a branch that is recorded in an + earlier version but not recorded in the current version since it + was unchanged between versions. Args: - pid (str): Identifier for the context of this folder hierarchy. - destination_path (str | Path): The physical path where the retrieved folder will be reconstructed. - child_path (Optional[str|Path|], optional): Path to a subfolder of the root folder. This is - normally None for the initial invocation of this method, and recursive calls will set the - child_path as needed. Defaults to None. + pid (str): The context (i.e. VMDAG version) within which this folder is being retrieved + path (str): Path within the context to the desired entry + Returns: + FolderEntries """ raise NotImplementedError() @@ -235,9 +244,9 @@ def get_hex_digest(self, pid, algorithm): raise NotImplementedError() @abstractmethod - def list_pids(self, pattern:Optional[str]=None) -> Generator: + def list_pids(self, pattern: Optional[str] = None) -> Generator: """Yields PIDs from the hashstore. - + :param str pattern: Optional regexp pattern to match. """ raise NotImplementedError() @@ -252,6 +261,7 @@ def get_object_status(self, pid) -> dict: """ raise NotImplementedError() + class HashStoreFactory: """A factory class for creating `HashStore`-like objects. From 880c355567d5fdc1252c9b697cadddf2e9116816 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:40:22 -0400 Subject: [PATCH 10/49] Dependency updates --- uv.lock | 329 ++++++++++++++++++++++---------------------------------- 1 file changed, 126 insertions(+), 203 deletions(-) diff --git a/uv.lock b/uv.lock index bcdc3492..2151d931 100644 --- a/uv.lock +++ b/uv.lock @@ -1,11 +1,10 @@ version = 1 revision = 3 -requires-python = ">=3.9, <4.0" +requires-python = ">=3.10, <4.0" resolution-markers = [ "python_full_version >= '3.12'", "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", - "python_full_version < '3.10'", + "python_full_version < '3.11'", ] [[package]] @@ -17,32 +16,12 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c9/7f/09065fd9e27da0eda08b4d6897f1c13535066174cc023af248fc2a8d5e5a/asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67", size = 105045, upload-time = "2022-03-15T14:46:51.055Z" }, ] -[[package]] -name = "astroid" -version = "3.3.11" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/74/dfb75f9ccd592bbedb175d4a32fc643cf569d7c218508bfbd6ea7ef9c091/astroid-3.3.11.tar.gz", hash = "sha256:1e5a5011af2920c7c67a53f65d536d65bfa7116feeaf2354d8b94f29573bb0ce", size = 400439, upload-time = "2025-07-13T18:04:23.177Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/af/0f/3b8fdc946b4d9cc8cc1e8af42c4e409468c84441b933d037e101b3d72d86/astroid-3.3.11-py3-none-any.whl", hash = "sha256:54c760ae8322ece1abd213057c4b5bba7c49818853fc901ef09719a60dbf9dec", size = 275612, upload-time = "2025-07-13T18:04:21.07Z" }, -] - [[package]] name = "astroid" version = "4.0.2" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", -] dependencies = [ - { name = "typing-extensions", marker = "python_full_version == '3.10.*'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b7/22/97df040e15d964e592d3a180598ace67e91b7c559d8298bdb3c949dc6e42/astroid-4.0.2.tar.gz", hash = "sha256:ac8fb7ca1c08eb9afec91ccc23edbd8ac73bb22cbdd7da1d488d9fb8d6579070", size = 405714, upload-time = "2025-11-09T21:21:18.373Z" } wheels = [ @@ -54,13 +33,11 @@ name = "black" version = "25.11.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "click", version = "8.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "click" }, { name = "mypy-extensions" }, { name = "packaging" }, { name = "pathspec" }, - { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "platformdirs" }, { name = "pytokens" }, { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "typing-extensions", marker = "python_full_version < '3.11'" }, @@ -87,39 +64,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/37/77/4297114d9e2fd2fc8ab0ab87192643cd49409eb059e2940391e7d2340e57/black-25.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dae49ef7369c6caa1a1833fd5efb7c3024bb7e4499bf64833f65ad27791b1545", size = 1612902, upload-time = "2025-11-10T01:59:33.382Z" }, { url = "https://files.pythonhosted.org/packages/de/63/d45ef97ada84111e330b2b2d45e1dd163e90bd116f00ac55927fb6bf8adb/black-25.11.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bd4a22a0b37401c8e492e994bce79e614f91b14d9ea911f44f36e262195fdda", size = 1680571, upload-time = "2025-11-10T01:57:04.239Z" }, { url = "https://files.pythonhosted.org/packages/ff/4b/5604710d61cdff613584028b4cb4607e56e148801ed9b38ee7970799dab6/black-25.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:aa211411e94fdf86519996b7f5f05e71ba34835d8f0c0f03c00a26271da02664", size = 1382599, upload-time = "2025-11-10T01:57:57.427Z" }, - { url = "https://files.pythonhosted.org/packages/d5/9a/5b2c0e3215fe748fcf515c2dd34658973a1210bf610e24de5ba887e4f1c8/black-25.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a3bb5ce32daa9ff0605d73b6f19da0b0e6c1f8f2d75594db539fdfed722f2b06", size = 1743063, upload-time = "2025-11-10T02:02:43.175Z" }, - { url = "https://files.pythonhosted.org/packages/a1/20/245164c6efc27333409c62ba54dcbfbe866c6d1957c9a6c0647786e950da/black-25.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9815ccee1e55717fe9a4b924cae1646ef7f54e0f990da39a34fc7b264fcf80a2", size = 1596867, upload-time = "2025-11-10T02:00:17.157Z" }, - { url = "https://files.pythonhosted.org/packages/ca/6f/1a3859a7da205f3d50cf3a8bec6bdc551a91c33ae77a045bb24c1f46ab54/black-25.11.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92285c37b93a1698dcbc34581867b480f1ba3a7b92acf1fe0467b04d7a4da0dc", size = 1655678, upload-time = "2025-11-10T01:57:09.028Z" }, - { url = "https://files.pythonhosted.org/packages/56/1a/6dec1aeb7be90753d4fcc273e69bc18bfd34b353223ed191da33f7519410/black-25.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:43945853a31099c7c0ff8dface53b4de56c41294fa6783c0441a8b1d9bf668bc", size = 1347452, upload-time = "2025-11-10T01:57:01.871Z" }, { url = "https://files.pythonhosted.org/packages/00/5d/aed32636ed30a6e7f9efd6ad14e2a0b0d687ae7c8c7ec4e4a557174b895c/black-25.11.0-py3-none-any.whl", hash = "sha256:e3f562da087791e96cefcd9dda058380a442ab322a02e222add53736451f604b", size = 204918, upload-time = "2025-11-10T01:53:48.917Z" }, ] -[[package]] -name = "click" -version = "8.1.8" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188, upload-time = "2024-12-21T18:38:41.666Z" }, -] - [[package]] name = "click" version = "8.3.1" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", -] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } wheels = [ @@ -158,31 +111,39 @@ wheels = [ [[package]] name = "hashstore" -version = "1.1.1" +version = "1.2.0" source = { editable = "." } dependencies = [ + { name = "click" }, { name = "pathlib" }, + { name = "pyarrow" }, { name = "pyyaml" }, + { name = "xattr-compat" }, ] [package.dev-dependencies] +cli = [ + { name = "rich" }, +] dev = [ { name = "black" }, { name = "exceptiongroup" }, { name = "pg8000" }, - { name = "pylint", version = "3.3.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "pylint", version = "4.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "pytest", version = "9.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pylint" }, + { name = "pytest" }, ] [package.metadata] requires-dist = [ + { name = "click", specifier = ">=8.3.1" }, { name = "pathlib", specifier = ">=1.0.1" }, + { name = "pyarrow", specifier = ">=24.0.0" }, { name = "pyyaml", specifier = ">=6.0" }, + { name = "xattr-compat", specifier = ">=1.0.0" }, ] [package.metadata.requires-dev] +cli = [{ name = "rich", specifier = ">=14.3.3" }] dev = [ { name = "black", specifier = ">=22.10.0" }, { name = "exceptiongroup", specifier = ">=1.1.0" }, @@ -191,39 +152,10 @@ dev = [ { name = "pytest", specifier = ">=7.2.0" }, ] -[[package]] -name = "importlib-metadata" -version = "8.7.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "zipp", marker = "python_full_version < '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/20/b0/36bd937216ec521246249be3bf9855081de4c5e06a0c9b4219dbeda50373/importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd", size = 27656, upload-time = "2025-04-27T15:29:00.214Z" }, -] - -[[package]] -name = "iniconfig" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, -] - [[package]] name = "iniconfig" version = "2.3.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, @@ -231,31 +163,23 @@ wheels = [ [[package]] name = "isort" -version = "6.1.0" +version = "7.0.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -dependencies = [ - { name = "importlib-metadata", marker = "python_full_version < '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1e/82/fa43935523efdfcce6abbae9da7f372b627b27142c3419fcf13bf5b0c397/isort-6.1.0.tar.gz", hash = "sha256:9b8f96a14cfee0677e78e941ff62f03769a06d412aabb9e2a90487b3b7e8d481", size = 824325, upload-time = "2025-10-01T16:26:45.027Z" } +sdist = { url = "https://files.pythonhosted.org/packages/63/53/4f3c058e3bace40282876f9b553343376ee687f3c35a525dc79dbd450f88/isort-7.0.0.tar.gz", hash = "sha256:5513527951aadb3ac4292a41a16cbc50dd1642432f5e8c20057d414bdafb4187", size = 805049, upload-time = "2025-10-11T13:30:59.107Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/cc/9b681a170efab4868a032631dea1e8446d8ec718a7f657b94d49d1a12643/isort-6.1.0-py3-none-any.whl", hash = "sha256:58d8927ecce74e5087aef019f778d4081a3b6c98f15a80ba35782ca8a2097784", size = 94329, upload-time = "2025-10-01T16:26:43.291Z" }, + { url = "https://files.pythonhosted.org/packages/7f/ed/e3705d6d02b4f7aea715a353c8ce193efd0b5db13e204df895d38734c244/isort-7.0.0-py3-none-any.whl", hash = "sha256:1bcabac8bc3c36c7fb7b98a76c8abb18e0f841a3ba81decac7691008592499c1", size = 94672, upload-time = "2025-10-11T13:30:57.665Z" }, ] [[package]] -name = "isort" -version = "7.0.0" +name = "markdown-it-py" +version = "4.0.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", +dependencies = [ + { name = "mdurl" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/63/53/4f3c058e3bace40282876f9b553343376ee687f3c35a525dc79dbd450f88/isort-7.0.0.tar.gz", hash = "sha256:5513527951aadb3ac4292a41a16cbc50dd1642432f5e8c20057d414bdafb4187", size = 805049, upload-time = "2025-10-11T13:30:59.107Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/f5/4ec618ed16cc4f8fb3b701563655a69816155e79e24a17b651541804721d/markdown_it_py-4.0.0.tar.gz", hash = "sha256:cb0a2b4aa34f932c007117b194e945bd74e0ec24133ceb5bac59009cda1cb9f3", size = 73070, upload-time = "2025-08-11T12:57:52.854Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/ed/e3705d6d02b4f7aea715a353c8ce193efd0b5db13e204df895d38734c244/isort-7.0.0-py3-none-any.whl", hash = "sha256:1bcabac8bc3c36c7fb7b98a76c8abb18e0f841a3ba81decac7691008592499c1", size = 94672, upload-time = "2025-10-11T13:30:57.665Z" }, + { url = "https://files.pythonhosted.org/packages/94/54/e7d793b573f298e1c9013b8c4dade17d481164aa517d1d7148619c2cedbf/markdown_it_py-4.0.0-py3-none-any.whl", hash = "sha256:87327c59b172c5011896038353a81343b6754500a08cd7a4973bb48c6d578147", size = 87321, upload-time = "2025-08-11T12:57:51.923Z" }, ] [[package]] @@ -267,6 +191,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, ] +[[package]] +name = "mdurl" +version = "0.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d6/54/cfe61301667036ec958cb99bd3efefba235e65cdeb9c84d24a8293ba1d90/mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba", size = 8729, upload-time = "2022-08-14T12:40:10.846Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, +] + [[package]] name = "mypy-extensions" version = "1.1.0" @@ -316,27 +249,10 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/45/07/5fd183858dff4d24840f07fc845f213cd371a19958558607ba22035dadd7/pg8000-1.31.5-py3-none-any.whl", hash = "sha256:0af2c1926b153307639868d2ee5cef6cd3a7d07448e12736989b10e1d491e201", size = 57816, upload-time = "2025-09-14T09:16:47.798Z" }, ] -[[package]] -name = "platformdirs" -version = "4.4.0" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -sdist = { url = "https://files.pythonhosted.org/packages/23/e8/21db9c9987b0e728855bd57bff6984f67952bea55d6f75e055c46b5383e8/platformdirs-4.4.0.tar.gz", hash = "sha256:ca753cf4d81dc309bc67b0ea38fd15dc97bc30ce419a7f58d13eb3bf14c4febf", size = 21634, upload-time = "2025-08-26T14:32:04.268Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/40/4b/2028861e724d3bd36227adfa20d3fd24c3fc6d52032f4a93c133be5d17ce/platformdirs-4.4.0-py3-none-any.whl", hash = "sha256:abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85", size = 18654, upload-time = "2025-08-26T14:32:02.735Z" }, -] - [[package]] name = "platformdirs" version = "4.5.0" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", -] sdist = { url = "https://files.pythonhosted.org/packages/61/33/9611380c2bdb1225fdef633e2a9610622310fed35ab11dac9620972ee088/platformdirs-4.5.0.tar.gz", hash = "sha256:70ddccdd7c99fc5942e9fc25636a8b34d04c24b335100223152c2803e4063312", size = 21632, upload-time = "2025-10-08T17:44:48.791Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/73/cb/ac7874b3e5d58441674fb70742e6c374b28b0c7cb988d37d991cde47166c/platformdirs-4.5.0-py3-none-any.whl", hash = "sha256:e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3", size = 18651, upload-time = "2025-10-08T17:44:47.223Z" }, @@ -351,6 +267,63 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] +[[package]] +name = "pyarrow" +version = "24.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a5/bf/a34fee1d624152124fa8355c42f34195ad5fe5233ce5bb87946432047d52/pyarrow-24.0.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:7c2b98645d576a0b9616892ead22b64a83a5f043c5e2ca15ebcefcb5b70c80cb", size = 35076681, upload-time = "2026-04-21T08:51:46.845Z" }, + { url = "https://files.pythonhosted.org/packages/1d/41/64180033d7027afce12dc96d0fe1f504c6fa112190582b458acea2399530/pyarrow-24.0.0-cp310-cp310-macosx_12_0_x86_64.whl", hash = "sha256:644a246325b8c69c595ad1dd4b463eba4b0cdb731370e4a86137d433208d6147", size = 36684260, upload-time = "2026-04-21T08:51:53.642Z" }, + { url = "https://files.pythonhosted.org/packages/57/02/9b9320e673dd8a99411fac78690f3df92f6dd6f59754c750110bca66d64e/pyarrow-24.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:3a577bd840ca83f646f0a625dbc571dba7044c43c2d1503afc378b570954345c", size = 45698566, upload-time = "2026-04-21T10:46:02.133Z" }, + { url = "https://files.pythonhosted.org/packages/67/33/f75e91b9a64c3f33c787e263c93b871ad91b8a4a68c1d5cebddd9840e835/pyarrow-24.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:e3268e43984d0b1a185c89b4cfff282a7ead12fc93f56cfd7088bdbcbe727041", size = 48835562, upload-time = "2026-04-21T10:46:10.278Z" }, + { url = "https://files.pythonhosted.org/packages/a5/63/097510448e47e4091faa41c43ba92f97cecaab8f4535b56a3d149578f634/pyarrow-24.0.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:2392d954fcb920f42d230284b677605e4e2fbb11f2821e823e642abd67fbb491", size = 49394997, upload-time = "2026-04-21T10:46:18.08Z" }, + { url = "https://files.pythonhosted.org/packages/60/6b/c047d6222ab279024a062742d1807e2fbaf27bba88a98637299ff47b9236/pyarrow-24.0.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:bec9373df11544592b0ba7ec2af0e35059e5f0e7647c6183a854dedd193298f1", size = 51911424, upload-time = "2026-04-21T10:46:25.347Z" }, + { url = "https://files.pythonhosted.org/packages/3a/ba/464cc70761c2a525d97ebd84e21c31ebd47f3ef4bdcee117009f51c46f24/pyarrow-24.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:c42ab9439498270139cc63e18847a02afe5c8b3ed9c931266533cfe378bd3591", size = 27251730, upload-time = "2026-04-21T10:46:30.913Z" }, + { url = "https://files.pythonhosted.org/packages/62/c9/a47ab7ece0d86cbe6678418a0fbd1ac4bb493b9184a3891dfa0e7f287ae0/pyarrow-24.0.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:b0e131f880cda8d04e076cee175a46fc0e8bc8b65c99c6c09dff6669335fde74", size = 35068898, upload-time = "2026-04-21T10:46:36.599Z" }, + { url = "https://files.pythonhosted.org/packages/d1/bc/8db86617a9a58008acf8913d6fed68ea2a46acb6de928db28d724c891a68/pyarrow-24.0.0-cp311-cp311-macosx_12_0_x86_64.whl", hash = "sha256:1b2fe7f9a5566401a0ef2571f197eb92358925c1f0c8dba305d6e43ea0871bb3", size = 36679915, upload-time = "2026-04-21T10:46:42.602Z" }, + { url = "https://files.pythonhosted.org/packages/eb/8e/fb178720400ef69db251eb4a9c3ccf4af269bc1feb5055529b8fc87170d1/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:0b3537c00fb8d384f15ac1e79b6eb6db04a16514c8c1d22e59a9b95c8ba42868", size = 45697931, upload-time = "2026-04-21T10:46:48.403Z" }, + { url = "https://files.pythonhosted.org/packages/f3/27/99c42abe8e21b44f4917f62631f3aa31404882a2c41d8a4cd5c110e13d52/pyarrow-24.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:14e31a3c9e35f1ab6356c6378f6f72830e6d2d5f1791df3774a7b097d18a6a1e", size = 48837449, upload-time = "2026-04-21T10:46:55.329Z" }, + { url = "https://files.pythonhosted.org/packages/36/b6/333749e2666e9032891125bf9c691146e92901bece62030ac1430e2e7c88/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:b7d9a514e73bc42711e6a35aaccf3587c520024fe0a25d830a1a8a27c15f4f57", size = 49395949, upload-time = "2026-04-21T10:47:01.869Z" }, + { url = "https://files.pythonhosted.org/packages/17/25/c5201706a2dd374e8ba6ee3fd7a8c89fb7ffc16eed5217a91fd2bd7f7626/pyarrow-24.0.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b196eb3f931862af3fa84c2a253514d859c08e0d8fe020e07be12e75a5a9780c", size = 51912986, upload-time = "2026-04-21T10:47:09.872Z" }, + { url = "https://files.pythonhosted.org/packages/f8/d2/4d1bbba65320b21a49678d6fbdc6ff7c649251359fdcfc03568c4136231d/pyarrow-24.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:35405aecb474e683fb36af650618fd5340ee5471fc65a21b36076a18bbc6c981", size = 27255371, upload-time = "2026-04-21T10:47:15.943Z" }, + { url = "https://files.pythonhosted.org/packages/b4/a9/9686d9f07837f91f775e8932659192e02c74f9d8920524b480b85212cc68/pyarrow-24.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:6233c9ed9ab9d1db47de57d9753256d9dcffbf42db341576099f0fd9f6bf4810", size = 34981559, upload-time = "2026-04-21T10:47:22.17Z" }, + { url = "https://files.pythonhosted.org/packages/80/b6/0ddf0e9b6ead3474ab087ae598c76b031fc45532bf6a63f3a553440fb258/pyarrow-24.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:f7616236ec1bc2b15bfdec22a71ab38851c86f8f05ff64f379e1278cf20c634a", size = 36663654, upload-time = "2026-04-21T10:47:28.315Z" }, + { url = "https://files.pythonhosted.org/packages/7c/3b/926382efe8ce27ba729071d3566ade6dfb86bdf112f366000196b2f5780a/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:1617043b99bd33e5318ae18eb2919af09c71322ef1ca46566cdafc6e6712fb66", size = 45679394, upload-time = "2026-04-21T10:47:34.821Z" }, + { url = "https://files.pythonhosted.org/packages/b3/7a/829f7d9dfd37c207206081d6dad474d81dde29952401f07f2ba507814818/pyarrow-24.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6165461f55ef6314f026de6638d661188e3455d3ec49834556a0ebbdbace18bb", size = 48863122, upload-time = "2026-04-21T10:47:42.056Z" }, + { url = "https://files.pythonhosted.org/packages/5f/e8/f88ce625fe8babaae64e8db2d417c7653adb3019b08aae85c5ed787dc816/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b13dedfe76a0ad2d1d859b0811b53827a4e9d93a0bcb05cf59333ab4980cc7e", size = 49376032, upload-time = "2026-04-21T10:47:48.967Z" }, + { url = "https://files.pythonhosted.org/packages/36/7a/82c363caa145fff88fb475da50d3bf52bb024f61917be5424c3392eaf878/pyarrow-24.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:25ea65d868eb04015cd18e6df2fbe98f07e5bda2abefabcb88fce39a947716f6", size = 51929490, upload-time = "2026-04-21T10:47:55.981Z" }, + { url = "https://files.pythonhosted.org/packages/66/1c/e3e72c8014ad2743ca64a701652c733cc5cbcee15c0463a32a8c55518d9e/pyarrow-24.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:295f0a7f2e242dabd513737cf076007dc5b2d59237e3eca37b05c0c6446f3826", size = 27355660, upload-time = "2026-04-21T10:48:01.718Z" }, + { url = "https://files.pythonhosted.org/packages/6f/d3/a1abf004482026ddc17f4503db227787fa3cfe41ec5091ff20e4fea55e57/pyarrow-24.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:02b001b3ed4723caa44f6cd1af2d5c86aa2cf9971dacc2ffa55b21237713dfba", size = 34976759, upload-time = "2026-04-21T10:48:07.258Z" }, + { url = "https://files.pythonhosted.org/packages/4f/4a/34f0a36d28a2dd32225301b79daad44e243dc1a2bb77d43b60749be255c4/pyarrow-24.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:04920d6a71aabd08a0417709efce97d45ea8e6fb733d9ca9ecffb13c67839f68", size = 36658471, upload-time = "2026-04-21T10:48:13.347Z" }, + { url = "https://files.pythonhosted.org/packages/1f/78/543b94712ae8bb1a6023bcc1acf1a740fbff8286747c289cd9468fced2a5/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a964266397740257f16f7bb2e4f08a0c81454004beab8ff59dd531b73610e9f2", size = 45675981, upload-time = "2026-04-21T10:48:20.201Z" }, + { url = "https://files.pythonhosted.org/packages/84/9f/8fb7c222b100d314137fa40ec050de56cd8c6d957d1cfff685ce72f15b17/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f066b179d68c413374294bc1735f68475457c933258df594443bb9d88ddc2a0", size = 48859172, upload-time = "2026-04-21T10:48:27.541Z" }, + { url = "https://files.pythonhosted.org/packages/a7/d3/1ea72538e6c8b3b475ed78d1049a2c518e655761ea50fe1171fc855fcab7/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1183baeb14c5f587b1ec52831e665718ce632caab84b7cd6b85fd44f96114495", size = 49385733, upload-time = "2026-04-21T10:48:34.7Z" }, + { url = "https://files.pythonhosted.org/packages/c3/be/c3d8b06a1ba35f2260f8e1f771abbee7d5e345c0937aab90675706b1690a/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:806f24b4085453c197a5078218d1ee08783ebbba271badd153d1ae22a3ee804f", size = 51934335, upload-time = "2026-04-21T10:48:42.099Z" }, + { url = "https://files.pythonhosted.org/packages/9c/62/89e07a1e7329d2cde3e3c6994ba0839a24977a2beda8be6005ea3d860b99/pyarrow-24.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:e4505fc6583f7b05ab854934896bcac8253b04ac1171a77dfb73efef92076d91", size = 27271748, upload-time = "2026-04-21T10:49:42.532Z" }, + { url = "https://files.pythonhosted.org/packages/17/1a/cff3a59f80b5b1658549d46611b67163f65e0664431c076ad728bf9d5af4/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1a4e45017efbf115032e4475ee876d525e0e36c742214fbe405332480ecd6275", size = 35238554, upload-time = "2026-04-21T10:48:48.526Z" }, + { url = "https://files.pythonhosted.org/packages/a8/99/cce0f42a327bfef2c420fb6078a3eb834826e5d6697bf3009fe11d2ad051/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:7986f1fa71cee060ad00758bcc79d3a93bab8559bf978fab9e53472a2e25a17b", size = 36782301, upload-time = "2026-04-21T10:48:55.181Z" }, + { url = "https://files.pythonhosted.org/packages/2a/66/8e560d5ff6793ca29aca213c53eec0dd482dd46cb93b2819e5aab52e4252/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d3e0b61e8efb24ed38898e5cdc5fffa9124be480008d401a1f8071500494ae42", size = 45721929, upload-time = "2026-04-21T10:49:03.676Z" }, + { url = "https://files.pythonhosted.org/packages/27/0c/a26e25505d030716e078d9f16eb74973cbf0b33b672884e9f9da1c83b871/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:55a3bc1e3df3b5567b7d27ef551b2283f0c68a5e86f1cd56abc569da4f31335b", size = 48825365, upload-time = "2026-04-21T10:49:11.714Z" }, + { url = "https://files.pythonhosted.org/packages/5f/eb/771f9ecb0c65e73fe9dccdd1717901b9594f08c4515d000c7c62df573811/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:641f795b361874ac9da5294f8f443dfdbee355cf2bd9e3b8d97aaac2306b9b37", size = 49451819, upload-time = "2026-04-21T10:49:21.474Z" }, + { url = "https://files.pythonhosted.org/packages/48/da/61ae89a88732f5a785646f3ec6125dbb640fa98a540eb2b9889caa561403/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8adc8e6ce5fccf5dc707046ae4914fd537def529709cc0d285d37a7f9cd442ca", size = 51909252, upload-time = "2026-04-21T10:49:31.164Z" }, + { url = "https://files.pythonhosted.org/packages/cb/1a/8dd5cafab7b66573fa91c03d06d213356ad4edd71813aa75e08ce2b3a844/pyarrow-24.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:9b18371ad2f44044b81a8d23bc2d8a9b6a6226dca775e8e16cfee640473d6c5d", size = 27388127, upload-time = "2026-04-21T10:49:37.334Z" }, + { url = "https://files.pythonhosted.org/packages/ad/80/d022a34ff05d2cbedd8ccf841fc1f532ecfa9eb5ed1711b56d0e0ea71fc9/pyarrow-24.0.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:1cc9057f0319e26333b357e17f3c2c022f1a83739b48a88b25bfd5fa2dc18838", size = 35007997, upload-time = "2026-04-21T10:49:48.796Z" }, + { url = "https://files.pythonhosted.org/packages/1a/ff/f01485fda6f4e5d441afb8dd5e7681e4db18826c1e271852f5d3957d6a80/pyarrow-24.0.0-cp314-cp314-macosx_12_0_x86_64.whl", hash = "sha256:e6f1278ee4785b6db21229374a1c9e54ec7c549de5d1efc9630b6207de7e170b", size = 36678720, upload-time = "2026-04-21T10:49:55.858Z" }, + { url = "https://files.pythonhosted.org/packages/9e/c2/2d2d5fea814237923f71b36495211f20b43a1576f9a4d6da7e751a64ec6f/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:adbbedc55506cbdabb830890444fb856bfb0060c46c6f8026c6c2f2cf86ae795", size = 45741852, upload-time = "2026-04-21T10:50:04.624Z" }, + { url = "https://files.pythonhosted.org/packages/8e/3a/28ba9c1c1ebdbb5f1b94dfebb46f207e52e6a554b7fe4132540fde29a3a0/pyarrow-24.0.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:ae8a1145af31d903fa9bb166824d7abe9b4681a000b0159c9fb99c11bc11ad26", size = 48889852, upload-time = "2026-04-21T10:50:12.293Z" }, + { url = "https://files.pythonhosted.org/packages/df/51/4a389acfd31dca009f8fb82d7f510bb4130f2b3a8e18cf00194d0687d8ac/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:d7027eba1df3b2069e2e8d80f644fa0918b68c46432af3d088ddd390d063ecde", size = 49445207, upload-time = "2026-04-21T10:50:20.677Z" }, + { url = "https://files.pythonhosted.org/packages/19/4b/0bab2b23d2ae901b1b9a03c0efd4b2d070256f8ce3fc43f6e58c167b2081/pyarrow-24.0.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e56a1ffe9bf7b727432b89104cc0849c21582949dd7bdcb34f17b2001a351a76", size = 51954117, upload-time = "2026-04-21T10:50:29.14Z" }, + { url = "https://files.pythonhosted.org/packages/29/88/f4e9145da0417b3d2c12035a8492b35ff4a3dbc653e614fcfb51d9dedb38/pyarrow-24.0.0-cp314-cp314-win_amd64.whl", hash = "sha256:38be1808cdd068605b787e6ca9119b27eb275a0234e50212c3492331680c3b1e", size = 28001155, upload-time = "2026-04-21T10:51:22.337Z" }, + { url = "https://files.pythonhosted.org/packages/79/4f/46a49a63f43526da895b1a45bbb51d5baf8e4d77159f8528fc3e5490007f/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:418e48ce50a45a6a6c73c454677203a9c75c966cb1e92ca3370959185f197a05", size = 35250387, upload-time = "2026-04-21T10:50:35.552Z" }, + { url = "https://files.pythonhosted.org/packages/a0/da/d5e0cd5ef00796922404806d5f00325cdadc3441ce2c13fe7115f2df9a64/pyarrow-24.0.0-cp314-cp314t-macosx_12_0_x86_64.whl", hash = "sha256:2f16197705a230a78270cdd4ea8a1d57e86b2fdcbc34a1f6aebc72e65c986f9a", size = 36797102, upload-time = "2026-04-21T10:50:42.417Z" }, + { url = "https://files.pythonhosted.org/packages/34/c7/5904145b0a593a05236c882933d439b5720f0a145381179063722fbfc123/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:fb24ac194bfc5e86839d7dcd52092ee31e5fe6733fe11f5e3b06ef0812b20072", size = 45745118, upload-time = "2026-04-21T10:50:49.324Z" }, + { url = "https://files.pythonhosted.org/packages/13/d3/cca42fe166d1c6e4d5b80e530b7949104d10e17508a90ae202dac205ce2a/pyarrow-24.0.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:9700ebd9a51f5895ce75ff4ac4b3c47a7d4b42bc618be8e713e5d56bacf5f931", size = 48844765, upload-time = "2026-04-21T10:50:55.579Z" }, + { url = "https://files.pythonhosted.org/packages/b0/49/942c3b79878ba928324d1e17c274ed84581db8c0a749b24bcf4cbdf15bd3/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:d8ddd2768da81d3ee08cfea9b597f4abb4e8e1dc8ae7e204b608d23a0d3ab699", size = 49471890, upload-time = "2026-04-21T10:51:02.439Z" }, + { url = "https://files.pythonhosted.org/packages/76/97/ff71431000a75d84135a1ace5ca4ba11726a231a8007bbb320a4c54075d5/pyarrow-24.0.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:61a3d7eaa97a14768b542f3d284dc6400dd2470d9f080708b13cd46b6ae18136", size = 51932250, upload-time = "2026-04-21T10:51:10.576Z" }, + { url = "https://files.pythonhosted.org/packages/51/be/6f79d55816d5c22557cf27533543d5d70dfe692adfbee4b99f2760674f38/pyarrow-24.0.0-cp314-cp314t-win_amd64.whl", hash = "sha256:c91d00057f23b8d353039520dc3a6c09d8608164c692e9f59a175a42b2ae0c19", size = 28131282, upload-time = "2026-04-21T10:51:16.815Z" }, +] + [[package]] name = "pygments" version = "2.19.2" @@ -360,91 +333,37 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] -[[package]] -name = "pylint" -version = "3.3.9" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -dependencies = [ - { name = "astroid", version = "3.3.11", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, - { name = "dill", marker = "python_full_version < '3.10'" }, - { name = "isort", version = "6.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "mccabe", marker = "python_full_version < '3.10'" }, - { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "tomli", marker = "python_full_version < '3.10'" }, - { name = "tomlkit", marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/04/9d/81c84a312d1fa8133b0db0c76148542a98349298a01747ab122f9314b04e/pylint-3.3.9.tar.gz", hash = "sha256:d312737d7b25ccf6b01cc4ac629b5dcd14a0fcf3ec392735ac70f137a9d5f83a", size = 1525946, upload-time = "2025-10-05T18:41:43.786Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/a7/69460c4a6af7575449e615144aa2205b89408dc2969b87bc3df2f262ad0b/pylint-3.3.9-py3-none-any.whl", hash = "sha256:01f9b0462c7730f94786c283f3e52a1fbdf0494bbe0971a78d7277ef46a751e7", size = 523465, upload-time = "2025-10-05T18:41:41.766Z" }, -] - [[package]] name = "pylint" version = "4.0.4" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", -] dependencies = [ - { name = "astroid", version = "4.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, - { name = "dill", marker = "python_full_version >= '3.10'" }, - { name = "isort", version = "7.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "mccabe", marker = "python_full_version >= '3.10'" }, - { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "tomli", marker = "python_full_version == '3.10.*'" }, - { name = "tomlkit", marker = "python_full_version >= '3.10'" }, + { name = "astroid" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "dill" }, + { name = "isort" }, + { name = "mccabe" }, + { name = "platformdirs" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "tomlkit" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5a/d2/b081da1a8930d00e3fc06352a1d449aaf815d4982319fab5d8cdb2e9ab35/pylint-4.0.4.tar.gz", hash = "sha256:d9b71674e19b1c36d79265b5887bf8e55278cbe236c9e95d22dc82cf044fdbd2", size = 1571735, upload-time = "2025-11-30T13:29:04.315Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/a6/92/d40f5d937517cc489ad848fc4414ecccc7592e4686b9071e09e64f5e378e/pylint-4.0.4-py3-none-any.whl", hash = "sha256:63e06a37d5922555ee2c20963eb42559918c20bd2b21244e4ef426e7c43b92e0", size = 536425, upload-time = "2025-11-30T13:29:02.53Z" }, ] -[[package]] -name = "pytest" -version = "8.4.2" -source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version < '3.10'", -] -dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, - { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "packaging", marker = "python_full_version < '3.10'" }, - { name = "pluggy", marker = "python_full_version < '3.10'" }, - { name = "pygments", marker = "python_full_version < '3.10'" }, - { name = "tomli", marker = "python_full_version < '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, -] - [[package]] name = "pytest" version = "9.0.1" source = { registry = "https://pypi.org/simple" } -resolution-markers = [ - "python_full_version >= '3.12'", - "python_full_version == '3.11.*'", - "python_full_version == '3.10.*'", -] dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, - { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, - { name = "iniconfig", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "packaging", marker = "python_full_version >= '3.10'" }, - { name = "pluggy", marker = "python_full_version >= '3.10'" }, - { name = "pygments", marker = "python_full_version >= '3.10'" }, - { name = "tomli", marker = "python_full_version == '3.10.*'" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/07/56/f013048ac4bc4c1d9be45afd4ab209ea62822fb1598f40687e6bf45dcea4/pytest-9.0.1.tar.gz", hash = "sha256:3e9c069ea73583e255c3b21cf46b8d3c56f6e3a1a8f6da94ccb0fcf57b9d73c8", size = 1564125, upload-time = "2025-11-12T13:05:09.333Z" } wheels = [ @@ -534,15 +453,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, - { url = "https://files.pythonhosted.org/packages/9f/62/67fc8e68a75f738c9200422bf65693fb79a4cd0dc5b23310e5202e978090/pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da", size = 184450, upload-time = "2025-09-25T21:33:00.618Z" }, - { url = "https://files.pythonhosted.org/packages/ae/92/861f152ce87c452b11b9d0977952259aa7df792d71c1053365cc7b09cc08/pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917", size = 174319, upload-time = "2025-09-25T21:33:02.086Z" }, - { url = "https://files.pythonhosted.org/packages/d0/cd/f0cfc8c74f8a030017a2b9c771b7f47e5dd702c3e28e5b2071374bda2948/pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9", size = 737631, upload-time = "2025-09-25T21:33:03.25Z" }, - { url = "https://files.pythonhosted.org/packages/ef/b2/18f2bd28cd2055a79a46c9b0895c0b3d987ce40ee471cecf58a1a0199805/pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5", size = 836795, upload-time = "2025-09-25T21:33:05.014Z" }, - { url = "https://files.pythonhosted.org/packages/73/b9/793686b2d54b531203c160ef12bec60228a0109c79bae6c1277961026770/pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a", size = 750767, upload-time = "2025-09-25T21:33:06.398Z" }, - { url = "https://files.pythonhosted.org/packages/a9/86/a137b39a611def2ed78b0e66ce2fe13ee701a07c07aebe55c340ed2a050e/pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926", size = 727982, upload-time = "2025-09-25T21:33:08.708Z" }, - { url = "https://files.pythonhosted.org/packages/dd/62/71c27c94f457cf4418ef8ccc71735324c549f7e3ea9d34aba50874563561/pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7", size = 755677, upload-time = "2025-09-25T21:33:09.876Z" }, - { url = "https://files.pythonhosted.org/packages/29/3d/6f5e0d58bd924fb0d06c3a6bad00effbdae2de5adb5cda5648006ffbd8d3/pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0", size = 142592, upload-time = "2025-09-25T21:33:10.983Z" }, - { url = "https://files.pythonhosted.org/packages/f0/0c/25113e0b5e103d7f1490c0e947e303fe4a696c10b501dea7a9f49d4e876c/pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007", size = 158777, upload-time = "2025-09-25T21:33:15.55Z" }, +] + +[[package]] +name = "rich" +version = "15.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markdown-it-py" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c0/8f/0722ca900cc807c13a6a0c696dacf35430f72e0ec571c4275d2371fca3e9/rich-15.0.0.tar.gz", hash = "sha256:edd07a4824c6b40189fb7ac9bc4c52536e9780fbbfbddf6f1e2502c31b068c36", size = 230680, upload-time = "2026-04-12T08:24:00.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/82/3b/64d4899d73f91ba49a8c18a8ff3f0ea8f1c1d75481760df8c68ef5235bf5/rich-15.0.0-py3-none-any.whl", hash = "sha256:33bd4ef74232fb73fe9279a257718407f169c09b78a87ad3d296f548e27de0bb", size = 310654, upload-time = "2026-04-12T08:24:02.83Z" }, ] [[package]] @@ -634,10 +557,10 @@ wheels = [ ] [[package]] -name = "zipp" -version = "3.23.0" +name = "xattr-compat" +version = "1.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } +sdist = { url = "https://files.pythonhosted.org/packages/75/82/5ffb78d69936e7148d6ee907dd44581b6478dbfc313cda32c122b264f505/xattr-compat-1.0.0.tar.gz", hash = "sha256:5ef02e523585276d78a4ebbc05f2d08f0ddc200116c847826b1a18d8aa71f5bc", size = 5300, upload-time = "2021-03-03T21:20:30.288Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, + { url = "https://files.pythonhosted.org/packages/06/51/4708370113789e0ab876c9bcb895a66fa116bb9e90e8940f6f6c932db5c2/xattr_compat-1.0.0-py3-none-any.whl", hash = "sha256:5fafcd360b274b362a46c88871473c26236a858071fbbe787a05f8df062899fa", size = 7130, upload-time = "2021-03-03T21:20:29.09Z" }, ] From e84f4de0eff551be880f12a0cc42e434bce49c84 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:43:17 -0400 Subject: [PATCH 11/49] Simplify folder methods, build recursion is responsibility of caller; relax pid check to match hashstore requirements; make find_object a public method --- src/hashstore/filehashstore.py | 347 +++++++++++++++------------------ 1 file changed, 153 insertions(+), 194 deletions(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 51e212ae..331c3807 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -19,6 +19,7 @@ import yaml +import hashstore.folderentry from hashstore import HashStore from hashstore.filehashstore_exceptions import ( CidRefsContentError, @@ -780,7 +781,7 @@ def retrieve_metadata(self, pid: str, format_id: Optional[str] = None) -> IO[byt return metadata_stream else: err_msg = f"No metadata found for pid: {pid}" - self.fhs_logger.warning(err_msg) + self.fhs_logger.warning(err_msg) raise KeyError(err_msg) def delete_object(self, pid: str) -> None: @@ -1055,219 +1056,172 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: logging.info(info_string) return hex_digest - def _store_container( - self, - pid:str, - name:str, - manifest:list[tuple[int, str, str]], - additional_algorithm: Optional[str] = None, - checksum: Optional[str] = None, - checksum_algorithm: Optional[str] = None, - ) -> "ObjectMetadata": - self.fhs_logger.debug(f"_store_container {pid=}") - manifest.sort(key=lambda x: (x[0], x[1])) - dest_stream = io.BytesIO() - dest_stream.name = name - dest_stream.write(f"container {len(manifest)}\n".encode("utf-8")) - for row in manifest: - dest_stream.write(f"{row[0]} {row[1]} {row[2]}\n".encode("utf-8")) - dest_stream.seek(0) - return self.store_object( - pid, - data=dest_stream, - additional_algorithm=additional_algorithm, - checksum=checksum, - checksum_algorithm=checksum_algorithm, - expected_object_size=None, - ) - - def _load_container(self, pid:str) -> list[tuple[int, str, str]]: - self.fhs_logger.debug(f"_load_container {pid=}") - obj_stream = self.retrieve_object(pid) - row = 0 - manifest = [] - with closing(obj_stream): - for line in obj_stream: - line = line.decode("utf-8").strip() - if row == 0: - if not line.startswith("container "): - msg = f"{pid} is not a container." - raise ValueError(msg) - else: - type_flag, cid, name = line.split(" ", 2) - manifest.append((int(type_flag), cid, name)) - row += 1 - return manifest - def store_folder( self, pid: str, - root_path: Union[str, Path], - child_path: Optional[Union[str, Path]] = None, + path: str, + entries: hashstore.folderentry.FolderEntries, additional_algorithm: Optional[str] = None, checksum: Optional[str] = None, checksum_algorithm: Optional[str] = None, - expected_object_size: Optional[int] = None, - pattern: Optional[str] = None, + verify_entry_cids: bool = True, ) -> Optional["ObjectMetadata"]: - """Store a folder (and subfolders) as container objects. + """Store a folder object. - Traverses the folder hierarchy in a depth first manner so that - the leaf elements are computed for inclusion in parent hashes. + A Folder is a list of entries that appear in a folder. Each entry + may be a file or a Folder. This method is used instead of store_object + because Folders have special requirements to ensure deterministic serialization. + + The Folder is tagged with an identifier that is "{PID} {path}", that is, the + PID followed by a single space, then the path. If the path portion is an empty + string, ".", or "/" then the Folder is the root Folder. + + Note that since the hash of a Folder is computed from hashes of its content, + a Folder hierarchy must be stored starting with the leaves. This method + will raise a ValueError if the hash of an entry does not already exist in + the hashstore. Hence the general pattern for storing a folder hierarchy is + to do a depth first traversal of the hierarchy, storing the files (ensuring + their hashes are available) and computing the hash for the containing folder + for use in the parent folder reference to the child. Args: pid (str): The context within which this folder is being stored - root_path (str): Path to the root of the folder. - child_path (str): Path to folder being stored relative to the root_path. If None, assumes root_path. - + path (str): Path to this folder relative to the root. + entries (list[FolderEntry]): A list of FolderEntry objects. + verify_entry_cids: If True then FolderEntry CID values are + verified to to ensure they exist in the hashstore. + Returns: - str: CID for the container + ObjectMetadata: The computed ObjectMetadata for this entry. """ - raise NotImplementedError - # Get the relative path for the object / folder - root_path = Path(root_path) - if child_path is None: - child_path = root_path - else: - child_path = Path(child_path) - relative_path = child_path.relative_to(root_path) - path_pid = pid - container_name = "root" - if str(relative_path) != ".": - path_pid = f"{pid}/{relative_path}" - container_name = str(relative_path) - - self.fhs_logger.debug("store_folder {path_pid=}") - # Check if this container already exists - try: - # resolve pid, path to CID. This raises if not found - _entry = self._find_object(path_pid) - size = os.path.getsize( - self._build_hashstore_data_object_path(_entry["cid"]) - ) - return ObjectMetadata( - pid=path_pid, cid=_entry["cid"], obj_size=size, hex_digests={} - ) - except PidNotFoundInCidRefsFile: - pass - except PidRefsDoesNotExist: - pass + if path in ("", ".", "/"): + path = "" + folder_pid = f"{pid} {path}" if path != "" else pid + self._check_string(folder_pid, "PID") + if verify_entry_cids: + # check that each entry CID is present in the hashstore. + for entry in entries: + if not self._exists("objects", entry.cid): + raise ValueError( + f"object {entry.name} cid {entry.cid} does not exist." + ) + # Sort the entries by cid + entries.sort(key=lambda entry: entry.cid) - # Container doesn't exist - manifest = [] - for item in child_path.iterdir(): - if item.is_dir(): - meta = self.store_folder( - pid, - root_path, - child_path=item.absolute(), - additional_algorithm=additional_algorithm, - checksum=checksum, - checksum_algorithm=checksum_algorithm, - expected_object_size=expected_object_size, - pattern=pattern, - ) - if meta is not None: - manifest.append((0, meta.cid, item.name)) - elif pattern is None and item.is_file(): - # If no pattern then grab all files, otherwise defer to - # globbing match later. - item_pid = f"{pid}/{item.relative_to(root_path)}" - self.fhs_logger.debug("store_folder {item_pid=}") - meta = self.store_object(item_pid, str(item.absolute())) - manifest.append((1, meta.cid, item.name)) - if pattern is not None: - # globbing pattern was specified. Grab the matching files here. - for item in child_path.glob(pattern): - if item.is_file(): - item_pid = f"{pid}/{item.relative_to(root_path)}" - meta = self.store_object(item_pid, str(item.absolute())) - manifest.append((1, meta.cid, item.name)) - if len(manifest) == 0: - return None - return self._store_container( - path_pid, - container_name, - manifest, - additional_algorithm=additional_algorithm, - checksum=checksum, - checksum_algorithm=checksum_algorithm - ) - - def folder_content(self, pid:str, depth:int=0, depth_first:bool=True) -> Generator: - """Yield the content of a folder, breadth first, recursively. - (depth, type, CID, name) - """ - # Retrieve the container object - manifest = self._load_container(pid) - if depth_first: - for entry in manifest: - if entry[0] == 0: - yield (depth, entry[0], entry[1], f"{pid}/{entry[2]}") - yield from self.folder_content(f"{pid}/{entry[2]}", depth=depth+1, depth_first=depth_first) - else: - yield (depth, entry[0], entry[1], f"{pid}/{entry[2]}") - else: - for entry in manifest: - yield (depth, entry[0], entry[1], f"{pid}/{entry[2]}") - for entry in manifest: - if entry[0] == 0: - # folder, recurse - yield from self.folder_content(f"{pid}/{entry[2]}", depth=depth+1, depth_first=depth_first) + hash_algorithms = { + self.algorithm: hashlib.new(self.algorithm), + } + if additional_algorithm is not None: + hash_algorithms[additional_algorithm] = hashlib.new(additional_algorithm) + + # Compute the CID for the folder by iterating over the + # list of CIDs and updating the hash being computed. This + # is about as fast as serializing the cids to a big string + # but much more memory efficient. + for entry in entries: + v = entry.cid.encode("utf-8") + for fhasher in hash_algorithms.values(): + fhasher.update(v) + hex_digests = { + name: hasher.hexdigest() for name, hasher in hash_algorithms.items() + } + folder_cid = hex_digests[self.algorithm] + + # Compute the physical path for the computed CID + cid_path = self._build_hashstore_data_object_path(folder_cid) + # ensure the folder path exists + self._create_path(Path(os.path.dirname(cid_path))) + + # Store the entries to disk. Records are serialized as parquet + # which is an indeterminate file format. Hence the need for + # computing the hashes seperately to the actual bytes on disk. + # The parquet format is very efficient especially as the number + # of folder entries increases. + obj_size = entries.to_parquet(cid_path, pid=folder_pid) + self.tag_object(folder_pid, folder_cid) + return ObjectMetadata( + pid=folder_pid, + cid=folder_cid, + hex_digests=hex_digests, + obj_size=obj_size, + ) - def retrieve_folder(self, pid:str, destination_path:Union[str, Path], child_path:Optional[Union[str, Path]]=None): - """Retrieve a folder (and subfolders) stored as container objects. + def retrieve_folder( + self, + pid: str, + path: str, + ) -> hashstore.folderentry.FolderEntries: + """Retrieve a FolderEntries instance from the hashstore. + + We first check to see if a CID is available for the combination of + "{PID} {path}", and if so, return that entry. Otherwise, we iterate + over path segments to find the correspoding FolderEntry, if any. + This iterative approach is necesary if since entire trees are not + stored when a new version of a folder hierarchy is stored. Hence, it + may be necessary to jump back to a branch that is recorded in an + earlier version but not recorded in the current version since it + was unchanged between versions. Args: - pid (str): The context within which this folder is being retrieved - destination_path (str|Path): Path to the root of the folder to create. - child_path (str|Path): Path to folder being retrieved relative to the destination_path. If None, assumes destination_path. + pid (str): The context (i.e. VMDAG version) within which this folder is being retrieved + path (str): Path within the context to the desired entry Returns: - None + FolderEntries """ - # TODO: Error handling - # TODO: read access control considerations - destination_path = Path(destination_path) - if child_path is None: - child_path = Path("") - else: - child_path = Path(child_path) - path_pid = pid - if str(child_path) != ".": - path_pid = f"{pid} {child_path}" - - # Retrieve the container object - obj_stream = self.retrieve_object(path_pid) - with closing(obj_stream): - for line in obj_stream: - line = line.decode("utf-8").strip() - type_flag, cid, name = line.split(" ", 2) - if type_flag == "0": - # Directory - (destination_path / child_path / name).mkdir(parents=True, exist_ok=True) - self.retrieve_folder( - pid, - destination_path, - child_path=child_path / name, - ) - elif type_flag == "1": - # File - item_pid = f"{pid} {child_path / name}" - file_stream = self.retrieve_object(item_pid) - with closing(file_stream): - dest_file_path = destination_path / child_path / name - dest_file_path.parent.mkdir(parents=True, exist_ok=True) - with open(dest_file_path, "wb") as dest_file: - shutil.copyfileobj(file_stream, dest_file) - - def list_pids(self, pattern: Optional[str]=None) -> Generator: + if path in ("", ".", "/"): + path = "" + folder_pid = f"{pid} {path}" if path != "" else pid + self._check_string(folder_pid, "PID") + # try direct reference to CID using folder_pid + try: + object_info_dict = self._find_object(folder_pid) + folder_cid = object_info_dict.get("cid") + if folder_cid is None: + raise PidRefsDoesNotExist("Entry has no cid?") + cid_path = object_info_dict.get("cid_object_path") + # self._build_hashstore_data_object_path(folder_cid) + return hashstore.folderentry.FolderEntries.from_parquet(cid_path) + except PidRefsDoesNotExist: + pass + + # otherwise, get the root, split the path, and start iterating. + # This will raise PidRefsDoesNotExist if the root PID isn't there + object_info_dict = self._find_object(pid) + folder_cid = object_info_dict.get("cid") + if folder_cid is None: + # Should never reach this... + raise PidRefsDoesNotExist("Entry has no cid?") + cid_path = object_info_dict.get("cid_object_path") + # self._build_hashstore_data_object_path(folder_cid) + current_folder = hashstore.folderentry.FolderEntries.from_parquet(cid_path) + path_segments = path.split("/") + # iterate over segments, saving last + for name in path_segments: + entry = current_folder.entry_by_name(name) + if entry is None: + raise KeyError(f"PID {pid} {path} not found.") + if entry.type == hashstore.folderentry.FTYPE_FILE: + # it's a file! + raise ValueError(f"Path {path} is a file.") + object_info_dict = self._find_object(pid) + folder_cid = object_info_dict.get("cid") + if folder_cid is None: + # Should never reach this... + raise PidRefsDoesNotExist("Entry has no cid?") + cid_path = object_info_dict.get("cid_object_path") + # cid_path = self._build_hashstore_data_object_path(folder_cid) + current_folder = hashstore.folderentry.FolderEntries.from_parquet(cid_path) + return current_folder + + def list_pids(self, pattern: Optional[str] = None) -> Generator: rpattern = None if pattern is not None: rpattern = re.compile(pattern) - ignore_names = [".DS_Store", ] - for cid_entry in self.cids.rglob('*'): + ignore_names = [ + ".DS_Store", + ] + for cid_entry in self.cids.rglob("*"): if cid_entry.is_file() and cid_entry.name not in ignore_names: - self.fhs_logger.debug(str(cid_entry)) for _, entry in enumerate(open(cid_entry, "r", encoding="utf-8")): pid = entry.strip() if len(pid) > 0: @@ -1276,7 +1230,6 @@ def list_pids(self, pattern: Optional[str]=None) -> Generator: yield pid else: yield pid - # FileHashStore Core Methods @@ -3018,8 +2971,14 @@ def _check_integer(file_size: int) -> None: @staticmethod def _check_string(string: str, arg: str) -> None: - """Check whether a string is None or empty - or if it contains an illegal character; - throws an exception if so. + """Raises ValueError if string is empty or has leading or trailing whitespace. + + Note: This checks string for valid use as a PID or CID in hashstore. It is + the responsibility of the calling application to ensure the string is a + valid PID or CID in the context of the application. For example, hashstore + allows whitespace in PIDs though Metacat / DataONE does not. This allows + for storing a path segment after a Metacat PID which is needed for referencing + entries within a folder identified by a PID. :param str string: Value to check. :param str arg: Name of the argument to check. From 067955b97645a2b2756b1183a667b0a91662c82d Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:43:51 -0400 Subject: [PATCH 12/49] Add structure for folder entries --- src/hashstore/folderentry.py | 118 +++++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 src/hashstore/folderentry.py diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py new file mode 100644 index 00000000..6c5dfa30 --- /dev/null +++ b/src/hashstore/folderentry.py @@ -0,0 +1,118 @@ +"""Implements FolderEntry class.""" + +import collections.abc +import dataclasses +import json +import logging +import os + +import pyarrow +import pyarrow.parquet + +FTYPE_FOLDER = 0 +"""FolderEntry is for a FolderEntry""" +FTYPE_FILE = 1 +"""FolderEntry is for a file.""" +# TODO: Ratify this key +PARQUET_METADATA_KEY = b"https://ns.dataone.org/types/FolderEntries" +"""Key in parquet file metadata pointing to dict of properties.""" +PARQUET_READ_BATCH_SIZE = 10000 +"""Number of entries to read at a time from FolderEntries parquet file.""" + + +def get_logger(): + return logging.getLogger("FolderEntry") + + +@dataclasses.dataclass +class FolderEntry: + """Represents a file or folder entry in a folder manifest.""" + + name: str + """The name portion of the path (not full path) for the file or folder.""" + cid: str + """The content hash (CID) for the entry.""" + type: int # '1' for file, '0' for directory + """The type of manifest entry: '1' for file, '0' for directory.""" + size: int = 0 + """Size of the file in bytes or number of entries for directories.""" + formatid: str | None = None + """Optional format identifier for files.""" + + def __post_init__(self): + if self.type not in (FTYPE_FILE, FTYPE_FOLDER): + raise ValueError(f"Invalid type: {self.type}") + + def __repr__(self) -> str: + # Representation of a FolderEntry + return json.dumps( + { + "cid": self.cid, + "type": self.type, + "name": self.name, + "size": self.size, + "formatid": self.formatid, + }, + ensure_ascii=False, + ) + + +class FolderEntries(list[FolderEntry]): + def entry_by_name(self, name) -> FolderEntry | None: + """Find the entry with name that matches.""" + for entry in self: + if entry.name == name: + return entry + return None + + def to_parquet( + self, pq_path: str, pid: str | None = None, writer_args: dict = {} + ) -> int: + """Writes the list of folder entries to a parquet file. + + See also: https://arrow.apache.org/docs/python/generated/pyarrow.parquet.write_table.html + + #TODO: There are quite a few options for tweaking the written parquet file, + # e.g. with respect to column sorting, a UI may prefer sorting by type or formatid + + args: + pq_path: path to destination parquet file + pid: Optional PID+path used to create this folder. + writer_args: optional dict of arguments for the parquet writer. + """ + # Add some metadata to the parquet file to help identify it as a list of FolderEntries + pq_metadata = { + "type": "FolderEntries", + "version": "1.0", + "pid": pid, + } + metadata_bytes = json.dumps(pq_metadata).encode("utf-8") + table = pyarrow.Table.from_pylist([dataclasses.asdict(entry) for entry in self]) + table = table.replace_schema_metadata({PARQUET_METADATA_KEY: metadata_bytes}) + pyarrow.parquet.write_table(table, pq_path, **writer_args) + return os.path.getsize(pq_path) + + @classmethod + def from_parquet(cls, pq_path) -> "FolderEntries": + """Create an instance of FolderEntries from a parquet source.""" + pq_metadata = pyarrow.parquet.read_metadata(pq_path) + try: + metadata = json.loads(pq_metadata.metadata[PARQUET_METADATA_KEY].decode()) + _ = metadata["version"] + except KeyError: + raise ValueError(f"File {pq_path} is not a FolderEntry list.") + + pq_file = pyarrow.parquet.ParquetFile(pq_path) + entries = cls() + for batch in pq_file.iter_batches(batch_size=PARQUET_READ_BATCH_SIZE): + for row in batch.to_pylist(): + entries.append( + FolderEntry( + name=row["name"], + cid=row["cid"], + type=row["type"], + size=row["size"], + formatid=row["formatid"], + ) + ) + return entries From 45ef848ac2bd8b71a92c8112fea0427ffddac9f2 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:47:24 -0400 Subject: [PATCH 13/49] Make find_object less noisy for common and expected cases --- src/hashstore/filehashstore.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 331c3807..d4aa6c6c 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -613,7 +613,7 @@ def get_object_status(self, pid: str) -> dict: object_status_dict = {} try: - object_info_dict = self._find_object(pid) + object_info_dict = self.find_object(pid) object_cid = object_info_dict.get("cid") if object_cid: obj_path = self._get_hashstore_data_object_path(object_cid) @@ -742,7 +742,7 @@ def retrieve_object(self, pid: str) -> IO[bytes]: self.fhs_logger.debug("Request to retrieve object for pid: %s", pid) self._check_string(pid, "pid") - object_info_dict = self._find_object(pid) + object_info_dict = self.find_object(pid) object_cid = object_info_dict.get("cid") entity = "objects" @@ -801,7 +801,7 @@ def delete_object(self, pid: str) -> None: self._synchronize_object_locked_pids(pid) try: - object_info_dict = self._find_object(pid) + object_info_dict = self.find_object(pid) cid = object_info_dict.get("cid") # Proceed with next steps - cid has been retrieved without any issues @@ -1044,7 +1044,7 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: entity = "objects" algorithm = self._clean_algorithm(algorithm) - object_cid = self._find_object(pid).get("cid") + object_cid = self.find_object(pid).get("cid") if not self._exists(entity, object_cid): err_msg = f"No object found for pid: {pid}" self.fhs_logger.error(err_msg) @@ -1174,7 +1174,7 @@ def retrieve_folder( self._check_string(folder_pid, "PID") # try direct reference to CID using folder_pid try: - object_info_dict = self._find_object(folder_pid) + object_info_dict = self.find_object(folder_pid) folder_cid = object_info_dict.get("cid") if folder_cid is None: raise PidRefsDoesNotExist("Entry has no cid?") @@ -1186,7 +1186,7 @@ def retrieve_folder( # otherwise, get the root, split the path, and start iterating. # This will raise PidRefsDoesNotExist if the root PID isn't there - object_info_dict = self._find_object(pid) + object_info_dict = self.find_object(pid) folder_cid = object_info_dict.get("cid") if folder_cid is None: # Should never reach this... @@ -1203,7 +1203,7 @@ def retrieve_folder( if entry.type == hashstore.folderentry.FTYPE_FILE: # it's a file! raise ValueError(f"Path {path} is a file.") - object_info_dict = self._find_object(pid) + object_info_dict = self.find_object(pid) folder_cid = object_info_dict.get("cid") if folder_cid is None: # Should never reach this... @@ -1233,7 +1233,7 @@ def list_pids(self, pattern: Optional[str] = None) -> Generator: # FileHashStore Core Methods - def _find_object(self, pid: str) -> Dict[str, str]: + def find_object(self, pid: str) -> Dict[str, str]: """Check if an object referenced by a pid exists and retrieve its content identifier. The `find_object` method validates the existence of an object based on the provided pid and returns the associated content identifier. @@ -1269,7 +1269,7 @@ def _find_object(self, pid: str) -> Dict[str, str]: self.fhs_logger.error(err_msg) raise RefsFileExistsButCidObjMissing(err_msg) else: - sysmeta_doc_name = self._computehash(pid + self.sysmeta_ns) + sysmeta_doc_name = self._computehash(f"{pid}{self.sysmeta_ns}") metadata_directory = self._computehash(pid) metadata_rel_path = Path(*self._shard(metadata_directory)) sysmeta_full_path = ( @@ -1310,7 +1310,8 @@ def _find_object(self, pid: str) -> Dict[str, str]: err_msg = ( f"Pid reference file not found for pid ({pid}): {pid_ref_abs_path}" ) - self.fhs_logger.error(err_msg) + # This can be an expected case, so not really an error. + self.fhs_logger.debug(err_msg) raise PidRefsDoesNotExist(err_msg) def _store_and_validate_data( @@ -1782,7 +1783,7 @@ def _untag_object(self, pid: str, cid: str) -> None: # which will throw custom exceptions if there is an issue with the reference files, # which help us determine the path to proceed with. try: - obj_info_dict = self._find_object(pid) + obj_info_dict = self.find_object(pid) cid_to_check = obj_info_dict["cid"] self._validate_and_check_cid_lock(pid, cid, cid_to_check) From 97ffe890c4f9666f5eede0c9f80ea746c9e5d2ae Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:52:52 -0400 Subject: [PATCH 14/49] Make find_object part of base hashstore, tweak hints --- src/hashstore/basehashstore.py | 21 +++++++++++++++++++++ src/hashstore/filehashstore.py | 8 ++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/src/hashstore/basehashstore.py b/src/hashstore/basehashstore.py index 6620c845..1b337545 100644 --- a/src/hashstore/basehashstore.py +++ b/src/hashstore/basehashstore.py @@ -261,6 +261,27 @@ def get_object_status(self, pid) -> dict: """ raise NotImplementedError() + @abstractmethod + def find_object(self, pid: str) -> dict[str, str]: + """Check if an object referenced by a pid exists and retrieve its content identifier. + + The `find_object` method validates the existence of an object based on the provided + pid and returns the associated content identifier and information about how to + retrieve various accoutrements. Note that the returned dict will contain values + relevant to the type of store, but will always contain a `cid` key if the object + is present. + + :param str pid: Authority-based or persistent identifier of the object. + + :return: obj_info_dict: + - cid: content identifier + - cid_object_path: path to the object + - cid_refs_path: path to the cid refs file + - pid_refs_path: path to the pid refs file + - sysmeta_path: path to the sysmeta file + """ + raise NotImplementedError() + class HashStoreFactory: """A factory class for creating `HashStore`-like objects. diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index d4aa6c6c..b815412d 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1233,10 +1233,14 @@ def list_pids(self, pattern: Optional[str] = None) -> Generator: # FileHashStore Core Methods - def find_object(self, pid: str) -> Dict[str, str]: + def find_object(self, pid: str) -> dict[str, str]: """Check if an object referenced by a pid exists and retrieve its content identifier. + The `find_object` method validates the existence of an object based on the provided - pid and returns the associated content identifier. + pid and returns the associated content identifier and information about how to + retrieve various accoutrements. Note that the returned dict will contain values + relevant to the type of store, but will always contain a `cid` key if the object + is present. :param str pid: Authority-based or persistent identifier of the object. From d324b9f4e516707014b3f38a1ce698cbf7648c69 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 7 May 2026 21:55:46 -0400 Subject: [PATCH 15/49] WIP: adjust cli for revised hashstore folder support --- src/hashstore/__main__.py | 202 +++++++++++++++++++++++++------------- 1 file changed, 135 insertions(+), 67 deletions(-) diff --git a/src/hashstore/__main__.py b/src/hashstore/__main__.py index d4b09654..4ffb1da3 100644 --- a/src/hashstore/__main__.py +++ b/src/hashstore/__main__.py @@ -5,10 +5,12 @@ import os import pathlib import sys + import click import rich import rich.tree import yaml + try: from yaml import CLoader as Loader except ImportError: @@ -20,9 +22,11 @@ HASHSTORE_FOLDER_NAME = ".hashstore" DEFAULT_HASHSTORE = f"./{HASHSTORE_FOLDER_NAME}" + def get_logger(): return logging.getLogger("hs") + def sizeof_fmt(num, suffix="B"): for unit in ("", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"): if abs(num) < 1024.0: @@ -30,26 +34,29 @@ def sizeof_fmt(num, suffix="B"): num /= 1024.0 return f"{num:.1f}Yi{suffix}" + def enumerate_dict(d): for key, value in d.items(): if isinstance(value, dict): for subkey, subvalue in enumerate_dict(value): yield f"{key}.{subkey}", subvalue - else: + else: yield key, value -def load_hashstore_properties(path:pathlib.Path)-> dict: +def load_hashstore_properties(path: pathlib.Path) -> dict: properties_file = path / "hashstore.yaml" if not properties_file.exists(): - raise FileNotFoundError(f"Hashstore properties file not found: {properties_file}") - with open(properties_file, 'r') as f: + raise FileNotFoundError( + f"Hashstore properties file not found: {properties_file}" + ) + with open(properties_file, "r") as f: properties = yaml.load(f, Loader=Loader) properties["store_path"] = str(path) return properties -def locate_hashstore(path:pathlib.Path)->pathlib.Path: +def locate_hashstore(path: pathlib.Path) -> pathlib.Path | None: # Iterate through the current directory and all its parents for directory in [path] + list(path.parents): # Use glob() to find files matching the pattern within the current directory @@ -61,36 +68,44 @@ def locate_hashstore(path:pathlib.Path)->pathlib.Path: @click.group() @click.option( - "--store", - type=click.Path(path_type=pathlib.Path, file_okay=False), - envvar="HASHSTORE_PATH", - default=None + "--store", + type=click.Path(path_type=pathlib.Path, file_okay=False), + envvar="HASHSTORE_PATH", + default=None, +) +@click.option( + "--config", + default="~/.config/hashstore/defaults.yml", + type=click.Path(), + help="Path to configuration file", ) -@click.option("--config", default='~/.config/hashstore/defaults.yml', type=click.Path(), help="Path to configuration file") @click.option("--log-level", default=None, help="Set the logging level (INFO)") @click.pass_context def main(ctx, store, config, log_level): - """Implements a high level file and folder operations CLI for hashstore. - """ + """Implements a high level file and folder operations CLI for hashstore.""" ctx.ensure_object(dict) # Load defaults from config file config = os.path.expanduser(config) if os.path.exists(config): - with open(config, 'r') as f: + with open(config, "r") as f: cfg = yaml.load(f, Loader=Loader) ctx.default_map = cfg if store is None: store = locate_hashstore(pathlib.Path.cwd()) if store is not None: - ctx.obj["hashstore_path"] = store + ctx.obj["hashstore_path"] = store else: store.expanduser() ctx.obj["hashstore_path"] = store - ctx.obj["module_name"] = ctx.default_map.get("module_name", "hashstore.filehashstore") + ctx.obj["module_name"] = ctx.default_map.get( + "module_name", "hashstore.filehashstore" + ) ctx.obj["class_name"] = ctx.default_map.get("class_name", "FileHashStore") # Set up logging if log_level is None: - log_level = ctx.default_map.get('log_level', 'INFO') if 'cfg' in locals() else 'INFO' + log_level = ( + ctx.default_map.get("log_level", "INFO") if "cfg" in locals() else "INFO" + ) logger = get_logger() numeric_level = getattr(logging, log_level.upper(), None) if not isinstance(numeric_level, int): @@ -117,8 +132,12 @@ def version(ctx): @click.pass_context @click.option("--depth", "-d", type=int, default=3, help="Hashstore hierarchy depth") @click.option("--width", "-w", type=int, default=2, help="Hashstore hierarchy width") -@click.option("--algorithm", "-a", type=str, default="SHA-256", help="Hashstore algorithm") -@click.option("--metadata_namespace", type=str, default=None, help="Hashstore metadata namespace") +@click.option( + "--algorithm", "-a", type=str, default="SHA-256", help="Hashstore algorithm" +) +@click.option( + "--metadata_namespace", type=str, default=None, help="Hashstore metadata namespace" +) def create(ctx, depth, width, algorithm, metadata_namespace): """Create a new hashstore at the specified root path.""" logger = get_logger() @@ -139,7 +158,9 @@ def create(ctx, depth, width, algorithm, metadata_namespace): } hashstore_factory = hashstore.HashStoreFactory() try: - hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + hash_store = hashstore_factory.get_hashstore( + ctx.obj["module_name"], ctx.obj["class_name"], properties + ) logger.info(f"Hashstore created at: {store}") except Exception as e: logger.error(f"Failed to create hashstore: {e}") @@ -150,38 +171,37 @@ def create(ctx, depth, width, algorithm, metadata_namespace): @main.command(name="add_folder") @click.pass_context @click.argument( - "root_path", - type=click.Path(path_type=pathlib.Path, file_okay=False, exists=True) + "root_path", type=click.Path(path_type=pathlib.Path, file_okay=False, exists=True) ) @click.argument("pid", type=str) @click.option( - "-s", - "--sysmeta_path", - type=click.Path(path_type=pathlib.Path), - default=None, - help="Path to system metadata XML file" + "-s", + "--sysmeta_path", + type=click.Path(path_type=pathlib.Path), + default=None, + help="Path to system metadata XML file", ) @click.option( "-p", "--pattern", type=str, default=None, - help="Glob pattern for files to include (defaults to all)" + help="Glob pattern for files to include (defaults to all)", ) def add_object( - ctx, - root_path:pathlib.Path, - pid: str|None, - sysmeta_path:pathlib.Path|None, - pattern:str|None, + ctx, + root_path: pathlib.Path, + pid: str | None, + sysmeta_path: pathlib.Path | None, + pattern: str | None, ): """Add a folder to the hashstore. PID is required and is used to reference the folder root. - - If object_path is a folder, a PID is generated for the folder contents added + + If object_path is a folder, a PID is generated for the folder contents added recursively using the object relative paths as suffix to the PID. - + if the file doesn't exist: add bytes store pid @@ -194,11 +214,15 @@ def add_object( """ logger = get_logger() store = ctx.obj["hashstore_path"] - logger.info(f"Adding folder at path: {store} with PID: {pid} and sysmeta: {sysmeta_path}") + logger.info( + f"Adding folder at path: {store} with PID: {pid} and sysmeta: {sysmeta_path}" + ) properties = load_hashstore_properties(store) hashstore_factory = hashstore.HashStoreFactory() try: - hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + hash_store = hashstore_factory.get_hashstore( + ctx.obj["module_name"], ctx.obj["class_name"], properties + ) logger.debug(f"Hashstore opened at: {store}") except Exception as e: logger.error(f"Failed to open hashstore: {e}") @@ -213,13 +237,19 @@ def add_object( root_path = root_path.absolute() if info is None: try: - info = hash_store.commit_folder(pid=pid, root_path=root_path, pattern=pattern) + info = hash_store.commit_folder( + pid=pid, root_path=root_path, pattern=pattern + ) print(json.dumps(dataclasses.asdict(info))) except Exception as e: logger.error(f"Failed to add object: {e}") return 1 if sysmeta_path is not None and sysmeta_path.is_file(): - hash_store.store_metadata(pid=pid, metadata=str(sysmeta_path), format_id=properties.get("store_metadata_namespace")) + hash_store.store_metadata( + pid=pid, + metadata=str(sysmeta_path), + format_id=properties.get("store_metadata_namespace"), + ) return 1 return 0 @@ -228,7 +258,9 @@ def add_object( @click.pass_context @click.argument("pid", type=str) @click.option("-s", "--stream", is_flag=True, help="Stream to stdout") -@click.option("-r", "--recursive", is_flag=True, help="Recurse into contents if PID is a folder.") +@click.option( + "-r", "--recursive", is_flag=True, help="Recurse into contents if PID is a folder." +) def get_object(ctx, pid, stream, recursive): """Retrieve an object or folder from hashstore.""" store = ctx.obj["hashstore_path"] @@ -237,17 +269,32 @@ def get_object(ctx, pid, stream, recursive): @main.command("ls") @click.pass_context -@click.option("-p", "--pattern", default=None, help="Optional regex pattern for PID matching.") -@click.option("-h", "--human-readable", is_flag=True, help="Display sizes in human readable format.") +@click.option( + "-p", "--pattern", default=None, help="Optional regex pattern for PID matching." +) +@click.option( + "-h", + "--human-readable", + is_flag=True, + help="Display sizes in human readable format.", +) +@click.option( + "-m", + "--show-metadata", + is_flag=True, + help="Show metadata for entry if available.", +) @click.option("-r", "--reference", is_flag=True, help="Include path references.") -def list_pids(ctx, pattern, human_readable, reference): +def list_pids(ctx, pattern, human_readable, show_metadata, reference): """List PIDs in the hashstore.""" logger = get_logger() store = ctx.obj["hashstore_path"] properties = load_hashstore_properties(store) hashstore_factory = hashstore.HashStoreFactory() try: - hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + hash_store = hashstore_factory.get_hashstore( + ctx.obj["module_name"], ctx.obj["class_name"], properties + ) logger.debug(f"Hashstore opened at: {store}") except Exception as e: logger.error(f"Failed to open hashstore: {e}") @@ -256,56 +303,77 @@ def list_pids(ctx, pattern, human_readable, reference): # iterate over the refs/cids folder, getting PIDs from each file. print(f"Hashstore: {str(store.relative_to(pathlib.Path.cwd(), walk_up=True))}") for pid in hash_store.list_pids(pattern=pattern): - try: + try: pid_stat = hash_store.get_object_status(pid) fsize = pid_stat.get("size", 0) total_objects += 1 t_modified = pid_stat.get("modtime", "-") if t_modified != "-": - t_modified = datetime.datetime.fromtimestamp(t_modified).isoformat(timespec="seconds") + t_modified = datetime.datetime.fromtimestamp(t_modified).isoformat( + timespec="seconds" + ) t_access = pid_stat.get("accesstime", "-") if t_access != "-": - t_access = datetime.datetime.fromtimestamp(t_access).isoformat(timespec="seconds") + t_access = datetime.datetime.fromtimestamp(t_access).isoformat( + timespec="seconds" + ) if human_readable: fsize = sizeof_fmt(fsize) - if fsize>0 or reference: + if fsize > 0 or reference: print(f"{fsize}\t{t_modified}\t{t_access}\t{pid}") - try: - meta = hash_store.retrieve_metadata(pid) - print(json.dumps(meta, indent=2)) - except KeyError: - pass + if show_metadata: + try: + meta = hash_store.retrieve_metadata(pid).read().decode() + print(meta) + # print(json.dumps(meta, indent=2)) + except KeyError: + pass except KeyError: logger.warning(f"PID status not available in hashstore: {pid}") print(f"Total {total_objects}") +def iterate_folder(hs, tree, pid, path=""): + current_folder = hs.retrieve_folder(pid, path=path) + for entry in current_folder: + branch = tree.add(entry.name) + if entry.type == 0: + _path = f"{path}/{entry.name}" if path != "" else entry.name + iterate_folder(hs, branch, pid, path=_path) + + @main.command("tree") @click.pass_context @click.argument("pid", type=str) -def get_container_tree(ctx, pid)->None: +def get_container_tree(ctx, pid) -> None: logger = get_logger() store = ctx.obj["hashstore_path"] properties = load_hashstore_properties(store) hashstore_factory = hashstore.HashStoreFactory() try: - hash_store = hashstore_factory.get_hashstore(ctx.obj["module_name"], ctx.obj["class_name"], properties) + hash_store = hashstore_factory.get_hashstore( + ctx.obj["module_name"], ctx.obj["class_name"], properties + ) logger.debug(f"Hashstore opened at: {store}") except Exception as e: logger.error(f"Failed to open hashstore: {e}") return 1 - tree = { - pid:rich.tree.Tree(pid), - } - for entry in hash_store.folder_content(pid, depth_first=True): - print(f"{entry[0]} {entry[1]} {entry[3]} {entry[2]}") - parts = entry[3].rsplit("/", 1) - if entry[1] == 0: - tree[entry[3]] = tree[parts[0]].add(entry[3]) - else: - # file - tree[parts[0]].add(entry[3]) - rich.print(tree[pid]) + parts = pid.split(" ", 1) + path = "" + if len(parts) > 1: + path = parts[1].strip() + tree = rich.tree.Tree(pid) + iterate_folder(hash_store, tree, parts[0], path=path) + # parts = [] + # for entry in iterate_folder(hash_store, pid): + # print(f"{entry.type} {entry.name} {entry.size} {entry.cid}") + # # parts = entry[3].rsplit("/", 1) + # if entry.type == 0: + # tree[entry.name] = tree[parts[0]].add(entry[3]) + # # else: + # # # file + # # tree[parts[0]].add(entry[3]) + rich.print(tree) if __name__ == "__main__": From 0970b1e31ebca306abeba46630050824067c7e33 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Fri, 8 May 2026 15:08:43 -0400 Subject: [PATCH 16/49] Added option to capture object creation events to an index file --- src/hashstore/filehashstore.py | 21 ++++++++++++-- src/hashstore/pidlogger.py | 51 ++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) create mode 100644 src/hashstore/pidlogger.py diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index b815412d..69ac8632 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -20,7 +20,7 @@ import yaml import hashstore.folderentry -from hashstore import HashStore +from hashstore import HashStore, pidlogger from hashstore.filehashstore_exceptions import ( CidRefsContentError, CidRefsFileNotFound, @@ -134,6 +134,8 @@ def __init__(self, properties=None): self._create_path(self.refs / "tmp") self._create_path(self.refs / "pids") self._create_path(self.refs / "cids") + # pidlog is used to create an index of cid - pid + self.pidlog = logging.getLogger("pid_logger") # Variables to orchestrate parallelization # Check to see whether a multiprocessing or threading sync lock should be used @@ -578,6 +580,7 @@ def store_object( cid = object_metadata.cid self.tag_object(pid, cid) self.fhs_logger.info("Successfully stored object for pid: %s", pid) + self.pidlog.info(cid, extra={"pid": pid}) finally: # Release pid self._release_object_locked_pids(pid) @@ -1068,6 +1071,8 @@ def store_folder( ) -> Optional["ObjectMetadata"]: """Store a folder object. + #TODO: important: Make thread and multi processing safe. + A Folder is a list of entries that appear in a folder. Each entry may be a file or a Folder. This method is used instead of store_object because Folders have special requirements to ensure deterministic serialization. @@ -1139,6 +1144,7 @@ def store_folder( # of folder entries increases. obj_size = entries.to_parquet(cid_path, pid=folder_pid) self.tag_object(folder_pid, folder_cid) + self.pidlog.info(folder_cid, extra={"pid": folder_pid}) return ObjectMetadata( pid=folder_pid, cid=folder_cid, @@ -1214,22 +1220,31 @@ def retrieve_folder( return current_folder def list_pids(self, pattern: Optional[str] = None) -> Generator: + """Yield create_timestamp, CID, PID. + + Iterates over all CID entries and yields the create timestamp, + CID value, and PID value for all entries or those PIDs that match + the optionally provided regexp pattern. + """ rpattern = None if pattern is not None: rpattern = re.compile(pattern) ignore_names = [ ".DS_Store", ] + cids_path = str(self.cids) for cid_entry in self.cids.rglob("*"): if cid_entry.is_file() and cid_entry.name not in ignore_names: + cid_value = str(cid_entry).replace(cids_path, "").replace("/", "") + ctime = cid_entry.stat().st_ctime for _, entry in enumerate(open(cid_entry, "r", encoding="utf-8")): pid = entry.strip() if len(pid) > 0: if rpattern is not None: if rpattern.fullmatch(pid): - yield pid + yield ctime, cid_value, pid else: - yield pid + yield ctime, cid_value, pid # FileHashStore Core Methods diff --git a/src/hashstore/pidlogger.py b/src/hashstore/pidlogger.py new file mode 100644 index 00000000..1091e9c0 --- /dev/null +++ b/src/hashstore/pidlogger.py @@ -0,0 +1,51 @@ +"""Implements a multi-process safe logger for generating a cid-pid index. + +This implementation writes an NDJSON file with each line containing a json array +with elements: + [0] timestamp + [1] cid + [2] pid + +The logging module is used because it is multi-thread and -process safe. + +The resulting ndjson file can be loaded into duckdb for example with: + + CREATE TABLE pids AS SELECT + to_timestamp(json[1]::DOUBLE) AS ctime, + json[2]->> '$' AS cid, + json[3]->>'$' AS pid + FROM read_json('pid_index.ndjson'); + +or create a parquet representation: + + duckdb -c "COPY (SELECT to_timestamp(json[1]::DOUBLE) AS ctime, + json[2]->> '\\$' AS cid, json[3]->>'\\$' AS pid FROM + read_json('pid_index.ndjson')) TO 'pid_index.parquet' (FORMAT parquet)" +""" + +import json +import logging + + +class PidIndexFormatter(logging.Formatter): + def format(self, record) -> str: + pid_record = ( + record.created, + record.getMessage().strip(), + record.pid if hasattr(record, "pid") else None, + ) + return json.dumps(pid_record) + + +def getPidLogger(log_file_name: str): + """Return a logger for the pid index. + + PID records are added to the index like: + logger.info(CID, extra={"pid":PID}) + """ + logger = logging.getLogger("pid_logger") + logger.setLevel(logging.INFO) + handler = logging.FileHandler(log_file_name) + handler.setFormatter(PidIndexFormatter()) + logger.addHandler(handler) + return logger From f7b24bec1528b3dca860bf979b3b0030f4d455e6 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Fri, 8 May 2026 16:44:14 -0400 Subject: [PATCH 17/49] Added folder inspection methods --- src/hashstore/__main__.py | 98 ++++++++++++++++++++++++++++++--------- 1 file changed, 76 insertions(+), 22 deletions(-) diff --git a/src/hashstore/__main__.py b/src/hashstore/__main__.py index 4ffb1da3..e791ba70 100644 --- a/src/hashstore/__main__.py +++ b/src/hashstore/__main__.py @@ -285,7 +285,8 @@ def get_object(ctx, pid, stream, recursive): help="Show metadata for entry if available.", ) @click.option("-r", "--reference", is_flag=True, help="Include path references.") -def list_pids(ctx, pattern, human_readable, show_metadata, reference): +@click.option("-l", "--list-only", is_flag=True, help="Just list the cid, pid values.") +def list_pids(ctx, pattern, human_readable, show_metadata, reference, list_only): """List PIDs in the hashstore.""" logger = get_logger() store = ctx.obj["hashstore_path"] @@ -301,8 +302,12 @@ def list_pids(ctx, pattern, human_readable, show_metadata, reference): return 1 total_objects = 0 # iterate over the refs/cids folder, getting PIDs from each file. - print(f"Hashstore: {str(store.relative_to(pathlib.Path.cwd(), walk_up=True))}") - for pid in hash_store.list_pids(pattern=pattern): + if not list_only: + print(f"Hashstore: {str(store.relative_to(pathlib.Path.cwd(), walk_up=True))}") + for ctime, cid, pid in hash_store.list_pids(pattern=pattern): + if list_only: + print(json.dumps((ctime, cid, pid))) + continue try: pid_stat = hash_store.get_object_status(pid) fsize = pid_stat.get("size", 0) @@ -330,22 +335,80 @@ def list_pids(ctx, pattern, human_readable, show_metadata, reference): pass except KeyError: logger.warning(f"PID status not available in hashstore: {pid}") - print(f"Total {total_objects}") + if not list_only: + print(f"Total {total_objects}") -def iterate_folder(hs, tree, pid, path=""): - current_folder = hs.retrieve_folder(pid, path=path) - for entry in current_folder: - branch = tree.add(entry.name) - if entry.type == 0: - _path = f"{path}/{entry.name}" if path != "" else entry.name - iterate_folder(hs, branch, pid, path=_path) +@main.command("finfo") +@click.pass_context +@click.argument("pid", type=str) +def get_folder_info(ctx, pid) -> None: + """Compute basic stats for a folder and sub-folders.""" + + def iterate_folder(hs, stats, pid, path="", depth: int = 1): + if depth > stats["max_depth"]: + stats["max_depth"] = depth + current_folder = hs.retrieve_folder(pid, path=path) + for entry in current_folder: + if entry.type == 0: + _path = f"{path}/{entry.name}" if path != "" else entry.name + stats["total_folders"] = stats["total_folders"] + 1 + iterate_folder(hs, stats, pid, path=_path, depth=depth + 1) + else: + stats["total_bytes"] = stats["total_bytes"] + entry.size + stats["total_files"] += 1 + + logger = get_logger() + store = ctx.obj["hashstore_path"] + properties = load_hashstore_properties(store) + hashstore_factory = hashstore.HashStoreFactory() + try: + hash_store = hashstore_factory.get_hashstore( + ctx.obj["module_name"], ctx.obj["class_name"], properties + ) + logger.debug(f"Hashstore opened at: {store}") + except Exception as e: + logger.error(f"Failed to open hashstore: {e}") + return 1 + + info = { + "total_files": 0, + "total_bytes": 0, + "total_folders": 0, + "max_depth": 0, + } + parts = pid.split(" ", 1) + path = "" + if len(parts) > 1: + path = parts[1].strip() + + iterate_folder(hash_store, info, parts[0], path=path, depth=0) + print(json.dumps(info, indent=2)) @main.command("tree") @click.pass_context @click.argument("pid", type=str) -def get_container_tree(ctx, pid) -> None: +@click.option("-n", "--no-files", is_flag=True, help="Show folders but not content.") +def get_folder_tree(ctx, pid: str, no_files: bool) -> None: + """Generate a tree representation of the folder and sub-folders.""" + + def iterate_folder(hs, tree, pid, path="", with_files: bool = True): + current_folder = hs.retrieve_folder(pid, path=path) + n = 0 + s = 0 + for entry in current_folder: + if entry.type == 0 or with_files: + branch = tree.add(entry.name) + if entry.type == 0: + _path = f"{path}/{entry.name}" if path != "" else entry.name + iterate_folder(hs, branch, pid, path=_path, with_files=with_files) + else: + n += 1 + s += entry.size + if not with_files: + branch = tree.add(f"{n:,} files, {s:,} bytes") + logger = get_logger() store = ctx.obj["hashstore_path"] properties = load_hashstore_properties(store) @@ -363,16 +426,7 @@ def get_container_tree(ctx, pid) -> None: if len(parts) > 1: path = parts[1].strip() tree = rich.tree.Tree(pid) - iterate_folder(hash_store, tree, parts[0], path=path) - # parts = [] - # for entry in iterate_folder(hash_store, pid): - # print(f"{entry.type} {entry.name} {entry.size} {entry.cid}") - # # parts = entry[3].rsplit("/", 1) - # if entry.type == 0: - # tree[entry.name] = tree[parts[0]].add(entry[3]) - # # else: - # # # file - # # tree[parts[0]].add(entry[3]) + iterate_folder(hash_store, tree, parts[0], path=path, with_files=not no_files) rich.print(tree) From 9b129311beeab6bb88a6e243b418d7b833d81a7e Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Mon, 11 May 2026 10:20:03 -0400 Subject: [PATCH 18/49] Enable folder creation without folderEntry cids --- src/hashstore/filehashstore.py | 19 +++++++++++++++---- src/hashstore/folderentry.py | 15 ++++++++++++++- 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 69ac8632..714bcbda 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1077,6 +1077,10 @@ def store_folder( may be a file or a Folder. This method is used instead of store_object because Folders have special requirements to ensure deterministic serialization. + If FolderEntries have cid = '' | None, then the cid is looked up using + + pid + " " + path + "/" + entry.name + The Folder is tagged with an identifier that is "{PID} {path}", that is, the PID followed by a single space, then the path. If the path portion is an empty string, ".", or "/" then the Folder is the root Folder. @@ -1106,10 +1110,17 @@ def store_folder( if verify_entry_cids: # check that each entry CID is present in the hashstore. for entry in entries: - if not self._exists("objects", entry.cid): - raise ValueError( - f"object {entry.name} cid {entry.cid} does not exist." - ) + if entry.cid is None or entry.cid == "": + # no cid provided, so look it up + _entry_pid = f"{folder_pid}/{entry.name}" + _meta = self.find_object(_entry_pid) + entry.cid = _meta["cid"] + else: + # verify provided cid is legit + if not self._exists("objects", entry.cid): + raise ValueError( + f"object {entry.name} cid {entry.cid} does not exist." + ) # Sort the entries by cid entries.sort(key=lambda entry: entry.cid) diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index 6c5dfa30..bae3dda1 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -56,6 +56,18 @@ def __repr__(self) -> str: ensure_ascii=False, ) + @classmethod + def parquet_schema(cls): + return pyarrow.schema( + ( + ("cid", pyarrow.string()), + ("type", pyarrow.bool_()), + ("name", pyarrow.string()), + ("size", pyarrow.int64()), + ("formatid", pyarrow.string()), + ) + ) + class FolderEntries(list[FolderEntry]): def entry_by_name(self, name) -> FolderEntry | None: @@ -86,10 +98,11 @@ def to_parquet( "version": "1.0", "pid": pid, } + pq_schema = FolderEntry.parquet_schema() metadata_bytes = json.dumps(pq_metadata).encode("utf-8") table = pyarrow.Table.from_pylist([dataclasses.asdict(entry) for entry in self]) table = table.replace_schema_metadata({PARQUET_METADATA_KEY: metadata_bytes}) - pyarrow.parquet.write_table(table, pq_path, **writer_args) + pyarrow.parquet.write_table(table, pq_path, schema=pq_schema, **writer_args) return os.path.getsize(pq_path) @classmethod From 9cf4f214ec923c04a2ff3ffc93ac10cb6419b610 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Mon, 11 May 2026 11:09:28 -0400 Subject: [PATCH 19/49] Change type to bool for efficiency --- src/hashstore/filehashstore.py | 4 ++-- src/hashstore/folderentry.py | 23 +++++++++-------------- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 714bcbda..8f414018 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1121,7 +1121,7 @@ def store_folder( raise ValueError( f"object {entry.name} cid {entry.cid} does not exist." ) - # Sort the entries by cid + # Sort the entries by cid for consistent hashing entries.sort(key=lambda entry: entry.cid) hash_algorithms = { @@ -1217,7 +1217,7 @@ def retrieve_folder( entry = current_folder.entry_by_name(name) if entry is None: raise KeyError(f"PID {pid} {path} not found.") - if entry.type == hashstore.folderentry.FTYPE_FILE: + if entry.is_file: # it's a file! raise ValueError(f"Path {path} is a file.") object_info_dict = self.find_object(pid) diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index bae3dda1..3797f169 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -9,10 +9,6 @@ import pyarrow import pyarrow.parquet -FTYPE_FOLDER = 0 -"""FolderEntry is for a FolderEntry""" -FTYPE_FILE = 1 -"""FolderEntry is for a file.""" # TODO: Ratify this key PARQUET_METADATA_KEY = b"https://ns.dataone.org/types/FolderEntries" """Key in parquet file metadata pointing to dict of properties.""" @@ -32,23 +28,19 @@ class FolderEntry: """The name portion of the path (not full path) for the file or folder.""" cid: str """The content hash (CID) for the entry.""" - type: int # '1' for file, '0' for directory + is_file: bool # True for file, False for Folder """The type of manifest entry: '1' for file, '0' for directory.""" size: int = 0 """Size of the file in bytes or number of entries for directories.""" formatid: str | None = None """Optional format identifier for files.""" - def __post_init__(self): - if self.type not in (FTYPE_FILE, FTYPE_FOLDER): - raise ValueError(f"Invalid type: {self.type}") - def __repr__(self) -> str: # Representation of a FolderEntry return json.dumps( { "cid": self.cid, - "type": self.type, + "is_file": self.is_file, "name": self.name, "size": self.size, "formatid": self.formatid, @@ -61,7 +53,7 @@ def parquet_schema(cls): return pyarrow.schema( ( ("cid", pyarrow.string()), - ("type", pyarrow.bool_()), + ("is_file", pyarrow.bool_()), ("name", pyarrow.string()), ("size", pyarrow.int64()), ("formatid", pyarrow.string()), @@ -100,9 +92,12 @@ def to_parquet( } pq_schema = FolderEntry.parquet_schema() metadata_bytes = json.dumps(pq_metadata).encode("utf-8") - table = pyarrow.Table.from_pylist([dataclasses.asdict(entry) for entry in self]) + table = pyarrow.Table.from_pylist( + [dataclasses.asdict(entry) for entry in self], + schema=pq_schema, + ) table = table.replace_schema_metadata({PARQUET_METADATA_KEY: metadata_bytes}) - pyarrow.parquet.write_table(table, pq_path, schema=pq_schema, **writer_args) + pyarrow.parquet.write_table(table, pq_path, **writer_args) return os.path.getsize(pq_path) @classmethod @@ -123,7 +118,7 @@ def from_parquet(cls, pq_path) -> "FolderEntries": FolderEntry( name=row["name"], cid=row["cid"], - type=row["type"], + is_file=row["is_file"], size=row["size"], formatid=row["formatid"], ) From 67e2668ee679eba610439c29d888dc0a4c3850bc Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Mon, 11 May 2026 16:55:11 -0400 Subject: [PATCH 20/49] Use alternate path delimiter --- src/hashstore/__main__.py | 23 +++++++++++++------- src/hashstore/filehashstore.py | 39 ++++++++++++++++++++++++++++------ src/hashstore/folderentry.py | 2 +- 3 files changed, 49 insertions(+), 15 deletions(-) diff --git a/src/hashstore/__main__.py b/src/hashstore/__main__.py index e791ba70..abad4d6e 100644 --- a/src/hashstore/__main__.py +++ b/src/hashstore/__main__.py @@ -18,6 +18,7 @@ import hashstore import hashstore.filehashstore_exceptions +from hashstore.folderentry import PATH_DELIMITER HASHSTORE_FOLDER_NAME = ".hashstore" DEFAULT_HASHSTORE = f"./{HASHSTORE_FOLDER_NAME}" @@ -350,13 +351,15 @@ def iterate_folder(hs, stats, pid, path="", depth: int = 1): stats["max_depth"] = depth current_folder = hs.retrieve_folder(pid, path=path) for entry in current_folder: - if entry.type == 0: - _path = f"{path}/{entry.name}" if path != "" else entry.name - stats["total_folders"] = stats["total_folders"] + 1 - iterate_folder(hs, stats, pid, path=_path, depth=depth + 1) - else: + if entry.is_file == 0: stats["total_bytes"] = stats["total_bytes"] + entry.size stats["total_files"] += 1 + else: + _path = ( + f"{path}{PATH_DELIMITER}{entry.name}" if path != "" else entry.name + ) + stats["total_folders"] = stats["total_folders"] + 1 + iterate_folder(hs, stats, pid, path=_path, depth=depth + 1) logger = get_logger() store = ctx.obj["hashstore_path"] @@ -398,10 +401,14 @@ def iterate_folder(hs, tree, pid, path="", with_files: bool = True): n = 0 s = 0 for entry in current_folder: - if entry.type == 0 or with_files: + if not entry.is_file or with_files: branch = tree.add(entry.name) - if entry.type == 0: - _path = f"{path}/{entry.name}" if path != "" else entry.name + if not entry.is_file: + _path = ( + f"{path}{PATH_DELIMITER}{entry.name}" + if path != "" + else entry.name + ) iterate_folder(hs, branch, pid, path=_path, with_files=with_files) else: n += 1 diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 8f414018..50d992d4 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1103,7 +1103,8 @@ def store_folder( Returns: ObjectMetadata: The computed ObjectMetadata for this entry. """ - if path in ("", ".", "/"): + delim = hashstore.folderentry.PATH_DELIMITER + if path in ("", ".", delim): path = "" folder_pid = f"{pid} {path}" if path != "" else pid self._check_string(folder_pid, "PID") @@ -1112,7 +1113,7 @@ def store_folder( for entry in entries: if entry.cid is None or entry.cid == "": # no cid provided, so look it up - _entry_pid = f"{folder_pid}/{entry.name}" + _entry_pid = f"{folder_pid}{delim}{entry.name}" _meta = self.find_object(_entry_pid) entry.cid = _meta["cid"] else: @@ -1163,6 +1164,17 @@ def store_folder( obj_size=obj_size, ) + def _make_pidpath(self, pid, path_segments): + if len(path_segments) < 1: + return pid + return f"{pid} {hashstore.folderentry.PATH_DELIMITER.join(path_segments)}" + + def _split_pidpath(self, pidpath: str) -> tuple[str, list[str]]: + parts = pidpath.split(" ") + if len(parts) == 1: + return (parts[0], []) + return (parts[0], parts[1].split(hashstore.folderentry.PATH_DELIMITER)) + def retrieve_folder( self, pid: str, @@ -1181,11 +1193,12 @@ def retrieve_folder( Args: pid (str): The context (i.e. VMDAG version) within which this folder is being retrieved - path (str): Path within the context to the desired entry + path (str): Path within the context to the desired entry, using folderentry.PATH_DELIMITER Returns: FolderEntries """ - if path in ("", ".", "/"): + delim = hashstore.folderentry.PATH_DELIMITER + if path in ("", ".", delim): path = "" folder_pid = f"{pid} {path}" if path != "" else pid self._check_string(folder_pid, "PID") @@ -1211,8 +1224,10 @@ def retrieve_folder( cid_path = object_info_dict.get("cid_object_path") # self._build_hashstore_data_object_path(folder_cid) current_folder = hashstore.folderentry.FolderEntries.from_parquet(cid_path) - path_segments = path.split("/") # iterate over segments, saving last + current_pid = pid + path_segments = path.split(delim) + segment_index = 0 for name in path_segments: entry = current_folder.entry_by_name(name) if entry is None: @@ -1220,7 +1235,19 @@ def retrieve_folder( if entry.is_file: # it's a file! raise ValueError(f"Path {path} is a file.") - object_info_dict = self.find_object(pid) + segment_index += 1 + try: + # Does name form a registered PID? + object_info_dict = self.find_object(name) + # if so, then return follow + return self.retrieve_folder( + name, delim.join(path_segments[segment_index:]) + ) + except PidRefsDoesNotExist: + # continue but trye with pid + path + pass + folder_pid = self._make_pidpath(current_pid, path_segments[:segment_index]) + object_info_dict = self.find_object(name) folder_cid = object_info_dict.get("cid") if folder_cid is None: # Should never reach this... diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index 3797f169..38e206b7 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -1,6 +1,5 @@ """Implements FolderEntry class.""" -import collections.abc import dataclasses import json import logging @@ -14,6 +13,7 @@ """Key in parquet file metadata pointing to dict of properties.""" PARQUET_READ_BATCH_SIZE = 10000 """Number of entries to read at a time from FolderEntries parquet file.""" +PATH_DELIMITER = "→" def get_logger(): From b1403b3f61832997089199daedffef5cc6eb3292 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Mon, 11 May 2026 22:28:31 -0400 Subject: [PATCH 21/49] switch path delim, path as list instead of str --- src/hashstore/filehashstore.py | 92 ++++++++++++---------------------- src/hashstore/folderentry.py | 20 ++++++-- 2 files changed, 46 insertions(+), 66 deletions(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 50d992d4..27e6e85a 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1061,8 +1061,7 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: def store_folder( self, - pid: str, - path: str, + pathpid: list[str], entries: hashstore.folderentry.FolderEntries, additional_algorithm: Optional[str] = None, checksum: Optional[str] = None, @@ -1079,11 +1078,12 @@ def store_folder( If FolderEntries have cid = '' | None, then the cid is looked up using - pid + " " + path + "/" + entry.name + pid + DELIM + path + DELIM + entry.name - The Folder is tagged with an identifier that is "{PID} {path}", that is, the - PID followed by a single space, then the path. If the path portion is an empty - string, ".", or "/" then the Folder is the root Folder. + The Folder is tagged with an identifier that is "{PID}{DELIM}{path}", that is, the + PID followed by a single delimiter, then the path. If the path portion is an empty + list or a list of length 1 with the first element ".", or DELIM then the Folder is + the root Folder within the context of PID. Note that since the hash of a Folder is computed from hashes of its content, a Folder hierarchy must be stored starting with the leaves. This method @@ -1095,7 +1095,7 @@ def store_folder( Args: pid (str): The context within which this folder is being stored - path (str): Path to this folder relative to the root. + path (list[str]): Path to this folder relative to the root. entries (list[FolderEntry]): A list of FolderEntry objects. verify_entry_cids: If True then FolderEntry CID values are verified to to ensure they exist in the hashstore. @@ -1103,17 +1103,23 @@ def store_folder( Returns: ObjectMetadata: The computed ObjectMetadata for this entry. """ - delim = hashstore.folderentry.PATH_DELIMITER - if path in ("", ".", delim): - path = "" - folder_pid = f"{pid} {path}" if path != "" else pid + # delim = hashstore.folderentry.PATH_DELIMITER + # if path in ("", ".", delim): + # path = "" + # folder_pid = f"{pid} {path}" if path != "" else pid + folder_pid = hashstore.folderentry.join_pathpid(pathpid) self._check_string(folder_pid, "PID") if verify_entry_cids: # check that each entry CID is present in the hashstore. for entry in entries: if entry.cid is None or entry.cid == "": # no cid provided, so look it up - _entry_pid = f"{folder_pid}{delim}{entry.name}" + _entry_pid = hashstore.folderentry.join_pathpid( + pathpid + + [ + entry.name, + ] + ) _meta = self.find_object(_entry_pid) entry.cid = _meta["cid"] else: @@ -1164,21 +1170,9 @@ def store_folder( obj_size=obj_size, ) - def _make_pidpath(self, pid, path_segments): - if len(path_segments) < 1: - return pid - return f"{pid} {hashstore.folderentry.PATH_DELIMITER.join(path_segments)}" - - def _split_pidpath(self, pidpath: str) -> tuple[str, list[str]]: - parts = pidpath.split(" ") - if len(parts) == 1: - return (parts[0], []) - return (parts[0], parts[1].split(hashstore.folderentry.PATH_DELIMITER)) - def retrieve_folder( self, - pid: str, - path: str, + pathpid: list[str], ) -> hashstore.folderentry.FolderEntries: """Retrieve a FolderEntries instance from the hashstore. @@ -1197,12 +1191,10 @@ def retrieve_folder( Returns: FolderEntries """ - delim = hashstore.folderentry.PATH_DELIMITER - if path in ("", ".", delim): - path = "" - folder_pid = f"{pid} {path}" if path != "" else pid + folder_pid = hashstore.folderentry.join_pathpid(pathpid) self._check_string(folder_pid, "PID") - # try direct reference to CID using folder_pid + # try direct reference to CID using folder_pid. This works if + # there is no branching to other PID contexts (typical case) try: object_info_dict = self.find_object(folder_pid) folder_cid = object_info_dict.get("cid") @@ -1214,46 +1206,24 @@ def retrieve_folder( except PidRefsDoesNotExist: pass - # otherwise, get the root, split the path, and start iterating. + # otherwise, iterate over the path, following a branch if needed. + # get the root, split the path, and start iterating. # This will raise PidRefsDoesNotExist if the root PID isn't there - object_info_dict = self.find_object(pid) + object_info_dict = self.find_object(pathpid[0]) folder_cid = object_info_dict.get("cid") if folder_cid is None: # Should never reach this... raise PidRefsDoesNotExist("Entry has no cid?") cid_path = object_info_dict.get("cid_object_path") # self._build_hashstore_data_object_path(folder_cid) + # Get the root folder, then find the next path element in the folder current_folder = hashstore.folderentry.FolderEntries.from_parquet(cid_path) - # iterate over segments, saving last - current_pid = pid - path_segments = path.split(delim) - segment_index = 0 - for name in path_segments: - entry = current_folder.entry_by_name(name) + for idx in range(1, len(pathpid)): + entry = current_folder.entry_by_name(pathpid[idx]) if entry is None: - raise KeyError(f"PID {pid} {path} not found.") - if entry.is_file: - # it's a file! - raise ValueError(f"Path {path} is a file.") - segment_index += 1 - try: - # Does name form a registered PID? - object_info_dict = self.find_object(name) - # if so, then return follow - return self.retrieve_folder( - name, delim.join(path_segments[segment_index:]) - ) - except PidRefsDoesNotExist: - # continue but trye with pid + path - pass - folder_pid = self._make_pidpath(current_pid, path_segments[:segment_index]) - object_info_dict = self.find_object(name) - folder_cid = object_info_dict.get("cid") - if folder_cid is None: - # Should never reach this... - raise PidRefsDoesNotExist("Entry has no cid?") - cid_path = object_info_dict.get("cid_object_path") - # cid_path = self._build_hashstore_data_object_path(folder_cid) + raise KeyError(f"PID {pathpid} not found.") + # given the entry, we have the cid. Use that to get the next folder + cid_path = self._get_hashstore_data_object_path(entry.cid) current_folder = hashstore.folderentry.FolderEntries.from_parquet(cid_path) return current_folder diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index 38e206b7..92eda30a 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -13,13 +13,25 @@ """Key in parquet file metadata pointing to dict of properties.""" PARQUET_READ_BATCH_SIZE = 10000 """Number of entries to read at a time from FolderEntries parquet file.""" -PATH_DELIMITER = "→" +PATH_DELIMITER = "⫽" def get_logger(): return logging.getLogger("FolderEntry") +def split_pathpid(pathpid: str) -> list[str]: + pathpid = pathpid.strip(PATH_DELIMITER) + parts = pathpid.split(PATH_DELIMITER) + return parts + + +def join_pathpid(path: list[str]) -> str: + # remove "", strings with only white space + cleaned = [s.strip() for s in path] + return PATH_DELIMITER.join(list(filter(str.strip, cleaned))) + + @dataclasses.dataclass class FolderEntry: """Represents a file or folder entry in a folder manifest.""" @@ -69,9 +81,7 @@ def entry_by_name(self, name) -> FolderEntry | None: return entry return None - def to_parquet( - self, pq_path: str, pid: str | None = None, writer_args: dict = {} - ) -> int: + def to_parquet(self, pq_path: str, pid: str, writer_args: dict = {}) -> int: """Writes the list of folder entries to a parquet file. See also: https://arrow.apache.org/docs/python/generated/pyarrow.parquet.write_table.html @@ -81,7 +91,7 @@ def to_parquet( args: pq_path: path to destination parquet file - pid: Optional PID+path used to create this folder. + pid: PID+path used to create this folder. writer_args: optional dict of arguments for the parquet writer. """ # Add some metadata to the parquet file to help identify it as a list of FolderEntries From 8e852e3320fedc035e79d80cb5202e6f69f64a3a Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 12 May 2026 07:55:42 -0400 Subject: [PATCH 22/49] Refactor to use path segments in api --- src/hashstore/__main__.py | 50 ++++++-------- src/hashstore/basehashstore.py | 43 ++++++++++-- src/hashstore/filehashstore.py | 121 ++++++++++++++++++++------------- src/hashstore/folderentry.py | 6 +- 4 files changed, 136 insertions(+), 84 deletions(-) diff --git a/src/hashstore/__main__.py b/src/hashstore/__main__.py index abad4d6e..e6a4d49f 100644 --- a/src/hashstore/__main__.py +++ b/src/hashstore/__main__.py @@ -18,7 +18,7 @@ import hashstore import hashstore.filehashstore_exceptions -from hashstore.folderentry import PATH_DELIMITER +import hashstore.folderentry HASHSTORE_FOLDER_NAME = ".hashstore" DEFAULT_HASHSTORE = f"./{HASHSTORE_FOLDER_NAME}" @@ -345,23 +345,25 @@ def list_pids(ctx, pattern, human_readable, show_metadata, reference, list_only) @click.argument("pid", type=str) def get_folder_info(ctx, pid) -> None: """Compute basic stats for a folder and sub-folders.""" + logger = get_logger() - def iterate_folder(hs, stats, pid, path="", depth: int = 1): + def iterate_folder(hs, stats, path, depth: int = 1): + logger.debug("Current path=%s", path) if depth > stats["max_depth"]: stats["max_depth"] = depth - current_folder = hs.retrieve_folder(pid, path=path) + current_folder = hs.retrieve_folder(path) for entry in current_folder: - if entry.is_file == 0: + logger.debug(str(entry)) + if entry.is_file: stats["total_bytes"] = stats["total_bytes"] + entry.size stats["total_files"] += 1 else: - _path = ( - f"{path}{PATH_DELIMITER}{entry.name}" if path != "" else entry.name - ) stats["total_folders"] = stats["total_folders"] + 1 - iterate_folder(hs, stats, pid, path=_path, depth=depth + 1) + _path = path + [ + entry.name, + ] + iterate_folder(hs, stats, _path, depth=depth + 1) - logger = get_logger() store = ctx.obj["hashstore_path"] properties = load_hashstore_properties(store) hashstore_factory = hashstore.HashStoreFactory() @@ -380,12 +382,9 @@ def iterate_folder(hs, stats, pid, path="", depth: int = 1): "total_folders": 0, "max_depth": 0, } - parts = pid.split(" ", 1) - path = "" - if len(parts) > 1: - path = parts[1].strip() + path = hashstore.folderentry.split_pidpath(pid) - iterate_folder(hash_store, info, parts[0], path=path, depth=0) + iterate_folder(hash_store, info, path, depth=0) print(json.dumps(info, indent=2)) @@ -395,28 +394,24 @@ def iterate_folder(hs, stats, pid, path="", depth: int = 1): @click.option("-n", "--no-files", is_flag=True, help="Show folders but not content.") def get_folder_tree(ctx, pid: str, no_files: bool) -> None: """Generate a tree representation of the folder and sub-folders.""" + logger = get_logger() - def iterate_folder(hs, tree, pid, path="", with_files: bool = True): - current_folder = hs.retrieve_folder(pid, path=path) + def iterate_folder(hs, tree, path, with_files: bool = True): + current_folder = hs.retrieve_folder(path) n = 0 s = 0 for entry in current_folder: if not entry.is_file or with_files: branch = tree.add(entry.name) if not entry.is_file: - _path = ( - f"{path}{PATH_DELIMITER}{entry.name}" - if path != "" - else entry.name - ) - iterate_folder(hs, branch, pid, path=_path, with_files=with_files) + _path = path + [entry.name] + iterate_folder(hs, branch, _path, with_files=with_files) else: n += 1 s += entry.size if not with_files: branch = tree.add(f"{n:,} files, {s:,} bytes") - logger = get_logger() store = ctx.obj["hashstore_path"] properties = load_hashstore_properties(store) hashstore_factory = hashstore.HashStoreFactory() @@ -428,12 +423,9 @@ def iterate_folder(hs, tree, pid, path="", with_files: bool = True): except Exception as e: logger.error(f"Failed to open hashstore: {e}") return 1 - parts = pid.split(" ", 1) - path = "" - if len(parts) > 1: - path = parts[1].strip() - tree = rich.tree.Tree(pid) - iterate_folder(hash_store, tree, parts[0], path=path, with_files=not no_files) + path = hashstore.folderentry.split_pidpath(pid) + tree = rich.tree.Tree(path[0]) + iterate_folder(hash_store, tree, path, with_files=not no_files) rich.print(tree) diff --git a/src/hashstore/basehashstore.py b/src/hashstore/basehashstore.py index 1b337545..93c50b34 100644 --- a/src/hashstore/basehashstore.py +++ b/src/hashstore/basehashstore.py @@ -4,7 +4,7 @@ import importlib.util from abc import ABC, abstractmethod from pathlib import Path -from typing import Generator, Optional, Union +from typing import IO, Generator, Optional, Union import hashstore.folderentry @@ -68,11 +68,33 @@ def store_object( """ raise NotImplementedError() + @abstractmethod + def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: + """Return object info dict given a path. + + A path may reference another path: + c_1 -> sub_1 -> c_0 -> sub_2 -> x + In such cases, the full path is not stored as a cidref, instead + we have: + path name + c_1 sub_1 + c_1, sub_1 c_0 <- change of context + c_0 sub_2 + c_0, sub_2 x + c_0, sub_2, x + + Hence it is necessary to walk the path to find the next context, + switch to that context, then continue looking for the target. + + An alternative strategy is to load the CID from each folder along + the path, but that is more IO and iterations to find the target. + """ + raise NotADirectoryError() + @abstractmethod def store_folder( self, - pid: str, - path: str, + pidpath: list[str], entries: hashstore.folderentry.FolderEntries, additional_algorithm: Optional[str] = None, checksum: Optional[str] = None, @@ -115,8 +137,7 @@ def store_folder( @abstractmethod def retrieve_folder( self, - pid: str, - path: str, + pidpath: list[str], ) -> hashstore.folderentry.FolderEntries: """Retrieve a FolderEntries instance from the hashstore. @@ -182,6 +203,18 @@ def retrieve_object(self, pid): """ raise NotImplementedError() + @abstractmethod + def retrieve_object_path(self, pidpath: list[str]) -> IO[bytes]: + """Retrieve an object from disk using a persistent identifier (pid). The `retrieve_object` + method opens and returns a buffered object stream ready for reading if the object + associated with the provided `pid` exists on disk. + + :param str pid: Authority-based identifier. + + :return: io.BufferedReader - Buffered stream of the data object. + """ + raise NotImplementedError() + @abstractmethod def retrieve_metadata(self, pid, format_id): """Retrieve the metadata object from disk using a persistent identifier (pid) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 27e6e85a..f0f21367 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1061,7 +1061,7 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: def store_folder( self, - pathpid: list[str], + pidpath: list[str], entries: hashstore.folderentry.FolderEntries, additional_algorithm: Optional[str] = None, checksum: Optional[str] = None, @@ -1107,15 +1107,15 @@ def store_folder( # if path in ("", ".", delim): # path = "" # folder_pid = f"{pid} {path}" if path != "" else pid - folder_pid = hashstore.folderentry.join_pathpid(pathpid) + folder_pid = hashstore.folderentry.join_pidpath(pidpath) self._check_string(folder_pid, "PID") if verify_entry_cids: # check that each entry CID is present in the hashstore. for entry in entries: if entry.cid is None or entry.cid == "": # no cid provided, so look it up - _entry_pid = hashstore.folderentry.join_pathpid( - pathpid + _entry_pid = hashstore.folderentry.join_pidpath( + pidpath + [ entry.name, ] @@ -1170,62 +1170,89 @@ def store_folder( obj_size=obj_size, ) + def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: + """Return object info dict given a path. + + A path may reference another path: + c_1 -> sub_1 -> c_0 -> sub_2 -> x + In such cases, the full path is not stored as a cidref, instead + we have: + path name + c_1 sub_1 + c_1, sub_1 c_0 <- change of context + c_0 sub_2 + c_0, sub_2 x + c_0, sub_2, x + + Hence it is necessary to walk the path to find the next context, + switch to that context, then continue looking for the target. + + An alternative strategy is to load the CID from each folder along + the path, but that is more IO and iterations to find the target. + """ + pid = hashstore.folderentry.join_pidpath(pidpath) + self._check_string(pid, "PID") + # first try the literal path + try: + return self.find_object(pid) + except PidRefsDoesNotExist as e: + # End of the line + if len(pidpath) == 1: + raise e + pass + # Path changes context at some point (or doesn't exist) + context_switch = 1 + for cpos in range(1, len(pidpath)): + # walk the path to find where context switches + try: + current_pid = hashstore.folderentry.join_pidpath(pidpath[:cpos]) + _ = self.find_object(current_pid) + except PidRefsDoesNotExist: + # context switch, start over with segment from current context onwards + context_switch = cpos + break + return self.resolve_pidpath(pidpath[context_switch - 1 :]) + + def retrieve_object_path(self, pidpath: list[str]) -> IO[bytes]: + object_info_dict = self.resolve_pidpath(pidpath) + object_cid = object_info_dict.get("cid") + entity = "objects" + if object_cid: + self.fhs_logger.debug( + "Metadata exists for pid: %s, retrieving object.", pidpath + ) + obj_stream = self._open(entity, object_cid) + else: + err_msg = f"No object found for pid: {pidpath}" + self.fhs_logger.error(err_msg) + raise ValueError(err_msg) + self.fhs_logger.info("Retrieved object for pid: %s", pidpath) + return obj_stream + def retrieve_folder( self, - pathpid: list[str], + pidpath: list[str], ) -> hashstore.folderentry.FolderEntries: """Retrieve a FolderEntries instance from the hashstore. - We first check to see if a CID is available for the combination of - "{PID} {path}", and if so, return that entry. Otherwise, we iterate - over path segments to find the correspoding FolderEntry, if any. - This iterative approach is necesary if since entire trees are not - stored when a new version of a folder hierarchy is stored. Hence, it - may be necessary to jump back to a branch that is recorded in an - earlier version but not recorded in the current version since it - was unchanged between versions. + Given a sequence of path segments to a FolderEntries object, + return the object. Args: - pid (str): The context (i.e. VMDAG version) within which this folder is being retrieved - path (str): Path within the context to the desired entry, using folderentry.PATH_DELIMITER + pidpath (list[str]): Path segments to the folder Returns: FolderEntries + Raises: + PidRefsDoesNotExist """ - folder_pid = hashstore.folderentry.join_pathpid(pathpid) - self._check_string(folder_pid, "PID") - # try direct reference to CID using folder_pid. This works if - # there is no branching to other PID contexts (typical case) - try: - object_info_dict = self.find_object(folder_pid) - folder_cid = object_info_dict.get("cid") - if folder_cid is None: - raise PidRefsDoesNotExist("Entry has no cid?") - cid_path = object_info_dict.get("cid_object_path") - # self._build_hashstore_data_object_path(folder_cid) - return hashstore.folderentry.FolderEntries.from_parquet(cid_path) - except PidRefsDoesNotExist: - pass - - # otherwise, iterate over the path, following a branch if needed. - # get the root, split the path, and start iterating. - # This will raise PidRefsDoesNotExist if the root PID isn't there - object_info_dict = self.find_object(pathpid[0]) + # this will raise if pidpath isn't found + object_info_dict = self.resolve_pidpath(pidpath) + # have info, load the folder object from the cid path folder_cid = object_info_dict.get("cid") if folder_cid is None: - # Should never reach this... - raise PidRefsDoesNotExist("Entry has no cid?") + raise PidRefsDoesNotExist(f"Entry {pidpath} has no cid?") cid_path = object_info_dict.get("cid_object_path") - # self._build_hashstore_data_object_path(folder_cid) - # Get the root folder, then find the next path element in the folder - current_folder = hashstore.folderentry.FolderEntries.from_parquet(cid_path) - for idx in range(1, len(pathpid)): - entry = current_folder.entry_by_name(pathpid[idx]) - if entry is None: - raise KeyError(f"PID {pathpid} not found.") - # given the entry, we have the cid. Use that to get the next folder - cid_path = self._get_hashstore_data_object_path(entry.cid) - current_folder = hashstore.folderentry.FolderEntries.from_parquet(cid_path) - return current_folder + return hashstore.folderentry.FolderEntries.from_parquet(cid_path) def list_pids(self, pattern: Optional[str] = None) -> Generator: """Yield create_timestamp, CID, PID. diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index 92eda30a..58470fcb 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -20,13 +20,13 @@ def get_logger(): return logging.getLogger("FolderEntry") -def split_pathpid(pathpid: str) -> list[str]: - pathpid = pathpid.strip(PATH_DELIMITER) +def split_pidpath(pidpath: str) -> list[str]: + pathpid = pidpath.strip(PATH_DELIMITER) parts = pathpid.split(PATH_DELIMITER) return parts -def join_pathpid(path: list[str]) -> str: +def join_pidpath(path: list[str]) -> str: # remove "", strings with only white space cleaned = [s.strip() for s in path] return PATH_DELIMITER.join(list(filter(str.strip, cleaned))) From 8c2a67346497fe542457e0f0132d2c44c4c4fe6d Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 12 May 2026 08:43:18 -0400 Subject: [PATCH 23/49] Fix recursion step, allow delimtier to be specified --- src/hashstore/__main__.py | 6 +++--- src/hashstore/filehashstore.py | 4 +++- src/hashstore/folderentry.py | 10 +++++----- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/hashstore/__main__.py b/src/hashstore/__main__.py index e6a4d49f..2e8c7bd7 100644 --- a/src/hashstore/__main__.py +++ b/src/hashstore/__main__.py @@ -382,7 +382,7 @@ def iterate_folder(hs, stats, path, depth: int = 1): "total_folders": 0, "max_depth": 0, } - path = hashstore.folderentry.split_pidpath(pid) + path = hashstore.folderentry.split_pidpath(pid, delimiter="|") iterate_folder(hash_store, info, path, depth=0) print(json.dumps(info, indent=2)) @@ -423,8 +423,8 @@ def iterate_folder(hs, tree, path, with_files: bool = True): except Exception as e: logger.error(f"Failed to open hashstore: {e}") return 1 - path = hashstore.folderentry.split_pidpath(pid) - tree = rich.tree.Tree(path[0]) + path = hashstore.folderentry.split_pidpath(pid, delimiter="|") + tree = rich.tree.Tree(pid) iterate_folder(hash_store, tree, path, with_files=not no_files) rich.print(tree) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index f0f21367..a1a2f725 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1190,6 +1190,7 @@ def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: An alternative strategy is to load the CID from each folder along the path, but that is more IO and iterations to find the target. """ + self.fhs_logger.debug("Resolve: %s", pidpath) pid = hashstore.folderentry.join_pidpath(pidpath) self._check_string(pid, "PID") # first try the literal path @@ -1202,8 +1203,9 @@ def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: pass # Path changes context at some point (or doesn't exist) context_switch = 1 - for cpos in range(1, len(pidpath)): + for cpos in range(1, len(pidpath) + 1): # walk the path to find where context switches + self.fhs_logger.debug("At: %s", pidpath[:cpos]) try: current_pid = hashstore.folderentry.join_pidpath(pidpath[:cpos]) _ = self.find_object(current_pid) diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index 58470fcb..511b45e0 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -20,16 +20,16 @@ def get_logger(): return logging.getLogger("FolderEntry") -def split_pidpath(pidpath: str) -> list[str]: - pathpid = pidpath.strip(PATH_DELIMITER) - parts = pathpid.split(PATH_DELIMITER) +def split_pidpath(pidpath: str, delimiter: str = PATH_DELIMITER) -> list[str]: + pathpid = pidpath.strip(delimiter) + parts = pathpid.split(delimiter) return parts -def join_pidpath(path: list[str]) -> str: +def join_pidpath(path: list[str], delimiter: str = PATH_DELIMITER) -> str: # remove "", strings with only white space cleaned = [s.strip() for s in path] - return PATH_DELIMITER.join(list(filter(str.strip, cleaned))) + return delimiter.join(list(filter(str.strip, cleaned))) @dataclasses.dataclass From 07e32a707c6f9f45c91ba5bfa19ca0d541365653 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 14 May 2026 11:08:34 -0400 Subject: [PATCH 24/49] Initial store - WIP --- src/hashstore/__init__.py | 4 +- src/hashstore/basehashstore.py | 6 +++ src/hashstore/filehashstore.py | 78 +++++++++++++++++++++++++--------- src/hashstore/folderentry.py | 4 +- src/hashstore/pidlogger.py | 28 ++++++------ 5 files changed, 83 insertions(+), 37 deletions(-) diff --git a/src/hashstore/__init__.py b/src/hashstore/__init__.py index 6d470289..6e58b02b 100644 --- a/src/hashstore/__init__.py +++ b/src/hashstore/__init__.py @@ -16,7 +16,7 @@ system. """ -from hashstore.basehashstore import HashStore, HashStoreFactory +from hashstore.basehashstore import HashStore, HashStoreFactory, PidObserver -__all__ = ("HashStore", "HashStoreFactory") +__all__ = ("HashStore", "HashStoreFactory", "PidObserver") __version__ = "1.1.0" diff --git a/src/hashstore/basehashstore.py b/src/hashstore/basehashstore.py index 93c50b34..14c40fc0 100644 --- a/src/hashstore/basehashstore.py +++ b/src/hashstore/basehashstore.py @@ -9,6 +9,12 @@ import hashstore.folderentry +class PidObserver(ABC): + @abstractmethod + def update(self, cid: str, pid: str | None = None): + pass + + class HashStore(ABC): """HashStore is a content-addressable file management system that utilizes an object's content identifier (hex digest/checksum) to address files.""" diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index a1a2f725..6461bcb2 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -20,7 +20,7 @@ import yaml import hashstore.folderentry -from hashstore import HashStore, pidlogger +from hashstore import HashStore, PidObserver from hashstore.filehashstore_exceptions import ( CidRefsContentError, CidRefsFileNotFound, @@ -85,6 +85,7 @@ class FileHashStore(HashStore): def __init__(self, properties=None): self.fhs_logger = logging.getLogger(__name__) + self.pid_watchers = [] # Now check properties if properties: # Validate properties against existing configuration if present @@ -134,8 +135,6 @@ def __init__(self, properties=None): self._create_path(self.refs / "tmp") self._create_path(self.refs / "pids") self._create_path(self.refs / "cids") - # pidlog is used to create an index of cid - pid - self.pidlog = logging.getLogger("pid_logger") # Variables to orchestrate parallelization # Check to see whether a multiprocessing or threading sync lock should be used @@ -518,6 +517,13 @@ def lookup_algo(algo_to_translate): # Public API / HashStore Interface Methods + def add_watcher(self, watcher: PidObserver) -> None: + self.pid_watchers.append(watcher) + + def notify(self, cid, pid): + for watcher in self.pid_watchers: + watcher.update(cid, pid) + def store_object( self, pid: Optional[str] = None, @@ -580,7 +586,7 @@ def store_object( cid = object_metadata.cid self.tag_object(pid, cid) self.fhs_logger.info("Successfully stored object for pid: %s", pid) - self.pidlog.info(cid, extra={"pid": pid}) + self.notify(cid, pid) finally: # Release pid self._release_object_locked_pids(pid) @@ -1162,7 +1168,7 @@ def store_folder( # of folder entries increases. obj_size = entries.to_parquet(cid_path, pid=folder_pid) self.tag_object(folder_pid, folder_cid) - self.pidlog.info(folder_cid, extra={"pid": folder_pid}) + self.notify(folder_cid, folder_pid) return ObjectMetadata( pid=folder_pid, cid=folder_cid, @@ -1173,6 +1179,10 @@ def store_folder( def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: """Return object info dict given a path. + #TODO: + # - write test cases for this. + # - consider adding a recursion trap + A path may reference another path: c_1 -> sub_1 -> c_0 -> sub_2 -> x In such cases, the full path is not stored as a cidref, instead @@ -1197,23 +1207,46 @@ def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: try: return self.find_object(pid) except PidRefsDoesNotExist as e: - # End of the line - if len(pidpath) == 1: + # End of the line ? + if len(pidpath) < 2: raise e - pass - # Path changes context at some point (or doesn't exist) - context_switch = 1 - for cpos in range(1, len(pidpath) + 1): - # walk the path to find where context switches - self.fhs_logger.debug("At: %s", pidpath[:cpos]) - try: - current_pid = hashstore.folderentry.join_pidpath(pidpath[:cpos]) - _ = self.find_object(current_pid) - except PidRefsDoesNotExist: - # context switch, start over with segment from current context onwards - context_switch = cpos - break - return self.resolve_pidpath(pidpath[context_switch - 1 :]) + # continue + # Does the root context exist? + try: + object_info_dict = self.find_object(pidpath[0]) + cid_object_path = object_info_dict.get("cid_object_path") + current_folder = hashstore.folderentry.FolderEntries.from_parquet( + cid_object_path + ) + except PidRefsDoesNotExist as e: + # nope + raise e + object_info_dict = { + "cid": None, + "cid_object_path": None, + "cid_refs_path": None, + "pid_refs_path": None, + "sysmeta_path": "Does not exist.", + } + entry = None + # walk the path, following cids referened by folder + for idx in range(1, len(pidpath)): + _name = pidpath[idx] + entry = current_folder.entry_by_name(_name) + if entry is None: + raise PidRefsDoesNotExist(f"PID not found: {pid}") + object_info_dict["cid"] = entry.cid + object_info_dict["cid_object_path"] = self._get_hashstore_data_object_path( + entry.cid + ) + object_info_dict["cid_ref_path"] = self._get_hashstore_cid_refs_path( + entry.cid + ) + if idx < len(pidpath): + current_folder = hashstore.folderentry.FolderEntries.from_parquet( + object_info_dict["cid_object_path"] + ) + return object_info_dict def retrieve_object_path(self, pidpath: list[str]) -> IO[bytes]: object_info_dict = self.resolve_pidpath(pidpath) @@ -1262,6 +1295,9 @@ def list_pids(self, pattern: Optional[str] = None) -> Generator: Iterates over all CID entries and yields the create timestamp, CID value, and PID value for all entries or those PIDs that match the optionally provided regexp pattern. + + Note that this can be really slow when there's a large number of + ref files. """ rpattern = None if pattern is not None: diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index 511b45e0..a4e4d15e 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -41,9 +41,9 @@ class FolderEntry: cid: str """The content hash (CID) for the entry.""" is_file: bool # True for file, False for Folder - """The type of manifest entry: '1' for file, '0' for directory.""" + """The type of manifest entry: False for folder, True for file.""" size: int = 0 - """Size of the file in bytes or number of entries for directories.""" + """Size of the file in bytes or number of entries for folders.""" formatid: str | None = None """Optional format identifier for files.""" diff --git a/src/hashstore/pidlogger.py b/src/hashstore/pidlogger.py index 1091e9c0..8901f7a7 100644 --- a/src/hashstore/pidlogger.py +++ b/src/hashstore/pidlogger.py @@ -26,6 +26,8 @@ import json import logging +from hashstore.basehashstore import PidObserver + class PidIndexFormatter(logging.Formatter): def format(self, record) -> str: @@ -37,15 +39,17 @@ def format(self, record) -> str: return json.dumps(pid_record) -def getPidLogger(log_file_name: str): - """Return a logger for the pid index. - - PID records are added to the index like: - logger.info(CID, extra={"pid":PID}) - """ - logger = logging.getLogger("pid_logger") - logger.setLevel(logging.INFO) - handler = logging.FileHandler(log_file_name) - handler.setFormatter(PidIndexFormatter()) - logger.addHandler(handler) - return logger +class PidLogObserver(PidObserver): + def __init__(self, log_file_name: str | None = None): + self.logger = logging.getLogger("pid_logger") + self.logger.handlers.clear() + self.logger.propagate = False + if log_file_name is None: + return + self.logger.setLevel(logging.INFO) + handler = logging.FileHandler(log_file_name) + handler.setFormatter(PidIndexFormatter()) + self.logger.addHandler(handler) + + def update(self, cid: str, pid: str | None = None): + self.logger.info(cid, extra={"pid": pid}) From fa1eb13a04b07f574a9422b4d3b73fa0ee8103aa Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 14 May 2026 11:58:47 -0400 Subject: [PATCH 25/49] Limit code changes to folder support --- src/hashstore/__init__.py | 4 +-- src/hashstore/basehashstore.py | 6 ---- src/hashstore/filehashstore.py | 44 ++++++++++----------------- src/hashstore/pidlogger.py | 55 ---------------------------------- 4 files changed, 17 insertions(+), 92 deletions(-) delete mode 100644 src/hashstore/pidlogger.py diff --git a/src/hashstore/__init__.py b/src/hashstore/__init__.py index 6e58b02b..6d470289 100644 --- a/src/hashstore/__init__.py +++ b/src/hashstore/__init__.py @@ -16,7 +16,7 @@ system. """ -from hashstore.basehashstore import HashStore, HashStoreFactory, PidObserver +from hashstore.basehashstore import HashStore, HashStoreFactory -__all__ = ("HashStore", "HashStoreFactory", "PidObserver") +__all__ = ("HashStore", "HashStoreFactory") __version__ = "1.1.0" diff --git a/src/hashstore/basehashstore.py b/src/hashstore/basehashstore.py index 14c40fc0..93c50b34 100644 --- a/src/hashstore/basehashstore.py +++ b/src/hashstore/basehashstore.py @@ -9,12 +9,6 @@ import hashstore.folderentry -class PidObserver(ABC): - @abstractmethod - def update(self, cid: str, pid: str | None = None): - pass - - class HashStore(ABC): """HashStore is a content-addressable file management system that utilizes an object's content identifier (hex digest/checksum) to address files.""" diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 6461bcb2..cfadf451 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -20,7 +20,7 @@ import yaml import hashstore.folderentry -from hashstore import HashStore, PidObserver +from hashstore import HashStore from hashstore.filehashstore_exceptions import ( CidRefsContentError, CidRefsFileNotFound, @@ -85,7 +85,6 @@ class FileHashStore(HashStore): def __init__(self, properties=None): self.fhs_logger = logging.getLogger(__name__) - self.pid_watchers = [] # Now check properties if properties: # Validate properties against existing configuration if present @@ -517,13 +516,6 @@ def lookup_algo(algo_to_translate): # Public API / HashStore Interface Methods - def add_watcher(self, watcher: PidObserver) -> None: - self.pid_watchers.append(watcher) - - def notify(self, cid, pid): - for watcher in self.pid_watchers: - watcher.update(cid, pid) - def store_object( self, pid: Optional[str] = None, @@ -586,7 +578,6 @@ def store_object( cid = object_metadata.cid self.tag_object(pid, cid) self.fhs_logger.info("Successfully stored object for pid: %s", pid) - self.notify(cid, pid) finally: # Release pid self._release_object_locked_pids(pid) @@ -1168,7 +1159,6 @@ def store_folder( # of folder entries increases. obj_size = entries.to_parquet(cid_path, pid=folder_pid) self.tag_object(folder_pid, folder_cid) - self.notify(folder_cid, folder_pid) return ObjectMetadata( pid=folder_pid, cid=folder_cid, @@ -1179,14 +1169,14 @@ def store_folder( def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: """Return object info dict given a path. - #TODO: - # - write test cases for this. - # - consider adding a recursion trap - A path may reference another path: c_1 -> sub_1 -> c_0 -> sub_2 -> x + + Similarly, a path rooted in on PID may reference a path rooted in another PID. + In such cases, the full path is not stored as a cidref, instead - we have: + we have in the example above: + path name c_1 sub_1 c_1, sub_1 c_0 <- change of context @@ -1194,11 +1184,8 @@ def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: c_0, sub_2 x c_0, sub_2, x - Hence it is necessary to walk the path to find the next context, - switch to that context, then continue looking for the target. - - An alternative strategy is to load the CID from each folder along - the path, but that is more IO and iterations to find the target. + Hence it is necessary to walk the path following folder entry CIDs + to locate the target if it is not directly resolvable by the full path. """ self.fhs_logger.debug("Resolve: %s", pidpath) pid = hashstore.folderentry.join_pidpath(pidpath) @@ -1210,8 +1197,14 @@ def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: # End of the line ? if len(pidpath) < 2: raise e - # continue # Does the root context exist? + object_info_dict = { + "cid": None, + "cid_object_path": None, + "cid_refs_path": None, + "pid_refs_path": None, + "sysmeta_path": "Does not exist.", + } try: object_info_dict = self.find_object(pidpath[0]) cid_object_path = object_info_dict.get("cid_object_path") @@ -1221,13 +1214,6 @@ def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: except PidRefsDoesNotExist as e: # nope raise e - object_info_dict = { - "cid": None, - "cid_object_path": None, - "cid_refs_path": None, - "pid_refs_path": None, - "sysmeta_path": "Does not exist.", - } entry = None # walk the path, following cids referened by folder for idx in range(1, len(pidpath)): diff --git a/src/hashstore/pidlogger.py b/src/hashstore/pidlogger.py deleted file mode 100644 index 8901f7a7..00000000 --- a/src/hashstore/pidlogger.py +++ /dev/null @@ -1,55 +0,0 @@ -"""Implements a multi-process safe logger for generating a cid-pid index. - -This implementation writes an NDJSON file with each line containing a json array -with elements: - [0] timestamp - [1] cid - [2] pid - -The logging module is used because it is multi-thread and -process safe. - -The resulting ndjson file can be loaded into duckdb for example with: - - CREATE TABLE pids AS SELECT - to_timestamp(json[1]::DOUBLE) AS ctime, - json[2]->> '$' AS cid, - json[3]->>'$' AS pid - FROM read_json('pid_index.ndjson'); - -or create a parquet representation: - - duckdb -c "COPY (SELECT to_timestamp(json[1]::DOUBLE) AS ctime, - json[2]->> '\\$' AS cid, json[3]->>'\\$' AS pid FROM - read_json('pid_index.ndjson')) TO 'pid_index.parquet' (FORMAT parquet)" -""" - -import json -import logging - -from hashstore.basehashstore import PidObserver - - -class PidIndexFormatter(logging.Formatter): - def format(self, record) -> str: - pid_record = ( - record.created, - record.getMessage().strip(), - record.pid if hasattr(record, "pid") else None, - ) - return json.dumps(pid_record) - - -class PidLogObserver(PidObserver): - def __init__(self, log_file_name: str | None = None): - self.logger = logging.getLogger("pid_logger") - self.logger.handlers.clear() - self.logger.propagate = False - if log_file_name is None: - return - self.logger.setLevel(logging.INFO) - handler = logging.FileHandler(log_file_name) - handler.setFormatter(PidIndexFormatter()) - self.logger.addHandler(handler) - - def update(self, cid: str, pid: str | None = None): - self.logger.info(cid, extra={"pid": pid}) From dfa8dc718d4c57c1c4edb278328c6776509446c1 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Wed, 27 May 2026 22:45:25 -0400 Subject: [PATCH 26/49] WIP folder mutations, reduce overly agressive type checking --- hashstore_layout.drawio | 57 ++++++- src/hashstore/__main__.py | 283 ++++++++++++++++++++++++++++----- src/hashstore/filehashstore.py | 13 +- src/hashstore/folderentry.py | 19 +++ 4 files changed, 326 insertions(+), 46 deletions(-) diff --git a/hashstore_layout.drawio b/hashstore_layout.drawio index 4b6c4c73..48e42c75 100644 --- a/hashstore_layout.drawio +++ b/hashstore_layout.drawio @@ -1,6 +1,6 @@ - + - + @@ -142,4 +142,57 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/hashstore/__main__.py b/src/hashstore/__main__.py index 2e8c7bd7..e6a991bd 100644 --- a/src/hashstore/__main__.py +++ b/src/hashstore/__main__.py @@ -1,10 +1,14 @@ +import concurrent.futures import dataclasses import datetime import json import logging import os import pathlib +import queue +import re import sys +import typing import click import rich @@ -67,6 +71,113 @@ def locate_hashstore(path: pathlib.Path) -> pathlib.Path | None: return None +# ctime, cid_value, pid +def _handle_cid_ref_file( + cids_path: str, cid_entry: os.DirEntry, rpattern: re.Pattern | None +) -> list[tuple[float, str, str]]: + result = [] + # print(cids_root, cids_path, cid_entries) + fname = cid_entry.path + try: + cid_value = fname.replace(cids_path, "").replace("/", "") + ctime = cid_entry.stat().st_ctime + for _, entry in enumerate(open(fname, "r", encoding="utf-8")): + pid = entry.strip() + if len(pid) > 0: + if rpattern is not None: + if rpattern.fullmatch(pid): + result.append( + ( + ctime, + cid_value, + pid, + ) + ) + else: + # print(pid) + result.append( + ( + ctime, + cid_value, + pid, + ) + ) + except Exception as e: + print(e) + return result + + +def enumerate_hs_files( + refs_cids_root: str, pattern: str | None = None, max_workers: int = 2 +) -> typing.Generator: + """Perform a depth first scan of the hashshore refs/cids to yield PIDs. + + This operation will perform a multi-threaded depth first traversal of the + hashstore refs/cids hierarchy, read each file, and yield a three-tuple of + create time, cid, pid + + This process is efficient, but still slow due to the number of files that + need to be processed in even modeterate sized hash stores. For example, + on an m5 mac running with 10 threads on a hashstore with 4 million entries, + the process takes about 20 minutes to complete. + """ + _L = get_logger() + rpattern = None + if pattern is not None: + rpattern = re.compile(pattern) + ignore_names = [ + ".DS_Store", + ] + # Use a LIFO queue for the dirs so we do depth first processing + dirs_queue = queue.LifoQueue() + # starting point is root of refs/cids + dirs_queue.put(refs_cids_root) + workers = {} + ticker = 0 + with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor: + # keep doing stuff untill both the dirs_queue and the workers dict are empty + while not dirs_queue.empty() or len(workers) > 0: + try: + current_dir = dirs_queue.get(timeout=0.01) + try: + with os.scandir(current_dir) as entries: + for entry in entries: + if entry.name in ignore_names: + continue + if entry.is_dir(follow_symlinks=False): + # entry is a folder, add it to the dirs_queue + dirs_queue.put(entry.path) + else: + # entry is a file. Create a task to process it on a worker + future = executor.submit( + _handle_cid_ref_file, + refs_cids_root, + entry, + rpattern, + ) + workers[future] = entry + except PermissionError: + _L.error("Permission denied for %s", current_dir) + except Exception as e: + _L.error("Error at %s: %s", current_dir, e) + except queue.Empty: + # no more dirs to process + pass + # simple UI feedback + if ticker % 1000 == 0: + print(f"{ticker:,} {dirs_queue.qsize():,} {len(workers):,}") + ticker += 1 + try: + # yield results from workers as they complete. Also remove completed + # from the workers dict to keep things compact + for worker in concurrent.futures.as_completed(workers, timeout=0.01): + for result in worker.result(): + yield result + del workers[worker] + except TimeoutError: + pass + + @click.group() @click.option( "--store", @@ -264,8 +375,21 @@ def add_object( ) def get_object(ctx, pid, stream, recursive): """Retrieve an object or folder from hashstore.""" + logger = get_logger() store = ctx.obj["hashstore_path"] - pass + properties = load_hashstore_properties(store) + hashstore_factory = hashstore.HashStoreFactory() + try: + hash_store = hashstore_factory.get_hashstore( + ctx.obj["module_name"], ctx.obj["class_name"], properties + ) + logger.debug(f"Hashstore opened at: {store}") + except Exception as e: + logger.error(f"Failed to open hashstore: {e}") + return + pidpath = hashstore.folderentry.split_pidpath(pid, delimiter="|") + info = hash_store.resolve_pidpath(pidpath) + print(info) @main.command("ls") @@ -287,8 +411,27 @@ def get_object(ctx, pid, stream, recursive): ) @click.option("-r", "--reference", is_flag=True, help="Include path references.") @click.option("-l", "--list-only", is_flag=True, help="Just list the cid, pid values.") -def list_pids(ctx, pattern, human_readable, show_metadata, reference, list_only): - """List PIDs in the hashstore.""" +@click.option("-w", "--workers", default=2, help="Number of workers.") +@click.option("-o", "--output", default=None, help="Write output to file.") +def list_pids( + ctx, pattern, human_readable, show_metadata, reference, list_only, workers, output +): + """List PIDs in the hashstore (async). + + The resulting ndjson file can be loaded into duckdb for example with: + + CREATE TABLE pids AS SELECT + to_timestamp(json[1]::DOUBLE) AS ctime, + json[2]->> '$' AS cid, + json[3]->>'$' AS pid + FROM read_json('pid_index.ndjson'); + + or create a parquet representation: + + duckdb -c "COPY (SELECT to_timestamp(json[1]::DOUBLE) AS ctime, + json[2]->> '\\$' AS cid, json[3]->>'\\$' AS pid FROM + read_json('pid_index.ndjson')) TO 'pid_index.parquet' (FORMAT parquet)" + """ logger = get_logger() store = ctx.obj["hashstore_path"] properties = load_hashstore_properties(store) @@ -305,45 +448,98 @@ def list_pids(ctx, pattern, human_readable, show_metadata, reference, list_only) # iterate over the refs/cids folder, getting PIDs from each file. if not list_only: print(f"Hashstore: {str(store.relative_to(pathlib.Path.cwd(), walk_up=True))}") - for ctime, cid, pid in hash_store.list_pids(pattern=pattern): - if list_only: - print(json.dumps((ctime, cid, pid))) - continue - try: - pid_stat = hash_store.get_object_status(pid) - fsize = pid_stat.get("size", 0) - total_objects += 1 - t_modified = pid_stat.get("modtime", "-") - if t_modified != "-": - t_modified = datetime.datetime.fromtimestamp(t_modified).isoformat( - timespec="seconds" - ) - t_access = pid_stat.get("accesstime", "-") - if t_access != "-": - t_access = datetime.datetime.fromtimestamp(t_access).isoformat( - timespec="seconds" + dest_file = sys.stdout + if output is not None: + dest_file = open(output, "w") + try: + for ctime, cid, pid in enumerate_hs_files( + str(hash_store.cids), pattern=pattern, max_workers=workers + ): + if list_only: + dest_file.write( + f"{json.dumps((ctime, cid, pid), ensure_ascii=False)}\n" ) - if human_readable: - fsize = sizeof_fmt(fsize) - if fsize > 0 or reference: - print(f"{fsize}\t{t_modified}\t{t_access}\t{pid}") - if show_metadata: - try: - meta = hash_store.retrieve_metadata(pid).read().decode() - print(meta) - # print(json.dumps(meta, indent=2)) - except KeyError: - pass - except KeyError: - logger.warning(f"PID status not available in hashstore: {pid}") - if not list_only: - print(f"Total {total_objects}") - - -@main.command("finfo") + continue + try: + pid_stat = hash_store.get_object_status(pid) + fsize = pid_stat.get("size", 0) + total_objects += 1 + t_modified = pid_stat.get("modtime", "-") + if t_modified != "-": + t_modified = datetime.datetime.fromtimestamp(t_modified).isoformat( + timespec="seconds" + ) + t_access = pid_stat.get("accesstime", "-") + if t_access != "-": + t_access = datetime.datetime.fromtimestamp(t_access).isoformat( + timespec="seconds" + ) + if human_readable: + fsize = sizeof_fmt(fsize) + if fsize > 0 or reference: + dest_file.write(f"{fsize}\t{t_modified}\t{t_access}\t{pid}\n") + if show_metadata: + try: + meta = hash_store.retrieve_metadata(pid).read().decode() + print(meta) + # print(json.dumps(meta, indent=2)) + except KeyError: + pass + except KeyError: + logger.warning(f"PID status not available in hashstore: {pid}") + if not list_only: + print(f"Total {total_objects}") + finally: + if output is not None: + dest_file.close() + + +@main.command("meta") +@click.pass_context +@click.argument("pid", type=str) +def get_system_metadata(ctx, pid) -> None: + """Retrieve system metadata for PID""" + logger = get_logger() + store = ctx.obj["hashstore_path"] + properties = load_hashstore_properties(store) + hashstore_factory = hashstore.HashStoreFactory() + try: + hash_store = hashstore_factory.get_hashstore( + ctx.obj["module_name"], ctx.obj["class_name"], properties + ) + logger.debug(f"Hashstore opened at: {store}") + except Exception as e: + logger.error(f"Failed to open hashstore: {e}") + return + + pidpath = hashstore.folderentry.split_pidpath(pid, delimiter="|") + + info = {} + sysm_found = False + sysm_pid_path = pidpath + while len(sysm_pid_path) > 0: + print(" | ".join(sysm_pid_path)) + info = hash_store.resolve_pidpath(sysm_pid_path) + print(info) + if info.get("sysmeta_path") not in (None, "Does not exist."): + sysm_found = True + break + sysm_pid_path.pop() + print("===") + if not sysm_found: + print(f"No system metadata found for {pid}") + sysm_pid = hashstore.folderentry.join_pidpath(sysm_pid_path) + with hash_store.retrieve_metadata( + sysm_pid, "https://ns.dataone.org/service/types/v2.0#SystemMetadata" + ) as sysmf: + sysm = sysmf.read() + print(sysm.decode("utf-8")) + + +@main.command("info") @click.pass_context @click.argument("pid", type=str) -def get_folder_info(ctx, pid) -> None: +def get_object_info(ctx, pid) -> None: """Compute basic stats for a folder and sub-folders.""" logger = get_logger() @@ -351,7 +547,14 @@ def iterate_folder(hs, stats, path, depth: int = 1): logger.debug("Current path=%s", path) if depth > stats["max_depth"]: stats["max_depth"] = depth - current_folder = hs.retrieve_folder(path) + target = hs.resolve_pidpath(path) + cid_object_path = target.get("cid_object_path") + if hashstore.folderentry.is_folder(cid_object_path): + current_folder = hs.retrieve_folder(path) + else: + stats["total_files"] += 1 + stats["total_bytes"] += cid_object_path.stat().st_size + return for entry in current_folder: logger.debug(str(entry)) if entry.is_file: diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index cfadf451..7c46464c 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1208,6 +1208,8 @@ def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: try: object_info_dict = self.find_object(pidpath[0]) cid_object_path = object_info_dict.get("cid_object_path") + if not hashstore.folderentry.is_folder(cid_object_path): + return object_info_dict current_folder = hashstore.folderentry.FolderEntries.from_parquet( cid_object_path ) @@ -1229,9 +1231,12 @@ def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: entry.cid ) if idx < len(pidpath): - current_folder = hashstore.folderentry.FolderEntries.from_parquet( - object_info_dict["cid_object_path"] - ) + if hashstore.folderentry.is_folder(object_info_dict["cid_object_path"]): + current_folder = hashstore.folderentry.FolderEntries.from_parquet( + object_info_dict["cid_object_path"] + ) + else: + return object_info_dict return object_info_dict def retrieve_object_path(self, pidpath: list[str]) -> IO[bytes]: @@ -3011,7 +3016,7 @@ def _check_arg_data(data: Union[str, os.PathLike, io.BufferedReader]) -> bool: if ( not isinstance(data, str) and not isinstance(data, Path) - and not isinstance(data, io.BufferedIOBase) + and not isinstance(data, io.IOBase) ): err_msg = ( "FileHashStore - _validate_arg_data: Data must be a path, string or buffered" diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index a4e4d15e..8ca72cd8 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -134,3 +134,22 @@ def from_parquet(cls, pq_path) -> "FolderEntries": ) ) return entries + + +def is_folder(path: str) -> bool: + """Test if the target of the path is a folder object. + + This test is fast, reading just a few bytes, but there + is of course associated file IO, so avoid use in loops etc. + """ + try: + pq_metadata = pyarrow.parquet.read_metadata(path) + try: + metadata = json.loads(pq_metadata.metadata[PARQUET_METADATA_KEY].decode()) + _ = metadata["version"] + except KeyError: + return False + return True + except Exception: + pass + return False From f9104cb1a3c77a77d89915a345d421797c28eb60 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Fri, 29 May 2026 16:01:44 -0400 Subject: [PATCH 27/49] Setting up types for folder mutation operations --- src/hashstore/folderentry.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/src/hashstore/folderentry.py b/src/hashstore/folderentry.py index 8ca72cd8..9e83ca27 100644 --- a/src/hashstore/folderentry.py +++ b/src/hashstore/folderentry.py @@ -153,3 +153,29 @@ def is_folder(path: str) -> bool: except Exception: pass return False + + +# == Folder Operation support + +OP_ADD = "add" +OP_DELETE = "delete" +OP_MOVE = "move" +OP_MODIFY = "modify" + + +@dataclasses.dataclass +class FolderOperation: + """Describes a single operation for folder hierarchy changes.""" + + operation: str + """Operation tyoe: 'add', 'delete', or 'move'.""" + a: str + """Path of item relative to a PID root.""" + b: str | None = None + """New path for move operations. Should be None for add/delete.""" + c: bytes | None = None + """Bytes content for add operations. Should be None for delete/move.""" + + def __post_init__(self): + if self.operation not in (OP_ADD, OP_DELETE, OP_MOVE): + raise ValueError(f"Invalid operation: {self.operation}") From 096727783f574a86b7723c1fbd07f95e351be3e9 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Fri, 29 May 2026 16:02:50 -0400 Subject: [PATCH 28/49] Overzealous parameter checking, revist after adding ful typehint support --- src/hashstore/filehashstore.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 7c46464c..d2b8acb5 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -537,7 +537,7 @@ def store_object( self.fhs_logger.debug("Request to store object for pid: %s", pid) # Validate input parameters self._check_string(pid, "pid") - self._check_arg_data(data) + # self._check_arg_data(data) self._check_integer(expected_object_size) ( additional_algorithm_checked, @@ -3103,8 +3103,10 @@ class Stream: """ def __init__(self, obj: Union[IO[bytes], str, Path]): + # is it a file like thing if hasattr(obj, "read"): pos = obj.tell() + # or a string elif os.path.isfile(obj): obj = io.open(obj, "rb") pos = None @@ -3114,7 +3116,9 @@ def __init__(self, obj: Union[IO[bytes], str, Path]): try: file_stat = os.stat(obj.name) buffer_size = file_stat.st_blksize - except (FileNotFoundError, PermissionError, OSError): + # except (FileNotFoundError, PermissionError, OSError, TypeError): + except Exception: + # We already know it's file like thing, so don't agonize buffer_size = 8192 self._obj = obj From fba35fd5ed4814f4a7cea69f104140cf94696b34 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Fri, 29 May 2026 16:03:53 -0400 Subject: [PATCH 29/49] Add start of clie delete action, folder delete todo --- src/hashstore/__main__.py | 78 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/src/hashstore/__main__.py b/src/hashstore/__main__.py index e6a991bd..61e1b4ee 100644 --- a/src/hashstore/__main__.py +++ b/src/hashstore/__main__.py @@ -26,6 +26,7 @@ HASHSTORE_FOLDER_NAME = ".hashstore" DEFAULT_HASHSTORE = f"./{HASHSTORE_FOLDER_NAME}" +DATAONE_SYSTEMMETADATA = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" def get_logger(): @@ -178,6 +179,41 @@ def enumerate_hs_files( pass +""" +def iterate_folder_hierarchy( + hash_store: hashstore.HashStore, start_path: list[str] +) -> typing.Generator: + + visited = set() + + def _drill(node_path: str): + if node_path in visited: + return + visited.add(node_path) + # Visit children first + path = hashstore.folderentry.split_pidpath(node_path) + neighbors = hash_store.retrieve_folder(path) + for neighbor in neighbors: + if neighbor.is_file: + yield neighbor + else: + yield from _drill(hashstore.folderentry.join_pidpath(path + [neighbor.name])) + yield path + + stack = [ + hashstore.folderentry.join_pidpath(start_path), + ] + while stack: + node_path = stack.pop() + if node_path not in visited: + visited.add(node_path) + + yield node_path + + # load node entries +""" + + @click.group() @click.option( "--store", @@ -366,6 +402,48 @@ def add_object( return 0 +@main.command("delete_object") +@click.pass_context +@click.argument("pid", type=str) +@click.option( + "-s", "--sysmeta", "del_sysmeta", is_flag=True, help="Delete system metadata too" +) +def delete_object_and_metadata(ctx: click.Context, pid: str, del_sysmeta: bool): + """Delete an object and system metadata. + + If the object is a folder, then all content is deleted as well. + + Note that deleting folders DOES NOT currently check for additional content references. + """ + logger = get_logger() + store = ctx.obj["hashstore_path"] + properties = load_hashstore_properties(store) + hashstore_factory = hashstore.HashStoreFactory() + try: + hash_store = hashstore_factory.get_hashstore( + ctx.obj["module_name"], ctx.obj["class_name"], properties + ) + logger.debug(f"Hashstore opened at: {store}") + except Exception as e: + logger.error(f"Failed to open hashstore: {e}") + return + pidpath = hashstore.folderentry.split_pidpath(pid, delimiter="|") + info = hash_store.resolve_pidpath(pidpath) + print(info) + if hashstore.folderentry.is_folder(info.get("cid_object_path")): + # TODO: Iterate over all entries and remove them + pass + try: + hash_store.delete_metadata(pid, DATAONE_SYSTEMMETADATA) + except Exception as e: + logger.error(e) + try: + hash_store.delete_object(pid) + print("Done") + except Exception as e: + logger.error(e) + + @main.command("get") @click.pass_context @click.argument("pid", type=str) From 7dde564b9d218e5631324d3f36e65223ca153e7d Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Sat, 30 May 2026 08:54:31 -0400 Subject: [PATCH 30/49] Fix unclosed file resource --- src/hashstore/filehashstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index a9415517..dedcca31 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1046,7 +1046,7 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: raise ValueError(err_msg) cid_stream = self._open(entity, object_cid) hex_digest = self._computehash(cid_stream, algorithm=algorithm) - + cid_stream.close() info_string = ( f"Successfully calculated hex digest for pid: {pid}. " f"Hex Digest: {hex_digest}" From 9011ba0664b306b31decbca0fd6ad492d45d6e7f Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Sat, 30 May 2026 08:55:39 -0400 Subject: [PATCH 31/49] Fix unclosed file resource --- src/hashstore/filehashstore.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index dedcca31..2c2f2555 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -2038,6 +2038,7 @@ def _verify_object_information( hex_digest_calculated = self._computehash( cid_stream, algorithm=checksum_algorithm ) + cid_stream.close() if hex_digest_calculated != checksum: err_msg = ( f"Checksum_algorithm ({checksum_algorithm}) cannot be found " From 2af0fb08f54efb390ec6d7638d1afc87f3dadb5c Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Sat, 30 May 2026 08:58:09 -0400 Subject: [PATCH 32/49] wrap open resource with try finally --- src/hashstore/filehashstore.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 2c2f2555..e68449b7 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1045,8 +1045,10 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: self.fhs_logger.error(err_msg) raise ValueError(err_msg) cid_stream = self._open(entity, object_cid) - hex_digest = self._computehash(cid_stream, algorithm=algorithm) - cid_stream.close() + try: + hex_digest = self._computehash(cid_stream, algorithm=algorithm) + finally: + cid_stream.close() info_string = ( f"Successfully calculated hex digest for pid: {pid}. " f"Hex Digest: {hex_digest}" @@ -2035,10 +2037,12 @@ def _verify_object_information( # Otherwise, a data object has been stored without a pid object_cid = hex_digests[self.algorithm] cid_stream = self._open(entity, object_cid) - hex_digest_calculated = self._computehash( - cid_stream, algorithm=checksum_algorithm - ) - cid_stream.close() + try: + hex_digest_calculated = self._computehash( + cid_stream, algorithm=checksum_algorithm + ) + finally: + cid_stream.close() if hex_digest_calculated != checksum: err_msg = ( f"Checksum_algorithm ({checksum_algorithm}) cannot be found " From f1213f2c27452995e0a647eb7fdd89533934d507 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Sat, 30 May 2026 08:58:37 -0400 Subject: [PATCH 33/49] use pathlib semantics --- tests/filehashstore/test_filehashstore_interface.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/filehashstore/test_filehashstore_interface.py b/tests/filehashstore/test_filehashstore_interface.py index 44bc50ba..e83e8a28 100644 --- a/tests/filehashstore/test_filehashstore_interface.py +++ b/tests/filehashstore/test_filehashstore_interface.py @@ -675,7 +675,7 @@ def test_store_object_interrupt_process(store): interrupting the process is cleaned up. """ file_size = 2 * 1024 * 1024 * 1024 # 2GB - file_path = store.root + "random_file_2.bin" + file_path = store.root / "random_file_2.bin" pid = "Testpid" # Generate a random file with the specified size From 7087e928670c2b0ff086cf21789d27cbcf75256b Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Mon, 1 Jun 2026 06:53:36 -0400 Subject: [PATCH 34/49] Tests for FileHashStoreProperties --- .pre-commit-config.yaml | 12 +++ .../test_filehashstoreproperties.py | 92 +++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 tests/filehashstore/test_filehashstoreproperties.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f4e29488..ffffc7f3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -30,3 +30,15 @@ repos: args: ["--fix", "--show-fixes"] # then, format - id: ruff-format + + - repo: https://github.com/pre-commit/mirrors-mypy + rev: "v1.19.1" + hooks: + - id: mypy + name: mypy + entry: uv run mypy + files: src + types: [python] + pass_filenames: true + args: [] + language: system diff --git a/tests/filehashstore/test_filehashstoreproperties.py b/tests/filehashstore/test_filehashstoreproperties.py new file mode 100644 index 00000000..1156db4e --- /dev/null +++ b/tests/filehashstore/test_filehashstoreproperties.py @@ -0,0 +1,92 @@ +"""Test cases for FileHashStoreProperties""" + +import pytest + +import hashstore.filehashstore + + +def test_defaults(): + p = hashstore.filehashstore.FileHashStoreProperties() + assert p.store_width == 2 + assert p.store_depth == 3 + assert p.store_algorithm == "sha256" + assert len(p.store_default_algo_list) >= 1 + assert p.store_algorithm in p.store_default_algo_list + + +def test_from_dict(): + props = { + "store_width": 2, + "store_depth": 3, + "store_algorithm": "SHA-256", + "store_default_algo_list": [ + "MD5", + ], + } + p = hashstore.filehashstore.FileHashStoreProperties.from_dict(props) + assert p.store_width == props["store_width"] + assert p.store_depth == props["store_depth"] + assert p.store_algorithm == hashstore.filehashstore.from_dataone_algorithm_name( + props["store_algorithm"] + ) + assert len(p.store_default_algo_list) == 2 + assert p.store_algorithm in p.store_default_algo_list + + +def test_from_yaml(tmp_path): + props = { + "store_width": 2, + "store_depth": 3, + "store_algorithm": "SHA-256", + "store_default_algo_list": [ + "MD5", + ], + } + p = hashstore.filehashstore.FileHashStoreProperties.from_dict(props) + yaml_path = tmp_path / "config.yaml" + p.to_yaml(yaml_path) + p2 = hashstore.filehashstore.FileHashStoreProperties.from_yaml(yaml_path) + assert p == p2 + + +def test_invalid_properties(): + props = { + "store_width": 256, + "store_depth": 3, + "store_algorithm": "SHA-256", + "store_default_algo_list": [ + "MD5", + ], + } + with pytest.raises(ValueError, match="store_width"): + hashstore.filehashstore.FileHashStoreProperties.from_dict(props) + props = { + "store_width": 2, + "store_depth": 30, + "store_algorithm": "SHA-256", + "store_default_algo_list": [ + "MD5", + ], + } + with pytest.raises(ValueError, match="store_depth"): + hashstore.filehashstore.FileHashStoreProperties.from_dict(props) + props = { + "store_width": 2, + "store_depth": 3, + "store_algorithm": "blake2s", + "store_default_algo_list": [ + "MD5", + ], + } + with pytest.raises(ValueError, match="algorithm"): + hashstore.filehashstore.FileHashStoreProperties.from_dict(props) + props = { + "store_width": 2, + "store_depth": 3, + "store_algorithm": "SHA-256", + "store_default_algo_list": [ + "foo", + ], + } + with pytest.raises(ValueError, match="not available"): + hashstore.filehashstore.FileHashStoreProperties.from_dict(props) From 44cdaaba77fba5c6cb74c7fae47fe27988e603cd Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Mon, 1 Jun 2026 09:58:29 -0400 Subject: [PATCH 35/49] WIP - working through tests. --- .pre-commit-config.yaml | 22 +- pyproject.toml | 42 +- src/hashstore/filehashstore.py | 709 ++++++--------- src/hashstore/hashstore.py | 7 +- tests/conftest.py | 31 +- tests/filehashstore/test_filehashstore.py | 176 +--- .../test_filehashstoreproperties.py | 15 + uv.lock | 835 ++++++++++++++---- 8 files changed, 1022 insertions(+), 815 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ffffc7f3..506e2b02 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -31,14 +31,14 @@ repos: # then, format - id: ruff-format - - repo: https://github.com/pre-commit/mirrors-mypy - rev: "v1.19.1" - hooks: - - id: mypy - name: mypy - entry: uv run mypy - files: src - types: [python] - pass_filenames: true - args: [] - language: system + # - repo: https://github.com/pre-commit/mirrors-mypy + # rev: "v1.19.1" + # hooks: + # - id: mypy + # name: mypy + # entry: uv run mypy + # files: src + # types: [python] + # pass_filenames: true + # args: [] + # language: system diff --git a/pyproject.toml b/pyproject.toml index d6ccb30a..14b04fbe 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,6 @@ classifiers = [ "Topic :: System :: Filesystems", ] dependencies = [ - "pathlib>=1.0.1", "pyyaml>=6.0", ] @@ -44,9 +43,10 @@ hashstore = "hashstore.hashstoreclient:main" dev = [ "pytest>=7.2.0", "exceptiongroup>=1.1.0", - "black>=22.10.0", - "pylint>=2.17.4", + "pre-commit", "pg8000>=1.29.8", + "pytest-cov>=7.1.0", + "mypy>=1.19.1", ] [build-system] @@ -55,6 +55,42 @@ build-backend = "hatchling.build" [tool.poetry_bumpversion.file."src/hashstore/__init__.py"] +[tool.pytest.ini_options] +minversion = "6.0" +addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"] +xfail_strict = true +filterwarnings = [ + "error", +] +log_cli_level = "INFO" +testpaths = [ + "tests", +] + +[tool.coverage] +run.source = ["synchronize_member_node"] +port.exclude_lines = [ + 'pragma: no cover', + '\.\.\.', + 'if typing.TYPE_CHECKING:', +] + +[tool.mypy] +files = ["src", "tests"] +python_version = "3.9" +show_error_codes = true +warn_unreachable = true +disallow_untyped_defs = false +disallow_incomplete_defs = false +check_untyped_defs = true +strict = true +enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] + +[[tool.mypy.overrides]] +module = "synchronize_member_node.*" +disallow_untyped_defs = true +disallow_incomplete_defs = true + [tool.ruff] src = ["src"] extend-exclude = ["tests/testdata/"] diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index e68449b7..682d4cb3 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1,6 +1,7 @@ """Core module for FileHashStore""" import atexit +import dataclasses import fcntl import hashlib import inspect @@ -37,6 +38,173 @@ UnsupportedAlgorithm, ) +DATAONE_ALGORITHM_TRANSLATION = { + "MD5": "md5", + "SHA-1": "sha1", + "SHA-256": "sha256", + "SHA-384": "sha384", + "SHA-512": "sha512", +} + +accepted_store_algorithms = ["MD5", "SHA-1", "SHA-256", "SHA-384", "SHA-512"] + + +def from_dataone_algorithm_name(algo: str) -> str: + """Translate from a DataONE algorithm name to a python hashlib name.""" + try: + return DATAONE_ALGORITHM_TRANSLATION[algo] + except KeyError: + pass + return algo + + +def to_dataone_algorithm_name(algo: str) -> str: + "Trnaslate from a python hashlib algorithm name to one used by DataONE." + for k, v in DATAONE_ALGORITHM_TRANSLATION.items(): + if algo == v: + return k + # no translation available, fail + msg = f"Algorithm {algo} is not used in DataONE." + raise KeyError(msg) + + +@dataclasses.dataclass +class FileHashStoreProperties: + """Configuration properties for a FileHashStore. + + Values are set to sensible defaults that correspond with the DataONE use. + + Once a hashstore is created, the properties are persisted to the + hashstore folder, so there's generally no need to use FileHashStoreProperties + except when creating a new HashStore instance. + + After intiailization, hash algorithm names are converted to the native + names used by python hashlib. When persisting or loading from YAML + config files, the DataONE hash names are expected. + """ + + # property names align with existing yaml config entries + store_depth: int = 3 + store_width: int = 2 + store_metadata_namespace: str = ( + "https://ns.dataone.org/service/types/v2.0#SystemMetadata" + ) + store_algorithm: str = "SHA-256" + store_default_algo_list: list[str] = dataclasses.field(default_factory=list) + + def __post_init__(self): + """Hash algorthm names are trnaslated from the DataONE names to the names + used by the python hashlib. + """ + # Impose reasonable defaults for folder path depth + if not (0 < self.store_depth <= 8): + msg = "store_depth not between 0 and 8" + raise ValueError(msg) + if not (1 <= self.store_width <= 4): + msg = "store_width not between 1 and 4" + raise ValueError(msg) + if ( + self.store_metadata_namespace is None + or len(self.store_metadata_namespace) < 1 + ): + msg = "Value is required for store_metadata_namespace." + raise ValueError(msg) + if self.store_algorithm not in DATAONE_ALGORITHM_TRANSLATION: + msg = ( + "store_algorithm must be one of " + f"{', '.join(DATAONE_ALGORITHM_TRANSLATION.keys())} " + f"not {self.store_algorithm}" + ) + raise ValueError(msg) + self.store_algorithm = from_dataone_algorithm_name(self.store_algorithm) + if self.store_algorithm not in hashlib.algorithms_available: + msg = f"store_algorithm: {self.store_algorithm} is not available." + raise ValueError(msg) + translated_algos = [] + for algo in self.store_default_algo_list: + translated_algo = from_dataone_algorithm_name(algo) + if translated_algo not in hashlib.algorithms_available: + msg = f"{algo} is not available." + raise ValueError(msg) + translated_algos.append(translated_algo) + self.store_default_algo_list = translated_algos + # Ensure that the store algorithm is included in the detault algorithm list + if self.store_algorithm not in self.store_default_algo_list: + self.store_default_algo_list.append(self.store_algorithm) + + @classmethod + def from_yaml(cls, source: Path) -> "FileHashStoreProperties": + """Load propertiees from yaml. + + Hash algorthm names are trnaslated from the DataONE names to the names + used by the python hashlib. + """ + with source.open("r") as data_source: + properties = yaml.safe_load(data_source) + return cls(**properties) + + @classmethod + def from_dict(cls, data: dict) -> "FileHashStoreProperties": + if data["store_algorithm"] not in DATAONE_ALGORITHM_TRANSLATION: + msg = ( + "store_algorithm must be one of " + f"{', '.join(DATAONE_ALGORITHM_TRANSLATION.keys())} " + f"not {data['store_algorithm']}" + ) + raise ValueError(msg) + return FileHashStoreProperties(**data) + + def to_yaml(self, dest_path: Path) -> None: + """Save properties to a yaml file. + + Hash algorithm names are translated to DataONE algorithm names. + """ + with dest_path.open("w") as dest: + dest.write("""# HashStore Configuration +# +############### Notes ############### +############### Directory Structure ############### +# store_depth +# - Desired amount of directories when sharding an object to form the permanent address +# - **WARNING**: DO NOT CHANGE UNLESS SETTING UP NEW HASHSTORE +# +# store_width +# - Width of directories created when sharding an object to form the permanent address +# - **WARNING**: DO NOT CHANGE UNLESS SETTING UP NEW HASHSTORE +# +# Example: +# Below, objects are shown listed in directories that are 3 levels deep (DIR_DEPTH=3), +# with each directory consisting of 2 characters (DIR_WIDTH=2). +# /var/filehashstore/objects +# ├── 7f +# │ └── 5c +# │ └── c1 +# │ └── 8f0b04e812a3b4c8f686ce34e6fec558804bf61e54b176742a7f6368d6 +# +############### Format of the Metadata ############### +# store_metadata_namespace +# - The default metadata format (ex. system metadata) +# +############### Hash Algorithms ############### +# store_algorithm +# - Hash algorithm to use when calculating object's hex digest for the permanent address +# +# store_default_algo_list +# - Algorithm values supported by python hashlib 3.9.0+ for File Hash Store (FHS) +# - The default algorithm list includes the hash algorithms calculated when storing an +# object to disk and returned to the caller after successful storage. + +""") + properties = dataclasses.asdict(self) + properties["store_algorithm"] = to_dataone_algorithm_name( + properties["store_algorithm"] + ) + translated_names = [] + for algo in properties.get("store_default_algo_list", []): + translated_names.append(to_dataone_algorithm_name(algo)) + properties["store_default_algo_list"] = translated_names + yaml.dump(properties, dest, default_flow_style=False) + class FileHashStore(HashStore): """FileHashStore is an object storage system that was extended from Derrick @@ -82,453 +250,117 @@ class FileHashStore(HashStore): "blake2s", ) - def __init__(self, properties=None): + def __init__(self, store_path: Path): self.fhs_logger = logging.getLogger(__name__) - # Now check properties - if properties: - # Validate properties against existing configuration if present - checked_properties = self._validate_properties(properties) - ( - prop_store_path, - prop_store_depth, - prop_store_width, - _, - prop_store_metadata_namespace, - ) = [ - checked_properties[property_name] - for property_name in self.property_required_keys - ] - - # Check to see if a configuration is present in the given store path - self.hashstore_configuration_yaml = Path(prop_store_path) / "hashstore.yaml" - self._verify_hashstore_properties(properties, prop_store_path) - - # If no exceptions thrown, FileHashStore ready for initialization - self.fhs_logger.debug("Initializing, properties verified.") - self.root = Path(prop_store_path) - self.depth = prop_store_depth - self.width = prop_store_width - self.sysmeta_ns = prop_store_metadata_namespace - # Write 'hashstore.yaml' to store path - if not os.path.isfile(self.hashstore_configuration_yaml): - # pylint: disable=W1201 - self.fhs_logger.debug( - "HashStore does not exist & configuration file not found." - " Writing configuration file." - ) - self._write_properties(properties) - # Default algorithm list for FileHashStore based on config file written - self._set_default_algorithms() - # Complete initialization/instantiation by setting and creating store - # directories - self.objects = self.root / "objects" - self.metadata = self.root / "metadata" - self.refs = self.root / "refs" - self.cids = self.refs / "cids" - self.pids = self.refs / "pids" - if not os.path.exists(self.objects): - self._create_path(self.objects / "tmp") - if not os.path.exists(self.metadata): - self._create_path(self.metadata / "tmp") - if not os.path.exists(self.refs): - self._create_path(self.refs / "tmp") - self._create_path(self.refs / "pids") - self._create_path(self.refs / "cids") - - # Variables to orchestrate parallelization - # Check to see whether a multiprocessing or threading sync lock should - # be used - self.use_multiprocessing = ( - os.getenv("USE_MULTIPROCESSING", "False") == "True" + config_path = FileHashStore.config_path(store_path) + if not config_path.exists() or not config_path.is_file(): + msg = f"No hashstore at: {store_path}" + raise RuntimeError(msg) + properties = FileHashStoreProperties.from_yaml(config_path) + + self.root = store_path + self.depth = properties.store_depth + self.width = properties.store_width + self.sysmeta_ns = properties.store_metadata_namespace + + self.algorithm = properties.store_algorithm + self.default_algo_list = properties.store_default_algo_list + # Sanity check, ensure algorithm is in the detaulr list + if self.algorithm not in self.default_algo_list: + self.default_algo_list.append(self.algorithm) + + self.objects = self.root / "objects" + self.metadata = self.root / "metadata" + self.refs = self.root / "refs" + self.cids = self.refs / "cids" + self.pids = self.refs / "pids" + if not os.path.exists(self.objects): + self._create_path(self.objects / "tmp") + if not os.path.exists(self.metadata): + self._create_path(self.metadata / "tmp") + if not os.path.exists(self.refs): + self._create_path(self.refs / "tmp") + self._create_path(self.refs / "pids") + self._create_path(self.refs / "cids") + + self.use_multiprocessing = os.getenv("USE_MULTIPROCESSING", "False") == "True" + + if self.use_multiprocessing == "True": + # Create multiprocessing synchronization variables + # Synchronization values for object locked pids + self.object_pid_lock_mp = multiprocessing.Lock() + self.object_pid_condition_mp = multiprocessing.Condition( + self.object_pid_lock_mp ) - if self.use_multiprocessing == "True": - # Create multiprocessing synchronization variables - # Synchronization values for object locked pids - self.object_pid_lock_mp = multiprocessing.Lock() - self.object_pid_condition_mp = multiprocessing.Condition( - self.object_pid_lock_mp - ) - self.object_locked_pids_mp = multiprocessing.Manager().list() - # Synchronization values for object locked cids - self.object_cid_lock_mp = multiprocessing.Lock() - self.object_cid_condition_mp = multiprocessing.Condition( - self.object_cid_lock_mp - ) - self.object_locked_cids_mp = multiprocessing.Manager().list() - # Synchronization values for metadata locked documents - self.metadata_lock_mp = multiprocessing.Lock() - self.metadata_condition_mp = multiprocessing.Condition( - self.metadata_lock_mp - ) - self.metadata_locked_docs_mp = multiprocessing.Manager().list() - # Synchronization values for reference locked pids - self.reference_pid_lock_mp = multiprocessing.Lock() - self.reference_pid_condition_mp = multiprocessing.Condition( - self.reference_pid_lock_mp - ) - self.reference_locked_pids_mp = multiprocessing.Manager().list() - else: - # Create threading synchronization variables - # Synchronization values for object locked pids - self.object_pid_lock_th = threading.Lock() - self.object_pid_condition_th = threading.Condition( - self.object_pid_lock_th - ) - self.object_locked_pids_th = [] - # Synchronization values for object locked cids - self.object_cid_lock_th = threading.Lock() - self.object_cid_condition_th = threading.Condition( - self.object_cid_lock_th - ) - self.object_locked_cids_th = [] - # Synchronization values for metadata locked documents - self.metadata_lock_th = threading.Lock() - self.metadata_condition_th = threading.Condition(self.metadata_lock_th) - self.metadata_locked_docs_th = [] - # Synchronization values for reference locked pids - self.reference_pid_lock_th = threading.Lock() - self.reference_pid_condition_th = threading.Condition( - self.metadata_lock_th - ) - self.reference_locked_pids_th = [] - - self.fhs_logger.debug("Initialization success. Store root: %s", self.root) - else: - # Cannot instantiate or initialize FileHashStore without config - err_msg = ( - "HashStore properties must be supplied." + f" Properties: {properties}" - ) - self.fhs_logger.debug(err_msg) - raise ValueError(err_msg) - - # Configuration and Related Methods - - @staticmethod - def _load_properties( - hashstore_yaml_path: Path, hashstore_required_prop_keys: list[str] - ) -> dict[str, Union[str, int]]: - """Get and return the contents of the current HashStore configuration. - - :return: HashStore properties with the following keys (and values): - - store_depth (int): Depth when sharding an object's hex digest. - - store_width (int): Width of directories when sharding an object's - hex digest. - - store_algorithm (str): Hash algo used for calculating the object's - hex digest. - - store_metadata_namespace (str): Namespace for the HashStore's system - metadata. - """ - if not os.path.isfile(hashstore_yaml_path): - err_msg = "'hashstore.yaml' not found in store root path." - logging.critical(err_msg) - raise FileNotFoundError(err_msg) - - # Open file - with open(hashstore_yaml_path, encoding="utf-8") as hs_yaml_file: - yaml_data = yaml.safe_load(hs_yaml_file) - - # Get hashstore properties - hashstore_yaml_dict = {} - for key in hashstore_required_prop_keys: - if key != "store_path": - hashstore_yaml_dict[key] = yaml_data[key] - logging.debug("Successfully retrieved 'hashstore.yaml' properties.") - return hashstore_yaml_dict - - def _write_properties(self, properties: dict[str, Union[str, int]]) -> None: - """Writes 'hashstore.yaml' to FileHashStore's root directory with the respective - properties object supplied. - - :param dict properties: A Python dictionary with the following keys - (and values): - - store_depth (int): Depth when sharding an object's hex digest. - - store_width (int): Width of directories when sharding an object's hex - digest. - - store_algorithm (str): Hash algo used for calculating the object's hex - digest. - - store_metadata_namespace (str): Namespace for the HashStore's system - metadata. - """ - # If hashstore.yaml already exists, must throw exception and proceed with - # caution - if os.path.isfile(self.hashstore_configuration_yaml): - err_msg = "Configuration file 'hashstore.yaml' already exists." - logging.error(err_msg) - raise FileExistsError(err_msg) - # Validate properties - checked_properties = self._validate_properties(properties) - - # Collect configuration properties from validated & supplied dictionary - ( - _, - store_depth, - store_width, - store_algorithm, - store_metadata_namespace, - ) = [ - checked_properties[property_name] - for property_name in self.property_required_keys - ] - - # Standardize algorithm value for cross-language compatibility - # Note, this must be declared here because HashStore has not yet been - # initialized - accepted_store_algorithms = ["MD5", "SHA-1", "SHA-256", "SHA-384", "SHA-512"] - if store_algorithm in accepted_store_algorithms: - checked_store_algorithm = store_algorithm - else: - err_msg = ( - f"Algorithm supplied ({store_algorithm}) cannot be used as default for" - f" HashStore. Must be one of: {', '.join(accepted_store_algorithms)}" - f" which are DataONE controlled algorithm values" + self.object_locked_pids_mp = multiprocessing.Manager().list() + # Synchronization values for object locked cids + self.object_cid_lock_mp = multiprocessing.Lock() + self.object_cid_condition_mp = multiprocessing.Condition( + self.object_cid_lock_mp ) - logging.error(err_msg) - raise ValueError(err_msg) - - # If given store path doesn't exist yet, create it. - if not os.path.exists(self.root): - self._create_path(self.root) - - # .yaml file to write - hashstore_configuration_yaml = self._build_hashstore_yaml_string( - store_depth, - store_width, - checked_store_algorithm, - store_metadata_namespace, - ) - # Write 'hashstore.yaml' - with open( - self.hashstore_configuration_yaml, "w", encoding="utf-8" - ) as hs_yaml_file: - hs_yaml_file.write(hashstore_configuration_yaml) - - logging.debug( - "Configuration file written to: %s", self.hashstore_configuration_yaml - ) - return - - @staticmethod - def _build_hashstore_yaml_string( - store_depth: int, - store_width: int, - store_algorithm: str, - store_metadata_namespace: str, - ) -> str: - """Build a YAML string representing the configuration for a HashStore. - - :param int store_depth: Depth when sharding an object's hex digest. - :param int store_width: Width of directories when sharding an object's hex - digest. - :param str store_algorithm: Hash algorithm used for calculating the object's hex - digest. - :param str store_metadata_namespace: Namespace for the HashStore's system - metadata. - - :return: A YAML string representing the configuration for a HashStore. - """ - hashstore_configuration = { - "store_depth": store_depth, - "store_width": store_width, - "store_metadata_namespace": store_metadata_namespace, - "store_algorithm": store_algorithm, - "store_default_algo_list": [ - "MD5", - "SHA-1", - "SHA-256", - "SHA-384", - "SHA-512", - ], - } - - # The tabbing here is intentional otherwise the created .yaml will have - # extra tabs - hashstore_configuration_comments = """ -# Default configuration variables for HashStore - -############### HashStore Config Notes ############### -############### Directory Structure ############### -# store_depth -# - Desired amount of directories when sharding an object to form the permanent address -# - **WARNING**: DO NOT CHANGE UNLESS SETTING UP NEW HASHSTORE -# -# store_width -# - Width of directories created when sharding an object to form the permanent address -# - **WARNING**: DO NOT CHANGE UNLESS SETTING UP NEW HASHSTORE -# -# Example: -# Below, objects are shown listed in directories that are 3 levels deep (DIR_DEPTH=3), -# with each directory consisting of 2 characters (DIR_WIDTH=2). -# /var/filehashstore/objects -# ├── 7f -# │ └── 5c -# │ └── c1 -# │ └── 8f0b04e812a3b4c8f686ce34e6fec558804bf61e54b176742a7f6368d6 - -############### Format of the Metadata ############### -# store_metadata_namespace -# - The default metadata format (ex. system metadata) - -############### Hash Algorithms ############### -# store_algorithm -# - Hash algorithm to use when calculating object's hex digest for the permanent address -# -# store_default_algo_list -# - Algorithm values supported by python hashlib 3.9.0+ for File Hash Store (FHS) -# - The default algorithm list includes the hash algorithms calculated when storing an -# - object to disk and returned to the caller after successful storage. - -""" - - return hashstore_configuration_comments + yaml.dump( - hashstore_configuration, sort_keys=False - ) - - def _verify_hashstore_properties( - self, properties: dict[str, Union[str, int]], prop_store_path: str - ) -> None: - """Determines whether FileHashStore can instantiate by validating a set - of arguments and throwing exceptions. HashStore will not instantiate if - an existing configuration file's properties (`hashstore.yaml`) are - different from what is supplied - or if an object store exists at the - given path, but it is missing the `hashstore.yaml` config file. - - If `hashstore.yaml` exists, it will retrieve its properties and compare - them with the given values; and if there is a mismatch, an exception - will be thrown. If not, it will look to see if any directories/files - exist in the given store path and throw an exception if any file or - directory is found. - - :param dict properties: HashStore properties. - :param str prop_store_path: Store path to check. - """ - if os.path.isfile(self.hashstore_configuration_yaml): - self.fhs_logger.debug( - "Config found (hashstore.yaml) at {%s}. Verifying properties.", - self.hashstore_configuration_yaml, + self.object_locked_cids_mp = multiprocessing.Manager().list() + # Synchronization values for metadata locked documents + self.metadata_lock_mp = multiprocessing.Lock() + self.metadata_condition_mp = multiprocessing.Condition( + self.metadata_lock_mp ) - # If 'hashstore.yaml' is found, verify given properties before init - hashstore_yaml_dict = self._load_properties( - self.hashstore_configuration_yaml, self.property_required_keys + self.metadata_locked_docs_mp = multiprocessing.Manager().list() + # Synchronization values for reference locked pids + self.reference_pid_lock_mp = multiprocessing.Lock() + self.reference_pid_condition_mp = multiprocessing.Condition( + self.reference_pid_lock_mp ) - for key in self.property_required_keys: - # 'store_path' is required to init HashStore but not saved in - # `hashstore.yaml` - if key != "store_path": - supplied_key = properties[key] - if key == "store_depth" or key == "store_width": - supplied_key = int(properties[key]) - if hashstore_yaml_dict[key] != supplied_key: - err_msg = ( - f"Given properties ({key}: {properties[key]}) does not " - f"match. HashStore configuration " - f"({key}: {hashstore_yaml_dict[key]}) found at: " - f"{self.hashstore_configuration_yaml}" - ) - self.fhs_logger.critical(err_msg) - raise ValueError(err_msg) + self.reference_locked_pids_mp = multiprocessing.Manager().list() else: - if os.path.exists(prop_store_path): - # Check if HashStore exists and throw exception if found - subfolders = ["objects", "metadata", "refs"] - if any( - os.path.isdir(os.path.join(prop_store_path, sub)) - for sub in subfolders - ): - err_msg = ( - "Unable to initialize HashStore. `hashstore.yaml` is not " - "present but conflicting HashStore directory exists. Please " - "delete '/objects', '/metadata' and/or '/refs' at the store " - "path or supply a new path." - ) - self.fhs_logger.critical(err_msg) - raise RuntimeError(err_msg) - - def _validate_properties( - self, properties: dict[str, Union[str, int]] - ) -> dict[str, Union[str, int]]: - """Validate a properties dictionary by checking if it contains all the - required keys and non-None values. + # Create threading synchronization variables + # Synchronization values for object locked pids + self.object_pid_lock_th = threading.Lock() + self.object_pid_condition_th = threading.Condition(self.object_pid_lock_th) + self.object_locked_pids_th = [] + # Synchronization values for object locked cids + self.object_cid_lock_th = threading.Lock() + self.object_cid_condition_th = threading.Condition(self.object_cid_lock_th) + self.object_locked_cids_th = [] + # Synchronization values for metadata locked documents + self.metadata_lock_th = threading.Lock() + self.metadata_condition_th = threading.Condition(self.metadata_lock_th) + self.metadata_locked_docs_th = [] + # Synchronization values for reference locked pids + self.reference_pid_lock_th = threading.Lock() + self.reference_pid_condition_th = threading.Condition(self.metadata_lock_th) + self.reference_locked_pids_th = [] - :param dict properties: Dictionary containing filehashstore properties. + @staticmethod + def config_path(root_path: Path) -> Path: + return root_path / "hashstore.yaml" - :raises KeyError: If key is missing from the required keys. - :raises ValueError: If value is missing for a required key. + @classmethod + def create_hashstore( + cls, store_path: Path, properties: FileHashStoreProperties + ) -> "FileHashStore": + """Creates a new empty FileHashstore at the specified folder. - :return: The given properties object (that has been validated). + The folder or config file must not already exist. """ - if not isinstance(properties, dict): - err_msg = "Invalid argument expected a dictionary." - self.fhs_logger.error(err_msg) - raise ValueError(err_msg) - - # New dictionary for validated properties - checked_properties = {} - - for key in self.property_required_keys: - if key not in properties: - err_msg = "Missing required key: {key}." - self.fhs_logger.error(err_msg) - raise KeyError(err_msg) - - value = properties.get(key) - if value is None: - err_msg = "Value for key: {key} is none." - self.fhs_logger.error(err_msg) - raise ValueError(err_msg) - - # Add key and values to checked_properties - if key == "store_depth" or key == "store_width": - # Ensure store depth and width are integers - try: - checked_properties[key] = int(value) - except Exception as err: - err_msg = ( - "Unexpected exception when attempting to ensure store depth " - f"and width are integers. Details: {err}" - ) - self.fhs_logger.error(err_msg) - raise ValueError(err_msg) from err - else: - checked_properties[key] = value - - return checked_properties - - def _set_default_algorithms(self): - """Set the default algorithms to calculate when storing objects.""" - - def lookup_algo(algo_to_translate): - """Translate DataONE controlled algorithms to python hashlib values: - https://dataoneorg.github.io/api-documentation/apis/Types.html#Types.ChecksumAlgorithm - """ - dataone_algo_translation = { - "MD5": "md5", - "SHA-1": "sha1", - "SHA-256": "sha256", - "SHA-384": "sha384", - "SHA-512": "sha512", - } - return dataone_algo_translation[algo_to_translate] - - if not os.path.isfile(self.hashstore_configuration_yaml): - err_msg = "hashstore.yaml not found in store root path." - self.fhs_logger.critical(err_msg) - raise FileNotFoundError(err_msg) - - with open(self.hashstore_configuration_yaml, encoding="utf-8") as hs_yaml_file: - yaml_data = yaml.safe_load(hs_yaml_file) - - # Set default store algorithm - self.algorithm = lookup_algo(yaml_data["store_algorithm"]) - # Takes DataOne controlled algorithm values and translates to hashlib supported - # values - yaml_store_default_algo_list = yaml_data["store_default_algo_list"] - translated_default_algo_list = [] - for algo in yaml_store_default_algo_list: - translated_default_algo_list.append(lookup_algo(algo)) - - # Set class variable - self.default_algo_list = translated_default_algo_list - return + config_path = FileHashStore.config_path(store_path) + # Fail if the config is already present, a bit redundant with + # what follows, but used to emphasize creation versus + # opening a hashstore. + if config_path.exists(): + msg = f"Hashstore already initialized at: {config_path}" + raise ValueError(msg) + # Force failure if the hashstore root folder already exists + store_path.mkdir(parents=True, exist_ok=False) + properties.to_yaml(config_path) + return cls(store_path) + + @classmethod + def clone_empty(cls, new_path: Path, existing: "FileHashStore") -> "FileHashStore": + """Create a new empty HashStore with the same configuration properties + as an existing one.""" + config_path = cls.config_path(existing.root) + properties = FileHashStoreProperties.from_yaml(config_path) + return cls.create_hashstore(new_path, properties) # Public API / HashStore Interface Methods @@ -1391,6 +1223,9 @@ def _write_to_tmp_file_and_get_hex_digests( algorithm_list_to_calculate = self._refine_algorithm_list( additional_algorithm, checksum_algorithm ) + self.fhs_logger.debug("chk algo: %s", checksum_algorithm) + self.fhs_logger.debug("add algo: %s", additional_algorithm) + self.fhs_logger.debug("algo list: %s", algorithm_list_to_calculate) tmp_root_path = self._get_store_path("objects") / "tmp" tmp = self._mktmpfile(tmp_root_path) diff --git a/src/hashstore/hashstore.py b/src/hashstore/hashstore.py index d741e433..c43f9716 100644 --- a/src/hashstore/hashstore.py +++ b/src/hashstore/hashstore.py @@ -2,6 +2,7 @@ import importlib.metadata import importlib.util +import pathlib from abc import ABC, abstractmethod @@ -11,15 +12,15 @@ class HashStore(ABC): address files.""" @staticmethod - def version(): + def version() -> str: """Return the version number""" return importlib.metadata.version("hashstore") @abstractmethod def store_object( self, - pid, - data, + pid: str, + data: str | pathlib.Path, additional_algorithm, checksum, checksum_algorithm, diff --git a/tests/conftest.py b/tests/conftest.py index e6efba3f..f2e9c8e7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,7 @@ import pytest -from hashstore.filehashstore import FileHashStore +from hashstore.filehashstore import FileHashStore, FileHashStoreProperties def pytest_addoption(parser): @@ -16,26 +16,25 @@ def pytest_addoption(parser): @pytest.fixture(name="props") -def init_props(tmp_path): +def init_props(): """Properties to initialize HashStore.""" - directory = tmp_path / "metacat" / "hashstore" - directory.mkdir(parents=True) - hashstore_path = directory.as_posix() - # Note, objects generated via tests are placed in a temporary folder - # with the 'directory' parameter above appended - return { - "store_path": hashstore_path, - "store_depth": 3, - "store_width": 2, - "store_algorithm": "SHA-256", - "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", - } + return FileHashStoreProperties( + store_depth=3, + store_width=2, + store_algorithm="SHA-256", + store_metadata_namespace="https://ns.dataone.org/service/types/v2.0#SystemMetadata", + ) + + +@pytest.fixture(name="hashstore_path") +def hastore_path(tmp_path): + return tmp_path / "metacat" / "hashstore" @pytest.fixture(name="store") -def init_store(props): +def init_store(hashstore_path, props): """Create FileHashStore instance for all tests.""" - return FileHashStore(props) + return FileHashStore.create_hashstore(hashstore_path, props) @pytest.fixture(name="pids") diff --git a/tests/filehashstore/test_filehashstore.py b/tests/filehashstore/test_filehashstore.py index 8613fe7b..156fa95a 100644 --- a/tests/filehashstore/test_filehashstore.py +++ b/tests/filehashstore/test_filehashstore.py @@ -8,7 +8,12 @@ import pytest -from hashstore.filehashstore import FileHashStore, ObjectMetadata, Stream +from hashstore.filehashstore import ( + FileHashStore, + FileHashStoreProperties, + ObjectMetadata, + Stream, +) from hashstore.filehashstore_exceptions import ( CidRefsContentError, CidRefsFileNotFound, @@ -42,191 +47,44 @@ def test_init_directories_created(store): assert os.path.exists(store.refs / "cids") -def test_init_existing_store_incorrect_algorithm_format(store): +def test_init_existing_store_incorrect_algorithm_format(): """Confirm that exception is thrown when store_algorithm is not a DataONE controlled value ( the string must exactly match the expected format). DataONE uses the library of congress vocabulary to standardize algorithm types.""" properties = { - "store_path": store.root / "incorrect_algo_format", "store_depth": 3, "store_width": 2, "store_algorithm": "sha256", "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", } - with pytest.raises(ValueError, match="Must be one of"): - FileHashStore(properties) + with pytest.raises(ValueError, match="must be one of"): + FileHashStoreProperties.from_dict(properties) -def test_init_existing_store_correct_algorithm_format(store): - """Confirm second instance of HashStore with DataONE controlled value.""" +def test_init_store_correct_algorithm_format(tmp_path): + """Confirm instance of HashStore with DataONE controlled value.""" properties = { - "store_path": store.root, "store_depth": 3, "store_width": 2, "store_algorithm": "SHA-256", "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", } - hashstore_instance = FileHashStore(properties) + hashstore_instance = FileHashStore.create_hashstore( + tmp_path / "new-store", FileHashStoreProperties(**properties) + ) assert isinstance(hashstore_instance, FileHashStore) def test_init_write_properties_hashstore_yaml_exists(store): """Verify config file present in store root directory.""" - assert os.path.exists(store.hashstore_configuration_yaml) - - -def test_init_with_existing_hashstore_mismatched_config_depth(store): - """Test init with existing HashStore raises a ValueError when supplied with - mismatching depth.""" - properties = { - "store_path": store.root, - "store_depth": 1, - "store_width": 2, - "store_algorithm": "SHA-256", - "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", - } - with pytest.raises(ValueError, match="depth"): - FileHashStore(properties) - - -def test_init_with_existing_hashstore_mismatched_config_width(store): - """Test init with existing HashStore raises a ValueError when supplied with - mismatching width.""" - properties = { - "store_path": store.root, - "store_depth": 3, - "store_width": 1, - "store_algorithm": "SHA-256", - "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", - } - with pytest.raises(ValueError, match="width"): - FileHashStore(properties) - - -def test_init_with_existing_hashstore_mismatched_config_algo(store): - """Test init with existing HashStore raises a ValueError when supplied with - mismatching default algorithm.""" - properties = { - "store_path": store.root, - "store_depth": 3, - "store_width": 1, - "store_algorithm": "SHA-512", - "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", - } - with pytest.raises(ValueError, match="configuration"): - FileHashStore(properties) - - -def test_init_with_existing_hashstore_mismatched_config_metadata_ns(store): - """Test init with existing HashStore raises a ValueError when supplied with - mismatching default name space.""" - properties = { - "store_path": store.root, - "store_depth": 3, - "store_width": 1, - "store_algorithm": "SHA-512", - "store_metadata_namespace": "http://ns.dataone.org/service/types/v5.0", - } - with pytest.raises(ValueError, match="configuration"): - FileHashStore(properties) - - -def test_init_with_existing_hashstore_missing_yaml(store, pids): - """Test init with existing store raises RuntimeError when hashstore.yaml - not found but objects exist.""" - test_dir = "tests/testdata/" - for pid in pids: - path = test_dir + pid.replace("/", "_") - store._store_and_validate_data(pid, path) - os.remove(store.hashstore_configuration_yaml) - properties = { - "store_path": store.root, - "store_depth": 3, - "store_width": 2, - "store_algorithm": "SHA-256", - "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", - } - with pytest.raises(RuntimeError): - FileHashStore(properties) - - -def test_load_properties(store): - """Verify dictionary returned from _load_properties matches initialization.""" - hashstore_yaml_dict = store._load_properties( - store.hashstore_configuration_yaml, store.property_required_keys - ) - assert hashstore_yaml_dict.get("store_depth") == 3 - assert hashstore_yaml_dict.get("store_width") == 2 - assert hashstore_yaml_dict.get("store_algorithm") == "SHA-256" - assert ( - hashstore_yaml_dict.get("store_metadata_namespace") - == "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - ) + assert os.path.exists(FileHashStore.config_path(store.root)) def test_load_properties_hashstore_yaml_missing(store): """Confirm FileNotFoundError is raised when hashstore.yaml does not exist.""" - os.remove(store.hashstore_configuration_yaml) - with pytest.raises(FileNotFoundError): - store._load_properties( - store.hashstore_configuration_yaml, store.property_required_keys - ) - - -def test_validate_properties(store): - """Confirm no exceptions are thrown when all key/values are supplied.""" - properties = { - "store_path": "/etc/test", - "store_depth": 3, - "store_width": 2, - "store_algorithm": "SHA-256", - "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", - } - assert store._validate_properties(properties) - - -def test_validate_properties_missing_key(store): - """Confirm exception raised when key missing in properties.""" - properties = { - "store_path": "/etc/test", - "store_depth": 3, - "store_width": 2, - "store_algorithm": "SHA-256", - } - with pytest.raises(KeyError): - store._validate_properties(properties) - - -def test_validate_properties_key_value_is_none(store): - """Confirm exception raised when a value from a key is 'None'.""" - properties = { - "store_path": "/etc/test", - "store_depth": 3, - "store_width": 2, - "store_algorithm": "SHA-256", - "store_metadata_namespace": None, - } - with pytest.raises(ValueError, match="Value for key"): - store._validate_properties(properties) - - -def test_validate_properties_incorrect_type(store): - """Confirm exception raised when a bad properties value is given.""" - properties = "etc/filehashstore/hashstore.yaml" - with pytest.raises(ValueError, match="Invalid"): - store._validate_properties(properties) - - -def test_set_default_algorithms_missing_yaml(store, pids): - """Confirm set_default_algorithms raises FileNotFoundError when hashstore.yaml - not found.""" - test_dir = "tests/testdata/" - for pid in pids: - path = test_dir + pid.replace("/", "_") - store._store_and_validate_data(pid, path) - os.remove(store.hashstore_configuration_yaml) + os.remove(FileHashStore.config_path(store.root)) with pytest.raises(FileNotFoundError): - store._set_default_algorithms() + _ = FileHashStoreProperties.from_yaml(FileHashStore.config_path(store.root)) # Tests for FileHashStore Core Methods diff --git a/tests/filehashstore/test_filehashstoreproperties.py b/tests/filehashstore/test_filehashstoreproperties.py index 1156db4e..2746f047 100644 --- a/tests/filehashstore/test_filehashstoreproperties.py +++ b/tests/filehashstore/test_filehashstoreproperties.py @@ -9,12 +9,14 @@ def test_defaults(): p = hashstore.filehashstore.FileHashStoreProperties() assert p.store_width == 2 assert p.store_depth == 3 + assert len(p.store_metadata_namespace) > 1 assert p.store_algorithm == "sha256" assert len(p.store_default_algo_list) >= 1 assert p.store_algorithm in p.store_default_algo_list def test_from_dict(): + """Veryify initialization from a dict of values.""" props = { "store_width": 2, "store_depth": 3, @@ -34,6 +36,7 @@ def test_from_dict(): def test_from_yaml(tmp_path): + """Verify storing and loading a YAML file""" props = { "store_width": 2, "store_depth": 3, @@ -50,6 +53,7 @@ def test_from_yaml(tmp_path): def test_invalid_properties(): + """Test initialization with various invalid property values.""" props = { "store_width": 256, "store_depth": 3, @@ -90,3 +94,14 @@ def test_invalid_properties(): } with pytest.raises(ValueError, match="not available"): hashstore.filehashstore.FileHashStoreProperties.from_dict(props) + props = { + "store_width": 2, + "store_depth": 3, + "store_algorithm": "SHA-256", + "store_default_algo_list": [ + "MD%", + ], + "store_metadata_namespace": None, + } + with pytest.raises(ValueError, match="store_metadata_namespace"): + hashstore.filehashstore.FileHashStoreProperties.from_dict(props) diff --git a/uv.lock b/uv.lock index bcdc3492..7aa7edbd 100644 --- a/uv.lock +++ b/uv.lock @@ -2,7 +2,8 @@ version = 1 revision = 3 requires-python = ">=3.9, <4.0" resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.15'", + "python_full_version >= '3.12' and python_full_version < '3.15'", "python_full_version == '3.11.*'", "python_full_version == '3.10.*'", "python_full_version < '3.10'", @@ -18,130 +19,331 @@ wheels = [ ] [[package]] -name = "astroid" -version = "3.3.11" +name = "ast-serialize" +version = "0.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/81/9d/09e27731bd5864a9ce04e3244074e674bb8936bf62b45e0357248717adac/ast_serialize-0.5.0.tar.gz", hash = "sha256:5880091bfe6f4f986f22866375c2e884843e7a0b6343ae41aeea659613d879b6", size = 61157, upload-time = "2026-05-17T17:48:29.429Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c0/9a/13dde51ba9e15f8b97957ab7cb0120d0e381524d651c6bd630b9c359227f/ast_serialize-0.5.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:8f5c14f169eb0972c0c21bada5358b23d6047c76583b005234f865b11f1fa00a", size = 1183520, upload-time = "2026-05-17T17:47:30.831Z" }, + { url = "https://files.pythonhosted.org/packages/37/de/5a7f0a9fe68944f536632a5af84676739c7d2582be42deb082634bf3a754/ast_serialize-0.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7d1a2de9de5be04652f0ed60738356ef94f66db37924a9499fffe98dc491aa0b", size = 1175779, upload-time = "2026-05-17T17:47:32.551Z" }, + { url = "https://files.pythonhosted.org/packages/9c/81/0bb853e76e4f6e9a1855d569003c59e19ffac45f7079d91505d1bb212f92/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:be5173fb66f9b49026d9d5a2ff0fc7c7009077107c0eb285b2d60fdf1fe10bd1", size = 1233750, upload-time = "2026-05-17T17:47:34.731Z" }, + { url = "https://files.pythonhosted.org/packages/e5/d3/4cf705beeccc08754d0bbda99aefff26110e209b9a07ac8a6b60eec48531/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f8015cd071ac1339924ee2b8098c93e00e155f30a16f40ec9816fcf84f4753f6", size = 1235942, upload-time = "2026-05-17T17:47:36.287Z" }, + { url = "https://files.pythonhosted.org/packages/26/c8/ee097e437ea27dd2b8b227865c875492b585650a5802a22d82b304c8201b/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5499e8797edff2a9186aa313ed382c6b422e798e9332d9953badcee6e69a88f2", size = 1442517, upload-time = "2026-05-17T17:47:38.17Z" }, + { url = "https://files.pythonhosted.org/packages/ff/bd/68063442838f1ba68ec72b5436430bc75b3bb17a1a3c3063f09b0c05ae2b/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6848f2a093fb5548751a9a09bff8fcd229e2bbeb0e3331f391b6ae6d26cd9903", size = 1254081, upload-time = "2026-05-17T17:47:39.826Z" }, + { url = "https://files.pythonhosted.org/packages/50/e2/1e520793bc6a4e4524a6ab022391e827825eaa0c3811828bfdc6852eca26/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:832d4c998e0b091fd60a6d6bceee535483c4d490de9ba85003af835225719261", size = 1259910, upload-time = "2026-05-17T17:47:41.369Z" }, + { url = "https://files.pythonhosted.org/packages/4e/e1/49b60f467979979cfe6913b43948ff25bca971ad0591d181812f163a988e/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:16db7c62ec0b8efe1d7afd283a388d8f74f2605d56032e5a37747d2de8dba027", size = 1250678, upload-time = "2026-05-17T17:47:43.702Z" }, + { url = "https://files.pythonhosted.org/packages/74/ba/66ab9555de6275677566f6574e5ef6c29cb185ea866f643bc06f8280a8ee/ast_serialize-0.5.0-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:baf5eb061eb5bccade4128ad42da33787d72f6013809cd1b590376ece8b3c937", size = 1301603, upload-time = "2026-05-17T17:47:46.256Z" }, + { url = "https://files.pythonhosted.org/packages/66/42/6aca9b9abc710014b2be9059689e5dd1679339e78f567ffb4d255a9e2050/ast_serialize-0.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:104e4a35bd7c124173c41760ef9aaea17ddb3f86c65cb643671d59afbe3ee94c", size = 1410332, upload-time = "2026-05-17T17:47:47.899Z" }, + { url = "https://files.pythonhosted.org/packages/47/68/2f76594432a22581ecf878b5e75a9b8601c24b2241cf0bbeb1e21fcf370c/ast_serialize-0.5.0-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:36be371028fc1675acb38a331bde160dbab7ff907fdf00b67eb6911aa106951b", size = 1509979, upload-time = "2026-05-17T17:47:50.942Z" }, + { url = "https://files.pythonhosted.org/packages/40/ac/a93c9b58292653f6c595752f677a08e608f903b710594909e9231a389b3b/ast_serialize-0.5.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:061ee58bdb52341c8201a6df41182a977736bae3b7ded87ca7176ca25a8a47ab", size = 1505002, upload-time = "2026-05-17T17:47:54.093Z" }, + { url = "https://files.pythonhosted.org/packages/14/2e/b278f68c497ee2f1d1576cbbef8db5281cd4a5f2db040537592ac9c8862e/ast_serialize-0.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b15219e9cdc9f53f6f4cb51c009203507228226148c05c5e8fe451c28b435eb3", size = 1456231, upload-time = "2026-05-17T17:47:56.311Z" }, + { url = "https://files.pythonhosted.org/packages/0b/43/419be1c566a4c504cd8fd60ce2f84e790f295495c0f327cfaeadf3d51012/ast_serialize-0.5.0-cp314-cp314t-win32.whl", hash = "sha256:842d1c004bb466c7df036f95fabef789570541922b10976b12f5592a69cf0b38", size = 1058668, upload-time = "2026-05-17T17:47:58.305Z" }, + { url = "https://files.pythonhosted.org/packages/03/6f/c9d4d549295ed05111aeb8853232d1afd9d0a179fddb01eeffbb3a4a6842/ast_serialize-0.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b0c06d760909b095cc466356dfccd05a1c7233a6ca191c020dca2c6a6f16c24c", size = 1101075, upload-time = "2026-05-17T17:48:00.35Z" }, + { url = "https://files.pythonhosted.org/packages/d0/8e/d00c5ab30c58222e07d62956fca86c59d91b9ad32997e633c38b526623a3/ast_serialize-0.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:787baedb0262cc49e8ce37cc15c00ae818e46a165a3b36f5e21ed174998104cb", size = 1075347, upload-time = "2026-05-17T17:48:01.753Z" }, + { url = "https://files.pythonhosted.org/packages/e0/9e/dc2530acb3a60dc6e46d65abf27d1d9f86721694757906a148d90a6860de/ast_serialize-0.5.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:0668aa9459cfa8c9c49ddd2163ebcf43088ba045ef7492af6fe22e0098303101", size = 1191380, upload-time = "2026-05-17T17:48:03.738Z" }, + { url = "https://files.pythonhosted.org/packages/26/0a/bd3d18a582f273d6c843d16bb9e22e9e16365ff7991e92f18f798e9f1224/ast_serialize-0.5.0-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:bf683d6363edf2b39eed6b6d4fe22d34b6203867a67e27134d9e2a2680c4bc4a", size = 1183879, upload-time = "2026-05-17T17:48:05.463Z" }, + { url = "https://files.pythonhosted.org/packages/40/ae/1f919100f8620887af58fcc381c61a1f218cdf89c6e155f87b213e61010a/ast_serialize-0.5.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cc22cf0c9be65e71cf88fda130af60d61eb4a79370ad4cfe7900d48a4aa2211", size = 1244529, upload-time = "2026-05-17T17:48:07.008Z" }, + { url = "https://files.pythonhosted.org/packages/c6/ca/6376559dcce707cdbc1d0d9a13c8d3baaaa501e949ce0ebdc4230cd881aa/ast_serialize-0.5.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f66173891548c9f2726bf27957b41cabce12fa679dc6da505ddbde4d4b3b31cf", size = 1240560, upload-time = "2026-05-17T17:48:08.46Z" }, + { url = "https://files.pythonhosted.org/packages/35/b2/a620e206b5aeb7efbf2710336df57d457cffbb3991076bbcc1147ef9abd4/ast_serialize-0.5.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e42d729ef2be96a14efbad355093284739e3670ece3e534f82cc8832790911d9", size = 1451172, upload-time = "2026-05-17T17:48:09.922Z" }, + { url = "https://files.pythonhosted.org/packages/fa/e0/4ad5c04c24a40481b2935ce9a0ccdb6023dc8b667167d06ae530cc3512f2/ast_serialize-0.5.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b725026bafa801dbd7310eb13a75f0a2e370e7e51b2cb225f9d21fcfadf919ee", size = 1265072, upload-time = "2026-05-17T17:48:11.469Z" }, + { url = "https://files.pythonhosted.org/packages/b2/71/4d1d479aa56d0101c40e17720c3d6ac2af7269ea0487a80b18e7bfd1a5b7/ast_serialize-0.5.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b54f60c1d78767a53b67eaa663f0dfac3afe606aa07f1301572f588b73d64809", size = 1270488, upload-time = "2026-05-17T17:48:13.575Z" }, + { url = "https://files.pythonhosted.org/packages/6d/4f/0de1bbe06f6edef9fde4ed12ca8e7b3ec7e6e2bd4e672c5af487f7957665/ast_serialize-0.5.0-cp39-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:27d51654fc240a1e87e742d353d98eb45b75f62f129086b3596ab53df2ac2a43", size = 1260702, upload-time = "2026-05-17T17:48:15.141Z" }, + { url = "https://files.pythonhosted.org/packages/75/61/e00872439cfdddcc3c1b6cdaa6e5d904ba8e26a18807c67c4e14409d0ca8/ast_serialize-0.5.0-cp39-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:2782c36237c46dd1674542f2109740ea5ea485a169bf1431939ada0434e17934", size = 1311182, upload-time = "2026-05-17T17:48:16.779Z" }, + { url = "https://files.pythonhosted.org/packages/76/8e/699a5b955f7926956c95e9e1d74132acad73c2fe7a426f94da89123c20aa/ast_serialize-0.5.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1943db345233cc7194a470f13afa9c59772c0b123dea0c9414c4d4ca54369759", size = 1421410, upload-time = "2026-05-17T17:48:18.527Z" }, + { url = "https://files.pythonhosted.org/packages/a9/ae/d5b7626874478997adc7a29ab28accf21e596fb590c944290401dfd0b29e/ast_serialize-0.5.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:df1c00022cbbcb064bfaa505aa9c9295362443ce5dacb459d1331d3da353f887", size = 1516587, upload-time = "2026-05-17T17:48:20.133Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ce/b59e02a82d9c4244d64cde502e0b00e83e38816abe19155ceb5437402c7f/ast_serialize-0.5.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:cae65289fc456fde04af979a2be09302ef5d8ab92ef23e596d6746dc267ada27", size = 1515171, upload-time = "2026-05-17T17:48:21.921Z" }, + { url = "https://files.pythonhosted.org/packages/8b/38/d8d90042747d05aa08d4efcf1c99035a5f670a6bf4c214d31644392afbca/ast_serialize-0.5.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:239a4c354e8d676e9d94631d1d4a64edc6b266f86ff3a5a80aedd344f342c01d", size = 1464668, upload-time = "2026-05-17T17:48:23.544Z" }, + { url = "https://files.pythonhosted.org/packages/dd/51/5b840c4df7334104cecffa28f23904fe81ca89ca223d2450e288de39fd3c/ast_serialize-0.5.0-cp39-abi3-win32.whl", hash = "sha256:143a4ef63285a075871908fda3672dc21864b83a8ec3ee12304aa3e4c5387b9a", size = 1068311, upload-time = "2026-05-17T17:48:25.027Z" }, + { url = "https://files.pythonhosted.org/packages/41/11/ca5672c7d491825bc4cd6702dea106a6b60d928707712ec257c7833ae476/ast_serialize-0.5.0-cp39-abi3-win_amd64.whl", hash = "sha256:cf25572c526add400f26a4750dc6ce0c3bb93fc1f75e7ae0cad4ce4f2cd5c590", size = 1108931, upload-time = "2026-05-17T17:48:26.591Z" }, + { url = "https://files.pythonhosted.org/packages/45/19/cc8bd127d28a43da249aa955cfd164cf8fd534e79e42cea96c4854d72fd0/ast_serialize-0.5.0-cp39-abi3-win_arm64.whl", hash = "sha256:92a31c9c20d25a076edaeec76b128a3535d74a24f340b9a8a7e96c9b86dc9642", size = 1081181, upload-time = "2026-05-17T17:48:28.122Z" }, +] + +[[package]] +name = "cfgv" +version = "3.4.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.10'", ] -dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/18/74/dfb75f9ccd592bbedb175d4a32fc643cf569d7c218508bfbd6ea7ef9c091/astroid-3.3.11.tar.gz", hash = "sha256:1e5a5011af2920c7c67a53f65d536d65bfa7116feeaf2354d8b94f29573bb0ce", size = 400439, upload-time = "2025-07-13T18:04:23.177Z" } +sdist = { url = "https://files.pythonhosted.org/packages/11/74/539e56497d9bd1d484fd863dd69cbbfa653cd2aa27abfe35653494d85e94/cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560", size = 7114, upload-time = "2023-08-12T20:38:17.776Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/af/0f/3b8fdc946b4d9cc8cc1e8af42c4e409468c84441b933d037e101b3d72d86/astroid-3.3.11-py3-none-any.whl", hash = "sha256:54c760ae8322ece1abd213057c4b5bba7c49818853fc901ef09719a60dbf9dec", size = 275612, upload-time = "2025-07-13T18:04:21.07Z" }, + { url = "https://files.pythonhosted.org/packages/c5/55/51844dd50c4fc7a33b653bfaba4c2456f06955289ca770a5dbd5fd267374/cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9", size = 7249, upload-time = "2023-08-12T20:38:16.269Z" }, ] [[package]] -name = "astroid" -version = "4.0.2" +name = "cfgv" +version = "3.5.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.15'", + "python_full_version >= '3.12' and python_full_version < '3.15'", "python_full_version == '3.11.*'", "python_full_version == '3.10.*'", ] -dependencies = [ - { name = "typing-extensions", marker = "python_full_version == '3.10.*'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b7/22/97df040e15d964e592d3a180598ace67e91b7c559d8298bdb3c949dc6e42/astroid-4.0.2.tar.gz", hash = "sha256:ac8fb7ca1c08eb9afec91ccc23edbd8ac73bb22cbdd7da1d488d9fb8d6579070", size = 405714, upload-time = "2025-11-09T21:21:18.373Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/b5/721b8799b04bf9afe054a3899c6cf4e880fcf8563cc71c15610242490a0c/cfgv-3.5.0.tar.gz", hash = "sha256:d5b1034354820651caa73ede66a6294d6e95c1b00acc5e9b098e917404669132", size = 7334, upload-time = "2025-11-19T20:55:51.612Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/93/ac/a85b4bfb4cf53221513e27f33cc37ad158fce02ac291d18bee6b49ab477d/astroid-4.0.2-py3-none-any.whl", hash = "sha256:d7546c00a12efc32650b19a2bb66a153883185d3179ab0d4868086f807338b9b", size = 276354, upload-time = "2025-11-09T21:21:16.54Z" }, + { url = "https://files.pythonhosted.org/packages/db/3c/33bac158f8ab7f89b2e59426d5fe2e4f63f7ed25df84c036890172b412b5/cfgv-3.5.0-py2.py3-none-any.whl", hash = "sha256:a8dc6b26ad22ff227d2634a65cb388215ce6cc96bbcc5cfde7641ae87e8dacc0", size = 7445, upload-time = "2025-11-19T20:55:50.744Z" }, ] [[package]] -name = "black" -version = "25.11.0" +name = "colorama" +version = "0.4.6" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "click", version = "8.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "mypy-extensions" }, - { name = "packaging" }, - { name = "pathspec" }, - { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "pytokens" }, - { name = "tomli", marker = "python_full_version < '3.11'" }, - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/8c/ad/33adf4708633d047950ff2dfdea2e215d84ac50ef95aff14a614e4b6e9b2/black-25.11.0.tar.gz", hash = "sha256:9a323ac32f5dc75ce7470501b887250be5005a01602e931a15e45593f70f6e08", size = 655669, upload-time = "2025-11-10T01:53:50.558Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/d2/6caccbc96f9311e8ec3378c296d4f4809429c43a6cd2394e3c390e86816d/black-25.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ec311e22458eec32a807f029b2646f661e6859c3f61bc6d9ffb67958779f392e", size = 1743501, upload-time = "2025-11-10T01:59:06.202Z" }, - { url = "https://files.pythonhosted.org/packages/69/35/b986d57828b3f3dccbf922e2864223197ba32e74c5004264b1c62bc9f04d/black-25.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1032639c90208c15711334d681de2e24821af0575573db2810b0763bcd62e0f0", size = 1597308, upload-time = "2025-11-10T01:57:58.633Z" }, - { url = "https://files.pythonhosted.org/packages/39/8e/8b58ef4b37073f52b64a7b2dd8c9a96c84f45d6f47d878d0aa557e9a2d35/black-25.11.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c0f7c461df55cf32929b002335883946a4893d759f2df343389c4396f3b6b37", size = 1656194, upload-time = "2025-11-10T01:57:10.909Z" }, - { url = "https://files.pythonhosted.org/packages/8d/30/9c2267a7955ecc545306534ab88923769a979ac20a27cf618d370091e5dd/black-25.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:f9786c24d8e9bd5f20dc7a7f0cdd742644656987f6ea6947629306f937726c03", size = 1347996, upload-time = "2025-11-10T01:57:22.391Z" }, - { url = "https://files.pythonhosted.org/packages/c4/62/d304786b75ab0c530b833a89ce7d997924579fb7484ecd9266394903e394/black-25.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:895571922a35434a9d8ca67ef926da6bc9ad464522a5fe0db99b394ef1c0675a", size = 1727891, upload-time = "2025-11-10T02:01:40.507Z" }, - { url = "https://files.pythonhosted.org/packages/82/5d/ffe8a006aa522c9e3f430e7b93568a7b2163f4b3f16e8feb6d8c3552761a/black-25.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cb4f4b65d717062191bdec8e4a442539a8ea065e6af1c4f4d36f0cdb5f71e170", size = 1581875, upload-time = "2025-11-10T01:57:51.192Z" }, - { url = "https://files.pythonhosted.org/packages/cb/c8/7c8bda3108d0bb57387ac41b4abb5c08782b26da9f9c4421ef6694dac01a/black-25.11.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d81a44cbc7e4f73a9d6ae449ec2317ad81512d1e7dce7d57f6333fd6259737bc", size = 1642716, upload-time = "2025-11-10T01:56:51.589Z" }, - { url = "https://files.pythonhosted.org/packages/34/b9/f17dea34eecb7cc2609a89627d480fb6caea7b86190708eaa7eb15ed25e7/black-25.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:7eebd4744dfe92ef1ee349dc532defbf012a88b087bb7ddd688ff59a447b080e", size = 1352904, upload-time = "2025-11-10T01:59:26.252Z" }, - { url = "https://files.pythonhosted.org/packages/7f/12/5c35e600b515f35ffd737da7febdb2ab66bb8c24d88560d5e3ef3d28c3fd/black-25.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:80e7486ad3535636657aa180ad32a7d67d7c273a80e12f1b4bfa0823d54e8fac", size = 1772831, upload-time = "2025-11-10T02:03:47Z" }, - { url = "https://files.pythonhosted.org/packages/1a/75/b3896bec5a2bb9ed2f989a970ea40e7062f8936f95425879bbe162746fe5/black-25.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6cced12b747c4c76bc09b4db057c319d8545307266f41aaee665540bc0e04e96", size = 1608520, upload-time = "2025-11-10T01:58:46.895Z" }, - { url = "https://files.pythonhosted.org/packages/f3/b5/2bfc18330eddbcfb5aab8d2d720663cd410f51b2ed01375f5be3751595b0/black-25.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cb2d54a39e0ef021d6c5eef442e10fd71fcb491be6413d083a320ee768329dd", size = 1682719, upload-time = "2025-11-10T01:56:55.24Z" }, - { url = "https://files.pythonhosted.org/packages/96/fb/f7dc2793a22cdf74a72114b5ed77fe3349a2e09ef34565857a2f917abdf2/black-25.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae263af2f496940438e5be1a0c1020e13b09154f3af4df0835ea7f9fe7bfa409", size = 1362684, upload-time = "2025-11-10T01:57:07.639Z" }, - { url = "https://files.pythonhosted.org/packages/ad/47/3378d6a2ddefe18553d1115e36aea98f4a90de53b6a3017ed861ba1bd3bc/black-25.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0a1d40348b6621cc20d3d7530a5b8d67e9714906dfd7346338249ad9c6cedf2b", size = 1772446, upload-time = "2025-11-10T02:02:16.181Z" }, - { url = "https://files.pythonhosted.org/packages/ba/4b/0f00bfb3d1f7e05e25bfc7c363f54dc523bb6ba502f98f4ad3acf01ab2e4/black-25.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:51c65d7d60bb25429ea2bf0731c32b2a2442eb4bd3b2afcb47830f0b13e58bfd", size = 1607983, upload-time = "2025-11-10T02:02:52.502Z" }, - { url = "https://files.pythonhosted.org/packages/99/fe/49b0768f8c9ae57eb74cc10a1f87b4c70453551d8ad498959721cc345cb7/black-25.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:936c4dd07669269f40b497440159a221ee435e3fddcf668e0c05244a9be71993", size = 1682481, upload-time = "2025-11-10T01:57:12.35Z" }, - { url = "https://files.pythonhosted.org/packages/55/17/7e10ff1267bfa950cc16f0a411d457cdff79678fbb77a6c73b73a5317904/black-25.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:f42c0ea7f59994490f4dccd64e6b2dd49ac57c7c84f38b8faab50f8759db245c", size = 1363869, upload-time = "2025-11-10T01:58:24.608Z" }, - { url = "https://files.pythonhosted.org/packages/67/c0/cc865ce594d09e4cd4dfca5e11994ebb51604328489f3ca3ae7bb38a7db5/black-25.11.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:35690a383f22dd3e468c85dc4b915217f87667ad9cce781d7b42678ce63c4170", size = 1771358, upload-time = "2025-11-10T02:03:33.331Z" }, - { url = "https://files.pythonhosted.org/packages/37/77/4297114d9e2fd2fc8ab0ab87192643cd49409eb059e2940391e7d2340e57/black-25.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dae49ef7369c6caa1a1833fd5efb7c3024bb7e4499bf64833f65ad27791b1545", size = 1612902, upload-time = "2025-11-10T01:59:33.382Z" }, - { url = "https://files.pythonhosted.org/packages/de/63/d45ef97ada84111e330b2b2d45e1dd163e90bd116f00ac55927fb6bf8adb/black-25.11.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bd4a22a0b37401c8e492e994bce79e614f91b14d9ea911f44f36e262195fdda", size = 1680571, upload-time = "2025-11-10T01:57:04.239Z" }, - { url = "https://files.pythonhosted.org/packages/ff/4b/5604710d61cdff613584028b4cb4607e56e148801ed9b38ee7970799dab6/black-25.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:aa211411e94fdf86519996b7f5f05e71ba34835d8f0c0f03c00a26271da02664", size = 1382599, upload-time = "2025-11-10T01:57:57.427Z" }, - { url = "https://files.pythonhosted.org/packages/d5/9a/5b2c0e3215fe748fcf515c2dd34658973a1210bf610e24de5ba887e4f1c8/black-25.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a3bb5ce32daa9ff0605d73b6f19da0b0e6c1f8f2d75594db539fdfed722f2b06", size = 1743063, upload-time = "2025-11-10T02:02:43.175Z" }, - { url = "https://files.pythonhosted.org/packages/a1/20/245164c6efc27333409c62ba54dcbfbe866c6d1957c9a6c0647786e950da/black-25.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9815ccee1e55717fe9a4b924cae1646ef7f54e0f990da39a34fc7b264fcf80a2", size = 1596867, upload-time = "2025-11-10T02:00:17.157Z" }, - { url = "https://files.pythonhosted.org/packages/ca/6f/1a3859a7da205f3d50cf3a8bec6bdc551a91c33ae77a045bb24c1f46ab54/black-25.11.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92285c37b93a1698dcbc34581867b480f1ba3a7b92acf1fe0467b04d7a4da0dc", size = 1655678, upload-time = "2025-11-10T01:57:09.028Z" }, - { url = "https://files.pythonhosted.org/packages/56/1a/6dec1aeb7be90753d4fcc273e69bc18bfd34b353223ed191da33f7519410/black-25.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:43945853a31099c7c0ff8dface53b4de56c41294fa6783c0441a8b1d9bf668bc", size = 1347452, upload-time = "2025-11-10T01:57:01.871Z" }, - { url = "https://files.pythonhosted.org/packages/00/5d/aed32636ed30a6e7f9efd6ad14e2a0b0d687ae7c8c7ec4e4a557174b895c/black-25.11.0-py3-none-any.whl", hash = "sha256:e3f562da087791e96cefcd9dda058380a442ab322a02e222add53736451f604b", size = 204918, upload-time = "2025-11-10T01:53:48.917Z" }, -] - -[[package]] -name = "click" -version = "8.1.8" + +[[package]] +name = "coverage" +version = "7.10.7" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.10'", ] -dependencies = [ - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } +sdist = { url = "https://files.pythonhosted.org/packages/51/26/d22c300112504f5f9a9fd2297ce33c35f3d353e4aeb987c8419453b2a7c2/coverage-7.10.7.tar.gz", hash = "sha256:f4ab143ab113be368a3e9b795f9cd7906c5ef407d6173fe9675a902e1fffc239", size = 827704, upload-time = "2025-09-21T20:03:56.815Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188, upload-time = "2024-12-21T18:38:41.666Z" }, + { url = "https://files.pythonhosted.org/packages/e5/6c/3a3f7a46888e69d18abe3ccc6fe4cb16cccb1e6a2f99698931dafca489e6/coverage-7.10.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fc04cc7a3db33664e0c2d10eb8990ff6b3536f6842c9590ae8da4c614b9ed05a", size = 217987, upload-time = "2025-09-21T20:00:57.218Z" }, + { url = "https://files.pythonhosted.org/packages/03/94/952d30f180b1a916c11a56f5c22d3535e943aa22430e9e3322447e520e1c/coverage-7.10.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e201e015644e207139f7e2351980feb7040e6f4b2c2978892f3e3789d1c125e5", size = 218388, upload-time = "2025-09-21T20:01:00.081Z" }, + { url = "https://files.pythonhosted.org/packages/50/2b/9e0cf8ded1e114bcd8b2fd42792b57f1c4e9e4ea1824cde2af93a67305be/coverage-7.10.7-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:240af60539987ced2c399809bd34f7c78e8abe0736af91c3d7d0e795df633d17", size = 245148, upload-time = "2025-09-21T20:01:01.768Z" }, + { url = "https://files.pythonhosted.org/packages/19/20/d0384ac06a6f908783d9b6aa6135e41b093971499ec488e47279f5b846e6/coverage-7.10.7-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:8421e088bc051361b01c4b3a50fd39a4b9133079a2229978d9d30511fd05231b", size = 246958, upload-time = "2025-09-21T20:01:03.355Z" }, + { url = "https://files.pythonhosted.org/packages/60/83/5c283cff3d41285f8eab897651585db908a909c572bdc014bcfaf8a8b6ae/coverage-7.10.7-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6be8ed3039ae7f7ac5ce058c308484787c86e8437e72b30bf5e88b8ea10f3c87", size = 248819, upload-time = "2025-09-21T20:01:04.968Z" }, + { url = "https://files.pythonhosted.org/packages/60/22/02eb98fdc5ff79f423e990d877693e5310ae1eab6cb20ae0b0b9ac45b23b/coverage-7.10.7-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e28299d9f2e889e6d51b1f043f58d5f997c373cc12e6403b90df95b8b047c13e", size = 245754, upload-time = "2025-09-21T20:01:06.321Z" }, + { url = "https://files.pythonhosted.org/packages/b4/bc/25c83bcf3ad141b32cd7dc45485ef3c01a776ca3aa8ef0a93e77e8b5bc43/coverage-7.10.7-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c4e16bd7761c5e454f4efd36f345286d6f7c5fa111623c355691e2755cae3b9e", size = 246860, upload-time = "2025-09-21T20:01:07.605Z" }, + { url = "https://files.pythonhosted.org/packages/3c/b7/95574702888b58c0928a6e982038c596f9c34d52c5e5107f1eef729399b5/coverage-7.10.7-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:b1c81d0e5e160651879755c9c675b974276f135558cf4ba79fee7b8413a515df", size = 244877, upload-time = "2025-09-21T20:01:08.829Z" }, + { url = "https://files.pythonhosted.org/packages/47/b6/40095c185f235e085df0e0b158f6bd68cc6e1d80ba6c7721dc81d97ec318/coverage-7.10.7-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:606cc265adc9aaedcc84f1f064f0e8736bc45814f15a357e30fca7ecc01504e0", size = 245108, upload-time = "2025-09-21T20:01:10.527Z" }, + { url = "https://files.pythonhosted.org/packages/c8/50/4aea0556da7a4b93ec9168420d170b55e2eb50ae21b25062513d020c6861/coverage-7.10.7-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:10b24412692df990dbc34f8fb1b6b13d236ace9dfdd68df5b28c2e39cafbba13", size = 245752, upload-time = "2025-09-21T20:01:11.857Z" }, + { url = "https://files.pythonhosted.org/packages/6a/28/ea1a84a60828177ae3b100cb6723838523369a44ec5742313ed7db3da160/coverage-7.10.7-cp310-cp310-win32.whl", hash = "sha256:b51dcd060f18c19290d9b8a9dd1e0181538df2ce0717f562fff6cf74d9fc0b5b", size = 220497, upload-time = "2025-09-21T20:01:13.459Z" }, + { url = "https://files.pythonhosted.org/packages/fc/1a/a81d46bbeb3c3fd97b9602ebaa411e076219a150489bcc2c025f151bd52d/coverage-7.10.7-cp310-cp310-win_amd64.whl", hash = "sha256:3a622ac801b17198020f09af3eaf45666b344a0d69fc2a6ffe2ea83aeef1d807", size = 221392, upload-time = "2025-09-21T20:01:14.722Z" }, + { url = "https://files.pythonhosted.org/packages/d2/5d/c1a17867b0456f2e9ce2d8d4708a4c3a089947d0bec9c66cdf60c9e7739f/coverage-7.10.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a609f9c93113be646f44c2a0256d6ea375ad047005d7f57a5c15f614dc1b2f59", size = 218102, upload-time = "2025-09-21T20:01:16.089Z" }, + { url = "https://files.pythonhosted.org/packages/54/f0/514dcf4b4e3698b9a9077f084429681bf3aad2b4a72578f89d7f643eb506/coverage-7.10.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:65646bb0359386e07639c367a22cf9b5bf6304e8630b565d0626e2bdf329227a", size = 218505, upload-time = "2025-09-21T20:01:17.788Z" }, + { url = "https://files.pythonhosted.org/packages/20/f6/9626b81d17e2a4b25c63ac1b425ff307ecdeef03d67c9a147673ae40dc36/coverage-7.10.7-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:5f33166f0dfcce728191f520bd2692914ec70fac2713f6bf3ce59c3deacb4699", size = 248898, upload-time = "2025-09-21T20:01:19.488Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ef/bd8e719c2f7417ba03239052e099b76ea1130ac0cbb183ee1fcaa58aaff3/coverage-7.10.7-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:35f5e3f9e455bb17831876048355dca0f758b6df22f49258cb5a91da23ef437d", size = 250831, upload-time = "2025-09-21T20:01:20.817Z" }, + { url = "https://files.pythonhosted.org/packages/a5/b6/bf054de41ec948b151ae2b79a55c107f5760979538f5fb80c195f2517718/coverage-7.10.7-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4da86b6d62a496e908ac2898243920c7992499c1712ff7c2b6d837cc69d9467e", size = 252937, upload-time = "2025-09-21T20:01:22.171Z" }, + { url = "https://files.pythonhosted.org/packages/0f/e5/3860756aa6f9318227443c6ce4ed7bf9e70bb7f1447a0353f45ac5c7974b/coverage-7.10.7-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6b8b09c1fad947c84bbbc95eca841350fad9cbfa5a2d7ca88ac9f8d836c92e23", size = 249021, upload-time = "2025-09-21T20:01:23.907Z" }, + { url = "https://files.pythonhosted.org/packages/26/0f/bd08bd042854f7fd07b45808927ebcce99a7ed0f2f412d11629883517ac2/coverage-7.10.7-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4376538f36b533b46f8971d3a3e63464f2c7905c9800db97361c43a2b14792ab", size = 250626, upload-time = "2025-09-21T20:01:25.721Z" }, + { url = "https://files.pythonhosted.org/packages/8e/a7/4777b14de4abcc2e80c6b1d430f5d51eb18ed1d75fca56cbce5f2db9b36e/coverage-7.10.7-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:121da30abb574f6ce6ae09840dae322bef734480ceafe410117627aa54f76d82", size = 248682, upload-time = "2025-09-21T20:01:27.105Z" }, + { url = "https://files.pythonhosted.org/packages/34/72/17d082b00b53cd45679bad682fac058b87f011fd8b9fe31d77f5f8d3a4e4/coverage-7.10.7-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:88127d40df529336a9836870436fc2751c339fbaed3a836d42c93f3e4bd1d0a2", size = 248402, upload-time = "2025-09-21T20:01:28.629Z" }, + { url = "https://files.pythonhosted.org/packages/81/7a/92367572eb5bdd6a84bfa278cc7e97db192f9f45b28c94a9ca1a921c3577/coverage-7.10.7-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ba58bbcd1b72f136080c0bccc2400d66cc6115f3f906c499013d065ac33a4b61", size = 249320, upload-time = "2025-09-21T20:01:30.004Z" }, + { url = "https://files.pythonhosted.org/packages/2f/88/a23cc185f6a805dfc4fdf14a94016835eeb85e22ac3a0e66d5e89acd6462/coverage-7.10.7-cp311-cp311-win32.whl", hash = "sha256:972b9e3a4094b053a4e46832b4bc829fc8a8d347160eb39d03f1690316a99c14", size = 220536, upload-time = "2025-09-21T20:01:32.184Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ef/0b510a399dfca17cec7bc2f05ad8bd78cf55f15c8bc9a73ab20c5c913c2e/coverage-7.10.7-cp311-cp311-win_amd64.whl", hash = "sha256:a7b55a944a7f43892e28ad4bc0561dfd5f0d73e605d1aa5c3c976b52aea121d2", size = 221425, upload-time = "2025-09-21T20:01:33.557Z" }, + { url = "https://files.pythonhosted.org/packages/51/7f/023657f301a276e4ba1850f82749bc136f5a7e8768060c2e5d9744a22951/coverage-7.10.7-cp311-cp311-win_arm64.whl", hash = "sha256:736f227fb490f03c6488f9b6d45855f8e0fd749c007f9303ad30efab0e73c05a", size = 220103, upload-time = "2025-09-21T20:01:34.929Z" }, + { url = "https://files.pythonhosted.org/packages/13/e4/eb12450f71b542a53972d19117ea5a5cea1cab3ac9e31b0b5d498df1bd5a/coverage-7.10.7-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7bb3b9ddb87ef7725056572368040c32775036472d5a033679d1fa6c8dc08417", size = 218290, upload-time = "2025-09-21T20:01:36.455Z" }, + { url = "https://files.pythonhosted.org/packages/37/66/593f9be12fc19fb36711f19a5371af79a718537204d16ea1d36f16bd78d2/coverage-7.10.7-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:18afb24843cbc175687225cab1138c95d262337f5473512010e46831aa0c2973", size = 218515, upload-time = "2025-09-21T20:01:37.982Z" }, + { url = "https://files.pythonhosted.org/packages/66/80/4c49f7ae09cafdacc73fbc30949ffe77359635c168f4e9ff33c9ebb07838/coverage-7.10.7-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:399a0b6347bcd3822be369392932884b8216d0944049ae22925631a9b3d4ba4c", size = 250020, upload-time = "2025-09-21T20:01:39.617Z" }, + { url = "https://files.pythonhosted.org/packages/a6/90/a64aaacab3b37a17aaedd83e8000142561a29eb262cede42d94a67f7556b/coverage-7.10.7-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:314f2c326ded3f4b09be11bc282eb2fc861184bc95748ae67b360ac962770be7", size = 252769, upload-time = "2025-09-21T20:01:41.341Z" }, + { url = "https://files.pythonhosted.org/packages/98/2e/2dda59afd6103b342e096f246ebc5f87a3363b5412609946c120f4e7750d/coverage-7.10.7-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c41e71c9cfb854789dee6fc51e46743a6d138b1803fab6cb860af43265b42ea6", size = 253901, upload-time = "2025-09-21T20:01:43.042Z" }, + { url = "https://files.pythonhosted.org/packages/53/dc/8d8119c9051d50f3119bb4a75f29f1e4a6ab9415cd1fa8bf22fcc3fb3b5f/coverage-7.10.7-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:bc01f57ca26269c2c706e838f6422e2a8788e41b3e3c65e2f41148212e57cd59", size = 250413, upload-time = "2025-09-21T20:01:44.469Z" }, + { url = "https://files.pythonhosted.org/packages/98/b3/edaff9c5d79ee4d4b6d3fe046f2b1d799850425695b789d491a64225d493/coverage-7.10.7-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a6442c59a8ac8b85812ce33bc4d05bde3fb22321fa8294e2a5b487c3505f611b", size = 251820, upload-time = "2025-09-21T20:01:45.915Z" }, + { url = "https://files.pythonhosted.org/packages/11/25/9a0728564bb05863f7e513e5a594fe5ffef091b325437f5430e8cfb0d530/coverage-7.10.7-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:78a384e49f46b80fb4c901d52d92abe098e78768ed829c673fbb53c498bef73a", size = 249941, upload-time = "2025-09-21T20:01:47.296Z" }, + { url = "https://files.pythonhosted.org/packages/e0/fd/ca2650443bfbef5b0e74373aac4df67b08180d2f184b482c41499668e258/coverage-7.10.7-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:5e1e9802121405ede4b0133aa4340ad8186a1d2526de5b7c3eca519db7bb89fb", size = 249519, upload-time = "2025-09-21T20:01:48.73Z" }, + { url = "https://files.pythonhosted.org/packages/24/79/f692f125fb4299b6f963b0745124998ebb8e73ecdfce4ceceb06a8c6bec5/coverage-7.10.7-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d41213ea25a86f69efd1575073d34ea11aabe075604ddf3d148ecfec9e1e96a1", size = 251375, upload-time = "2025-09-21T20:01:50.529Z" }, + { url = "https://files.pythonhosted.org/packages/5e/75/61b9bbd6c7d24d896bfeec57acba78e0f8deac68e6baf2d4804f7aae1f88/coverage-7.10.7-cp312-cp312-win32.whl", hash = "sha256:77eb4c747061a6af8d0f7bdb31f1e108d172762ef579166ec84542f711d90256", size = 220699, upload-time = "2025-09-21T20:01:51.941Z" }, + { url = "https://files.pythonhosted.org/packages/ca/f3/3bf7905288b45b075918d372498f1cf845b5b579b723c8fd17168018d5f5/coverage-7.10.7-cp312-cp312-win_amd64.whl", hash = "sha256:f51328ffe987aecf6d09f3cd9d979face89a617eacdaea43e7b3080777f647ba", size = 221512, upload-time = "2025-09-21T20:01:53.481Z" }, + { url = "https://files.pythonhosted.org/packages/5c/44/3e32dbe933979d05cf2dac5e697c8599cfe038aaf51223ab901e208d5a62/coverage-7.10.7-cp312-cp312-win_arm64.whl", hash = "sha256:bda5e34f8a75721c96085903c6f2197dc398c20ffd98df33f866a9c8fd95f4bf", size = 220147, upload-time = "2025-09-21T20:01:55.2Z" }, + { url = "https://files.pythonhosted.org/packages/9a/94/b765c1abcb613d103b64fcf10395f54d69b0ef8be6a0dd9c524384892cc7/coverage-7.10.7-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:981a651f543f2854abd3b5fcb3263aac581b18209be49863ba575de6edf4c14d", size = 218320, upload-time = "2025-09-21T20:01:56.629Z" }, + { url = "https://files.pythonhosted.org/packages/72/4f/732fff31c119bb73b35236dd333030f32c4bfe909f445b423e6c7594f9a2/coverage-7.10.7-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:73ab1601f84dc804f7812dc297e93cd99381162da39c47040a827d4e8dafe63b", size = 218575, upload-time = "2025-09-21T20:01:58.203Z" }, + { url = "https://files.pythonhosted.org/packages/87/02/ae7e0af4b674be47566707777db1aa375474f02a1d64b9323e5813a6cdd5/coverage-7.10.7-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:a8b6f03672aa6734e700bbcd65ff050fd19cddfec4b031cc8cf1c6967de5a68e", size = 249568, upload-time = "2025-09-21T20:01:59.748Z" }, + { url = "https://files.pythonhosted.org/packages/a2/77/8c6d22bf61921a59bce5471c2f1f7ac30cd4ac50aadde72b8c48d5727902/coverage-7.10.7-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:10b6ba00ab1132a0ce4428ff68cf50a25efd6840a42cdf4239c9b99aad83be8b", size = 252174, upload-time = "2025-09-21T20:02:01.192Z" }, + { url = "https://files.pythonhosted.org/packages/b1/20/b6ea4f69bbb52dac0aebd62157ba6a9dddbfe664f5af8122dac296c3ee15/coverage-7.10.7-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c79124f70465a150e89340de5963f936ee97097d2ef76c869708c4248c63ca49", size = 253447, upload-time = "2025-09-21T20:02:02.701Z" }, + { url = "https://files.pythonhosted.org/packages/f9/28/4831523ba483a7f90f7b259d2018fef02cb4d5b90bc7c1505d6e5a84883c/coverage-7.10.7-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:69212fbccdbd5b0e39eac4067e20a4a5256609e209547d86f740d68ad4f04911", size = 249779, upload-time = "2025-09-21T20:02:04.185Z" }, + { url = "https://files.pythonhosted.org/packages/a7/9f/4331142bc98c10ca6436d2d620c3e165f31e6c58d43479985afce6f3191c/coverage-7.10.7-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:7ea7c6c9d0d286d04ed3541747e6597cbe4971f22648b68248f7ddcd329207f0", size = 251604, upload-time = "2025-09-21T20:02:06.034Z" }, + { url = "https://files.pythonhosted.org/packages/ce/60/bda83b96602036b77ecf34e6393a3836365481b69f7ed7079ab85048202b/coverage-7.10.7-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b9be91986841a75042b3e3243d0b3cb0b2434252b977baaf0cd56e960fe1e46f", size = 249497, upload-time = "2025-09-21T20:02:07.619Z" }, + { url = "https://files.pythonhosted.org/packages/5f/af/152633ff35b2af63977edd835d8e6430f0caef27d171edf2fc76c270ef31/coverage-7.10.7-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:b281d5eca50189325cfe1f365fafade89b14b4a78d9b40b05ddd1fc7d2a10a9c", size = 249350, upload-time = "2025-09-21T20:02:10.34Z" }, + { url = "https://files.pythonhosted.org/packages/9d/71/d92105d122bd21cebba877228990e1646d862e34a98bb3374d3fece5a794/coverage-7.10.7-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:99e4aa63097ab1118e75a848a28e40d68b08a5e19ce587891ab7fd04475e780f", size = 251111, upload-time = "2025-09-21T20:02:12.122Z" }, + { url = "https://files.pythonhosted.org/packages/a2/9e/9fdb08f4bf476c912f0c3ca292e019aab6712c93c9344a1653986c3fd305/coverage-7.10.7-cp313-cp313-win32.whl", hash = "sha256:dc7c389dce432500273eaf48f410b37886be9208b2dd5710aaf7c57fd442c698", size = 220746, upload-time = "2025-09-21T20:02:13.919Z" }, + { url = "https://files.pythonhosted.org/packages/b1/b1/a75fd25df44eab52d1931e89980d1ada46824c7a3210be0d3c88a44aaa99/coverage-7.10.7-cp313-cp313-win_amd64.whl", hash = "sha256:cac0fdca17b036af3881a9d2729a850b76553f3f716ccb0360ad4dbc06b3b843", size = 221541, upload-time = "2025-09-21T20:02:15.57Z" }, + { url = "https://files.pythonhosted.org/packages/14/3a/d720d7c989562a6e9a14b2c9f5f2876bdb38e9367126d118495b89c99c37/coverage-7.10.7-cp313-cp313-win_arm64.whl", hash = "sha256:4b6f236edf6e2f9ae8fcd1332da4e791c1b6ba0dc16a2dc94590ceccb482e546", size = 220170, upload-time = "2025-09-21T20:02:17.395Z" }, + { url = "https://files.pythonhosted.org/packages/bb/22/e04514bf2a735d8b0add31d2b4ab636fc02370730787c576bb995390d2d5/coverage-7.10.7-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:a0ec07fd264d0745ee396b666d47cef20875f4ff2375d7c4f58235886cc1ef0c", size = 219029, upload-time = "2025-09-21T20:02:18.936Z" }, + { url = "https://files.pythonhosted.org/packages/11/0b/91128e099035ece15da3445d9015e4b4153a6059403452d324cbb0a575fa/coverage-7.10.7-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:dd5e856ebb7bfb7672b0086846db5afb4567a7b9714b8a0ebafd211ec7ce6a15", size = 219259, upload-time = "2025-09-21T20:02:20.44Z" }, + { url = "https://files.pythonhosted.org/packages/8b/51/66420081e72801536a091a0c8f8c1f88a5c4bf7b9b1bdc6222c7afe6dc9b/coverage-7.10.7-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f57b2a3c8353d3e04acf75b3fed57ba41f5c0646bbf1d10c7c282291c97936b4", size = 260592, upload-time = "2025-09-21T20:02:22.313Z" }, + { url = "https://files.pythonhosted.org/packages/5d/22/9b8d458c2881b22df3db5bb3e7369e63d527d986decb6c11a591ba2364f7/coverage-7.10.7-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1ef2319dd15a0b009667301a3f84452a4dc6fddfd06b0c5c53ea472d3989fbf0", size = 262768, upload-time = "2025-09-21T20:02:24.287Z" }, + { url = "https://files.pythonhosted.org/packages/f7/08/16bee2c433e60913c610ea200b276e8eeef084b0d200bdcff69920bd5828/coverage-7.10.7-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83082a57783239717ceb0ad584de3c69cf581b2a95ed6bf81ea66034f00401c0", size = 264995, upload-time = "2025-09-21T20:02:26.133Z" }, + { url = "https://files.pythonhosted.org/packages/20/9d/e53eb9771d154859b084b90201e5221bca7674ba449a17c101a5031d4054/coverage-7.10.7-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:50aa94fb1fb9a397eaa19c0d5ec15a5edd03a47bf1a3a6111a16b36e190cff65", size = 259546, upload-time = "2025-09-21T20:02:27.716Z" }, + { url = "https://files.pythonhosted.org/packages/ad/b0/69bc7050f8d4e56a89fb550a1577d5d0d1db2278106f6f626464067b3817/coverage-7.10.7-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:2120043f147bebb41c85b97ac45dd173595ff14f2a584f2963891cbcc3091541", size = 262544, upload-time = "2025-09-21T20:02:29.216Z" }, + { url = "https://files.pythonhosted.org/packages/ef/4b/2514b060dbd1bc0aaf23b852c14bb5818f244c664cb16517feff6bb3a5ab/coverage-7.10.7-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:2fafd773231dd0378fdba66d339f84904a8e57a262f583530f4f156ab83863e6", size = 260308, upload-time = "2025-09-21T20:02:31.226Z" }, + { url = "https://files.pythonhosted.org/packages/54/78/7ba2175007c246d75e496f64c06e94122bdb914790a1285d627a918bd271/coverage-7.10.7-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:0b944ee8459f515f28b851728ad224fa2d068f1513ef6b7ff1efafeb2185f999", size = 258920, upload-time = "2025-09-21T20:02:32.823Z" }, + { url = "https://files.pythonhosted.org/packages/c0/b3/fac9f7abbc841409b9a410309d73bfa6cfb2e51c3fada738cb607ce174f8/coverage-7.10.7-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:4b583b97ab2e3efe1b3e75248a9b333bd3f8b0b1b8e5b45578e05e5850dfb2c2", size = 261434, upload-time = "2025-09-21T20:02:34.86Z" }, + { url = "https://files.pythonhosted.org/packages/ee/51/a03bec00d37faaa891b3ff7387192cef20f01604e5283a5fabc95346befa/coverage-7.10.7-cp313-cp313t-win32.whl", hash = "sha256:2a78cd46550081a7909b3329e2266204d584866e8d97b898cd7fb5ac8d888b1a", size = 221403, upload-time = "2025-09-21T20:02:37.034Z" }, + { url = "https://files.pythonhosted.org/packages/53/22/3cf25d614e64bf6d8e59c7c669b20d6d940bb337bdee5900b9ca41c820bb/coverage-7.10.7-cp313-cp313t-win_amd64.whl", hash = "sha256:33a5e6396ab684cb43dc7befa386258acb2d7fae7f67330ebb85ba4ea27938eb", size = 222469, upload-time = "2025-09-21T20:02:39.011Z" }, + { url = "https://files.pythonhosted.org/packages/49/a1/00164f6d30d8a01c3c9c48418a7a5be394de5349b421b9ee019f380df2a0/coverage-7.10.7-cp313-cp313t-win_arm64.whl", hash = "sha256:86b0e7308289ddde73d863b7683f596d8d21c7d8664ce1dee061d0bcf3fbb4bb", size = 220731, upload-time = "2025-09-21T20:02:40.939Z" }, + { url = "https://files.pythonhosted.org/packages/23/9c/5844ab4ca6a4dd97a1850e030a15ec7d292b5c5cb93082979225126e35dd/coverage-7.10.7-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:b06f260b16ead11643a5a9f955bd4b5fd76c1a4c6796aeade8520095b75de520", size = 218302, upload-time = "2025-09-21T20:02:42.527Z" }, + { url = "https://files.pythonhosted.org/packages/f0/89/673f6514b0961d1f0e20ddc242e9342f6da21eaba3489901b565c0689f34/coverage-7.10.7-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:212f8f2e0612778f09c55dd4872cb1f64a1f2b074393d139278ce902064d5b32", size = 218578, upload-time = "2025-09-21T20:02:44.468Z" }, + { url = "https://files.pythonhosted.org/packages/05/e8/261cae479e85232828fb17ad536765c88dd818c8470aca690b0ac6feeaa3/coverage-7.10.7-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:3445258bcded7d4aa630ab8296dea4d3f15a255588dd535f980c193ab6b95f3f", size = 249629, upload-time = "2025-09-21T20:02:46.503Z" }, + { url = "https://files.pythonhosted.org/packages/82/62/14ed6546d0207e6eda876434e3e8475a3e9adbe32110ce896c9e0c06bb9a/coverage-7.10.7-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:bb45474711ba385c46a0bfe696c695a929ae69ac636cda8f532be9e8c93d720a", size = 252162, upload-time = "2025-09-21T20:02:48.689Z" }, + { url = "https://files.pythonhosted.org/packages/ff/49/07f00db9ac6478e4358165a08fb41b469a1b053212e8a00cb02f0d27a05f/coverage-7.10.7-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:813922f35bd800dca9994c5971883cbc0d291128a5de6b167c7aa697fcf59360", size = 253517, upload-time = "2025-09-21T20:02:50.31Z" }, + { url = "https://files.pythonhosted.org/packages/a2/59/c5201c62dbf165dfbc91460f6dbbaa85a8b82cfa6131ac45d6c1bfb52deb/coverage-7.10.7-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:93c1b03552081b2a4423091d6fb3787265b8f86af404cff98d1b5342713bdd69", size = 249632, upload-time = "2025-09-21T20:02:51.971Z" }, + { url = "https://files.pythonhosted.org/packages/07/ae/5920097195291a51fb00b3a70b9bbd2edbfe3c84876a1762bd1ef1565ebc/coverage-7.10.7-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:cc87dd1b6eaf0b848eebb1c86469b9f72a1891cb42ac7adcfbce75eadb13dd14", size = 251520, upload-time = "2025-09-21T20:02:53.858Z" }, + { url = "https://files.pythonhosted.org/packages/b9/3c/a815dde77a2981f5743a60b63df31cb322c944843e57dbd579326625a413/coverage-7.10.7-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:39508ffda4f343c35f3236fe8d1a6634a51f4581226a1262769d7f970e73bffe", size = 249455, upload-time = "2025-09-21T20:02:55.807Z" }, + { url = "https://files.pythonhosted.org/packages/aa/99/f5cdd8421ea656abefb6c0ce92556709db2265c41e8f9fc6c8ae0f7824c9/coverage-7.10.7-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:925a1edf3d810537c5a3abe78ec5530160c5f9a26b1f4270b40e62cc79304a1e", size = 249287, upload-time = "2025-09-21T20:02:57.784Z" }, + { url = "https://files.pythonhosted.org/packages/c3/7a/e9a2da6a1fc5d007dd51fca083a663ab930a8c4d149c087732a5dbaa0029/coverage-7.10.7-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2c8b9a0636f94c43cd3576811e05b89aa9bc2d0a85137affc544ae5cb0e4bfbd", size = 250946, upload-time = "2025-09-21T20:02:59.431Z" }, + { url = "https://files.pythonhosted.org/packages/ef/5b/0b5799aa30380a949005a353715095d6d1da81927d6dbed5def2200a4e25/coverage-7.10.7-cp314-cp314-win32.whl", hash = "sha256:b7b8288eb7cdd268b0304632da8cb0bb93fadcfec2fe5712f7b9cc8f4d487be2", size = 221009, upload-time = "2025-09-21T20:03:01.324Z" }, + { url = "https://files.pythonhosted.org/packages/da/b0/e802fbb6eb746de006490abc9bb554b708918b6774b722bb3a0e6aa1b7de/coverage-7.10.7-cp314-cp314-win_amd64.whl", hash = "sha256:1ca6db7c8807fb9e755d0379ccc39017ce0a84dcd26d14b5a03b78563776f681", size = 221804, upload-time = "2025-09-21T20:03:03.4Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e8/71d0c8e374e31f39e3389bb0bd19e527d46f00ea8571ec7ec8fd261d8b44/coverage-7.10.7-cp314-cp314-win_arm64.whl", hash = "sha256:097c1591f5af4496226d5783d036bf6fd6cd0cbc132e071b33861de756efb880", size = 220384, upload-time = "2025-09-21T20:03:05.111Z" }, + { url = "https://files.pythonhosted.org/packages/62/09/9a5608d319fa3eba7a2019addeacb8c746fb50872b57a724c9f79f146969/coverage-7.10.7-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:a62c6ef0d50e6de320c270ff91d9dd0a05e7250cac2a800b7784bae474506e63", size = 219047, upload-time = "2025-09-21T20:03:06.795Z" }, + { url = "https://files.pythonhosted.org/packages/f5/6f/f58d46f33db9f2e3647b2d0764704548c184e6f5e014bef528b7f979ef84/coverage-7.10.7-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:9fa6e4dd51fe15d8738708a973470f67a855ca50002294852e9571cdbd9433f2", size = 219266, upload-time = "2025-09-21T20:03:08.495Z" }, + { url = "https://files.pythonhosted.org/packages/74/5c/183ffc817ba68e0b443b8c934c8795553eb0c14573813415bd59941ee165/coverage-7.10.7-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8fb190658865565c549b6b4706856d6a7b09302c797eb2cf8e7fe9dabb043f0d", size = 260767, upload-time = "2025-09-21T20:03:10.172Z" }, + { url = "https://files.pythonhosted.org/packages/0f/48/71a8abe9c1ad7e97548835e3cc1adbf361e743e9d60310c5f75c9e7bf847/coverage-7.10.7-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:affef7c76a9ef259187ef31599a9260330e0335a3011732c4b9effa01e1cd6e0", size = 262931, upload-time = "2025-09-21T20:03:11.861Z" }, + { url = "https://files.pythonhosted.org/packages/84/fd/193a8fb132acfc0a901f72020e54be5e48021e1575bb327d8ee1097a28fd/coverage-7.10.7-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6e16e07d85ca0cf8bafe5f5d23a0b850064e8e945d5677492b06bbe6f09cc699", size = 265186, upload-time = "2025-09-21T20:03:13.539Z" }, + { url = "https://files.pythonhosted.org/packages/b1/8f/74ecc30607dd95ad50e3034221113ccb1c6d4e8085cc761134782995daae/coverage-7.10.7-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:03ffc58aacdf65d2a82bbeb1ffe4d01ead4017a21bfd0454983b88ca73af94b9", size = 259470, upload-time = "2025-09-21T20:03:15.584Z" }, + { url = "https://files.pythonhosted.org/packages/0f/55/79ff53a769f20d71b07023ea115c9167c0bb56f281320520cf64c5298a96/coverage-7.10.7-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1b4fd784344d4e52647fd7857b2af5b3fbe6c239b0b5fa63e94eb67320770e0f", size = 262626, upload-time = "2025-09-21T20:03:17.673Z" }, + { url = "https://files.pythonhosted.org/packages/88/e2/dac66c140009b61ac3fc13af673a574b00c16efdf04f9b5c740703e953c0/coverage-7.10.7-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:0ebbaddb2c19b71912c6f2518e791aa8b9f054985a0769bdb3a53ebbc765c6a1", size = 260386, upload-time = "2025-09-21T20:03:19.36Z" }, + { url = "https://files.pythonhosted.org/packages/a2/f1/f48f645e3f33bb9ca8a496bc4a9671b52f2f353146233ebd7c1df6160440/coverage-7.10.7-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:a2d9a3b260cc1d1dbdb1c582e63ddcf5363426a1a68faa0f5da28d8ee3c722a0", size = 258852, upload-time = "2025-09-21T20:03:21.007Z" }, + { url = "https://files.pythonhosted.org/packages/bb/3b/8442618972c51a7affeead957995cfa8323c0c9bcf8fa5a027421f720ff4/coverage-7.10.7-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:a3cc8638b2480865eaa3926d192e64ce6c51e3d29c849e09d5b4ad95efae5399", size = 261534, upload-time = "2025-09-21T20:03:23.12Z" }, + { url = "https://files.pythonhosted.org/packages/b2/dc/101f3fa3a45146db0cb03f5b4376e24c0aac818309da23e2de0c75295a91/coverage-7.10.7-cp314-cp314t-win32.whl", hash = "sha256:67f8c5cbcd3deb7a60b3345dffc89a961a484ed0af1f6f73de91705cc6e31235", size = 221784, upload-time = "2025-09-21T20:03:24.769Z" }, + { url = "https://files.pythonhosted.org/packages/4c/a1/74c51803fc70a8a40d7346660379e144be772bab4ac7bb6e6b905152345c/coverage-7.10.7-cp314-cp314t-win_amd64.whl", hash = "sha256:e1ed71194ef6dea7ed2d5cb5f7243d4bcd334bfb63e59878519be558078f848d", size = 222905, upload-time = "2025-09-21T20:03:26.93Z" }, + { url = "https://files.pythonhosted.org/packages/12/65/f116a6d2127df30bcafbceef0302d8a64ba87488bf6f73a6d8eebf060873/coverage-7.10.7-cp314-cp314t-win_arm64.whl", hash = "sha256:7fe650342addd8524ca63d77b2362b02345e5f1a093266787d210c70a50b471a", size = 220922, upload-time = "2025-09-21T20:03:28.672Z" }, + { url = "https://files.pythonhosted.org/packages/a3/ad/d1c25053764b4c42eb294aae92ab617d2e4f803397f9c7c8295caa77a260/coverage-7.10.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fff7b9c3f19957020cac546c70025331113d2e61537f6e2441bc7657913de7d3", size = 217978, upload-time = "2025-09-21T20:03:30.362Z" }, + { url = "https://files.pythonhosted.org/packages/52/2f/b9f9daa39b80ece0b9548bbb723381e29bc664822d9a12c2135f8922c22b/coverage-7.10.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:bc91b314cef27742da486d6839b677b3f2793dfe52b51bbbb7cf736d5c29281c", size = 218370, upload-time = "2025-09-21T20:03:32.147Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6e/30d006c3b469e58449650642383dddf1c8fb63d44fdf92994bfd46570695/coverage-7.10.7-cp39-cp39-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:567f5c155eda8df1d3d439d40a45a6a5f029b429b06648235f1e7e51b522b396", size = 244802, upload-time = "2025-09-21T20:03:33.919Z" }, + { url = "https://files.pythonhosted.org/packages/b0/49/8a070782ce7e6b94ff6a0b6d7c65ba6bc3091d92a92cef4cd4eb0767965c/coverage-7.10.7-cp39-cp39-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:2af88deffcc8a4d5974cf2d502251bc3b2db8461f0b66d80a449c33757aa9f40", size = 246625, upload-time = "2025-09-21T20:03:36.09Z" }, + { url = "https://files.pythonhosted.org/packages/6a/92/1c1c5a9e8677ce56d42b97bdaca337b2d4d9ebe703d8c174ede52dbabd5f/coverage-7.10.7-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c7315339eae3b24c2d2fa1ed7d7a38654cba34a13ef19fbcb9425da46d3dc594", size = 248399, upload-time = "2025-09-21T20:03:38.342Z" }, + { url = "https://files.pythonhosted.org/packages/c0/54/b140edee7257e815de7426d5d9846b58505dffc29795fff2dfb7f8a1c5a0/coverage-7.10.7-cp39-cp39-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:912e6ebc7a6e4adfdbb1aec371ad04c68854cd3bf3608b3514e7ff9062931d8a", size = 245142, upload-time = "2025-09-21T20:03:40.591Z" }, + { url = "https://files.pythonhosted.org/packages/e4/9e/6d6b8295940b118e8b7083b29226c71f6154f7ff41e9ca431f03de2eac0d/coverage-7.10.7-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:f49a05acd3dfe1ce9715b657e28d138578bc40126760efb962322c56e9ca344b", size = 246284, upload-time = "2025-09-21T20:03:42.355Z" }, + { url = "https://files.pythonhosted.org/packages/db/e5/5e957ca747d43dbe4d9714358375c7546cb3cb533007b6813fc20fce37ad/coverage-7.10.7-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:cce2109b6219f22ece99db7644b9622f54a4e915dad65660ec435e89a3ea7cc3", size = 244353, upload-time = "2025-09-21T20:03:44.218Z" }, + { url = "https://files.pythonhosted.org/packages/9a/45/540fc5cc92536a1b783b7ef99450bd55a4b3af234aae35a18a339973ce30/coverage-7.10.7-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:f3c887f96407cea3916294046fc7dab611c2552beadbed4ea901cbc6a40cc7a0", size = 244430, upload-time = "2025-09-21T20:03:46.065Z" }, + { url = "https://files.pythonhosted.org/packages/75/0b/8287b2e5b38c8fe15d7e3398849bb58d382aedc0864ea0fa1820e8630491/coverage-7.10.7-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:635adb9a4507c9fd2ed65f39693fa31c9a3ee3a8e6dc64df033e8fdf52a7003f", size = 245311, upload-time = "2025-09-21T20:03:48.19Z" }, + { url = "https://files.pythonhosted.org/packages/0c/1d/29724999984740f0c86d03e6420b942439bf5bd7f54d4382cae386a9d1e9/coverage-7.10.7-cp39-cp39-win32.whl", hash = "sha256:5a02d5a850e2979b0a014c412573953995174743a3f7fa4ea5a6e9a3c5617431", size = 220500, upload-time = "2025-09-21T20:03:50.024Z" }, + { url = "https://files.pythonhosted.org/packages/43/11/4b1e6b129943f905ca54c339f343877b55b365ae2558806c1be4f7476ed5/coverage-7.10.7-cp39-cp39-win_amd64.whl", hash = "sha256:c134869d5ffe34547d14e174c866fd8fe2254918cc0a95e99052903bc1543e07", size = 221408, upload-time = "2025-09-21T20:03:51.803Z" }, + { url = "https://files.pythonhosted.org/packages/ec/16/114df1c291c22cac3b0c127a73e0af5c12ed7bbb6558d310429a0ae24023/coverage-7.10.7-py3-none-any.whl", hash = "sha256:f7941f6f2fe6dd6807a1208737b8a0cbcf1cc6d7b07d24998ad2d63590868260", size = 209952, upload-time = "2025-09-21T20:03:53.918Z" }, +] + +[package.optional-dependencies] +toml = [ + { name = "tomli", marker = "python_full_version < '3.10'" }, ] [[package]] -name = "click" -version = "8.3.1" +name = "coverage" +version = "7.14.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.15'", + "python_full_version >= '3.12' and python_full_version < '3.15'", "python_full_version == '3.11.*'", "python_full_version == '3.10.*'", ] -dependencies = [ - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +sdist = { url = "https://files.pythonhosted.org/packages/54/fd/0ab2772530e946e1be1abd0bc09e647ec9b02e88f0867857601fefca8953/coverage-7.14.1.tar.gz", hash = "sha256:30c08f7d90415aa98b3c990385dea2939b0da55f38515e5b369b83655f8523be", size = 920132, upload-time = "2026-05-26T20:41:36.783Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, + { url = "https://files.pythonhosted.org/packages/92/69/0d2ef01ff4b8fcecd4cba920d11e92fa4f96ae412441d3b56a90a258e69b/coverage-7.14.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3e3680291c4a1d0dadfa84a2c459576a4af5133abb617905714339a0c73138cf", size = 219722, upload-time = "2026-05-26T20:38:14.002Z" }, + { url = "https://files.pythonhosted.org/packages/f8/ae/9afdeaa31b9d9ce98124b6abf8bb49119bf71aecae04f8567c189d91299f/coverage-7.14.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a5274669f37f2343635a347b91a60777621341ab3378e9c6ac9335eee704bddf", size = 220240, upload-time = "2026-05-26T20:38:17.424Z" }, + { url = "https://files.pythonhosted.org/packages/51/69/c998589871df7ea7dba865cc5ee32b5a3e1d47ba6c68ef91104c7c46fa5e/coverage-7.14.1-cp310-cp310-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:cfe5a5fec635799ef33428f1e5e61bafa45a92a96190ba731561ba558ccc214d", size = 246981, upload-time = "2026-05-26T20:38:19.266Z" }, + { url = "https://files.pythonhosted.org/packages/fc/10/1c7d04c13040dac531d21b712bbe08f902e6dd9b58f5d77875c4d030f8f2/coverage-7.14.1-cp310-cp310-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:62a9f70b52e0b5a95cfef4a5c5641b06983cadc5e538a3feeb5c00211f523ac2", size = 248812, upload-time = "2026-05-26T20:38:20.75Z" }, + { url = "https://files.pythonhosted.org/packages/c1/65/2a38a4607ef27cadcfbcee034dba5830ae2569f90144a0f4c7dbf47d30b0/coverage-7.14.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c18ebc343e15be53049b3a2dce38fe82d58f37e20ab9094b3a39c0aa4f6bb47", size = 250675, upload-time = "2026-05-26T20:38:22.159Z" }, + { url = "https://files.pythonhosted.org/packages/c9/a2/a446ed9752a4a59b79e0fb6cbb319f6facb2183045c0725462625e66f87e/coverage-7.14.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b84ffdf877644e7096aa936991efeed873f7f3df57b9cd001312b7668ab08550", size = 252590, upload-time = "2026-05-26T20:38:23.63Z" }, + { url = "https://files.pythonhosted.org/packages/9e/fd/e81fbd7ba752365546e9842b1cbdaad3d6919d2a522c590aef16a281ec5e/coverage-7.14.1-cp310-cp310-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e854312c4103f2ad4c0dc023b69b77ebfd2c89db5f86c4c94dc2353f9a92167e", size = 247691, upload-time = "2026-05-26T20:38:25.057Z" }, + { url = "https://files.pythonhosted.org/packages/53/35/f3c26fdaae9ea937d154ca4d372e5ea0a4167ff70d36c6074ac2eacb2f83/coverage-7.14.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c643734307300234fafa36bf2a040a7235f8f177ea1fd6ec1423aea6fb7b929f", size = 248716, upload-time = "2026-05-26T20:38:26.406Z" }, + { url = "https://files.pythonhosted.org/packages/2e/14/940b6c49551fd343e8507ee2b0ba7af5d0aa04ed5bf768285cb7c72a9884/coverage-7.14.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:84ac9499e48700399a5dd0ea7085b5091961fec52c68d66b4ec0d3cf7f4441b1", size = 246721, upload-time = "2026-05-26T20:38:28.282Z" }, + { url = "https://files.pythonhosted.org/packages/aa/2c/40fc0634186c28292a662dff578866b3913983d6c375a3c2a74020938719/coverage-7.14.1-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:7f02d09f70776579b926d889a4c9c235070a1f47c40458aeaca563fae5acfdb5", size = 250533, upload-time = "2026-05-26T20:38:29.753Z" }, + { url = "https://files.pythonhosted.org/packages/de/e3/2c26bf1e811f9df991ff2a9bdddebdd13ee0665d564df7d05979f9146297/coverage-7.14.1-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:ce66d8e46da2bb5ee313a745cbd2e391d319176c1f7a9451bfcd3a2fb920859b", size = 246990, upload-time = "2026-05-26T20:38:31.516Z" }, + { url = "https://files.pythonhosted.org/packages/a8/b0/060260ef56bd92363ebdce0c7095ce422b06e69aae71828efeca473ab1ca/coverage-7.14.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:c912c259304cfb5ee584481cfb7ce1ff932b4d61e6c9140b8f19cb7b5ed82332", size = 247593, upload-time = "2026-05-26T20:38:33.065Z" }, + { url = "https://files.pythonhosted.org/packages/63/f3/501502046efeb0d6d94b5ca54941d95f1184183dd6bdb7f283985783bb4a/coverage-7.14.1-cp310-cp310-win32.whl", hash = "sha256:1238cb94638e610e972c60dac68e813f868dc7d6e982535270558443058d9d59", size = 222330, upload-time = "2026-05-26T20:38:35.36Z" }, + { url = "https://files.pythonhosted.org/packages/a0/5d/1bf99f2c558f128faf7906817ccbdb576ba815d3b41ce2ac1719b70a3663/coverage-7.14.1-cp310-cp310-win_amd64.whl", hash = "sha256:fc459e5d73be2d6332fcfe8dbf3d8994671fe33c700f4565988ecfa511547253", size = 223261, upload-time = "2026-05-26T20:38:37.196Z" }, + { url = "https://files.pythonhosted.org/packages/7d/d7/477ad149490e6cb849f28abea1dabb9c823cea72e7500c81b4240ce619c0/coverage-7.14.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:478b5bcd63c2e1357c5c7e16c070690df7b07f676b1c114d7b93e533c664309f", size = 219848, upload-time = "2026-05-26T20:38:38.715Z" }, + { url = "https://files.pythonhosted.org/packages/91/82/a5eb47257c50601bb7b9a9d2857c67b7a3a85ad74180eb2c98bb1fbe0ce5/coverage-7.14.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a24a81f9715ee42ef59a316cc11611c98fe23920f7c81861315c9f3ff4a230f4", size = 220354, upload-time = "2026-05-26T20:38:40.232Z" }, + { url = "https://files.pythonhosted.org/packages/43/8b/78419b5391a5cb706b6544390507e469d83ffc9a8248b02c4011aceb9365/coverage-7.14.1-cp311-cp311-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:196a13319ad88d6d8ef5ab489ec4f44ddde2143c0c7d5b27786f6c3ffd56a7e1", size = 250771, upload-time = "2026-05-26T20:38:41.782Z" }, + { url = "https://files.pythonhosted.org/packages/77/63/e77aaacd491182210d639636b7a8bba23ffffa9b82aa3762da9431855fa9/coverage-7.14.1-cp311-cp311-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3d452fd08b5c72c5167c93e6867b5c08500bd40f2a21e1e854a500550b6cc36f", size = 252683, upload-time = "2026-05-26T20:38:43.305Z" }, + { url = "https://files.pythonhosted.org/packages/65/1c/a022e3cfbec2ac241640003cb3a817e161d9c7f5aa9b49173756cdc03204/coverage-7.14.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:23bf7fa51ac02e07fc7c96849b82946da47ae862dc8f86d183b2a4864fc38129", size = 254791, upload-time = "2026-05-26T20:38:45.361Z" }, + { url = "https://files.pythonhosted.org/packages/61/d6/967e408aca4c1ceb88cb0cc677169110ae7f5995fb5eaf5fb1f5a1bb8f5d/coverage-7.14.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bcaa50684dcaadfa599ac48f81103c756d791cfd85c97203d2217c593d48b860", size = 256748, upload-time = "2026-05-26T20:38:46.91Z" }, + { url = "https://files.pythonhosted.org/packages/b8/be/869188f7fe28638078ec479331ace6dc5f7b40b7153eb616f47ab79404d8/coverage-7.14.1-cp311-cp311-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4ea1c034f95c9b056e856b794630b17f9fa3d57e4800ff1e503d3be0f9c9078c", size = 250907, upload-time = "2026-05-26T20:38:48.493Z" }, + { url = "https://files.pythonhosted.org/packages/07/aa/adb7d3b4278d690e68703abcd76ab1b948242e3668d921711551b78f9ddb/coverage-7.14.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c7e057326434e441306226fbeb5d1aaf14a2637efe97ba668306635835f32ad7", size = 252483, upload-time = "2026-05-26T20:38:50.074Z" }, + { url = "https://files.pythonhosted.org/packages/43/61/331c74103c62dcb0c4b9b3a0de9a61aca016208b0a90f109592a9f9ecc28/coverage-7.14.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:59baf88468dbc8d63b1887afd92bda52e40bb1561696e5819670601403810cec", size = 250545, upload-time = "2026-05-26T20:38:51.613Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b6/c5dae3c104d89be04828f61810e6b3473825482e4c288cc4ed04553e08ae/coverage-7.14.1-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:d34d75f892b3ab73ba11cab5442cce7b3e168fd64162b16f0e1e0d09c508edef", size = 254310, upload-time = "2026-05-26T20:38:53.503Z" }, + { url = "https://files.pythonhosted.org/packages/ad/a1/2b9d5863e3b83c01ad8199e3c597802fbb3a9dc90b058885804c20296d31/coverage-7.14.1-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3a56abc20a472baf0304c455721bc601477440d28ecfde8a03dde79ede07e0df", size = 250266, upload-time = "2026-05-26T20:38:55.414Z" }, + { url = "https://files.pythonhosted.org/packages/7f/5e/0e511fbdb269359be26fe678a1c3fa1f2aa2a01573cc3f54268c8d6d4797/coverage-7.14.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6a3cb83d1552c0cd1b4906655b6a33fd4a8473229633a901c6b73bf86914dee9", size = 251174, upload-time = "2026-05-26T20:38:57.141Z" }, + { url = "https://files.pythonhosted.org/packages/85/10/e55307b622b3dd9671cb321824502dc10f93e72f2802b9946159a8edadeb/coverage-7.14.1-cp311-cp311-win32.whl", hash = "sha256:10274a1fbeb8ec5d72966e17bb198a3104257aca4ac09d98667c5f8aca8c8548", size = 222354, upload-time = "2026-05-26T20:38:58.727Z" }, + { url = "https://files.pythonhosted.org/packages/71/cf/107421693cfb71e4f1ca5bf70443f64d4161878068d07a3e51c7ad21d17b/coverage-7.14.1-cp311-cp311-win_amd64.whl", hash = "sha256:87ebdf787d4888e3f3f2d523eadc6e18c6d18c6d0eb173801a189641627fb37e", size = 223290, upload-time = "2026-05-26T20:39:00.413Z" }, + { url = "https://files.pythonhosted.org/packages/b8/1d/3e3644585eb29e9dafefb19555078529a4d7cce12bd21929664eea989277/coverage-7.14.1-cp311-cp311-win_arm64.whl", hash = "sha256:dd34767fa19848d35659ffc0a75314f58c7af3f1cd87ec521e8292a1238398a3", size = 221953, upload-time = "2026-05-26T20:39:02.159Z" }, + { url = "https://files.pythonhosted.org/packages/3d/b7/bdbb725ba02c5b42825b200c940f38b7a54fcad24627b7192f78f8110d76/coverage-7.14.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a06c76364a9360e33d6d23769aefdf7f66f38e2ffb60ceb1baaa4989d83b695c", size = 220022, upload-time = "2026-05-26T20:39:03.702Z" }, + { url = "https://files.pythonhosted.org/packages/72/81/fdc0898a55c6219223291ec1a1fe89966ef212ce82276aa0899df84b5de0/coverage-7.14.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fad54e871165f6ec2f536063ac74c3104508a12963e64072ba44bd822de52b0c", size = 220379, upload-time = "2026-05-26T20:39:05.381Z" }, + { url = "https://files.pythonhosted.org/packages/de/72/de048c4a25e13bce59ac6a339351c10bdf2515e07459afcdaf04dc3143a2/coverage-7.14.1-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:84b535f00655ecafe1d929d1fb00ed5d6fa3051ea643ab2c161a3887b86f294b", size = 251888, upload-time = "2026-05-26T20:39:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/28/30/300c343f68beb9d4cbb64ec81e58c5b6b80b56927f72d2b38654ac26e013/coverage-7.14.1-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:6b6b0853b895fe0e98cbfc580d1ec3393d9302b4b1e96a77b3f5c91fdab899e6", size = 254624, upload-time = "2026-05-26T20:39:09.037Z" }, + { url = "https://files.pythonhosted.org/packages/b1/ed/7b25642496e8170b6bac14adce00537c6e5fa2d586159401a4de3e8b49e6/coverage-7.14.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:442cc9c952b2df400cda54bb04ab87330cf2cd08a8692cbbea36773531eb6f37", size = 255739, upload-time = "2026-05-26T20:39:10.889Z" }, + { url = "https://files.pythonhosted.org/packages/7f/a2/abd210b8c4e29c24e4624916db97bb519097a91034aaeb767f937e7da794/coverage-7.14.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:8270544c361ed405a27a060dbc9ed2c124b084d96dfdc2d9a2510482aef981ad", size = 257998, upload-time = "2026-05-26T20:39:12.722Z" }, + { url = "https://files.pythonhosted.org/packages/7f/24/7c50beed3792fe62f6ce0545c6686ce83379719e2c0276179333d97eae92/coverage-7.14.1-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:48b283b1dd6372e8de2a7a9a4c4d5dc06f4d4fd209b876f3c88a7a205a0c8f84", size = 252296, upload-time = "2026-05-26T20:39:14.259Z" }, + { url = "https://files.pythonhosted.org/packages/15/05/0f874628ebcbfc77ead559ff210281ef06a97db08481832e7dd39274a135/coverage-7.14.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:5b0c99ba93a07d56f6df340bb79be53202a082b2fdb81bfe6190b741a3470d54", size = 253658, upload-time = "2026-05-26T20:39:15.923Z" }, + { url = "https://files.pythonhosted.org/packages/99/6f/ca6ad067364b337ef997802115e7ecad2abd2248b05471464b0dea02b4d4/coverage-7.14.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e471bc5769ff073b058cfadb0d736b56ce067c8560eabeb0da88462df98c23e7", size = 251803, upload-time = "2026-05-26T20:39:17.537Z" }, + { url = "https://files.pythonhosted.org/packages/c0/30/b9b4d377cd9f40baf228068f5a81faf8450c6228503011bd499708483a50/coverage-7.14.1-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:f497a1ea81d4cd7c10ddcaa685135b9aabd291af3d55775a9ddf3cb7a364cdd9", size = 255873, upload-time = "2026-05-26T20:39:19.414Z" }, + { url = "https://files.pythonhosted.org/packages/3c/21/7c721a9e5e6bb88547d30a787aefb97512d3f54c1324c7488d9b3743f7f9/coverage-7.14.1-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:2222be86d0b54f5dd5a38f45f17f315f737245e857bf0bdedc70734f84a13c02", size = 251372, upload-time = "2026-05-26T20:39:21.169Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f8ae5a2200130e1503cd7661a6cd3b2b7bacef98277fbf3571fb13f8b766/coverage-7.14.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:85e85586565842f6932abebd4c18bcb1074223dc0b3576e7d173ca710622813a", size = 253245, upload-time = "2026-05-26T20:39:23.097Z" }, + { url = "https://files.pythonhosted.org/packages/34/62/70a9024672a5f6910517d9628c52c9afbdd3cf8f46426af52bb148a56fff/coverage-7.14.1-cp312-cp312-win32.whl", hash = "sha256:4a28fd227808366b196a75476dced2eb35b351d6766ba9c858dc93319e87f4f1", size = 222567, upload-time = "2026-05-26T20:39:24.868Z" }, + { url = "https://files.pythonhosted.org/packages/f6/81/8b7cd386839b039ebe1855733b9f9449a8dec5d79564018234f185a7fa70/coverage-7.14.1-cp312-cp312-win_amd64.whl", hash = "sha256:54acdb6674a4661768d7bf7db32dfb9f46ab1d764f8aba6df75ce1a6a088724e", size = 223372, upload-time = "2026-05-26T20:39:26.603Z" }, + { url = "https://files.pythonhosted.org/packages/ae/ba/b44d472022f620d289d95fa830143235c0c36461c6f2437ea8d51e5481ed/coverage-7.14.1-cp312-cp312-win_arm64.whl", hash = "sha256:99cd41ff91afd94896fea3bc002706b6ae4ce95727d06e4a0f39c0a8d8bd8b1a", size = 221989, upload-time = "2026-05-26T20:39:28.242Z" }, + { url = "https://files.pythonhosted.org/packages/8a/9e/5f6d56327c62b185225d145191c607e07515294a0aa6338e58805cd4a5ac/coverage-7.14.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:be9f2c802dcfce3f71298303aa5dad0dce440a76c52f2f60dacd8656dab78793", size = 220044, upload-time = "2026-05-26T20:39:29.902Z" }, + { url = "https://files.pythonhosted.org/packages/75/92/e82aca356744cbbc0f77a0b623e38918c1872361963413a3bab5d0340393/coverage-7.14.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:6223a72fd0e4c7156353ec0f08a5f93623e1d3034d0e2683b9bb8ea674131b1d", size = 220412, upload-time = "2026-05-26T20:39:31.561Z" }, + { url = "https://files.pythonhosted.org/packages/27/c9/385bde0bf7ed0f4bf3a7ee5367060a86b5d218718cfd6fb943c0f836b34f/coverage-7.14.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7279d2110a28cebc738b6459ecda2771735a4c18465fbbd36b3288fe5ed92247", size = 251412, upload-time = "2026-05-26T20:39:33.337Z" }, + { url = "https://files.pythonhosted.org/packages/51/8c/23faf6a2343a0d17f960a4bd56c43bc7eb4cf312f774dd6ceebd82c7d8fc/coverage-7.14.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9eeb3fcbc13ba40dfbdb22d01d196a28e9cef9ed4c29b60061a1e0e823a9929d", size = 254008, upload-time = "2026-05-26T20:39:35.009Z" }, + { url = "https://files.pythonhosted.org/packages/42/06/36f4aa9ca8a815e6036156e80706a67828bb97bd826948244f6996dda957/coverage-7.14.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5f0cfc27c539f07cf5c0a4cfe211d0b6cae039f8f40526dbaa71944e64b50a7b", size = 255241, upload-time = "2026-05-26T20:39:36.71Z" }, + { url = "https://files.pythonhosted.org/packages/ca/79/95266316352f90f6b1c6736bb413302edfde2453fb32422d3911642691b3/coverage-7.14.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:221c70f316241a78e77e607c227cefc8808d4e08f28d99c04f35694690e940be", size = 257373, upload-time = "2026-05-26T20:39:38.412Z" }, + { url = "https://files.pythonhosted.org/packages/e3/9c/58316d1f66c488b5fca8a0eb3e98348807813efa8a0d0833b9021be27488/coverage-7.14.1-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:da028256b04ec30e5e0114b6f76172938c313991f0a2d3d894271315cf5d5e43", size = 251635, upload-time = "2026-05-26T20:39:40.268Z" }, + { url = "https://files.pythonhosted.org/packages/ef/5a/ca2398a568e16fed7bb713e84ba3603a7164fb65779abe645c565ec890d5/coverage-7.14.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:76a085d7005236a767e3426148b2c407e53ad61695c562f8a81da2d373324901", size = 253373, upload-time = "2026-05-26T20:39:42.145Z" }, + { url = "https://files.pythonhosted.org/packages/6e/2c/0396562c32deaebe7be51d865b3a41e9a87d7561acafe1a28f53b07e019a/coverage-7.14.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:b553d04b5e778a8e56d57eb134aff42a92718ecba45e79c4764ecfa40efd92ff", size = 251341, upload-time = "2026-05-26T20:39:43.907Z" }, + { url = "https://files.pythonhosted.org/packages/fd/8f/a94f9221184c9cae1ee115820e3798e48b6b17777a9f19e46fb9a0c8dc74/coverage-7.14.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:46f714d2fb8ae2f4f29f23ada7f1e79b759fff5a70f94a1dac23af204c3ec9e4", size = 255497, upload-time = "2026-05-26T20:39:46.166Z" }, + { url = "https://files.pythonhosted.org/packages/71/69/505d70e47db1eaebcd002c39759707621ef184cd6b1ae084d9f41293f323/coverage-7.14.1-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:1896f5e19ff3f0431c7ce2172adc54890fd97f86b59ced8ca1649145d9ffe35d", size = 251159, upload-time = "2026-05-26T20:39:48.03Z" }, + { url = "https://files.pythonhosted.org/packages/e0/aa/58681c383aa33a9d2ed40a02d7a22fbf780d1fa4d575396365777828198c/coverage-7.14.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:62fd185ef9df3c33d1c8178c5af105f762afbad96038de9a4ae100aa6297ca33", size = 252934, upload-time = "2026-05-26T20:39:49.872Z" }, + { url = "https://files.pythonhosted.org/packages/eb/fd/11c928cd6bdffc7074bb5965c173d9ebf517fb00205e1da524b98d29ef92/coverage-7.14.1-cp313-cp313-win32.whl", hash = "sha256:ab4af6352741a604c431c6072fce5bee33bf0f20dc7a56618d6bf6bb89e9810c", size = 222584, upload-time = "2026-05-26T20:39:51.68Z" }, + { url = "https://files.pythonhosted.org/packages/6f/92/fb416fc26d340dcba19518c418d6048e913186e17243982c5e435e41fa7a/coverage-7.14.1-cp313-cp313-win_amd64.whl", hash = "sha256:7af486dabe8954d03b087f0021540897afe084f04e16ff5579e08cc46f871416", size = 223394, upload-time = "2026-05-26T20:39:53.472Z" }, + { url = "https://files.pythonhosted.org/packages/73/c6/02d56e3867972f77d5036de924643f26c056e848f00452cafb4dbc3c29b4/coverage-7.14.1-cp313-cp313-win_arm64.whl", hash = "sha256:2224f89ffd0c5605ccce1ed7a584da162bc7c55f601ab1c946bc9de31a486b42", size = 222015, upload-time = "2026-05-26T20:39:55.374Z" }, + { url = "https://files.pythonhosted.org/packages/4d/9e/fcc77914050df73f7662fa1f00902774c79c075a8388ab334074574bf77e/coverage-7.14.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:de286598cc65d2b489411174b1faec2f5a7775fb3201fd925db2a76b4030f37d", size = 220733, upload-time = "2026-05-26T20:39:57.189Z" }, + { url = "https://files.pythonhosted.org/packages/f7/67/2963cbdaf5cbadec44efa3a1e39eaa1f02df4079585f05387607a221e126/coverage-7.14.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:042c46ded7c288aeb07cf14a28b6c1e10b78fcba40171c3fa1e939377eeef0b5", size = 221086, upload-time = "2026-05-26T20:39:59.019Z" }, + { url = "https://files.pythonhosted.org/packages/c8/c5/8701645574e11881f2f47d8930f98bc48b5d43b25eb5b4430dfc4a2f9f48/coverage-7.14.1-cp313-cp313t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:f4ddbe407477f04c45115d1a4e5bc480f753553b534d338d4c3358b1cdd0ea52", size = 262381, upload-time = "2026-05-26T20:40:00.822Z" }, + { url = "https://files.pythonhosted.org/packages/7c/28/7a64d73598263e0c5abd5084211a8474488d31b3c552ff531c719dfcff62/coverage-7.14.1-cp313-cp313t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:d13e6725992e2d2fd7d81d4f5241952d13740121dfd501da09201be39b2c003a", size = 264458, upload-time = "2026-05-26T20:40:02.506Z" }, + { url = "https://files.pythonhosted.org/packages/fa/d8/4969179db9f7eb4df218e69540adf829d1c835f59452513d065d15446802/coverage-7.14.1-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f747dc8edcfe740130f28f32f3995e955494285717e86ee25af51db2219df08a", size = 266884, upload-time = "2026-05-26T20:40:04.421Z" }, + { url = "https://files.pythonhosted.org/packages/a6/78/a45d5794dbc9bafd97afc96a4377c86c7820d78b6cf51b89bc1d4e919275/coverage-7.14.1-cp313-cp313t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ced2f09ef276fd58611a1ef502164ad266d2b75174e5a40cabbdb4033f9f6cf2", size = 268022, upload-time = "2026-05-26T20:40:06.298Z" }, + { url = "https://files.pythonhosted.org/packages/21/cb/4f5e354e9e3e67af96bd4e57113e6db6b22298c7168b13eec408a549903d/coverage-7.14.1-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b84800013769a78ccb9ef4659402e26d06867e337b61ec365f77ad008adea80e", size = 261631, upload-time = "2026-05-26T20:40:08.226Z" }, + { url = "https://files.pythonhosted.org/packages/ec/49/eced49af4cb996d5d8b7e94e736175c513e4facd3398507b89892b4326d8/coverage-7.14.1-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:ea8cd6ca0ee9f616aaef3afc6882e32c2cbf18b00d96313ffd76af650574034d", size = 264443, upload-time = "2026-05-26T20:40:10.137Z" }, + { url = "https://files.pythonhosted.org/packages/f1/d8/5603a88a7c5913a6b54f6cb1a8c46f7b39cbb30f27cd3f492908da09b2d7/coverage-7.14.1-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:aa5e304a873fabddc11e484e9b6b738bd38bd7bed17b09aa84eecf5332e8b8bb", size = 262069, upload-time = "2026-05-26T20:40:11.999Z" }, + { url = "https://files.pythonhosted.org/packages/f0/59/2ae3cb79da554a06c8619d6c88ea19dd1e4aed4b834b6a83bb1fa243bdc5/coverage-7.14.1-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:5a1c5215be81035e629d5bc756650634d0bf31991038db7a0eccb90f025ce16d", size = 265780, upload-time = "2026-05-26T20:40:13.858Z" }, + { url = "https://files.pythonhosted.org/packages/af/5f/b130c1dc999031f2648bd25317fbce505ad8d5562079b4ed81e736a84967/coverage-7.14.1-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:79058c47dae6788504b5effb319961bcd72d7240551464b91d474bc0ed186d69", size = 260970, upload-time = "2026-05-26T20:40:16.142Z" }, + { url = "https://files.pythonhosted.org/packages/87/d1/ec13ccddeb48ec963bdfa72a11224bac2584bd045ba13beca82f8113e9c7/coverage-7.14.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:370c5afae3fa0658e11694a32b24c2778f6bc2d17718121f94ee185e69f26b54", size = 263157, upload-time = "2026-05-26T20:40:18.382Z" }, + { url = "https://files.pythonhosted.org/packages/cf/c2/cd91ead503045161092d3845f7bb95ea2f25131ce96d3e314dd835d91b9c/coverage-7.14.1-cp313-cp313t-win32.whl", hash = "sha256:3758dd0a7f1fa57365ef2e781df0f0731d38b6e3772259d13dae4bd8a958d4b1", size = 223259, upload-time = "2026-05-26T20:40:20.381Z" }, + { url = "https://files.pythonhosted.org/packages/71/9f/1e28d97e6bd2c76b07f38b7c02870f1371255ff6717f54eca578fcbbdd0e/coverage-7.14.1-cp313-cp313t-win_amd64.whl", hash = "sha256:6ff665fb023a77386fe11685190cee1f60a7d635994a30d9b0a061533d470fce", size = 224320, upload-time = "2026-05-26T20:40:22.316Z" }, + { url = "https://files.pythonhosted.org/packages/a9/e0/d936e908f0e1efa55e52b91e01b52f1055cef5e1ab2718493390ed8e2fb8/coverage-7.14.1-cp313-cp313t-win_arm64.whl", hash = "sha256:17a5a241e5997621a956a7f402a7433ef4221e5152809b785bec79e2323799f1", size = 222577, upload-time = "2026-05-26T20:40:24.894Z" }, + { url = "https://files.pythonhosted.org/packages/d6/34/fc2f101b151af3799a101f0550b0454aa008afdc0add677394ec4aa8ea10/coverage-7.14.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d5ed429d0b8edaac649e889b4ffcedb6c80b06629a3f93050e3dddfb99235bee", size = 220091, upload-time = "2026-05-26T20:40:27.249Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a7/1ebae2ab5b961b5c79bb09fe7b3ac99edb190d8be4a8c510b2cf66f46468/coverage-7.14.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:8011224a62280e50dab346960c03cf47aca1a1e09e608c0fb33fd6e0cc8e9500", size = 220421, upload-time = "2026-05-26T20:40:30.084Z" }, + { url = "https://files.pythonhosted.org/packages/5e/90/92aca9cf0acc95123c96cd1eb1f08917897a7f5dee01e15738922971ec31/coverage-7.14.1-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:12c42ec1e14f553c4f817e989365982e646e27211f10a0f717855b94a79c8906", size = 251466, upload-time = "2026-05-26T20:40:32.542Z" }, + { url = "https://files.pythonhosted.org/packages/26/2b/78048cbe3b999f6cbf9cc0d90abba6a88a3e0863a8c1c6cbc762f3f8802f/coverage-7.14.1-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:06144cd511cf2624873a035c5069cf297144f6e77a73ee3d7a55b605ec5efb42", size = 253973, upload-time = "2026-05-26T20:40:34.473Z" }, + { url = "https://files.pythonhosted.org/packages/8e/21/c2e33b29d1cfde484a19d437afc343c6cd30b08d78cbbf9f5aff14e57b2b/coverage-7.14.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a311d8e1da24be5c1ccf85cbfb06315dbaa1703d5a1eab3f6432c72b837917c8", size = 255318, upload-time = "2026-05-26T20:40:38.154Z" }, + { url = "https://files.pythonhosted.org/packages/8e/ee/aad2f108d63b769121005302f16bf66db8625c88ceaba466942e09a2607e/coverage-7.14.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c79cead5b5bc584d9c71451cb984d0e3a84e0c0937379c8efcbf27c8d661b851", size = 257633, upload-time = "2026-05-26T20:40:40.164Z" }, + { url = "https://files.pythonhosted.org/packages/c2/f8/11a2c29b4fd76d9849f81d0bb812ec0017a9396df3217214e38934a8c837/coverage-7.14.1-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:dcbf65f1f66a26cdd88c35cf68fb4729c5d1cd2e88added72420541dfb212034", size = 251488, upload-time = "2026-05-26T20:40:42.631Z" }, + { url = "https://files.pythonhosted.org/packages/c9/b8/9a5820de4b8ac2b71d85e3b5fb49108d7469c665f0e2ad0dd7569023e305/coverage-7.14.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fd86572566fb40189a8260446158235159bc7a82dfbc87a3b39cf4fb57fcec1c", size = 253329, upload-time = "2026-05-26T20:40:45.208Z" }, + { url = "https://files.pythonhosted.org/packages/6b/ff/f33e4823667e27548e8fd8df44217515303f9808d0ff29817db56f87d990/coverage-7.14.1-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:7771b601718fdde84832c3a434ca9bbf4ae9adbc49d84198b4110700c3c77c36", size = 251291, upload-time = "2026-05-26T20:40:47.502Z" }, + { url = "https://files.pythonhosted.org/packages/68/9b/489db0ebb209054766b90a9014a45f6d26eb724c02ec21311c3733b5a644/coverage-7.14.1-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:39b21e212c55af06fa375e3dbf90a8a8e38792f3a910c580066d23563830ddd5", size = 255564, upload-time = "2026-05-26T20:40:49.372Z" }, + { url = "https://files.pythonhosted.org/packages/27/b5/16bc2d4c2409b23c7737edb68c83bc89e345f378050549fe1d75ac7d34d5/coverage-7.14.1-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f2302660e32562a532b442480121aef8aa61a5bdb20b30bf0adab29f10a5a4b4", size = 251107, upload-time = "2026-05-26T20:40:51.677Z" }, + { url = "https://files.pythonhosted.org/packages/7d/0c/2629997469a00cd069d588a41c9dc887610f2775ae89d250c4791e65272a/coverage-7.14.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:03a6f93c1ec3b7f2e77b5dbcc5573a2c21f12529a5c6bbe0f16f72303cc2fa4d", size = 252764, upload-time = "2026-05-26T20:40:54.267Z" }, + { url = "https://files.pythonhosted.org/packages/d2/ee/f78d63c8f079e0d7211c7e2401fa17e311514534ba61bae03e4b287ce4ab/coverage-7.14.1-cp314-cp314-win32.whl", hash = "sha256:8a3ce026d73290f42f08dafecbd82c193a74df280461fbf97300fec51fd133ee", size = 222837, upload-time = "2026-05-26T20:40:56.496Z" }, + { url = "https://files.pythonhosted.org/packages/dc/b9/be539854f93a70dfbeec69117f33ec70dc42ff0b65b5b07ab8d40d04228e/coverage-7.14.1-cp314-cp314-win_amd64.whl", hash = "sha256:114c95ef29302423b87d159075805f4ab973254a2638a5d7d046c94887cc87d7", size = 223650, upload-time = "2026-05-26T20:40:58.351Z" }, + { url = "https://files.pythonhosted.org/packages/fe/9e/24e2842fef40f35ac82ba3a7719c8023d011bf3bf652d0675316a9d088a1/coverage-7.14.1-cp314-cp314-win_arm64.whl", hash = "sha256:a07891c3f4805442b31b71e84ba3cf29ed1aa9a428284e06deeb4b23e5b46343", size = 222218, upload-time = "2026-05-26T20:41:00.321Z" }, + { url = "https://files.pythonhosted.org/packages/0a/1d/ac0a9df5fe31c1e8bdd658074905fc12844a05c1a7e3fdb8417e97c31e23/coverage-7.14.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:1101a5ebb083aecb625ebb6209d4105b58f647b093cb2dc8122d7b33f743cfe1", size = 220822, upload-time = "2026-05-26T20:41:02.281Z" }, + { url = "https://files.pythonhosted.org/packages/32/cf/f964fd9aff20323f9f1a726c97135f8a76bcd87b92dad141a456a43f3c64/coverage-7.14.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:851b9e1e4e8a4608e77c79714b2e77c0970d2ed7202a05e92ae407817481887b", size = 221084, upload-time = "2026-05-26T20:41:04.593Z" }, + { url = "https://files.pythonhosted.org/packages/d8/5e/7e5ef2aba844de2b80d678619fcf0841b42e3f37f16411226f3fe4c1016f/coverage-7.14.1-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:d5b89cdfb2ee051b71e8c3c70bd81a9eff81100f736a269136fe1a68efe00474", size = 262454, upload-time = "2026-05-26T20:41:06.641Z" }, + { url = "https://files.pythonhosted.org/packages/64/62/75809bded87015cc4935524218a2a8ed8dd1a8498bfed30a2f4f7a4b4d34/coverage-7.14.1-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:0177614a0370f227888b4e436a7c55686d6a9f90eb1ade2b624ba685a1686e86", size = 264578, upload-time = "2026-05-26T20:41:08.556Z" }, + { url = "https://files.pythonhosted.org/packages/f3/42/d33392dc14633525012d2d504fa1a33b05538bf535f5c1d64675e5754b78/coverage-7.14.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2d69af5dea2de76fc485a83032a630523f985198b7e25be901ec60181587b01e", size = 266981, upload-time = "2026-05-26T20:41:10.824Z" }, + { url = "https://files.pythonhosted.org/packages/2a/49/0157c4428c2aca7f1e09d5565930586fd5ae36f1655f08b0daa7cf1fcae1/coverage-7.14.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:35ab22d91de736e8966b980dc355cbcdd2c6dbbcfe275f9a2991bc8a91b3df65", size = 268112, upload-time = "2026-05-26T20:41:12.966Z" }, + { url = "https://files.pythonhosted.org/packages/96/26/86b9ce71f4092b1ed325ce1421698081df1286b833400b6836912834d6e0/coverage-7.14.1-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:357d4e32935c36588aaba057d734fa32428c360c9fc2e4442afbf1b646beee6e", size = 261558, upload-time = "2026-05-26T20:41:15Z" }, + { url = "https://files.pythonhosted.org/packages/20/4c/c311210c5472cf5401d8422b0d7812cdd520f24417673afabda6c323faca/coverage-7.14.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:51bd64741cc6fa065abd300ede1afe5a5291ece9c31da8b24884deda48bcc3f8", size = 264447, upload-time = "2026-05-26T20:41:17.369Z" }, + { url = "https://files.pythonhosted.org/packages/fb/71/59513f8710ed3e6b0ac0a050a5b7e977bb9c9e880354863b5d00d8809256/coverage-7.14.1-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:9132cd363a68a4c3daa7c8704a654b1e39d3360f6f5b8ddd470608a945236c07", size = 262048, upload-time = "2026-05-26T20:41:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/84/8d/bceed32dc494f5bbf50f775cd2e78ca814953942b5ea28d3c1c3ac316f14/coverage-7.14.1-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:07c6290b1697b862c0478eab545eec949a0d0e4d6d03497f446d706da3b4f2de", size = 265781, upload-time = "2026-05-26T20:41:21.559Z" }, + { url = "https://files.pythonhosted.org/packages/e7/c5/9348fe40dbfd4991aaf78df2c6c3098bfb2cc834d1fd362a64b4efef855a/coverage-7.14.1-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:5ea0c297e27133853b4d8a3eb799bff5a2dbd9f2f41537a240d337ac9b4df890", size = 260896, upload-time = "2026-05-26T20:41:23.428Z" }, + { url = "https://files.pythonhosted.org/packages/ca/92/1ea0f03929da7cf87206b1fa24f4c8e9c158be0455481af29ec0a1f3503f/coverage-7.14.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:01b7733daad0237daa01ef80fe2dfceffc911e6a17fa7b55d14aa8214eaaaecd", size = 263214, upload-time = "2026-05-26T20:41:25.419Z" }, + { url = "https://files.pythonhosted.org/packages/f6/a9/b2493c054c0e01a643266742ab45e15744e60743f9260cd930c7142b1124/coverage-7.14.1-cp314-cp314t-win32.whl", hash = "sha256:6adc5a36984624a70bf11d7184e20fa0a49aa7c47ffab43804106a1a695ea22e", size = 223624, upload-time = "2026-05-26T20:41:27.795Z" }, + { url = "https://files.pythonhosted.org/packages/fc/bd/3e1e6a57fccd2d7c83fcdf338e93ba98eb85c6e877dd34731ac585375490/coverage-7.14.1-cp314-cp314t-win_amd64.whl", hash = "sha256:ddf799247318f34dbcd2efa8c95a8d0642674e926bb1774cf9b63dfd2a389d1c", size = 224728, upload-time = "2026-05-26T20:41:30.098Z" }, + { url = "https://files.pythonhosted.org/packages/bb/d7/31066cf1d2f0c6c797fce911bcfa01dd35642dc6da992a950256097c5860/coverage-7.14.1-cp314-cp314t-win_arm64.whl", hash = "sha256:145986fe66647eb489f18d9a997567a3fd358584c4b5a808769113abc07466af", size = 222752, upload-time = "2026-05-26T20:41:32.123Z" }, + { url = "https://files.pythonhosted.org/packages/8a/3c/1a983b9a745d7f83d53f057bcc5bf79ba6a2bbc08266b3f0c7d6fe630c9b/coverage-7.14.1-py3-none-any.whl", hash = "sha256:a252f21c27e38347e60111a3266b03827422a7d5525951aceee313aa68bab1d2", size = 211815, upload-time = "2026-05-26T20:41:34.078Z" }, ] -[[package]] -name = "colorama" -version = "0.4.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +[package.optional-dependencies] +toml = [ + { name = "tomli", marker = "python_full_version >= '3.10' and python_full_version <= '3.11'" }, ] [[package]] -name = "dill" +name = "distlib" version = "0.4.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/12/80/630b4b88364e9a8c8c5797f4602d0f76ef820909ee32f0bacb9f90654042/dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0", size = 186976, upload-time = "2025-04-16T00:41:48.867Z" } +sdist = { url = "https://files.pythonhosted.org/packages/96/8e/709914eb2b5749865801041647dc7f4e6d00b549cfe88b65ca192995f07c/distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d", size = 614605, upload-time = "2025-07-17T16:52:00.465Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668, upload-time = "2025-04-16T00:41:47.671Z" }, + { url = "https://files.pythonhosted.org/packages/33/6b/e0547afaf41bf2c42e52430072fa5658766e3d65bd4b03a563d1b6336f57/distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16", size = 469047, upload-time = "2025-07-17T16:51:58.613Z" }, ] [[package]] @@ -156,51 +358,92 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, ] +[[package]] +name = "filelock" +version = "3.19.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +sdist = { url = "https://files.pythonhosted.org/packages/40/bb/0ab3e58d22305b6f5440629d20683af28959bf793d98d11950e305c1c326/filelock-3.19.1.tar.gz", hash = "sha256:66eda1888b0171c998b35be2bcc0f6d75c388a7ce20c3f3f37aa8e96c2dddf58", size = 17687, upload-time = "2025-08-14T16:56:03.016Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" }, +] + +[[package]] +name = "filelock" +version = "3.29.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15'", + "python_full_version >= '3.12' and python_full_version < '3.15'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/b5/fe/997687a931ab51049acce6fa1f23e8f01216374ea81374ddee763c493db5/filelock-3.29.0.tar.gz", hash = "sha256:69974355e960702e789734cb4871f884ea6fe50bd8404051a3530bc07809cf90", size = 57571, upload-time = "2026-04-19T15:39:10.068Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/47/dd9a212ef6e343a6857485ffe25bba537304f1913bdbed446a23f7f592e1/filelock-3.29.0-py3-none-any.whl", hash = "sha256:96f5f6344709aa1572bbf631c640e4ebeeb519e08da902c39a001882f30ac258", size = 39812, upload-time = "2026-04-19T15:39:08.752Z" }, +] + [[package]] name = "hashstore" version = "1.1.1" source = { editable = "." } dependencies = [ - { name = "pathlib" }, { name = "pyyaml" }, ] [package.dev-dependencies] dev = [ - { name = "black" }, { name = "exceptiongroup" }, + { name = "mypy", version = "1.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "mypy", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pg8000" }, - { name = "pylint", version = "3.3.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "pylint", version = "4.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pre-commit", version = "4.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "pre-commit", version = "4.6.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "pytest", version = "9.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pytest-cov" }, ] [package.metadata] -requires-dist = [ - { name = "pathlib", specifier = ">=1.0.1" }, - { name = "pyyaml", specifier = ">=6.0" }, -] +requires-dist = [{ name = "pyyaml", specifier = ">=6.0" }] [package.metadata.requires-dev] dev = [ - { name = "black", specifier = ">=22.10.0" }, { name = "exceptiongroup", specifier = ">=1.1.0" }, + { name = "mypy", specifier = ">=1.19.1" }, { name = "pg8000", specifier = ">=1.29.8" }, - { name = "pylint", specifier = ">=2.17.4" }, + { name = "pre-commit" }, { name = "pytest", specifier = ">=7.2.0" }, + { name = "pytest-cov", specifier = ">=7.1.0" }, ] [[package]] -name = "importlib-metadata" -version = "8.7.0" +name = "identify" +version = "2.6.15" source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "zipp", marker = "python_full_version < '3.10'" }, +resolution-markers = [ + "python_full_version < '3.10'", ] -sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/e7/685de97986c916a6d93b3876139e00eef26ad5bbbd61925d670ae8013449/identify-2.6.15.tar.gz", hash = "sha256:e4f4864b96c6557ef2a1e1c951771838f4edc9df3a72ec7118b338801b11c7bf", size = 99311, upload-time = "2025-10-02T17:43:40.631Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/b0/36bd937216ec521246249be3bf9855081de4c5e06a0c9b4219dbeda50373/importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd", size = 27656, upload-time = "2025-04-27T15:29:00.214Z" }, + { url = "https://files.pythonhosted.org/packages/0f/1c/e5fd8f973d4f375adb21565739498e2e9a1e54c858a97b9a8ccfdc81da9b/identify-2.6.15-py2.py3-none-any.whl", hash = "sha256:1181ef7608e00704db228516541eb83a88a9f94433a8c80bb9b5bd54b1d81757", size = 99183, upload-time = "2025-10-02T17:43:39.137Z" }, +] + +[[package]] +name = "identify" +version = "2.6.19" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15'", + "python_full_version >= '3.12' and python_full_version < '3.15'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/52/63/51723b5f116cc04b061cb6f5a561790abf249d25931d515cd375e063e0f4/identify-2.6.19.tar.gz", hash = "sha256:6be5020c38fcb07da56c53733538a3081ea5aa70d36a156f83044bfbf9173842", size = 99567, upload-time = "2026-04-17T18:39:50.265Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/84/d9273cd09688070a6523c4aee4663a8538721b2b755c4962aafae0011e72/identify-2.6.19-py2.py3-none-any.whl", hash = "sha256:20e6a87f786f768c092a721ad107fc9df0eb89347be9396cadf3f4abbd1fb78a", size = 99397, upload-time = "2026-04-17T18:39:49.221Z" }, ] [[package]] @@ -220,7 +463,8 @@ name = "iniconfig" version = "2.3.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.15'", + "python_full_version >= '3.12' and python_full_version < '3.15'", "python_full_version == '3.11.*'", "python_full_version == '3.10.*'", ] @@ -230,41 +474,220 @@ wheels = [ ] [[package]] -name = "isort" -version = "6.1.0" +name = "librt" +version = "0.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/40/08/9e7f6b5d2b5bed6ad055cdd5925f192bb403a51280f86b56554d9d0699a2/librt-0.11.0.tar.gz", hash = "sha256:075dc3ef4458a278e0195cbf6ac9d38808d9b906c5a6c7f7f79c3888276a3fb1", size = 200139, upload-time = "2026-05-10T18:17:25.138Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/83/10/37fd9e9ba96cb0bd742dfb20fc3d082e54bdbec759d7300df927f360ef07/librt-0.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6e94ebfcfa2d5e9926d6c3b9aa4617ffc42a845b4321fb84021b872358c82a0f", size = 141706, upload-time = "2026-05-10T18:15:16.129Z" }, + { url = "https://files.pythonhosted.org/packages/cf/72/1b1466f358e4a0b728051f69bc27e67b432c6eaa2e05b88db49d3785ae0d/librt-0.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ae627397a2f351560440d872d6f7c8dbb4072e57868e7b2fc5b8b430fe489d45", size = 142605, upload-time = "2026-05-10T18:15:18.148Z" }, + { url = "https://files.pythonhosted.org/packages/ca/85/ed26dd2f6bc9a0baf48306433e579e8d354d70b2bcb78134ed950a5d0e1e/librt-0.11.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:dc329359321b67d24efdf4bc69012b0597001649544db662c001db5a0184794c", size = 476555, upload-time = "2026-05-10T18:15:19.569Z" }, + { url = "https://files.pythonhosted.org/packages/66/fe/11891191c0e0a3fd617724e891f6e67a71a7658974a892b9a9a97fdb2977/librt-0.11.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:7e82e642ab0f7608ce2fe53d76ca2280a9ee33a1b06556142c7c6fe80a86fc33", size = 468434, upload-time = "2026-05-10T18:15:20.87Z" }, + { url = "https://files.pythonhosted.org/packages/6f/50/5ec949d7f9ce1a07af903aa3e13abb98b717923bdead6e719b2f824ccc07/librt-0.11.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:88145c15c67731d54283d135b03244028c750cc9edc334a96a4f5950ebdb2884", size = 496918, upload-time = "2026-05-10T18:15:22.616Z" }, + { url = "https://files.pythonhosted.org/packages/ea/c4/177336c7524e34875a38bf668e88b193a6723a4eb4045d07f74df6e1506c/librt-0.11.0-cp310-cp310-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9d36a51b3d93320b686588e27123f4995804dbf1bce81df78c02fc3c6eea9280", size = 490334, upload-time = "2026-05-10T18:15:24.2Z" }, + { url = "https://files.pythonhosted.org/packages/13/1f/da3112f7569eda3b49f9a2629bae1fe059812b6085df16c885f6454dff49/librt-0.11.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d00f3ac06a2a8b246327f11e186a53a100a4d5c7ed52346367e5ec751d51586c", size = 511287, upload-time = "2026-05-10T18:15:26.226Z" }, + { url = "https://files.pythonhosted.org/packages/fa/94/03fec301522e172d105581431223be56b27594ff46440ebfbb658a3735d5/librt-0.11.0-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:461bbceede621f1ffb8839755f8663e886087ee7af16294cab7fb4d782c62eeb", size = 517202, upload-time = "2026-05-10T18:15:27.965Z" }, + { url = "https://files.pythonhosted.org/packages/b7/6e/339f6e5a7b413ce014f1917a756dae630fe59cc99f34153205b1cb540901/librt-0.11.0-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:0cad8a4d6a8ff03c9b76f9414caccd78e7cfbc8a2e12fa334d8e1d9932753783", size = 497517, upload-time = "2026-05-10T18:15:29.614Z" }, + { url = "https://files.pythonhosted.org/packages/cd/43/acdd5ce317cb46e8253ca9bfbdb8b12e68a24d745949336a7f3d5fb79ba0/librt-0.11.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f37aa505b3cf60701562eddb32df74b12a9e380c207fd8b06dd157a943ac7ea0", size = 538878, upload-time = "2026-05-10T18:15:30.928Z" }, + { url = "https://files.pythonhosted.org/packages/29/b5/7a25bb12e3172839f647f196b3e988318b7bb1ca7501732a225c4dce2ec0/librt-0.11.0-cp310-cp310-win32.whl", hash = "sha256:94663a21534637f0e787ec2a2a756022df6e5b7b2335a5cdd7d8e33d68a2af89", size = 100070, upload-time = "2026-05-10T18:15:32.551Z" }, + { url = "https://files.pythonhosted.org/packages/c6/0d/ebbcf4d77999c02c937b05d2b90ff4cd4dcc7e9a365ba132329ac1fe7a0f/librt-0.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:dec7db73758c2b54953fd8b7fe348c45188fe26b39ee18446196edd08453a5d4", size = 117918, upload-time = "2026-05-10T18:15:33.678Z" }, + { url = "https://files.pythonhosted.org/packages/fe/87/2bf31fe17587b29e3f93ec31421e2b1e1c3e349b8bf6c7c313dbad1d5340/librt-0.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:93d95bd45b7d58343d8b90d904450a545144eec19a002511163426f8ab1fae29", size = 141092, upload-time = "2026-05-10T18:15:34.795Z" }, + { url = "https://files.pythonhosted.org/packages/cf/08/5c5bf772920b7ebac6e32bc91a643e0ab3870199c0b542356d3baa83970a/librt-0.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:4ee278c769a713638cdacd4c0436d72156e75df3ebc0166ab2b9dc43acc386c9", size = 142035, upload-time = "2026-05-10T18:15:36.242Z" }, + { url = "https://files.pythonhosted.org/packages/06/20/662a03d254e5b000d838e8b345d83303ddb768c080fd488e40634c0fa66b/librt-0.11.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f230cb1cbc9faaa616f9a678f530ebcf186e414b6bcbd88b960e4ba1b92428d5", size = 475022, upload-time = "2026-05-10T18:15:37.56Z" }, + { url = "https://files.pythonhosted.org/packages/de/f3/aa81523e45184c6ec23dc7f63263362ec55f80a09d424c012359ecbe7e35/librt-0.11.0-cp311-cp311-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:5d63c855d86938d9de93e265c9bd8c705b51ec494de5738340ee93767a686e4b", size = 467273, upload-time = "2026-05-10T18:15:39.182Z" }, + { url = "https://files.pythonhosted.org/packages/6b/6f/59c74b560ca8853834d5501d589c8a2519f4184f273a085ffd0f37a1cc47/librt-0.11.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:993f028be9e96a08d31df3479ac80d99be374d17f3b78e4796b3fd3c913d4e89", size = 497083, upload-time = "2026-05-10T18:15:40.634Z" }, + { url = "https://files.pythonhosted.org/packages/fe/7b/5aa4d2c9600a719401160bf7055417df0b2a47439b9d88286ce45e56b65f/librt-0.11.0-cp311-cp311-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:258d73a0aa66a055e65b2e4d1b8cdb23b9d132c5bb915d9547d804fcaed116cc", size = 489139, upload-time = "2026-05-10T18:15:41.934Z" }, + { url = "https://files.pythonhosted.org/packages/d6/31/9143803d7da6856a69153785768c4936864430eec0fd9461c3ea527d9922/librt-0.11.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0827efe7854718f04aaddf6496e96960a956e676fe1d0f04eb41511fd8ad06d5", size = 508442, upload-time = "2026-05-10T18:15:43.206Z" }, + { url = "https://files.pythonhosted.org/packages/2f/5a/bce08184488426bda4ccc2c4964ac048c8f68ae89bd7120082eef4233cfd/librt-0.11.0-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:7753e57d6e12d019c0d8786f1c09c709f4c3fcc57c3887b24e36e6c06ec938b7", size = 514230, upload-time = "2026-05-10T18:15:44.761Z" }, + { url = "https://files.pythonhosted.org/packages/89/8c/bb5e213d254b7505a0e658da199d8ab719086632ce09eef311ab27976523/librt-0.11.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:11bd19822431cc21af9f27374e7ae2e58103c7d98bda823536a6c47f6bb2bb3d", size = 494231, upload-time = "2026-05-10T18:15:46.308Z" }, + { url = "https://files.pythonhosted.org/packages/9d/fb/541cdad5b1ab1300398c74c4c9a497b88e5074c21b1244c8f49731d3a284/librt-0.11.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:22bdf239b219d3993761a148ffa134b19e52e9989c84f845d5d7b71d70a17412", size = 537585, upload-time = "2026-05-10T18:15:47.629Z" }, + { url = "https://files.pythonhosted.org/packages/8f/f2/464bb69295c320cb06bddb4f14a4ec67934ee14b2bffb12b19fb7ab287ba/librt-0.11.0-cp311-cp311-win32.whl", hash = "sha256:46c60b61e308eb535fbd6fa622b1ee1bb2815691c1ad9c98bf7b84952ec3bc8d", size = 100509, upload-time = "2026-05-10T18:15:49.157Z" }, + { url = "https://files.pythonhosted.org/packages/6d/e7/a17ee1788f9e4fbf548c19f4afa07c92089b9e24fef6cb2410863781ef4c/librt-0.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:902e546ff044f579ff1c953ff5fce97b636fe9e3943996b2177710c6ef076f73", size = 118628, upload-time = "2026-05-10T18:15:50.345Z" }, + { url = "https://files.pythonhosted.org/packages/cc/c7/6c766214f9f9903bcfcfbef97d807af8d8f5aa3502d247858ab17582d212/librt-0.11.0-cp311-cp311-win_arm64.whl", hash = "sha256:65ac3bc20f78aa0ee5ae84baa68917f89fef4af63e941084dd019a0d0e749f0c", size = 103122, upload-time = "2026-05-10T18:15:52.068Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d0/07c77e067f0838949b43bd89232c29d72efebb9d2801a9750184eb706b71/librt-0.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b87504f1690a23b9a2cca841191a04f83895d4fc2dd04df91d82b1a04ca2ad46", size = 144147, upload-time = "2026-05-10T18:15:53.227Z" }, + { url = "https://files.pythonhosted.org/packages/7a/24/8493538fa4f62f982686398a5b8f68008138a75086abdea19ade64bf4255/librt-0.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40071fc5fe0ce8daa6de616702314a01e1250711682b0523d6ab8d4525910cb3", size = 143614, upload-time = "2026-05-10T18:15:54.657Z" }, + { url = "https://files.pythonhosted.org/packages/ff/1e/f8bad050810d9171f34a1648ed910e56814c2ba61639f2bd53c6377ae24b/librt-0.11.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:137e79445c896a0ea7b265f52d23954e05b64222ee1af69e2cb34219067cbb67", size = 485538, upload-time = "2026-05-10T18:15:56.117Z" }, + { url = "https://files.pythonhosted.org/packages/c0/fe/3594ebfbaf03084ba4b120c9ba5c3183fd938a48725e9bbe6ff0a5159ad8/librt-0.11.0-cp312-cp312-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:cca6644054e78746d8d4ef238681f9c34ff8b584fe6b988ecebb8db3b15e622a", size = 479623, upload-time = "2026-05-10T18:15:57.544Z" }, + { url = "https://files.pythonhosted.org/packages/b0/da/5d1876984b3746c85dbd219dbfcb73c85f54ee263fd32e5b2a632ec14571/librt-0.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d5b0eea49f5562861ee8d757a32ef7d559c1d35be2aaaa1ec28941d74c9ffc8a", size = 513082, upload-time = "2026-05-10T18:15:58.805Z" }, + { url = "https://files.pythonhosted.org/packages/19/6e/55bdf5d5ca00c3e18430690bf2c953d8d3ffd3c337418173d33dec985dc9/librt-0.11.0-cp312-cp312-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0d1029d7e1ae1a7e647ed6fb5df8c4ce2dffefb7a9f5fd1376a4554d96dac09f", size = 508105, upload-time = "2026-05-10T18:16:00.2Z" }, + { url = "https://files.pythonhosted.org/packages/07/10/f1f23a7c595ee90ece4d35c851e5d104b1311a887ed1b4ac4c35bbd13da8/librt-0.11.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bc3ce6b33c5828d9e80592011a5c584cb2ce86edbc4088405f70da47dc1d1b3b", size = 522268, upload-time = "2026-05-10T18:16:01.708Z" }, + { url = "https://files.pythonhosted.org/packages/b6/02/5720f5697a7f54b78b3aefbe20df3a48cedcff1276618c4aa481177942ed/librt-0.11.0-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:936c5995f3514a42111f20099397d8177c79b4d7e70961e396c6f5a0a3566766", size = 527348, upload-time = "2026-05-10T18:16:03.496Z" }, + { url = "https://files.pythonhosted.org/packages/50/db/b4a47c6f91db4ff76348a0b3dd0cc65e090a078b765a810a62ff9434c3d3/librt-0.11.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:9bc0ca6ad9381cbe8e4aa6e5726e4c80c78115a6e9723c599ed1d73e092bc49d", size = 516294, upload-time = "2026-05-10T18:16:05.173Z" }, + { url = "https://files.pythonhosted.org/packages/9e/58/9384b2f4eb1ed1d273d40948a7c5c4b2360213b402ef3be4641c06299f9c/librt-0.11.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:070aa8c26c0a74774317a72df8851facc7f0f012a5b406557ac56992d92e1ec8", size = 553608, upload-time = "2026-05-10T18:16:06.839Z" }, + { url = "https://files.pythonhosted.org/packages/21/7b/5aa8848a7c6a9278c79375146da1812e695754ceec5f005e6043461a7315/librt-0.11.0-cp312-cp312-win32.whl", hash = "sha256:6bf14feb84b05ae945277395451998c89c54d0def4070eb5c08de544930b245a", size = 101879, upload-time = "2026-05-10T18:16:08.103Z" }, + { url = "https://files.pythonhosted.org/packages/37/33/8a745436944947575b584231750a41417de1a38cf6a2e9251d1065651c09/librt-0.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:75672f0bc524ede266287d532d7923dbce94c7514ad07627bac3d0c6d92cc4d9", size = 119831, upload-time = "2026-05-10T18:16:09.174Z" }, + { url = "https://files.pythonhosted.org/packages/59/67/a6739ac96e28b7855808bdb0370e250606104a859750d209e5a0716fe7ab/librt-0.11.0-cp312-cp312-win_arm64.whl", hash = "sha256:2f10cf143e4a9bb0f4f5af568a00df94a2d69ef41c2579584454bb0fe5cc642c", size = 103470, upload-time = "2026-05-10T18:16:10.369Z" }, + { url = "https://files.pythonhosted.org/packages/82/61/e59168d4d0bf2bf90f4f0caf7a001bfc60254c3af4586013b04dc3ef517b/librt-0.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:78dc31f7fdfe9c9d0eb0e8f42d139db230e826415bbcabd9f0e9faaaee909894", size = 144119, upload-time = "2026-05-10T18:16:11.771Z" }, + { url = "https://files.pythonhosted.org/packages/61/fd/caa1d60b12f7dd79ccea23054e06eeaebe266a5f52c40a6b651069200ce5/librt-0.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:fa475675db22290c3158e1d42326d0f5a65f04f44a0e68c3630a25b53560fb9c", size = 143565, upload-time = "2026-05-10T18:16:13.334Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a9/dc744f5c2b4978d48db970be29f22716d3413d28b14ad99740817315cf2c/librt-0.11.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:621db29691044bdeda22e789e482e1b0f3a985d90e3426c9c6d17606416205ea", size = 485395, upload-time = "2026-05-10T18:16:14.729Z" }, + { url = "https://files.pythonhosted.org/packages/8f/21/7f8e97a1e4dae952a5a95948f6f8507a173bc1e669f54340bba6ca1ca31b/librt-0.11.0-cp313-cp313-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:a9010e2ed5b3a9e158c5fd966b3ab7e834bb3d3aacc8f66c91dd4b57a3799230", size = 479383, upload-time = "2026-05-10T18:16:16.321Z" }, + { url = "https://files.pythonhosted.org/packages/a6/6d/d8ee9c114bebf2c50e29ec2aa940826fccb62a645c3e4c18760987d0e16d/librt-0.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7c39513d8b7477a2e1ed8c43fc21c524e8d5a0f8d4e8b7b074dbdbe7820a08e2", size = 513010, upload-time = "2026-05-10T18:16:17.647Z" }, + { url = "https://files.pythonhosted.org/packages/f0/43/0b5708af2bd30a46400e72ba6bdaa8f066f15fb9a688527e34220e8d6c06/librt-0.11.0-cp313-cp313-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:7aef3cf1d5af86e770ab04bfd993dfc4ae8b8c17f66fb77dd4a7d50de7bbb1a3", size = 508433, upload-time = "2026-05-10T18:16:19.309Z" }, + { url = "https://files.pythonhosted.org/packages/4a/50/356187247d09013490481033183b3532b58acf8028bcb34b2b56a375c9b2/librt-0.11.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:557183ddc36babe46b27dd60facbd5adb4492181a5be887587d57cda6e092f21", size = 522595, upload-time = "2026-05-10T18:16:20.642Z" }, + { url = "https://files.pythonhosted.org/packages/40/e7/c6ac4240899c7f3248079d5a9900debe0dadb3fdeaf856684c987105ba47/librt-0.11.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:83d3e1f72bd42f6c5c0b7daec530c3f829bd02db42c70b8ddf0c2d90a2459930", size = 527255, upload-time = "2026-05-10T18:16:22.352Z" }, + { url = "https://files.pythonhosted.org/packages/eb/b5/a81322dbeedeeaf9c1ee6f001734d28a09d8383ac9e6779bc24bbd0743c6/librt-0.11.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:4ce1f21fbe589bc1afd7872dece84fb0e1144f794a288e58a10d2c54a55c43be", size = 516847, upload-time = "2026-05-10T18:16:23.627Z" }, + { url = "https://files.pythonhosted.org/packages/ae/66/6e6323787d592b55204a42595ff1102da5115601b53a7e9ddebc889a6da5/librt-0.11.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:970b09f7044ea2b64c9da42fd3d335666518cfd1c6e8a182c95da73d0214b41e", size = 553920, upload-time = "2026-05-10T18:16:25.025Z" }, + { url = "https://files.pythonhosted.org/packages/9c/21/623f8ca230857102066d9ca8c6c1734995908c4d0d1bee7bb2ef0021cb33/librt-0.11.0-cp313-cp313-win32.whl", hash = "sha256:78fddc31cd4d3caa897ad5d31f856b1faadc9474021ad6cb182b9018793e254e", size = 101898, upload-time = "2026-05-10T18:16:26.649Z" }, + { url = "https://files.pythonhosted.org/packages/b3/1d/b4ebd44dd723f768469007515cb92251e0ae286c94c140f374801140fa74/librt-0.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:8ca8aa88751a775870b764e93bad5135385f563cb8dcee399abf034ea4d3cb47", size = 119812, upload-time = "2026-05-10T18:16:27.859Z" }, + { url = "https://files.pythonhosted.org/packages/3b/e4/b2f4ca7965ca373b491cdb4bc25cdb30c1649ca81a8782056a83850292a9/librt-0.11.0-cp313-cp313-win_arm64.whl", hash = "sha256:96f044bb325fd9cf1a723015638c219e9143f0dfbc0ca54c565df2b7fc748b44", size = 103448, upload-time = "2026-05-10T18:16:29.066Z" }, + { url = "https://files.pythonhosted.org/packages/29/eb/dbce197da4e227779e56b5735f2decc3eb36e55a1cdbf1bd65d6639d76c1/librt-0.11.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:4a017a95e5837dc15a8c5661d60e05daa96b90908b1aa6b7acdf443cd25c8ebd", size = 143345, upload-time = "2026-05-10T18:16:30.674Z" }, + { url = "https://files.pythonhosted.org/packages/76/a3/254bebd0c11c8ba684018efb8006ff22e466abce445215cca6c778e7d9de/librt-0.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:b1ecbd9819deccc39b7542bf4d2a740d8a620694d39989e58661d3763458f8d4", size = 143131, upload-time = "2026-05-10T18:16:32.037Z" }, + { url = "https://files.pythonhosted.org/packages/f1/3f/f77d6122d21ac7bf6ae8a7dfced1bd2a7ac545d3273ebdcaf8042f6d619f/librt-0.11.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7da327dacd7be8f8ec36547373550744a3cc0e536d54665cd83f8bcd961200e8", size = 477024, upload-time = "2026-05-10T18:16:33.493Z" }, + { url = "https://files.pythonhosted.org/packages/ac/0a/2c996dadebaa7d9bbbd43ef2d4f3e66b6da545f838a41694ef6172cebec8/librt-0.11.0-cp314-cp314-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:0dc56b1f8d06e60db362cc3fdae206681817f86ce4725d34511473487f12a34b", size = 474221, upload-time = "2026-05-10T18:16:34.864Z" }, + { url = "https://files.pythonhosted.org/packages/0a/7e/f5d92af8486b8272c23b3e686b46ff72d89c8169585eb61eef01a2ac7147/librt-0.11.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:05fb8fb2ab90e21c8d12ea240d744ad514da9baf381ebfa70d91d20d21713175", size = 505174, upload-time = "2026-05-10T18:16:36.705Z" }, + { url = "https://files.pythonhosted.org/packages/af/1a/cb0734fe86398eb33193ab753b7326255c74cac5eb09e76b9b16536e7adb/librt-0.11.0-cp314-cp314-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:cae74872be221df4374d10fec61f93ed1513b9546ea84f2c0bf73ab3e9bd0b03", size = 497216, upload-time = "2026-05-10T18:16:38.418Z" }, + { url = "https://files.pythonhosted.org/packages/18/06/094820f91558b66e29943c0ec41c9914f460f48dd51fc503c3101e10842d/librt-0.11.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:32bcc918c0148eb7e3d57385125bac7e5f9e4359d05f07448b09f6f778c2f31c", size = 513921, upload-time = "2026-05-10T18:16:39.848Z" }, + { url = "https://files.pythonhosted.org/packages/0b/c2/00de9018871a282f530cacb457d5ec0428f6ac7e6fedde9aff7468d9fb04/librt-0.11.0-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:f9743fc99135d5f78d2454435615f6dec0473ca507c26ce9d92b10b562a280d3", size = 520850, upload-time = "2026-05-10T18:16:41.471Z" }, + { url = "https://files.pythonhosted.org/packages/51/9d/64631832348fd1834fb3a61b996434edddaaf25a31d03b0a76273159d2cf/librt-0.11.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:5ba067f4aadae8fda802d91d2124c90c42195ff32d9161d3549e6d05cfe26f96", size = 504237, upload-time = "2026-05-10T18:16:43.15Z" }, + { url = "https://files.pythonhosted.org/packages/a5/ec/ae5525eb16edc827a044e7bb8777a455ff95d4bca9379e7e6bddd7383647/librt-0.11.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:de3bf945454d032f9e390b85c4072e0a0570bf825421c8be0e71209fa65e1abe", size = 546261, upload-time = "2026-05-10T18:16:44.408Z" }, + { url = "https://files.pythonhosted.org/packages/5a/09/adce371f27ca039411da9659f7430fcc2ba6cd0c7b3e4467a0f091be7fa9/librt-0.11.0-cp314-cp314-win32.whl", hash = "sha256:d2277a05f6dcb9fd13db9566aac4fabd68c3ea1ea46ee5567d4eef8efa495a2f", size = 96965, upload-time = "2026-05-10T18:16:46.039Z" }, + { url = "https://files.pythonhosted.org/packages/d6/ee/8ac720d98548f173c7ce2e632a7ca94673f74cacd5c8162a84af5b35958a/librt-0.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:ab73e8db5e3f564d812c1f5c3a175930a5f9bc96ccb5e3b22a34d7858b401cf7", size = 115151, upload-time = "2026-05-10T18:16:47.133Z" }, + { url = "https://files.pythonhosted.org/packages/94/20/c900cf14efeb09b6bef2b2dff20779f73464b97fd58d1c6bccc379588ae3/librt-0.11.0-cp314-cp314-win_arm64.whl", hash = "sha256:aea3caa317752e3a466fa8af45d91ee0ea8c7fdd96e42b0a8dd9b76a7931eba1", size = 98850, upload-time = "2026-05-10T18:16:48.597Z" }, + { url = "https://files.pythonhosted.org/packages/0c/71/944bfe4b64e12abffcd3c15e1cce07f72f3d55655083786285f4dedeb532/librt-0.11.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:d1b36540d7aaf9b9101b3a6f376c8d8e9f7a9aec93ed05918f2c69d493ffef72", size = 151138, upload-time = "2026-05-10T18:16:49.839Z" }, + { url = "https://files.pythonhosted.org/packages/b6/10/99e64a5c86989357fda078c8143c533389585f6473b7439172dd8f3b3b2d/librt-0.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:efbb343ab2ce3540f4ecbe6315d677ed70f37cd9a72b1e58066c918ca83acbaa", size = 151976, upload-time = "2026-05-10T18:16:51.062Z" }, + { url = "https://files.pythonhosted.org/packages/21/31/5072ad880946d83e5ea4147d6d018c78eefce85b77819b19bdd0ee229435/librt-0.11.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aa0dd688aab3f7914d3e6e5e3554978e0383312fb8e771d84be008a35b9ee548", size = 557927, upload-time = "2026-05-10T18:16:52.632Z" }, + { url = "https://files.pythonhosted.org/packages/5e/8d/70b5fb7cfbab60edbe7381614ab985da58e144fbf465c86d44c95f43cdca/librt-0.11.0-cp314-cp314t-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:f5fb36b8c6c63fdcbb1d526d94c0d1331610d43f4118cc1beb4efef4f3faacb2", size = 539698, upload-time = "2026-05-10T18:16:53.934Z" }, + { url = "https://files.pythonhosted.org/packages/fa/a3/ba3495a0b3edbd24a4cae0d1d3c64f39a9fc45d06e812101289b50c1a619/librt-0.11.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4a9a237d13addb93715b6fee74023d5ee3469b53fce527626c0e088aa585805f", size = 577162, upload-time = "2026-05-10T18:16:55.589Z" }, + { url = "https://files.pythonhosted.org/packages/f7/db/36e25fb81f99937ff1b96612a1dc9fd66f039cb9cc3aee12c01fac31aab9/librt-0.11.0-cp314-cp314t-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5ddd17bd87b2c56ddd60e546a7984a2e64c4e8eab92fb4cf3830a48ad5469d51", size = 566494, upload-time = "2026-05-10T18:16:56.975Z" }, + { url = "https://files.pythonhosted.org/packages/33/0d/3f622b47f0b013eeb9cf4cc07ae9bfe378d832a4eec998b2b209fe84244d/librt-0.11.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:bd43992b4473d42f12ff9e68326079f0696d9d4e6000e8f39a0238d482ba6ee2", size = 596858, upload-time = "2026-05-10T18:16:58.374Z" }, + { url = "https://files.pythonhosted.org/packages/a9/02/71b90bc93039c46a2000651f6ad60122b114c8f54c4ad306e0e96f5b75ad/librt-0.11.0-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:f8e3e8056dd674e279741485e2e512d6e9a751c7455809d0114e6ebf8d781085", size = 590318, upload-time = "2026-05-10T18:16:59.676Z" }, + { url = "https://files.pythonhosted.org/packages/04/04/418cb3f75621e2b761fb1ab0f017f4d70a1a72a6e7c74ee4f7e8d198c2f3/librt-0.11.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:c1f708d8ae9c56cf38a903c44297243d2ec83fd82b396b977e0144a3e76217e3", size = 575115, upload-time = "2026-05-10T18:17:01.007Z" }, + { url = "https://files.pythonhosted.org/packages/cc/2c/5a2183ac58dd911f26b5d7e7d7d8f1d87fcecdddd99d6c12169a258ff62c/librt-0.11.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:0add982e0e7b9fc14cf4b33789d5f13f66581889b88c2f58099f6ce8f92617bd", size = 617918, upload-time = "2026-05-10T18:17:02.682Z" }, + { url = "https://files.pythonhosted.org/packages/15/1f/dc6771a52592a4451be6effa200cbfc9cec61e4393d3033d81a9d307961d/librt-0.11.0-cp314-cp314t-win32.whl", hash = "sha256:2b481d846ac894c4e8403c5fd0e87c5d11d6499e404b474602508a224ff531c8", size = 103562, upload-time = "2026-05-10T18:17:03.99Z" }, + { url = "https://files.pythonhosted.org/packages/62/4a/7d1415567027286a75ba1093ec4aca11f073e0f559c530cf3e0a757ad55c/librt-0.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:28edb433edde181112a908c78907af28f964eabc15f4dd16c9d66c834302677c", size = 124327, upload-time = "2026-05-10T18:17:05.465Z" }, + { url = "https://files.pythonhosted.org/packages/ce/62/b40b382fa0c66fee1478073eb8db352a4a6beda4a1adccf1df911d8c289c/librt-0.11.0-cp314-cp314t-win_arm64.whl", hash = "sha256:dee008f20b542e3cd162ba338a7f9ec0f6d23d395f66fe8aeeec3c9d067ea253", size = 102572, upload-time = "2026-05-10T18:17:06.809Z" }, + { url = "https://files.pythonhosted.org/packages/66/54/5d5f27cc840d2d8a64d60e0650dba14044a95d85a875e42af2eb104ac8b9/librt-0.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6bd72d903911d995ab666dbd1871f8b1e80925a699af8063fbf50053329fb05f", size = 142475, upload-time = "2026-05-10T18:17:08.263Z" }, + { url = "https://files.pythonhosted.org/packages/f9/72/535efe79cf47f70975e0b14ceb3b7984bb7e8b97fb2867d3979771be0b6a/librt-0.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ef69ac715f3cd8e5cd252cb2aebfa72c015492aacc339d5d7bf8fef3c62c677", size = 143365, upload-time = "2026-05-10T18:17:09.565Z" }, + { url = "https://files.pythonhosted.org/packages/83/cc/4130d462aeaf190357517d2a48a0a25030fbfd604230f6c45908452fff9c/librt-0.11.0-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:624a40c4a4ad7773315c287276cd024509b2c66ff5904f504bfc08d2c70293ab", size = 475743, upload-time = "2026-05-10T18:17:10.822Z" }, + { url = "https://files.pythonhosted.org/packages/62/e8/3c8000edefeb443fd2139692fb966f6c5556cb1032c44f734550896df3b9/librt-0.11.0-cp39-cp39-manylinux2014_i686.manylinux_2_17_i686.manylinux_2_28_i686.whl", hash = "sha256:41dc19fe150b69716c8ece4f76773a9e8813fe3e35e032a58b4d46423fb8d7c0", size = 467088, upload-time = "2026-05-10T18:17:12.273Z" }, + { url = "https://files.pythonhosted.org/packages/b2/a1/6de754256493924874e5fa6c0f4f990d8b101c38d974589020d9dc3d02af/librt-0.11.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4e8bd98ea9c47ae90b319a087ab28dac493f1ffbc1ecd1f28fcdbf3b7e1108d1", size = 496277, upload-time = "2026-05-10T18:17:13.662Z" }, + { url = "https://files.pythonhosted.org/packages/92/fb/c34cb5358d6f993f85014045decd6dccd089a6f11d188660e062ee6262ff/librt-0.11.0-cp39-cp39-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:84308fc49423ce6475d1c5d1985cd69a8ca9f0325fc7d5f81bb690a3f3625d4e", size = 489320, upload-time = "2026-05-10T18:17:15.232Z" }, + { url = "https://files.pythonhosted.org/packages/48/65/7761d70841bac875be9627496546b2eccbdeb07da3e42431bc4a40cf0819/librt-0.11.0-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:ff0fbaf5f44a21beeb0110f2ab64f45135a9536a834b79c0d1ef018f2786bbfa", size = 510221, upload-time = "2026-05-10T18:17:16.595Z" }, + { url = "https://files.pythonhosted.org/packages/cb/8d/af9d4ac1057cd4e472b89553924b528b3d34afa6b7167645b7e6db39596b/librt-0.11.0-cp39-cp39-musllinux_1_2_i686.whl", hash = "sha256:9c028a9442a18e266955d364ce42259136e79a7ba14d773e0d778d5f70cd56f1", size = 516650, upload-time = "2026-05-10T18:17:18.245Z" }, + { url = "https://files.pythonhosted.org/packages/86/f4/08faaf48ce0833d3717ebe0a0054c09a05df1bc83ee2715113c9901cc147/librt-0.11.0-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:9f1692105a02bcf853f355032a5fdc5494358ef83d8fd22d16de375c85cec3f5", size = 496622, upload-time = "2026-05-10T18:17:19.857Z" }, + { url = "https://files.pythonhosted.org/packages/0d/11/ec3e390627f70477093909875a38843c826ee2ff554d1649645c7cc59248/librt-0.11.0-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:7a80a71e1fda83cc752a9141e87aae7fef279538597564d670e9ce513f286192", size = 538049, upload-time = "2026-05-10T18:17:21.221Z" }, + { url = "https://files.pythonhosted.org/packages/cd/a7/649401dae7ea8645dd218aa2d9c351afa7b9e0645f07dc8776a1972c0cad/librt-0.11.0-cp39-cp39-win32.whl", hash = "sha256:140695816ddf3c86eb972981a26f35efd871c44b0c3aed44c8cd01749386617f", size = 100360, upload-time = "2026-05-10T18:17:22.537Z" }, + { url = "https://files.pythonhosted.org/packages/2a/7e/6a9711d78f338445e36992a90071962294f5bab388b554ef8a313e6412dd/librt-0.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:92f7ff819c197fc30473190a12c2856f325ac90aabfccbeb2072d28cc2e234e3", size = 118407, upload-time = "2026-05-10T18:17:24.01Z" }, +] + +[[package]] +name = "mypy" +version = "1.19.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "importlib-metadata", marker = "python_full_version < '3.10'" }, + { name = "librt", marker = "python_full_version < '3.10' and platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions", marker = "python_full_version < '3.10'" }, + { name = "pathspec", marker = "python_full_version < '3.10'" }, + { name = "tomli", marker = "python_full_version < '3.10'" }, + { name = "typing-extensions", marker = "python_full_version < '3.10'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1e/82/fa43935523efdfcce6abbae9da7f372b627b27142c3419fcf13bf5b0c397/isort-6.1.0.tar.gz", hash = "sha256:9b8f96a14cfee0677e78e941ff62f03769a06d412aabb9e2a90487b3b7e8d481", size = 824325, upload-time = "2025-10-01T16:26:45.027Z" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/db/4efed9504bc01309ab9c2da7e352cc223569f05478012b5d9ece38fd44d2/mypy-1.19.1.tar.gz", hash = "sha256:19d88bb05303fe63f71dd2c6270daca27cb9401c4ca8255fe50d1d920e0eb9ba", size = 3582404, upload-time = "2025-12-15T05:03:48.42Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/cc/9b681a170efab4868a032631dea1e8446d8ec718a7f657b94d49d1a12643/isort-6.1.0-py3-none-any.whl", hash = "sha256:58d8927ecce74e5087aef019f778d4081a3b6c98f15a80ba35782ca8a2097784", size = 94329, upload-time = "2025-10-01T16:26:43.291Z" }, + { url = "https://files.pythonhosted.org/packages/2f/63/e499890d8e39b1ff2df4c0c6ce5d371b6844ee22b8250687a99fd2f657a8/mypy-1.19.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:5f05aa3d375b385734388e844bc01733bd33c644ab48e9684faa54e5389775ec", size = 13101333, upload-time = "2025-12-15T05:03:03.28Z" }, + { url = "https://files.pythonhosted.org/packages/72/4b/095626fc136fba96effc4fd4a82b41d688ab92124f8c4f7564bffe5cf1b0/mypy-1.19.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:022ea7279374af1a5d78dfcab853fe6a536eebfda4b59deab53cd21f6cd9f00b", size = 12164102, upload-time = "2025-12-15T05:02:33.611Z" }, + { url = "https://files.pythonhosted.org/packages/0c/5b/952928dd081bf88a83a5ccd49aaecfcd18fd0d2710c7ff07b8fb6f7032b9/mypy-1.19.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee4c11e460685c3e0c64a4c5de82ae143622410950d6be863303a1c4ba0e36d6", size = 12765799, upload-time = "2025-12-15T05:03:28.44Z" }, + { url = "https://files.pythonhosted.org/packages/2a/0d/93c2e4a287f74ef11a66fb6d49c7a9f05e47b0a4399040e6719b57f500d2/mypy-1.19.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:de759aafbae8763283b2ee5869c7255391fbc4de3ff171f8f030b5ec48381b74", size = 13522149, upload-time = "2025-12-15T05:02:36.011Z" }, + { url = "https://files.pythonhosted.org/packages/7b/0e/33a294b56aaad2b338d203e3a1d8b453637ac36cb278b45005e0901cf148/mypy-1.19.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:ab43590f9cd5108f41aacf9fca31841142c786827a74ab7cc8a2eacb634e09a1", size = 13810105, upload-time = "2025-12-15T05:02:40.327Z" }, + { url = "https://files.pythonhosted.org/packages/0e/fd/3e82603a0cb66b67c5e7abababce6bf1a929ddf67bf445e652684af5c5a0/mypy-1.19.1-cp310-cp310-win_amd64.whl", hash = "sha256:2899753e2f61e571b3971747e302d5f420c3fd09650e1951e99f823bc3089dac", size = 10057200, upload-time = "2025-12-15T05:02:51.012Z" }, + { url = "https://files.pythonhosted.org/packages/ef/47/6b3ebabd5474d9cdc170d1342fbf9dddc1b0ec13ec90bf9004ee6f391c31/mypy-1.19.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d8dfc6ab58ca7dda47d9237349157500468e404b17213d44fc1cb77bce532288", size = 13028539, upload-time = "2025-12-15T05:03:44.129Z" }, + { url = "https://files.pythonhosted.org/packages/5c/a6/ac7c7a88a3c9c54334f53a941b765e6ec6c4ebd65d3fe8cdcfbe0d0fd7db/mypy-1.19.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e3f276d8493c3c97930e354b2595a44a21348b320d859fb4a2b9f66da9ed27ab", size = 12083163, upload-time = "2025-12-15T05:03:37.679Z" }, + { url = "https://files.pythonhosted.org/packages/67/af/3afa9cf880aa4a2c803798ac24f1d11ef72a0c8079689fac5cfd815e2830/mypy-1.19.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2abb24cf3f17864770d18d673c85235ba52456b36a06b6afc1e07c1fdcd3d0e6", size = 12687629, upload-time = "2025-12-15T05:02:31.526Z" }, + { url = "https://files.pythonhosted.org/packages/2d/46/20f8a7114a56484ab268b0ab372461cb3a8f7deed31ea96b83a4e4cfcfca/mypy-1.19.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a009ffa5a621762d0c926a078c2d639104becab69e79538a494bcccb62cc0331", size = 13436933, upload-time = "2025-12-15T05:03:15.606Z" }, + { url = "https://files.pythonhosted.org/packages/5b/f8/33b291ea85050a21f15da910002460f1f445f8007adb29230f0adea279cb/mypy-1.19.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f7cee03c9a2e2ee26ec07479f38ea9c884e301d42c6d43a19d20fb014e3ba925", size = 13661754, upload-time = "2025-12-15T05:02:26.731Z" }, + { url = "https://files.pythonhosted.org/packages/fd/a3/47cbd4e85bec4335a9cd80cf67dbc02be21b5d4c9c23ad6b95d6c5196bac/mypy-1.19.1-cp311-cp311-win_amd64.whl", hash = "sha256:4b84a7a18f41e167f7995200a1d07a4a6810e89d29859df936f1c3923d263042", size = 10055772, upload-time = "2025-12-15T05:03:26.179Z" }, + { url = "https://files.pythonhosted.org/packages/06/8a/19bfae96f6615aa8a0604915512e0289b1fad33d5909bf7244f02935d33a/mypy-1.19.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a8174a03289288c1f6c46d55cef02379b478bfbc8e358e02047487cad44c6ca1", size = 13206053, upload-time = "2025-12-15T05:03:46.622Z" }, + { url = "https://files.pythonhosted.org/packages/a5/34/3e63879ab041602154ba2a9f99817bb0c85c4df19a23a1443c8986e4d565/mypy-1.19.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffcebe56eb09ff0c0885e750036a095e23793ba6c2e894e7e63f6d89ad51f22e", size = 12219134, upload-time = "2025-12-15T05:03:24.367Z" }, + { url = "https://files.pythonhosted.org/packages/89/cc/2db6f0e95366b630364e09845672dbee0cbf0bbe753a204b29a944967cd9/mypy-1.19.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b64d987153888790bcdb03a6473d321820597ab8dd9243b27a92153c4fa50fd2", size = 12731616, upload-time = "2025-12-15T05:02:44.725Z" }, + { url = "https://files.pythonhosted.org/packages/00/be/dd56c1fd4807bc1eba1cf18b2a850d0de7bacb55e158755eb79f77c41f8e/mypy-1.19.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c35d298c2c4bba75feb2195655dfea8124d855dfd7343bf8b8c055421eaf0cf8", size = 13620847, upload-time = "2025-12-15T05:03:39.633Z" }, + { url = "https://files.pythonhosted.org/packages/6d/42/332951aae42b79329f743bf1da088cd75d8d4d9acc18fbcbd84f26c1af4e/mypy-1.19.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:34c81968774648ab5ac09c29a375fdede03ba253f8f8287847bd480782f73a6a", size = 13834976, upload-time = "2025-12-15T05:03:08.786Z" }, + { url = "https://files.pythonhosted.org/packages/6f/63/e7493e5f90e1e085c562bb06e2eb32cae27c5057b9653348d38b47daaecc/mypy-1.19.1-cp312-cp312-win_amd64.whl", hash = "sha256:b10e7c2cd7870ba4ad9b2d8a6102eb5ffc1f16ca35e3de6bfa390c1113029d13", size = 10118104, upload-time = "2025-12-15T05:03:10.834Z" }, + { url = "https://files.pythonhosted.org/packages/de/9f/a6abae693f7a0c697dbb435aac52e958dc8da44e92e08ba88d2e42326176/mypy-1.19.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e3157c7594ff2ef1634ee058aafc56a82db665c9438fd41b390f3bde1ab12250", size = 13201927, upload-time = "2025-12-15T05:02:29.138Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a4/45c35ccf6e1c65afc23a069f50e2c66f46bd3798cbe0d680c12d12935caa/mypy-1.19.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:bdb12f69bcc02700c2b47e070238f42cb87f18c0bc1fc4cdb4fb2bc5fd7a3b8b", size = 12206730, upload-time = "2025-12-15T05:03:01.325Z" }, + { url = "https://files.pythonhosted.org/packages/05/bb/cdcf89678e26b187650512620eec8368fded4cfd99cfcb431e4cdfd19dec/mypy-1.19.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f859fb09d9583a985be9a493d5cfc5515b56b08f7447759a0c5deaf68d80506e", size = 12724581, upload-time = "2025-12-15T05:03:20.087Z" }, + { url = "https://files.pythonhosted.org/packages/d1/32/dd260d52babf67bad8e6770f8e1102021877ce0edea106e72df5626bb0ec/mypy-1.19.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c9a6538e0415310aad77cb94004ca6482330fece18036b5f360b62c45814c4ef", size = 13616252, upload-time = "2025-12-15T05:02:49.036Z" }, + { url = "https://files.pythonhosted.org/packages/71/d0/5e60a9d2e3bd48432ae2b454b7ef2b62a960ab51292b1eda2a95edd78198/mypy-1.19.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:da4869fc5e7f62a88f3fe0b5c919d1d9f7ea3cef92d3689de2823fd27e40aa75", size = 13840848, upload-time = "2025-12-15T05:02:55.95Z" }, + { url = "https://files.pythonhosted.org/packages/98/76/d32051fa65ecf6cc8c6610956473abdc9b4c43301107476ac03559507843/mypy-1.19.1-cp313-cp313-win_amd64.whl", hash = "sha256:016f2246209095e8eda7538944daa1d60e1e8134d98983b9fc1e92c1fc0cb8dd", size = 10135510, upload-time = "2025-12-15T05:02:58.438Z" }, + { url = "https://files.pythonhosted.org/packages/de/eb/b83e75f4c820c4247a58580ef86fcd35165028f191e7e1ba57128c52782d/mypy-1.19.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:06e6170bd5836770e8104c8fdd58e5e725cfeb309f0a6c681a811f557e97eac1", size = 13199744, upload-time = "2025-12-15T05:03:30.823Z" }, + { url = "https://files.pythonhosted.org/packages/94/28/52785ab7bfa165f87fcbb61547a93f98bb20e7f82f90f165a1f69bce7b3d/mypy-1.19.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:804bd67b8054a85447c8954215a906d6eff9cabeabe493fb6334b24f4bfff718", size = 12215815, upload-time = "2025-12-15T05:02:42.323Z" }, + { url = "https://files.pythonhosted.org/packages/0a/c6/bdd60774a0dbfb05122e3e925f2e9e846c009e479dcec4821dad881f5b52/mypy-1.19.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:21761006a7f497cb0d4de3d8ef4ca70532256688b0523eee02baf9eec895e27b", size = 12740047, upload-time = "2025-12-15T05:03:33.168Z" }, + { url = "https://files.pythonhosted.org/packages/32/2a/66ba933fe6c76bd40d1fe916a83f04fed253152f451a877520b3c4a5e41e/mypy-1.19.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:28902ee51f12e0f19e1e16fbe2f8f06b6637f482c459dd393efddd0ec7f82045", size = 13601998, upload-time = "2025-12-15T05:03:13.056Z" }, + { url = "https://files.pythonhosted.org/packages/e3/da/5055c63e377c5c2418760411fd6a63ee2b96cf95397259038756c042574f/mypy-1.19.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:481daf36a4c443332e2ae9c137dfee878fcea781a2e3f895d54bd3002a900957", size = 13807476, upload-time = "2025-12-15T05:03:17.977Z" }, + { url = "https://files.pythonhosted.org/packages/cd/09/4ebd873390a063176f06b0dbf1f7783dd87bd120eae7727fa4ae4179b685/mypy-1.19.1-cp314-cp314-win_amd64.whl", hash = "sha256:8bb5c6f6d043655e055be9b542aa5f3bdd30e4f3589163e85f93f3640060509f", size = 10281872, upload-time = "2025-12-15T05:03:05.549Z" }, + { url = "https://files.pythonhosted.org/packages/b5/f7/88436084550ca9af5e610fa45286be04c3b63374df3e021c762fe8c4369f/mypy-1.19.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7bcfc336a03a1aaa26dfce9fff3e287a3ba99872a157561cbfcebe67c13308e3", size = 13102606, upload-time = "2025-12-15T05:02:46.833Z" }, + { url = "https://files.pythonhosted.org/packages/ca/a5/43dfad311a734b48a752790571fd9e12d61893849a01bff346a54011957f/mypy-1.19.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b7951a701c07ea584c4fe327834b92a30825514c868b1f69c30445093fdd9d5a", size = 12164496, upload-time = "2025-12-15T05:03:41.947Z" }, + { url = "https://files.pythonhosted.org/packages/88/f0/efbfa391395cce2f2771f937e0620cfd185ec88f2b9cd88711028a768e96/mypy-1.19.1-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b13cfdd6c87fc3efb69ea4ec18ef79c74c3f98b4e5498ca9b85ab3b2c2329a67", size = 12772068, upload-time = "2025-12-15T05:02:53.689Z" }, + { url = "https://files.pythonhosted.org/packages/25/05/58b3ba28f5aed10479e899a12d2120d582ba9fa6288851b20bf1c32cbb4f/mypy-1.19.1-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f28f99c824ecebcdaa2e55d82953e38ff60ee5ec938476796636b86afa3956e", size = 13520385, upload-time = "2025-12-15T05:02:38.328Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a0/c006ccaff50b31e542ae69b92fe7e2f55d99fba3a55e01067dd564325f85/mypy-1.19.1-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:c608937067d2fc5a4dd1a5ce92fd9e1398691b8c5d012d66e1ddd430e9244376", size = 13796221, upload-time = "2025-12-15T05:03:22.147Z" }, + { url = "https://files.pythonhosted.org/packages/b2/ff/8bdb051cd710f01b880472241bd36b3f817a8e1c5d5540d0b761675b6de2/mypy-1.19.1-cp39-cp39-win_amd64.whl", hash = "sha256:409088884802d511ee52ca067707b90c883426bd95514e8cfda8281dc2effe24", size = 10055456, upload-time = "2025-12-15T05:03:35.169Z" }, + { url = "https://files.pythonhosted.org/packages/8d/f4/4ce9a05ce5ded1de3ec1c1d96cf9f9504a04e54ce0ed55cfa38619a32b8d/mypy-1.19.1-py3-none-any.whl", hash = "sha256:f1235f5ea01b7db5468d53ece6aaddf1ad0b88d9e7462b86ef96fe04995d7247", size = 2471239, upload-time = "2025-12-15T05:03:07.248Z" }, ] [[package]] -name = "isort" -version = "7.0.0" +name = "mypy" +version = "2.1.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.15'", + "python_full_version >= '3.12' and python_full_version < '3.15'", "python_full_version == '3.11.*'", "python_full_version == '3.10.*'", ] -sdist = { url = "https://files.pythonhosted.org/packages/63/53/4f3c058e3bace40282876f9b553343376ee687f3c35a525dc79dbd450f88/isort-7.0.0.tar.gz", hash = "sha256:5513527951aadb3ac4292a41a16cbc50dd1642432f5e8c20057d414bdafb4187", size = 805049, upload-time = "2025-10-11T13:30:59.107Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/ed/e3705d6d02b4f7aea715a353c8ce193efd0b5db13e204df895d38734c244/isort-7.0.0-py3-none-any.whl", hash = "sha256:1bcabac8bc3c36c7fb7b98a76c8abb18e0f841a3ba81decac7691008592499c1", size = 94672, upload-time = "2025-10-11T13:30:57.665Z" }, +dependencies = [ + { name = "ast-serialize", marker = "python_full_version >= '3.10'" }, + { name = "librt", marker = "python_full_version >= '3.10' and platform_python_implementation != 'PyPy'" }, + { name = "mypy-extensions", marker = "python_full_version >= '3.10'" }, + { name = "pathspec", marker = "python_full_version >= '3.10'" }, + { name = "tomli", marker = "python_full_version == '3.10.*'" }, + { name = "typing-extensions", marker = "python_full_version >= '3.10'" }, ] - -[[package]] -name = "mccabe" -version = "0.7.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } +sdist = { url = "https://files.pythonhosted.org/packages/82/15/cca9d88503549ed6fedeaa1d448cdddd542ee8a490232d732e278036fbf2/mypy-2.1.0.tar.gz", hash = "sha256:81e76ad12c2d804512e9b13240d1588316531bfba07558286078bfbce9613633", size = 3898359, upload-time = "2026-05-11T18:37:36.237Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, + { url = "https://files.pythonhosted.org/packages/a4/71/d351dca3e9b30da2328ee9d445c88b8388072808ebfbc49eb69d30b67749/mypy-2.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:11a6beb180257a805961aea9ec591bbd0bd17f1e18d35b8456d57aee5bedfedc", size = 14778792, upload-time = "2026-05-11T18:36:23.605Z" }, + { url = "https://files.pythonhosted.org/packages/2f/45/7d51594b644c17c0bcf74ed8cd5fc33b324276d708e8506f220b70dab9d9/mypy-2.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8ef78c1d306bbf9a8a12f526c44902c9c28dffd6c52c52bf6a72641ce18d3849", size = 13645739, upload-time = "2026-05-11T18:37:22.752Z" }, + { url = "https://files.pythonhosted.org/packages/65/01/455c31b170e9468265074840bf18863a8482a24103fdaabe4e199392aa5f/mypy-2.1.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c209a90853081ff01d01ee895cafe10f7db1474e0d95beaeef0f6c1db9119bbd", size = 14074199, upload-time = "2026-05-11T18:35:09.292Z" }, + { url = "https://files.pythonhosted.org/packages/41/5a/93093f0b29a9e982deafde698f740a2eb2e05886e79ccf0594c7fd5413a3/mypy-2.1.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:47cebf61abde7c088a4e27718a8b13a81655686b2e9c251f5c0915a802248166", size = 14953128, upload-time = "2026-05-11T18:31:57.678Z" }, + { url = "https://files.pythonhosted.org/packages/7f/2f/a196f5331d96170ad3d28f144d2aba690d4b2911381f68d51e489c7ab82a/mypy-2.1.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:d57a90ae5e872138a425ec328edbc9b235d1934c4377881a33ec05b341acc9a8", size = 15249378, upload-time = "2026-05-11T18:33:00.101Z" }, + { url = "https://files.pythonhosted.org/packages/54/de/94d321cc12da9f71341ac0c270efbed5c725750c7b4c334d957de9a087d9/mypy-2.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:aea7f7a8a55b459c34275fc468ada6ca7c173a5e43a68f5dbe588a563d8a06b8", size = 11060994, upload-time = "2026-05-11T18:33:18.848Z" }, + { url = "https://files.pythonhosted.org/packages/e1/62/0c27ca55219a7c764a7fb88c7bb2b7b2f9780ade8bbf16bc8ed8400eef6b/mypy-2.1.0-cp310-cp310-win_arm64.whl", hash = "sha256:c989640253f0d76843e9c6c1bbf4bd48c5e85ada61bde4beb37cb3eca035685e", size = 9976743, upload-time = "2026-05-11T18:31:25.554Z" }, + { url = "https://files.pythonhosted.org/packages/0a/a1/639f3024794a2a15899cb90707fe02e044c4412794c39c5769fd3df2e2ef/mypy-2.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:a683016b16fe2f572dc04c72be7ee0504ac1605a265d0200f5cea695fb788f41", size = 14691685, upload-time = "2026-05-11T18:33:27.973Z" }, + { url = "https://files.pythonhosted.org/packages/3b/08/9a585dea4325f20d8b80dc78623fa50d1fd2173b710f6237afd6ba6ab39b/mypy-2.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1a293c534adb55271fef24a26da04b855540a8c13cc07bc5917b9fd2c394f2ca", size = 13555165, upload-time = "2026-05-11T18:32:16.107Z" }, + { url = "https://files.pythonhosted.org/packages/81/dc/7c42cc9c6cb01e8eb09961f1f738741d3e9c7e9d5c5b30ec69222625cd5f/mypy-2.1.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7406f4d048e71e576f5356d317e5b0a9e666dfd966bd99f9d14ca06e1a341538", size = 13994376, upload-time = "2026-05-11T18:32:39.256Z" }, + { url = "https://files.pythonhosted.org/packages/d4/fa/285946c33bce716e082c11dfeee9ee196eaf1f5042efb3581a31f9f205e4/mypy-2.1.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e0210d626fc8b31ccc90233754c7bc90e1f43205e85d96387f7db1285b55c398", size = 14864618, upload-time = "2026-05-11T18:34:49.765Z" }, + { url = "https://files.pythonhosted.org/packages/2b/83/82397f48af6c27e295d57979ded8490c9829040152cf7571b2f026aeb9a0/mypy-2.1.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:3712c20deed54e814eaaa825603bada8ea1c390670a397c95b98405347acc563", size = 15102063, upload-time = "2026-05-11T18:34:05.855Z" }, + { url = "https://files.pythonhosted.org/packages/40/68/b02dec39057b88eb03dc0aa854732e26e8361f34f9d0e20c7614967d1eba/mypy-2.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:fcaa0e479066e31f7cceb6a3bea39cb22b2ff51a6b2f24f193d19179ba17c389", size = 11060564, upload-time = "2026-05-11T18:35:36.494Z" }, + { url = "https://files.pythonhosted.org/packages/cf/a8/ea3dcbef31f99b634f2ee23bb0321cbc8c1b388b76a861eb849f13c347dc/mypy-2.1.0-cp311-cp311-win_arm64.whl", hash = "sha256:0b1a5260c95aa443083f9ed3592662941951bca3d4ca224a5dc517c38b7cf666", size = 9966983, upload-time = "2026-05-11T18:37:14.139Z" }, + { url = "https://files.pythonhosted.org/packages/95/b1/55861beb5c339b44f9a2ba92df9e2cb1eeb4ae1eee674cdf7772c797778b/mypy-2.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:244358bf1c0da7722230bce60683d52e8e9fd030554926f15b747a84efb5b3af", size = 14874381, upload-time = "2026-05-11T18:37:31.784Z" }, + { url = "https://files.pythonhosted.org/packages/0b/b3/b7f770114b7d0ac92d0f76e8d93c2780844a70488a90e91821927850da86/mypy-2.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4ec7c57657493c7a75534df2751c8ae2cda383c16ecc55d2106c54476b1b16f6", size = 13665501, upload-time = "2026-05-11T18:34:23.063Z" }, + { url = "https://files.pythonhosted.org/packages/b6/f3/8ae2037967e2126689a0c11d99e2b707134a565191e92c60ca2572aec60a/mypy-2.1.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d8161b6ff4392410023224f0969d17db93e1e154bc3e4ba62598e720723ae211", size = 14045750, upload-time = "2026-05-11T18:31:48.151Z" }, + { url = "https://files.pythonhosted.org/packages/a0/32/615eb5911859e43d054941b0d0a7d06cfa2870eba86529cf385b052b111c/mypy-2.1.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf03e12003084a67395184d3eb8cbd6a489dc3655b5664b28c210a9e2403ab0b", size = 15061630, upload-time = "2026-05-11T18:37:06.898Z" }, + { url = "https://files.pythonhosted.org/packages/d4/03/4eafbfff8bfab1b87082741eae6e6a624028c984e6708b73bce2a8570c9d/mypy-2.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:20509760fd791c51579d573153407d226385ec1f8bcce55d730b354f3336bc22", size = 15288831, upload-time = "2026-05-11T18:31:18.07Z" }, + { url = "https://files.pythonhosted.org/packages/99/ee/919661478e5891a3c96e549c036e467e64563ab85995b10c53c8358e16a3/mypy-2.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:6753d0c1fdd6b1a23b9e4f283ce80b2153b724adcb2653b20b85a8a28ac6436b", size = 11135228, upload-time = "2026-05-11T18:34:31.23Z" }, + { url = "https://files.pythonhosted.org/packages/24/0a/6a12b9782ca0831a553192f351679f4548abc9d19a7cc93bb7feb02084c7/mypy-2.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:98ebb6589bb3b6d0c6f0c459d53ca55b8091fbc13d277c4041c885392e8195e8", size = 10040684, upload-time = "2026-05-11T18:36:48.199Z" }, + { url = "https://files.pythonhosted.org/packages/6e/dd/c7191469c777f07689c032a8f7326e393ea34c92d6d76eb7ce5ba57ea66d/mypy-2.1.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:35aac3bb114e03888f535d5eb51b8bafbb3266586b599da1940f9b1be3ec5bd5", size = 14852174, upload-time = "2026-05-11T18:31:38.929Z" }, + { url = "https://files.pythonhosted.org/packages/55/8c/aed55408879043d72bb9135f4d0d19a02b886dd569631e113e3d2706cb8d/mypy-2.1.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:8de55a8c861f2a49331f807be98d90caeceeef520bde13d43a160207f8af613e", size = 13651542, upload-time = "2026-05-11T18:36:04.636Z" }, + { url = "https://files.pythonhosted.org/packages/3a/8e/f371a824b1f1fa8ea6e3dbb8703d232977d572be2329554a3bc4d960302f/mypy-2.1.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5fdf2941a07434af755837d9880f7d7d25f1dacb1af9dcd4b9b66f2220a3024e", size = 14033929, upload-time = "2026-05-11T18:35:55.742Z" }, + { url = "https://files.pythonhosted.org/packages/94/21/f54be870d6dd53a82c674407e0f8eed7174b05ec78d42e5abd7b42e84fd5/mypy-2.1.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e195b817c13f02352a9c124301f9f30f078405444679b6753c1b96b6eed37285", size = 15039200, upload-time = "2026-05-11T18:33:10.281Z" }, + { url = "https://files.pythonhosted.org/packages/17/99/bf21748626a40ce59fd29a39386ab46afec88b7bd2f0fa6c3a97c995523f/mypy-2.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5431d42af987ebd92ba2f71d45c85ed41d8e6ca9f5fd209a69f68f707d2469e5", size = 15272690, upload-time = "2026-05-11T18:32:07.205Z" }, + { url = "https://files.pythonhosted.org/packages/d6/d7/9e90d2cf47100bea550ed2bc7b0d4de3a62181d84d5e37da0003e8462637/mypy-2.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:767fe8c66dc3e01e19e1737d4c38ebefead16125e1b8e58ad421903b376f5c65", size = 11147435, upload-time = "2026-05-11T18:33:56.477Z" }, + { url = "https://files.pythonhosted.org/packages/ec/46/e5c449e858798e35ffc90946282a27c62a77be743fe17480e4977374eb91/mypy-2.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:ecfe70d43775ab99562ab128ce49854a362044c9f894961f68f898c23cb7429d", size = 10035052, upload-time = "2026-05-11T18:32:30.049Z" }, + { url = "https://files.pythonhosted.org/packages/b0/ca/b279a672e874aedd5498ae25f722dacc8aa86bbffb939b3f97cbb1cf6686/mypy-2.1.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:7354c5a7f69d9345c3d6e69921d57088eea3ddeeb6b20d34c1b3855b02c36ec2", size = 14848422, upload-time = "2026-05-11T18:35:45.984Z" }, + { url = "https://files.pythonhosted.org/packages/27/e6/3efe56c631d959b9b4454e208b0ac4b7f4f58b404c89f8bec7b49efdfc21/mypy-2.1.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:49890d4f76ac9e06ec117f9e09f3174da70a620a0c300953d8595c926e80947f", size = 13677374, upload-time = "2026-05-11T18:36:57.188Z" }, + { url = "https://files.pythonhosted.org/packages/84/7f/8107ea87a44fd1f1b59882442f033c9c3488c127201b1d1d15f1cbd6022e/mypy-2.1.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:761be68e023ef5d94678772396a8af1220030f80837a3afd8d0aef3b419666f4", size = 14055743, upload-time = "2026-05-11T18:35:18.361Z" }, + { url = "https://files.pythonhosted.org/packages/51/4d/b6d34db183133b83761b9199a82d31557cdbb70a380d8c3b3438e11882a3/mypy-2.1.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c90345fc182dc363b891350457ec69c35140858538f38b4540845afcc32b1aef", size = 15020937, upload-time = "2026-05-11T18:34:59.618Z" }, + { url = "https://files.pythonhosted.org/packages/ff/d7/f08360c691d758acb02f45022c34d98b92892f4ea756644e1000d4b9f3d8/mypy-2.1.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b84802e7b5a6daf1f5e15bc9fcd7ddae77be13981ffab037f1c67bb84d67d135", size = 15253371, upload-time = "2026-05-11T18:36:41.081Z" }, + { url = "https://files.pythonhosted.org/packages/67/1b/09460a13719530a19bce27bd3bc8449e83569dd2ba7faf51c9c3c30c0b61/mypy-2.1.0-cp314-cp314-win_amd64.whl", hash = "sha256:022c771234936ceac541ebaf836fe9e2abeb3f5e09aff21588fe543ff006fe21", size = 11326429, upload-time = "2026-05-11T18:34:13.526Z" }, + { url = "https://files.pythonhosted.org/packages/40/62/75dbf0f82f7b6680340efc614af29dd0b3c17b8a4f1cd09b8bd2fd6bc814/mypy-2.1.0-cp314-cp314-win_arm64.whl", hash = "sha256:498207db725cec88829a6a5c2fc771205fd043719ef98bc49aba8fb9fc4e6d57", size = 10218799, upload-time = "2026-05-11T18:32:23.491Z" }, + { url = "https://files.pythonhosted.org/packages/b2/66/caca04ed7d972fb6eb6dd1ccd6df1de5c38fae8c5b3dc1c4e8e0d85ee6b9/mypy-2.1.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7d5e5cad0efeba72b93cd17490cc0d69c5ac9ca132994fe3fb0314808aeeb83e", size = 15923458, upload-time = "2026-05-11T18:35:28.64Z" }, + { url = "https://files.pythonhosted.org/packages/ed/52/2d90cbe49d014b13ed7ff337930c30bad35893fe38a1e4641e756bb62191/mypy-2.1.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ff715050c127d724fd260a2e666e7747fdd83511c0c47d449d98238970aef780", size = 14757697, upload-time = "2026-05-11T18:36:14.208Z" }, + { url = "https://files.pythonhosted.org/packages/ac/37/d98f4a14e081b238992d0ed96b6d39c7cc0148c9699eb71eaa68629665ea/mypy-2.1.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:82208da9e09414d520e912d3e462d454854bed0810b71540bb016dcbca7308fd", size = 15405638, upload-time = "2026-05-11T18:33:48.249Z" }, + { url = "https://files.pythonhosted.org/packages/a3/c2/15c46613b24a84fad2aea1248bf9619b99c2767ae9071fe224c179a0b7d4/mypy-2.1.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e79ebc1b904b84f0310dff7469655a9c36c7a68bddb37bdd42b67a332df61d08", size = 16215852, upload-time = "2026-05-11T18:32:50.296Z" }, + { url = "https://files.pythonhosted.org/packages/5c/90/9c16a57f482c76d25f6379762b56bbf65c711d8158cf271fb2802cfb0640/mypy-2.1.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e583edc957cfb0deb142079162ae826f58449b116c1d442f2d91c69d9fced081", size = 16452695, upload-time = "2026-05-11T18:33:38.182Z" }, + { url = "https://files.pythonhosted.org/packages/0f/4c/215a4eeb63cacc5f17f516691ea7285d11e249802b942476bff15922a314/mypy-2.1.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b33b6cd332695bba180d55e717a79d3038e479a2c49cc5eb3d53603409b9a5d7", size = 12866622, upload-time = "2026-05-11T18:34:39.945Z" }, + { url = "https://files.pythonhosted.org/packages/4b/50/1043e1db5f455ffe4c9ab22747cd8ca2bc492b1e4f4e21b130a44ee2b217/mypy-2.1.0-cp314-cp314t-win_arm64.whl", hash = "sha256:4f910fe825376a7b66ef7ca8c98e5a149e8cd64c19ae71d84047a74ee060d4e6", size = 10610798, upload-time = "2026-05-11T18:36:31.444Z" }, + { url = "https://files.pythonhosted.org/packages/0d/2a/13ca1f292f6db1b98ff495ef3467736b331621c5917cad984b7043e7348d/mypy-2.1.0-py3-none-any.whl", hash = "sha256:a663814603a5c563fb87a4f96fb473eeb30d1f5a4885afcf44f9db000a366289", size = 2693302, upload-time = "2026-05-11T18:31:29.246Z" }, ] [[package]] @@ -277,30 +700,30 @@ wheels = [ ] [[package]] -name = "packaging" -version = "25.0" +name = "nodeenv" +version = "1.10.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +sdist = { url = "https://files.pythonhosted.org/packages/24/bf/d1bda4f6168e0b2e9e5958945e01910052158313224ada5ce1fb2e1113b8/nodeenv-1.10.0.tar.gz", hash = "sha256:996c191ad80897d076bdfba80a41994c2b47c68e224c542b48feba42ba00f8bb", size = 55611, upload-time = "2025-12-20T14:08:54.006Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, + { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] [[package]] -name = "pathlib" -version = "1.0.1" +name = "packaging" +version = "25.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ac/aa/9b065a76b9af472437a0059f77e8f962fe350438b927cb80184c32f075eb/pathlib-1.0.1.tar.gz", hash = "sha256:6940718dfc3eff4258203ad5021090933e5c04707d5ca8cc9e73c94a7894ea9f", size = 49298, upload-time = "2014-09-03T15:41:57.18Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/78/f9/690a8600b93c332de3ab4a344a4ac34f00c8f104917061f779db6a918ed6/pathlib-1.0.1-py3-none-any.whl", hash = "sha256:f35f95ab8b0f59e6d354090350b44a80a80635d22efdedfa84c7ad1cf0a74147", size = 14363, upload-time = "2022-05-04T13:37:20.585Z" }, + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] [[package]] name = "pathspec" -version = "0.12.1" +version = "1.1.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, + { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, ] [[package]] @@ -309,7 +732,8 @@ version = "1.31.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "python-dateutil" }, - { name = "scramp" }, + { name = "scramp", version = "1.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "scramp", version = "1.4.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/c8/9a/077ab21e700051e03d8c5232b6bcb9a1a4d4b6242c9a0226df2cfa306414/pg8000-1.31.5.tar.gz", hash = "sha256:46ebb03be52b7a77c03c725c79da2ca281d6e8f59577ca66b17c9009618cae78", size = 118933, upload-time = "2025-09-14T09:16:49.748Z" } wheels = [ @@ -333,7 +757,8 @@ name = "platformdirs" version = "4.5.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.15'", + "python_full_version >= '3.12' and python_full_version < '3.15'", "python_full_version == '3.11.*'", "python_full_version == '3.10.*'", ] @@ -352,59 +777,53 @@ wheels = [ ] [[package]] -name = "pygments" -version = "2.19.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, -] - -[[package]] -name = "pylint" -version = "3.3.9" +name = "pre-commit" +version = "4.3.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ "python_full_version < '3.10'", ] dependencies = [ - { name = "astroid", version = "3.3.11", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, - { name = "dill", marker = "python_full_version < '3.10'" }, - { name = "isort", version = "6.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "mccabe", marker = "python_full_version < '3.10'" }, - { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, - { name = "tomli", marker = "python_full_version < '3.10'" }, - { name = "tomlkit", marker = "python_full_version < '3.10'" }, - { name = "typing-extensions", marker = "python_full_version < '3.10'" }, + { name = "cfgv", version = "3.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "identify", version = "2.6.15", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "nodeenv", marker = "python_full_version < '3.10'" }, + { name = "pyyaml", marker = "python_full_version < '3.10'" }, + { name = "virtualenv", marker = "python_full_version < '3.10'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/04/9d/81c84a312d1fa8133b0db0c76148542a98349298a01747ab122f9314b04e/pylint-3.3.9.tar.gz", hash = "sha256:d312737d7b25ccf6b01cc4ac629b5dcd14a0fcf3ec392735ac70f137a9d5f83a", size = 1525946, upload-time = "2025-10-05T18:41:43.786Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ff/29/7cf5bbc236333876e4b41f56e06857a87937ce4bf91e117a6991a2dbb02a/pre_commit-4.3.0.tar.gz", hash = "sha256:499fe450cc9d42e9d58e606262795ecb64dd05438943c62b66f6a8673da30b16", size = 193792, upload-time = "2025-08-09T18:56:14.651Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/a7/69460c4a6af7575449e615144aa2205b89408dc2969b87bc3df2f262ad0b/pylint-3.3.9-py3-none-any.whl", hash = "sha256:01f9b0462c7730f94786c283f3e52a1fbdf0494bbe0971a78d7277ef46a751e7", size = 523465, upload-time = "2025-10-05T18:41:41.766Z" }, + { url = "https://files.pythonhosted.org/packages/5b/a5/987a405322d78a73b66e39e4a90e4ef156fd7141bf71df987e50717c321b/pre_commit-4.3.0-py2.py3-none-any.whl", hash = "sha256:2b0747ad7e6e967169136edffee14c16e148a778a54e4f967921aa1ebf2308d8", size = 220965, upload-time = "2025-08-09T18:56:13.192Z" }, ] [[package]] -name = "pylint" -version = "4.0.4" +name = "pre-commit" +version = "4.6.0" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.15'", + "python_full_version >= '3.12' and python_full_version < '3.15'", "python_full_version == '3.11.*'", "python_full_version == '3.10.*'", ] dependencies = [ - { name = "astroid", version = "4.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, - { name = "dill", marker = "python_full_version >= '3.10'" }, - { name = "isort", version = "7.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "mccabe", marker = "python_full_version >= '3.10'" }, - { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, - { name = "tomli", marker = "python_full_version == '3.10.*'" }, - { name = "tomlkit", marker = "python_full_version >= '3.10'" }, + { name = "cfgv", version = "3.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "identify", version = "2.6.19", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "nodeenv", marker = "python_full_version >= '3.10'" }, + { name = "pyyaml", marker = "python_full_version >= '3.10'" }, + { name = "virtualenv", marker = "python_full_version >= '3.10'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5a/d2/b081da1a8930d00e3fc06352a1d449aaf815d4982319fab5d8cdb2e9ab35/pylint-4.0.4.tar.gz", hash = "sha256:d9b71674e19b1c36d79265b5887bf8e55278cbe236c9e95d22dc82cf044fdbd2", size = 1571735, upload-time = "2025-11-30T13:29:04.315Z" } +sdist = { url = "https://files.pythonhosted.org/packages/8e/22/2de9408ac81acbb8a7d05d4cc064a152ccf33b3d480ebe0cd292153db239/pre_commit-4.6.0.tar.gz", hash = "sha256:718d2208cef53fdc38206e40524a6d4d9576d103eb16f0fec11c875e7716e9d9", size = 198525, upload-time = "2026-04-21T20:31:41.613Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/92/d40f5d937517cc489ad848fc4414ecccc7592e4686b9071e09e64f5e378e/pylint-4.0.4-py3-none-any.whl", hash = "sha256:63e06a37d5922555ee2c20963eb42559918c20bd2b21244e4ef426e7c43b92e0", size = 536425, upload-time = "2025-11-30T13:29:02.53Z" }, + { url = "https://files.pythonhosted.org/packages/80/6e/4b28b62ecb6aae56769c34a8ff1d661473ec1e9519e2d5f8b2c150086b26/pre_commit-4.6.0-py2.py3-none-any.whl", hash = "sha256:e2cf246f7299edcabcf15f9b0571fdce06058527f0a06535068a86d38089f29b", size = 226472, upload-time = "2026-04-21T20:31:40.092Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, ] [[package]] @@ -433,7 +852,8 @@ name = "pytest" version = "9.0.1" source = { registry = "https://pypi.org/simple" } resolution-markers = [ - "python_full_version >= '3.12'", + "python_full_version >= '3.15'", + "python_full_version >= '3.12' and python_full_version < '3.15'", "python_full_version == '3.11.*'", "python_full_version == '3.10.*'", ] @@ -451,6 +871,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/8b/6300fb80f858cda1c51ffa17075df5d846757081d11ab4aa35cef9e6258b/pytest-9.0.1-py3-none-any.whl", hash = "sha256:67be0030d194df2dfa7b556f2e56fb3c3315bd5c8822c6951162b92b32ce7dad", size = 373668, upload-time = "2025-11-12T13:05:07.379Z" }, ] +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage", version = "7.10.7", source = { registry = "https://pypi.org/simple" }, extra = ["toml"], marker = "python_full_version < '3.10'" }, + { name = "coverage", version = "7.14.1", source = { registry = "https://pypi.org/simple" }, extra = ["toml"], marker = "python_full_version >= '3.10'" }, + { name = "pluggy" }, + { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "pytest", version = "9.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + [[package]] name = "python-dateutil" version = "2.9.0.post0" @@ -464,12 +900,18 @@ wheels = [ ] [[package]] -name = "pytokens" -version = "0.3.0" +name = "python-discovery" +version = "1.4.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4e/8d/a762be14dae1c3bf280202ba3172020b2b0b4c537f94427435f19c413b72/pytokens-0.3.0.tar.gz", hash = "sha256:2f932b14ed08de5fcf0b391ace2642f858f1394c0857202959000b68ed7a458a", size = 17644, upload-time = "2025-11-05T13:36:35.34Z" } +dependencies = [ + { name = "filelock", version = "3.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "filelock", version = "3.29.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/12/38c1a0b1e64806780c9563e3fc9f6e472251839662587cfbe9bfaf2ae10a/python_discovery-1.4.0.tar.gz", hash = "sha256:eb8bc7daad3c226c147e45bb4e970a1feb1bf4048ee178e6db59e197b8010ce3", size = 68455, upload-time = "2026-05-28T01:15:37.639Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/84/25/d9db8be44e205a124f6c98bc0324b2bb149b7431c53877fc6d1038dddaf5/pytokens-0.3.0-py3-none-any.whl", hash = "sha256:95b2b5eaf832e469d141a378872480ede3f251a5a5041b8ec6e581d3ac71bbf3", size = 12195, upload-time = "2025-11-05T13:36:33.183Z" }, + { url = "https://files.pythonhosted.org/packages/c8/8d/3d316429f65029532bb1e28ff77b797d86b5ac3915bb44ca4e19aa283d43/python_discovery-1.4.0-py3-none-any.whl", hash = "sha256:26ed78d703e234879a66244c7d4114563fb13ec5cd30a2d1357e5fb4850782da", size = 33217, upload-time = "2026-05-28T01:15:36.573Z" }, ] [[package]] @@ -549,14 +991,35 @@ wheels = [ name = "scramp" version = "1.4.6" source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] dependencies = [ - { name = "asn1crypto" }, + { name = "asn1crypto", marker = "python_full_version < '3.10'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/58/77/6db18bab446c12cfbee22ca8f65d5b187966bd8f900aeb65db9e60d4be3d/scramp-1.4.6.tar.gz", hash = "sha256:fe055ebbebf4397b9cb323fcc4b299f219cd1b03fd673ca40c97db04ac7d107e", size = 16306, upload-time = "2025-07-05T14:44:03.977Z" } wheels = [ { url = "https://files.pythonhosted.org/packages/69/bf/54b5d40bea1c1805175ead2d496c267f05eec87561687dd73ab76869d8d9/scramp-1.4.6-py3-none-any.whl", hash = "sha256:a0cf9d2b4624b69bac5432dd69fecfc55a542384fe73c3a23ed9b138cda484e1", size = 12812, upload-time = "2025-07-05T14:44:02.345Z" }, ] +[[package]] +name = "scramp" +version = "1.4.8" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.15'", + "python_full_version >= '3.12' and python_full_version < '3.15'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "asn1crypto", marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/98/52/a866f1ac9ae9025ec7f9bea803bba9d54796f8a84236165a700831f61b27/scramp-1.4.8.tar.gz", hash = "sha256:bd018fabfe46343cceeb9f1c3e8d23f55770271e777e3accbfaee3ff0a316e71", size = 16630, upload-time = "2026-01-06T21:01:01.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/07/a962d2477331abfdb2c6a8251b65c673dbb07ad707d1882d61562b8b9147/scramp-1.4.8-py3-none-any.whl", hash = "sha256:87c2f15976845a2872fe5490a06097f0d01813cceb53774ea168c911f2ad025c", size = 13121, upload-time = "2026-01-06T21:00:59.474Z" }, +] + [[package]] name = "six" version = "1.17.0" @@ -615,15 +1078,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/b8/0135fadc89e73be292b473cb820b4f5a08197779206b33191e801feeae40/tomli-2.3.0-py3-none-any.whl", hash = "sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b", size = 14408, upload-time = "2025-10-08T22:01:46.04Z" }, ] -[[package]] -name = "tomlkit" -version = "0.13.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/cc/18/0bbf3884e9eaa38819ebe46a7bd25dcd56b67434402b66a58c4b8e552575/tomlkit-0.13.3.tar.gz", hash = "sha256:430cf247ee57df2b94ee3fbe588e71d362a941ebb545dec29b53961d61add2a1", size = 185207, upload-time = "2025-06-05T07:13:44.947Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bd/75/8539d011f6be8e29f339c42e633aae3cb73bffa95dd0f9adec09b9c58e85/tomlkit-0.13.3-py3-none-any.whl", hash = "sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0", size = 38901, upload-time = "2025-06-05T07:13:43.546Z" }, -] - [[package]] name = "typing-extensions" version = "4.15.0" @@ -634,10 +1088,19 @@ wheels = [ ] [[package]] -name = "zipp" -version = "3.23.0" +name = "virtualenv" +version = "21.4.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } +dependencies = [ + { name = "distlib" }, + { name = "filelock", version = "3.19.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "filelock", version = "3.29.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "python-discovery" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/95/f0/b47ecf438211a25a97f8f0e4b23c22bc2496ebfea18dd6ec16210f09cc36/virtualenv-21.4.1.tar.gz", hash = "sha256:2ca543c713b72840ceffd94e9bdedfbd09a661defa1f7f69e5429ad4059442e2", size = 7613344, upload-time = "2026-05-28T04:12:49.905Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, + { url = "https://files.pythonhosted.org/packages/ff/dc/ac4f3a987a87e1a18556896f257c4e15c95ed157b7975347ec6b313b75ce/virtualenv-21.4.1-py3-none-any.whl", hash = "sha256:caf4ff72d1b4039057f41d8e8466e859513d67c0400d9c6b62c02c9d1ebc3e12", size = 7594078, upload-time = "2026-05-28T04:12:47.686Z" }, ] From 013461a4b58cfdb26b75668586085704f0027e74 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Mon, 1 Jun 2026 10:34:32 -0400 Subject: [PATCH 36/49] Make properties defaults match original --- src/hashstore/filehashstore.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 682d4cb3..bba1e52b 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -90,7 +90,15 @@ class FileHashStoreProperties: "https://ns.dataone.org/service/types/v2.0#SystemMetadata" ) store_algorithm: str = "SHA-256" - store_default_algo_list: list[str] = dataclasses.field(default_factory=list) + store_default_algo_list: list[str] = dataclasses.field( + default_factory=lambda: [ + "MD5", + "SHA-1", + "SHA-256", + "SHA-384", + "SHA-512", + ] + ) def __post_init__(self): """Hash algorthm names are trnaslated from the DataONE names to the names From 3088d559b82220cd79876c8aa63997c07a7380fa Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:46:50 -0400 Subject: [PATCH 37/49] Generalize hashstore properties, enforce property validation, path consistency --- src/hashstore/__init__.py | 4 +- src/hashstore/filehashstore.py | 245 ++++-------------- src/hashstore/hashstore.py | 190 +++++++++++++- src/hashstore/hashstoreclient.py | 54 ++-- tests/conftest.py | 34 ++- tests/filehashstore/test_filehashstore.py | 8 +- .../test_filehashstoreproperties.py | 107 -------- tests/test_hashstore.py | 113 ++++---- 8 files changed, 345 insertions(+), 410 deletions(-) delete mode 100644 tests/filehashstore/test_filehashstoreproperties.py diff --git a/src/hashstore/__init__.py b/src/hashstore/__init__.py index be656f5e..92ede09a 100644 --- a/src/hashstore/__init__.py +++ b/src/hashstore/__init__.py @@ -16,7 +16,7 @@ system. """ -from hashstore.hashstore import HashStore, HashStoreFactory +from hashstore.hashstore import HashStore, HashStoreFactory, HashStoreProperties -__all__ = ("HashStore", "HashStoreFactory") +__all__ = ("HashStore", "HashStoreFactory", "HashStoreProperties") __version__ = "1.1.0" diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index bba1e52b..4469026c 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1,7 +1,6 @@ """Core module for FileHashStore""" import atexit -import dataclasses import fcntl import hashlib import inspect @@ -17,9 +16,7 @@ from tempfile import NamedTemporaryFile from typing import IO, Any, Optional, Union -import yaml - -from hashstore import HashStore +from hashstore import HashStore, HashStoreProperties from hashstore.filehashstore_exceptions import ( CidRefsContentError, CidRefsFileNotFound, @@ -38,181 +35,6 @@ UnsupportedAlgorithm, ) -DATAONE_ALGORITHM_TRANSLATION = { - "MD5": "md5", - "SHA-1": "sha1", - "SHA-256": "sha256", - "SHA-384": "sha384", - "SHA-512": "sha512", -} - -accepted_store_algorithms = ["MD5", "SHA-1", "SHA-256", "SHA-384", "SHA-512"] - - -def from_dataone_algorithm_name(algo: str) -> str: - """Translate from a DataONE algorithm name to a python hashlib name.""" - try: - return DATAONE_ALGORITHM_TRANSLATION[algo] - except KeyError: - pass - return algo - - -def to_dataone_algorithm_name(algo: str) -> str: - "Trnaslate from a python hashlib algorithm name to one used by DataONE." - for k, v in DATAONE_ALGORITHM_TRANSLATION.items(): - if algo == v: - return k - # no translation available, fail - msg = f"Algorithm {algo} is not used in DataONE." - raise KeyError(msg) - - -@dataclasses.dataclass -class FileHashStoreProperties: - """Configuration properties for a FileHashStore. - - Values are set to sensible defaults that correspond with the DataONE use. - - Once a hashstore is created, the properties are persisted to the - hashstore folder, so there's generally no need to use FileHashStoreProperties - except when creating a new HashStore instance. - - After intiailization, hash algorithm names are converted to the native - names used by python hashlib. When persisting or loading from YAML - config files, the DataONE hash names are expected. - """ - - # property names align with existing yaml config entries - store_depth: int = 3 - store_width: int = 2 - store_metadata_namespace: str = ( - "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - ) - store_algorithm: str = "SHA-256" - store_default_algo_list: list[str] = dataclasses.field( - default_factory=lambda: [ - "MD5", - "SHA-1", - "SHA-256", - "SHA-384", - "SHA-512", - ] - ) - - def __post_init__(self): - """Hash algorthm names are trnaslated from the DataONE names to the names - used by the python hashlib. - """ - # Impose reasonable defaults for folder path depth - if not (0 < self.store_depth <= 8): - msg = "store_depth not between 0 and 8" - raise ValueError(msg) - if not (1 <= self.store_width <= 4): - msg = "store_width not between 1 and 4" - raise ValueError(msg) - if ( - self.store_metadata_namespace is None - or len(self.store_metadata_namespace) < 1 - ): - msg = "Value is required for store_metadata_namespace." - raise ValueError(msg) - if self.store_algorithm not in DATAONE_ALGORITHM_TRANSLATION: - msg = ( - "store_algorithm must be one of " - f"{', '.join(DATAONE_ALGORITHM_TRANSLATION.keys())} " - f"not {self.store_algorithm}" - ) - raise ValueError(msg) - self.store_algorithm = from_dataone_algorithm_name(self.store_algorithm) - if self.store_algorithm not in hashlib.algorithms_available: - msg = f"store_algorithm: {self.store_algorithm} is not available." - raise ValueError(msg) - translated_algos = [] - for algo in self.store_default_algo_list: - translated_algo = from_dataone_algorithm_name(algo) - if translated_algo not in hashlib.algorithms_available: - msg = f"{algo} is not available." - raise ValueError(msg) - translated_algos.append(translated_algo) - self.store_default_algo_list = translated_algos - # Ensure that the store algorithm is included in the detault algorithm list - if self.store_algorithm not in self.store_default_algo_list: - self.store_default_algo_list.append(self.store_algorithm) - - @classmethod - def from_yaml(cls, source: Path) -> "FileHashStoreProperties": - """Load propertiees from yaml. - - Hash algorthm names are trnaslated from the DataONE names to the names - used by the python hashlib. - """ - with source.open("r") as data_source: - properties = yaml.safe_load(data_source) - return cls(**properties) - - @classmethod - def from_dict(cls, data: dict) -> "FileHashStoreProperties": - if data["store_algorithm"] not in DATAONE_ALGORITHM_TRANSLATION: - msg = ( - "store_algorithm must be one of " - f"{', '.join(DATAONE_ALGORITHM_TRANSLATION.keys())} " - f"not {data['store_algorithm']}" - ) - raise ValueError(msg) - return FileHashStoreProperties(**data) - - def to_yaml(self, dest_path: Path) -> None: - """Save properties to a yaml file. - - Hash algorithm names are translated to DataONE algorithm names. - """ - with dest_path.open("w") as dest: - dest.write("""# HashStore Configuration -# -############### Notes ############### -############### Directory Structure ############### -# store_depth -# - Desired amount of directories when sharding an object to form the permanent address -# - **WARNING**: DO NOT CHANGE UNLESS SETTING UP NEW HASHSTORE -# -# store_width -# - Width of directories created when sharding an object to form the permanent address -# - **WARNING**: DO NOT CHANGE UNLESS SETTING UP NEW HASHSTORE -# -# Example: -# Below, objects are shown listed in directories that are 3 levels deep (DIR_DEPTH=3), -# with each directory consisting of 2 characters (DIR_WIDTH=2). -# /var/filehashstore/objects -# ├── 7f -# │ └── 5c -# │ └── c1 -# │ └── 8f0b04e812a3b4c8f686ce34e6fec558804bf61e54b176742a7f6368d6 -# -############### Format of the Metadata ############### -# store_metadata_namespace -# - The default metadata format (ex. system metadata) -# -############### Hash Algorithms ############### -# store_algorithm -# - Hash algorithm to use when calculating object's hex digest for the permanent address -# -# store_default_algo_list -# - Algorithm values supported by python hashlib 3.9.0+ for File Hash Store (FHS) -# - The default algorithm list includes the hash algorithms calculated when storing an -# object to disk and returned to the caller after successful storage. - -""") - properties = dataclasses.asdict(self) - properties["store_algorithm"] = to_dataone_algorithm_name( - properties["store_algorithm"] - ) - translated_names = [] - for algo in properties.get("store_default_algo_list", []): - translated_names.append(to_dataone_algorithm_name(algo)) - properties["store_default_algo_list"] = translated_names - yaml.dump(properties, dest, default_flow_style=False) - class FileHashStore(HashStore): """FileHashStore is an object storage system that was extended from Derrick @@ -235,14 +57,6 @@ class FileHashStore(HashStore): - store_metadata_namespace (str): Namespace for the HashStore's system metadata. """ - # Property (hashstore configuration) requirements - property_required_keys = ( - "store_path", - "store_depth", - "store_width", - "store_algorithm", - "store_metadata_namespace", - ) # Permissions settings for writing files and creating directories f_mode = 0o664 d_mode = 0o755 @@ -258,21 +72,54 @@ class FileHashStore(HashStore): "blake2s", ) - def __init__(self, store_path: Path): + def __init__( + self, + store_path: str | Path, + store_properties: HashStoreProperties | dict | None = None, + ): + """Open or create a FileHashStore. + + - It is an error to include store_properties when opening an existing store. + - If the path exists, then config is loaded from the store_path. + - If path does not exist, then a store is created at that path with the + provided properties. + """ self.fhs_logger = logging.getLogger(__name__) + store_path = Path(store_path) config_path = FileHashStore.config_path(store_path) - if not config_path.exists() or not config_path.is_file(): - msg = f"No hashstore at: {store_path}" - raise RuntimeError(msg) - properties = FileHashStoreProperties.from_yaml(config_path) + if config_path.is_file(): + if store_properties is not None: + msg = "Do not include store_properties for an existing hashstore." + raise RuntimeError(msg) + store_properties = HashStoreProperties.from_yaml(config_path) + else: + if store_properties is None: + msg = f"No hashstore at: {store_path}" + raise RuntimeError(msg) + # verify reserved folders are not in store_path + _reserved_paths = ("objects", "metadata", "refs") + for _reserved_path in _reserved_paths: + if (store_path / _reserved_path).exists(): + msg = ( + "Cannot create FileHashstnore in existing folder " + "'{store_path}' with reserved subfolder: {_reserved_path}." + ) + raise RuntimeError(msg) + if isinstance(store_properties, dict): + store_properties = HashStoreProperties.from_dict(store_properties) + store_path.mkdir(parents=True, exist_ok=True) + store_properties.to_yaml(config_path) + + # sanity check + assert store_properties is not None self.root = store_path - self.depth = properties.store_depth - self.width = properties.store_width - self.sysmeta_ns = properties.store_metadata_namespace + self.depth = store_properties.store_depth + self.width = store_properties.store_width + self.sysmeta_ns = store_properties.store_metadata_namespace - self.algorithm = properties.store_algorithm - self.default_algo_list = properties.store_default_algo_list + self.algorithm = store_properties.store_algorithm + self.default_algo_list = store_properties.store_default_algo_list # Sanity check, ensure algorithm is in the detaulr list if self.algorithm not in self.default_algo_list: self.default_algo_list.append(self.algorithm) @@ -344,7 +191,7 @@ def config_path(root_path: Path) -> Path: @classmethod def create_hashstore( - cls, store_path: Path, properties: FileHashStoreProperties + cls, store_path: Path, properties: HashStoreProperties ) -> "FileHashStore": """Creates a new empty FileHashstore at the specified folder. @@ -367,7 +214,7 @@ def clone_empty(cls, new_path: Path, existing: "FileHashStore") -> "FileHashStor """Create a new empty HashStore with the same configuration properties as an existing one.""" config_path = cls.config_path(existing.root) - properties = FileHashStoreProperties.from_yaml(config_path) + properties = HashStoreProperties.from_yaml(config_path) return cls.create_hashstore(new_path, properties) # Public API / HashStore Interface Methods diff --git a/src/hashstore/hashstore.py b/src/hashstore/hashstore.py index c43f9716..ec9d26c7 100644 --- a/src/hashstore/hashstore.py +++ b/src/hashstore/hashstore.py @@ -1,10 +1,191 @@ """Hashstore Interface""" +import dataclasses +import hashlib import importlib.metadata import importlib.util import pathlib from abc import ABC, abstractmethod +import yaml + +DATAONE_ALGORITHM_TRANSLATION = { + "MD5": "md5", + "SHA-1": "sha1", + "SHA-256": "sha256", + "SHA-384": "sha384", + "SHA-512": "sha512", +} + + +def from_dataone_algorithm_name(algo: str) -> str: + """Translate from a DataONE algorithm name to a python hashlib name.""" + try: + return DATAONE_ALGORITHM_TRANSLATION[algo] + except KeyError: + pass + return algo + + +def to_dataone_algorithm_name(algo: str) -> str: + "Trnaslate from a python hashlib algorithm name to one used by DataONE." + for k, v in DATAONE_ALGORITHM_TRANSLATION.items(): + if algo == v: + return k + # no translation available, fail + msg = f"Algorithm {algo} is not used in DataONE." + raise KeyError(msg) + + +@dataclasses.dataclass +class HashStoreProperties: + """Configuration properties for a HashStore. + + Values are set to sensible defaults that correspond with the DataONE use. + + Once a hashstore is created, the properties are persisted to the + hashstore folder, so there's generally no need to use HashStoreProperties + except when creating a new HashStore instance. + + After intiailization, hash algorithm names are converted to the native + names used by python hashlib. When persisting or loading from YAML + config files, the DataONE hash names are expected. + """ + + # property names align with existing yaml config entries + store_depth: int = 3 + store_width: int = 2 + store_metadata_namespace: str = ( + "https://ns.dataone.org/service/types/v2.0#SystemMetadata" + ) + store_algorithm: str = "SHA-256" + store_default_algo_list: list[str] = dataclasses.field( + default_factory=lambda: [ + "MD5", + "SHA-1", + "SHA-256", + "SHA-384", + "SHA-512", + ] + ) + + def __post_init__(self): + """Hash algorthm names are trnaslated from the DataONE names to the names + used by the python hashlib. + """ + if isinstance(self.store_depth, str): + self.store_depth = int(self.store_depth) + if isinstance(self.store_width, str): + self.store_width = int(self.store_width) + # Impose reasonable defaults for folder path depth + if not (0 < self.store_depth <= 8): + msg = "store_depth not between 0 and 8" + raise ValueError(msg) + if not (1 <= self.store_width <= 4): + msg = "store_width not between 1 and 4" + raise ValueError(msg) + if ( + self.store_metadata_namespace is None + or len(self.store_metadata_namespace) < 1 + ): + msg = "Value is required for store_metadata_namespace." + raise ValueError(msg) + if self.store_algorithm not in DATAONE_ALGORITHM_TRANSLATION: + msg = ( + "store_algorithm must be one of " + f"{', '.join(DATAONE_ALGORITHM_TRANSLATION.keys())} " + f"not {self.store_algorithm}" + ) + raise ValueError(msg) + self.store_algorithm = from_dataone_algorithm_name(self.store_algorithm) + if self.store_algorithm not in hashlib.algorithms_available: + msg = f"store_algorithm: {self.store_algorithm} is not available." + raise ValueError(msg) + translated_algos = [] + for algo in self.store_default_algo_list: + translated_algo = from_dataone_algorithm_name(algo) + if translated_algo not in hashlib.algorithms_available: + msg = f"{algo} is not available." + raise ValueError(msg) + translated_algos.append(translated_algo) + self.store_default_algo_list = translated_algos + # Ensure that the store algorithm is included in the detault algorithm list + if self.store_algorithm not in self.store_default_algo_list: + self.store_default_algo_list.append(self.store_algorithm) + + @classmethod + def from_yaml(cls, source: pathlib.Path) -> "HashStoreProperties": + """Load propertiees from yaml. + + Hash algorthm names are trnaslated from the DataONE names to the names + used by the python hashlib. + """ + with source.open("r") as data_source: + properties = yaml.safe_load(data_source) + return cls(**properties) + + @classmethod + def from_dict(cls, data: dict) -> "HashStoreProperties": + if data["store_algorithm"] not in DATAONE_ALGORITHM_TRANSLATION: + msg = ( + "store_algorithm must be one of " + f"{', '.join(DATAONE_ALGORITHM_TRANSLATION.keys())} " + f"not {data['store_algorithm']}" + ) + raise ValueError(msg) + return HashStoreProperties(**data) + + def to_yaml(self, dest_path: pathlib.Path) -> None: + """Save properties to a yaml file. + + Hash algorithm names are translated to DataONE algorithm names. + """ + with dest_path.open("w") as dest: + dest.write("""# HashStore Configuration +# +############### Notes ############### +############### Directory Structure ############### +# store_depth +# - Desired amount of directories when sharding an object to form the permanent address +# - **WARNING**: DO NOT CHANGE UNLESS SETTING UP NEW HASHSTORE +# +# store_width +# - Width of directories created when sharding an object to form the permanent address +# - **WARNING**: DO NOT CHANGE UNLESS SETTING UP NEW HASHSTORE +# +# Example: +# Below, objects are shown listed in directories that are 3 levels deep (DIR_DEPTH=3), +# with each directory consisting of 2 characters (DIR_WIDTH=2). +# /var/filehashstore/objects +# ├── 7f +# │ └── 5c +# │ └── c1 +# │ └── 8f0b04e812a3b4c8f686ce34e6fec558804bf61e54b176742a7f6368d6 +# +############### Format of the Metadata ############### +# store_metadata_namespace +# - The default metadata format (ex. system metadata) +# +############### Hash Algorithms ############### +# store_algorithm +# - Hash algorithm to use when calculating object's hex digest for the permanent address +# +# store_default_algo_list +# - Algorithm values supported by python hashlib 3.9.0+ for File Hash Store (FHS) +# - The default algorithm list includes the hash algorithms calculated when storing an +# object to disk and returned to the caller after successful storage. + +""") + properties = dataclasses.asdict(self) + properties["store_algorithm"] = to_dataone_algorithm_name( + properties["store_algorithm"] + ) + translated_names = [] + for algo in properties.get("store_default_algo_list", []): + translated_names.append(to_dataone_algorithm_name(algo)) + properties["store_default_algo_list"] = translated_names + yaml.dump(properties, dest, default_flow_style=False) + class HashStore(ABC): """HashStore is a content-addressable file management @@ -196,7 +377,9 @@ class HashStoreFactory: name (e.g., "FileHashStore").""" @staticmethod - def get_hashstore(module_name, class_name, properties=None): + def get_hashstore( + module_name: str, class_name: str, properties: dict | None = None + ): """Get a `HashStore`-like object based on the specified `module_name` and `class_name`. @@ -231,9 +414,12 @@ def get_hashstore(module_name, class_name, properties=None): # Get HashStore imported_module = importlib.import_module(module_name) + if properties is None: + properties = {} + # If class is not part of module, raise error if hasattr(imported_module, class_name): hashstore_class = getattr(imported_module, class_name) - return hashstore_class(properties=properties) + return hashstore_class(**properties) msg = f"Class name '{class_name}' is not an attribute of module '{module_name}'" raise AttributeError(msg) diff --git a/src/hashstore/hashstoreclient.py b/src/hashstore/hashstoreclient.py index a6f989d8..536aee79 100644 --- a/src/hashstore/hashstoreclient.py +++ b/src/hashstore/hashstoreclient.py @@ -10,7 +10,7 @@ import pg8000 import yaml -from hashstore import HashStoreFactory +from hashstore import HashStoreFactory, HashStoreProperties class HashStoreParser: @@ -195,7 +195,7 @@ def __init__(self): ) @staticmethod - def load_store_properties(hashstore_yaml): + def load_store_properties(hashstore_yaml: str): """Get and return the contents of the current HashStore config file. :return: HashStore properties with the following keys (and values): @@ -208,31 +208,13 @@ def load_store_properties(hashstore_yaml): metadata. :rtype: dict """ - property_required_keys = [ - "store_depth", - "store_width", - "store_algorithm", - "store_metadata_namespace", - ] - if not os.path.exists(hashstore_yaml): exception_string = ( "HashStoreParser - load_store_properties: hashstore.yaml not found" " in store root path." ) raise FileNotFoundError(exception_string) - # Open file - with open(hashstore_yaml, encoding="utf-8") as file: - yaml_data = yaml.safe_load(file) - - # Get hashstore properties - hashstore_yaml_dict = {} - for key in property_required_keys: - checked_property = yaml_data[key] - if key == "store_depth" or key == "store_width": - checked_property = int(yaml_data[key]) - hashstore_yaml_dict[key] = checked_property - return hashstore_yaml_dict + return HashStoreProperties.from_yaml(Path(hashstore_yaml)) def get_parser_args(self): """Get command line arguments.""" @@ -245,7 +227,12 @@ class HashStoreClient: OBJ_TYPE = "object" MET_TYPE = "metadata" - def __init__(self, properties, testflag=None): + def __init__( + self, + store_path: Path, + store_properties: HashStoreProperties | None = None, + testflag: bool | None = None, + ): """Initialize the HashStoreClient with optional flag to test with the test server at 'test.arcticdata.io' @@ -268,6 +255,7 @@ def __init__(self, properties, testflag=None): "HashStoreClient - use_multiprocessing (bool): %s", use_multiprocessing ) + properties = {"store_path": store_path, "store_properties": store_properties} # Instance attributes self.hashstore = factory.get_hashstore(module_name, class_name, properties) logging.info("HashStoreClient - HashStore initialized.") @@ -744,14 +732,13 @@ def main(): if args.create_hashstore: # Create HashStore if -chs flag is true in a given directory # Get store attributes, HashStore will validate properties - props = { - "store_path": args.store_path, - "store_depth": int(args.depth), - "store_width": int(args.width), - "store_algorithm": args.algorithm, - "store_metadata_namespace": args.formatid, - } - HashStoreClient(props) + props = HashStoreProperties( + store_depth=int(args.depth), + store_width=int(args.width), + store_algorithm=args.algorithm, + store_metadata_namespace=args.formatid, + ) + HashStoreClient(args.store_path, props) # Can't use client app without first initializing HashStore store_path = args.store_path store_path_config_yaml = store_path + "/hashstore.yaml" @@ -795,10 +782,9 @@ def main(): formatid = default_formatid knbvm_test = args.knbvm_flag # Instantiate HashStore Client - props = parser.load_store_properties(store_path_config_yaml) - # Reminder: 'hashstore.yaml' only contains 4 of the required 5 properties - props["store_path"] = store_path - hashstore_c = HashStoreClient(props, knbvm_test) + # props = parser.load_store_properties(store_path_config_yaml) + props = None + hashstore_c = HashStoreClient(store_path, props, knbvm_test) if knbvm_test: directory_to_convert = args.source_directory # Check if the directory to convert exists diff --git a/tests/conftest.py b/tests/conftest.py index f2e9c8e7..44ddb4e9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,7 +2,8 @@ import pytest -from hashstore.filehashstore import FileHashStore, FileHashStoreProperties +from hashstore import HashStoreProperties +from hashstore.filehashstore import FileHashStore def pytest_addoption(parser): @@ -15,26 +16,31 @@ def pytest_addoption(parser): ) -@pytest.fixture(name="props") -def init_props(): - """Properties to initialize HashStore.""" - return FileHashStoreProperties( - store_depth=3, - store_width=2, - store_algorithm="SHA-256", - store_metadata_namespace="https://ns.dataone.org/service/types/v2.0#SystemMetadata", - ) - - @pytest.fixture(name="hashstore_path") def hastore_path(tmp_path): return tmp_path / "metacat" / "hashstore" +@pytest.fixture(name="props") +def init_props(hashstore_path): + """Properties to initialize HashStore.""" + return { + "store_path": hashstore_path, + "store_properties": HashStoreProperties( + store_depth=3, + store_width=2, + store_algorithm="SHA-256", + store_metadata_namespace="https://ns.dataone.org/service/types/v2.0#SystemMetadata", + ), + } + + @pytest.fixture(name="store") -def init_store(hashstore_path, props): +def init_store(props): """Create FileHashStore instance for all tests.""" - return FileHashStore.create_hashstore(hashstore_path, props) + return FileHashStore( + props["store_path"], store_properties=props["store_properties"] + ) @pytest.fixture(name="pids") diff --git a/tests/filehashstore/test_filehashstore.py b/tests/filehashstore/test_filehashstore.py index 156fa95a..1754bfd7 100644 --- a/tests/filehashstore/test_filehashstore.py +++ b/tests/filehashstore/test_filehashstore.py @@ -8,9 +8,9 @@ import pytest +from hashstore import HashStoreProperties from hashstore.filehashstore import ( FileHashStore, - FileHashStoreProperties, ObjectMetadata, Stream, ) @@ -58,7 +58,7 @@ def test_init_existing_store_incorrect_algorithm_format(): "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", } with pytest.raises(ValueError, match="must be one of"): - FileHashStoreProperties.from_dict(properties) + HashStoreProperties.from_dict(properties) def test_init_store_correct_algorithm_format(tmp_path): @@ -70,7 +70,7 @@ def test_init_store_correct_algorithm_format(tmp_path): "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", } hashstore_instance = FileHashStore.create_hashstore( - tmp_path / "new-store", FileHashStoreProperties(**properties) + tmp_path / "new-store", HashStoreProperties(**properties) ) assert isinstance(hashstore_instance, FileHashStore) @@ -84,7 +84,7 @@ def test_load_properties_hashstore_yaml_missing(store): """Confirm FileNotFoundError is raised when hashstore.yaml does not exist.""" os.remove(FileHashStore.config_path(store.root)) with pytest.raises(FileNotFoundError): - _ = FileHashStoreProperties.from_yaml(FileHashStore.config_path(store.root)) + _ = HashStoreProperties.from_yaml(FileHashStore.config_path(store.root)) # Tests for FileHashStore Core Methods diff --git a/tests/filehashstore/test_filehashstoreproperties.py b/tests/filehashstore/test_filehashstoreproperties.py deleted file mode 100644 index 2746f047..00000000 --- a/tests/filehashstore/test_filehashstoreproperties.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Test cases for FileHashStoreProperties""" - -import pytest - -import hashstore.filehashstore - - -def test_defaults(): - p = hashstore.filehashstore.FileHashStoreProperties() - assert p.store_width == 2 - assert p.store_depth == 3 - assert len(p.store_metadata_namespace) > 1 - assert p.store_algorithm == "sha256" - assert len(p.store_default_algo_list) >= 1 - assert p.store_algorithm in p.store_default_algo_list - - -def test_from_dict(): - """Veryify initialization from a dict of values.""" - props = { - "store_width": 2, - "store_depth": 3, - "store_algorithm": "SHA-256", - "store_default_algo_list": [ - "MD5", - ], - } - p = hashstore.filehashstore.FileHashStoreProperties.from_dict(props) - assert p.store_width == props["store_width"] - assert p.store_depth == props["store_depth"] - assert p.store_algorithm == hashstore.filehashstore.from_dataone_algorithm_name( - props["store_algorithm"] - ) - assert len(p.store_default_algo_list) == 2 - assert p.store_algorithm in p.store_default_algo_list - - -def test_from_yaml(tmp_path): - """Verify storing and loading a YAML file""" - props = { - "store_width": 2, - "store_depth": 3, - "store_algorithm": "SHA-256", - "store_default_algo_list": [ - "MD5", - ], - } - p = hashstore.filehashstore.FileHashStoreProperties.from_dict(props) - yaml_path = tmp_path / "config.yaml" - p.to_yaml(yaml_path) - p2 = hashstore.filehashstore.FileHashStoreProperties.from_yaml(yaml_path) - assert p == p2 - - -def test_invalid_properties(): - """Test initialization with various invalid property values.""" - props = { - "store_width": 256, - "store_depth": 3, - "store_algorithm": "SHA-256", - "store_default_algo_list": [ - "MD5", - ], - } - with pytest.raises(ValueError, match="store_width"): - hashstore.filehashstore.FileHashStoreProperties.from_dict(props) - props = { - "store_width": 2, - "store_depth": 30, - "store_algorithm": "SHA-256", - "store_default_algo_list": [ - "MD5", - ], - } - with pytest.raises(ValueError, match="store_depth"): - hashstore.filehashstore.FileHashStoreProperties.from_dict(props) - props = { - "store_width": 2, - "store_depth": 3, - "store_algorithm": "blake2s", - "store_default_algo_list": [ - "MD5", - ], - } - with pytest.raises(ValueError, match="algorithm"): - hashstore.filehashstore.FileHashStoreProperties.from_dict(props) - props = { - "store_width": 2, - "store_depth": 3, - "store_algorithm": "SHA-256", - "store_default_algo_list": [ - "foo", - ], - } - with pytest.raises(ValueError, match="not available"): - hashstore.filehashstore.FileHashStoreProperties.from_dict(props) - props = { - "store_width": 2, - "store_depth": 3, - "store_algorithm": "SHA-256", - "store_default_algo_list": [ - "MD%", - ], - "store_metadata_namespace": None, - } - with pytest.raises(ValueError, match="store_metadata_namespace"): - hashstore.filehashstore.FileHashStoreProperties.from_dict(props) diff --git a/tests/test_hashstore.py b/tests/test_hashstore.py index a2a28569..d2697b9c 100644 --- a/tests/test_hashstore.py +++ b/tests/test_hashstore.py @@ -1,7 +1,5 @@ """Test module for HashStore's HashStoreFactory and ObjectMetadata class.""" -import os - import pytest from hashstore.filehashstore import FileHashStore @@ -44,36 +42,42 @@ def test_factory_get_hashstore_unsupported_module(factory): factory.get_hashstore(module_name, class_name) -def test_factory_get_hashstore_filehashstore_unsupported_algorithm(factory): +def test_factory_get_hashstore_filehashstore_unsupported_algorithm(tmp_path, factory): """Check factory raises exception with store algorithm value that is not part of the default list.""" module_name = "hashstore.filehashstore" class_name = "FileHashStore" properties = { - "store_path": os.getcwd() + "/metacat/hashstore", - "store_depth": 3, - "store_width": 2, - "store_algorithm": "MD2", - "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", + "store_path": tmp_path / "metacat/hashstore", + "store_properties": { + "store_depth": 3, + "store_width": 2, + "store_algorithm": "MD2", + "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", + }, } - with pytest.raises(ValueError, match="Must be one of"): + with pytest.raises(ValueError, match="store_algorithm"): factory.get_hashstore(module_name, class_name, properties) -def test_factory_get_hashstore_filehashstore_incorrect_algorithm_format(factory): +def test_factory_get_hashstore_filehashstore_incorrect_algorithm_format( + tmp_path, factory +): """Check factory raises exception with incorrectly formatted algorithm value.""" module_name = "hashstore.filehashstore" class_name = "FileHashStore" properties = { - "store_path": os.getcwd() + "/metacat/hashstore", - "store_depth": 3, - "store_width": 2, - "store_algorithm": "dou_algo", - "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", + "store_path": tmp_path / "metacat/hashstore", + "store_properties": { + "store_depth": 3, + "store_width": 2, + "store_algorithm": "dou_algo", + "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", + }, } - with pytest.raises(ValueError, match="Must be one of"): + with pytest.raises(ValueError, match="store_algorithm"): factory.get_hashstore(module_name, class_name, properties) @@ -88,10 +92,12 @@ def test_factory_get_hashstore_filehashstore_conflicting_obj_dir(factory, tmp_pa properties = { "store_path": douhspath, - "store_depth": 3, - "store_width": 2, - "store_algorithm": "SHA-256", - "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", + "store_properties": { + "store_depth": 3, + "store_width": 2, + "store_algorithm": "SHA-256", + "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", + }, } with pytest.raises(RuntimeError): factory.get_hashstore(module_name, class_name, properties) @@ -110,10 +116,12 @@ def test_factory_get_hashstore_filehashstore_conflicting_metadata_dir( properties = { "store_path": douhspath, - "store_depth": 3, - "store_width": 2, - "store_algorithm": "SHA-256", - "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", + "store_properties": { + "store_depth": 3, + "store_width": 2, + "store_algorithm": "SHA-256", + "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", + }, } with pytest.raises(RuntimeError): factory.get_hashstore(module_name, class_name, properties) @@ -130,10 +138,12 @@ def test_factory_get_hashstore_filehashstore_conflicting_refs_dir(factory, tmp_p properties = { "store_path": douhspath, - "store_depth": 3, - "store_width": 2, - "store_algorithm": "SHA-256", - "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", + "store_properties": { + "store_depth": 3, + "store_width": 2, + "store_algorithm": "SHA-256", + "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", + }, } with pytest.raises(RuntimeError): factory.get_hashstore(module_name, class_name, properties) @@ -150,16 +160,37 @@ def test_factory_get_hashstore_filehashstore_nonconflicting_dir(factory, tmp_pat properties = { "store_path": douhspath, - "store_depth": 3, - "store_width": 2, - "store_algorithm": "SHA-256", - "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", + "store_properties": { + "store_depth": 3, + "store_width": 2, + "store_algorithm": "SHA-256", + "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", + }, } factory.get_hashstore(module_name, class_name, properties) -def test_factory_get_hashstore_filehashstore_string_int_prop(factory, tmp_path): +int_properties_as_strings = ( + { + "store_depth": "3", + "store_width": "2", + "store_algorithm": "SHA-256", + "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", + }, + { + "store_depth": str(3), + "store_width": str(2), + "store_algorithm": "SHA-256", + "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", + }, +) + + +@pytest.mark.parametrize("str_properties", int_properties_as_strings) +def test_factory_get_hashstore_filehashstore_string_int_prop( + factory, tmp_path, str_properties +): """Check factory does not raise exception when an integer is passed as a string in a properties object.""" module_name = "hashstore.filehashstore" @@ -171,20 +202,6 @@ def test_factory_get_hashstore_filehashstore_string_int_prop(factory, tmp_path): properties = { "store_path": douhspath, - "store_depth": "3", - "store_width": "2", - "store_algorithm": "SHA-256", - "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", + "store_properties": str_properties, } - - factory.get_hashstore(module_name, class_name, properties) - - properties = { - "store_path": douhspath, - "store_depth": str(3), - "store_width": str(2), - "store_algorithm": "SHA-256", - "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", - } - factory.get_hashstore(module_name, class_name, properties) From 0dcf91a6ed79c460b9a435de56e25886e410bdbd Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Mon, 1 Jun 2026 14:55:10 -0400 Subject: [PATCH 38/49] Add missing tests --- tests/test_hashstoreproperties.py | 108 ++++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 tests/test_hashstoreproperties.py diff --git a/tests/test_hashstoreproperties.py b/tests/test_hashstoreproperties.py new file mode 100644 index 00000000..020fd1d2 --- /dev/null +++ b/tests/test_hashstoreproperties.py @@ -0,0 +1,108 @@ +"""Test cases for FileHashStoreProperties""" + +import pytest + +import hashstore +import hashstore.hashstore + + +def test_defaults(): + p = hashstore.HashStoreProperties() + assert p.store_width == 2 + assert p.store_depth == 3 + assert len(p.store_metadata_namespace) > 1 + assert p.store_algorithm == "sha256" + assert len(p.store_default_algo_list) >= 1 + assert p.store_algorithm in p.store_default_algo_list + + +def test_from_dict(): + """Veryify initialization from a dict of values.""" + props = { + "store_width": 2, + "store_depth": 3, + "store_algorithm": "SHA-256", + "store_default_algo_list": [ + "MD5", + ], + } + p = hashstore.HashStoreProperties.from_dict(props) + assert p.store_width == props["store_width"] + assert p.store_depth == props["store_depth"] + assert p.store_algorithm == hashstore.hashstore.from_dataone_algorithm_name( + props["store_algorithm"] + ) + assert len(p.store_default_algo_list) == 2 + assert p.store_algorithm in p.store_default_algo_list + + +def test_from_yaml(tmp_path): + """Verify storing and loading a YAML file""" + props = { + "store_width": 2, + "store_depth": 3, + "store_algorithm": "SHA-256", + "store_default_algo_list": [ + "MD5", + ], + } + p = hashstore.HashStoreProperties.from_dict(props) + yaml_path = tmp_path / "config.yaml" + p.to_yaml(yaml_path) + p2 = hashstore.HashStoreProperties.from_yaml(yaml_path) + assert p == p2 + + +def test_invalid_properties(): + """Test initialization with various invalid property values.""" + props = { + "store_width": 256, + "store_depth": 3, + "store_algorithm": "SHA-256", + "store_default_algo_list": [ + "MD5", + ], + } + with pytest.raises(ValueError, match="store_width"): + hashstore.HashStoreProperties.from_dict(props) + props = { + "store_width": 2, + "store_depth": 30, + "store_algorithm": "SHA-256", + "store_default_algo_list": [ + "MD5", + ], + } + with pytest.raises(ValueError, match="store_depth"): + hashstore.HashStoreProperties.from_dict(props) + props = { + "store_width": 2, + "store_depth": 3, + "store_algorithm": "blake2s", + "store_default_algo_list": [ + "MD5", + ], + } + with pytest.raises(ValueError, match="algorithm"): + hashstore.HashStoreProperties.from_dict(props) + props = { + "store_width": 2, + "store_depth": 3, + "store_algorithm": "SHA-256", + "store_default_algo_list": [ + "foo", + ], + } + with pytest.raises(ValueError, match="not available"): + hashstore.HashStoreProperties.from_dict(props) + props = { + "store_width": 2, + "store_depth": 3, + "store_algorithm": "SHA-256", + "store_default_algo_list": [ + "MD%", + ], + "store_metadata_namespace": None, + } + with pytest.raises(ValueError, match="store_metadata_namespace"): + hashstore.HashStoreProperties.from_dict(props) From f2ea6265671302906a7a1d03034013c5a6f95628 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 2 Jun 2026 09:02:31 -0400 Subject: [PATCH 39/49] Enable ruff pre-commit --- .pre-commit-config.yaml | 18 +- src/hashstore/basehashstore.py | 37 ++- src/hashstore/filehashstore.py | 387 +++++++++++----------- tests/filehashstore/test_filehashstore.py | 28 +- 4 files changed, 228 insertions(+), 242 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7ee51d79..f4e29488 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,12 +21,12 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - # - repo: https://github.com/astral-sh/ruff-pre-commit - # rev: "v0.14.13" - # hooks: - # # first, lint + autofix - # - id: ruff - # types_or: [python, pyi, jupyter] - # args: ["--fix", "--show-fixes"] - # # then, format - # - id: ruff-format + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.14.13" + hooks: + # first, lint + autofix + - id: ruff + types_or: [python, pyi, jupyter] + args: ["--fix", "--show-fixes"] + # then, format + - id: ruff-format diff --git a/src/hashstore/basehashstore.py b/src/hashstore/basehashstore.py index aa645068..a3f91477 100644 --- a/src/hashstore/basehashstore.py +++ b/src/hashstore/basehashstore.py @@ -4,7 +4,7 @@ import importlib.util from abc import ABC, abstractmethod from collections.abc import Generator -from typing import IO, Optional +from typing import IO import hashstore.folderentry @@ -100,9 +100,9 @@ def store_folder( self, pidpath: list[str], entries: hashstore.folderentry.FolderEntries, - additional_algorithm: Optional[str] = None, - checksum: Optional[str] = None, - checksum_algorithm: Optional[str] = None, + additional_algorithm: str | None = None, + checksum: str | None = None, + checksum_algorithm: str | None = None, verify_entry_cids: bool = True, ): """Store a folder object. @@ -155,7 +155,8 @@ def retrieve_folder( was unchanged between versions. Args: - pid (str): The context (i.e. VMDAG version) within which this folder is being retrieved + pid (str): The context (i.e. VMDAG version) within which this folder is + being retrieved path (str): Path within the context to the desired entry Returns: FolderEntries @@ -213,9 +214,9 @@ def retrieve_object(self, pid): @abstractmethod def retrieve_object_path(self, pidpath: list[str]) -> IO[bytes]: - """Retrieve an object from disk using a persistent identifier (pid). The `retrieve_object` - method opens and returns a buffered object stream ready for reading if the object - associated with the provided `pid` exists on disk. + """Retrieve an object from disk using a persistent identifier (pid). The + `retrieve_object` method opens and returns a buffered object stream ready + for reading if the object associated with the provided `pid` exists on disk. :param str pid: Authority-based identifier. @@ -289,7 +290,7 @@ def get_hex_digest(self, pid, algorithm): raise NotImplementedError() @abstractmethod - def list_pids(self, pattern: Optional[str] = None) -> Generator: + def list_pids(self, pattern: str | None = None) -> Generator: """Yields PIDs from the hashstore. :param str pattern: Optional regexp pattern to match. @@ -298,7 +299,8 @@ def list_pids(self, pattern: Optional[str] = None) -> Generator: @abstractmethod def get_object_status(self, pid) -> dict: - """Returns a dictionary of the object size, modtime, accesstime for the given pid. + """Returns a dictionary of the object size, modtime, accesstime for the given + pid. :param str pid: Object identifier @@ -308,13 +310,14 @@ def get_object_status(self, pid) -> dict: @abstractmethod def find_object(self, pid: str) -> dict[str, str]: - """Check if an object referenced by a pid exists and retrieve its content identifier. - - The `find_object` method validates the existence of an object based on the provided - pid and returns the associated content identifier and information about how to - retrieve various accoutrements. Note that the returned dict will contain values - relevant to the type of store, but will always contain a `cid` key if the object - is present. + """Check if an object referenced by a pid exists and retrieve its content + identifier. + + The `find_object` method validates the existence of an object based on the + provided pid and returns the associated content identifier and information + about how to retrieve various accoutrements. Note that the returned dict will + contain values relevant to the type of store, but will always contain a `cid` + key if the object is present. :param str pid: Authority-based or persistent identifier of the object. diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 95b8846d..b56ef283 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -11,11 +11,12 @@ import re import shutil import threading +from collections.abc import Generator from contextlib import closing from dataclasses import dataclass from pathlib import Path from tempfile import NamedTemporaryFile -from typing import IO, Any, Dict, Generator, List, Optional, Set, Tuple, Union +from typing import IO, Any, Optional, Union import yaml @@ -208,7 +209,7 @@ def __init__(self, properties=None): @staticmethod def _load_properties( hashstore_yaml_path: Path, hashstore_required_prop_keys: list[str] - ) -> dict[str, Union[str, int]]: + ) -> dict[str, str | int]: """Get and return the contents of the current HashStore configuration. :return: HashStore properties with the following keys (and values): @@ -237,7 +238,7 @@ def _load_properties( logging.debug("Successfully retrieved 'hashstore.yaml' properties.") return hashstore_yaml_dict - def _write_properties(self, properties: dict[str, Union[str, int]]) -> None: + def _write_properties(self, properties: dict[str, str | int]) -> None: """Writes 'hashstore.yaml' to FileHashStore's root directory with the respective properties object supplied. @@ -386,7 +387,7 @@ def _build_hashstore_yaml_string( ) def _verify_hashstore_properties( - self, properties: dict[str, Union[str, int]], prop_store_path: str + self, properties: dict[str, str | int], prop_store_path: str ) -> None: """Determines whether FileHashStore can instantiate by validating a set of arguments and throwing exceptions. HashStore will not instantiate if @@ -446,8 +447,8 @@ def _verify_hashstore_properties( raise RuntimeError(err_msg) def _validate_properties( - self, properties: dict[str, Union[str, int]] - ) -> dict[str, Union[str, int]]: + self, properties: dict[str, str | int] + ) -> dict[str, str | int]: """Validate a properties dictionary by checking if it contains all the required keys and non-None values. @@ -536,77 +537,74 @@ def lookup_algo(algo_to_translate): def store_object( self, - pid: Optional[str] = None, - data: Optional[Union[str, bytes]] = None, - additional_algorithm: Optional[str] = None, - checksum: Optional[str] = None, - checksum_algorithm: Optional[str] = None, - expected_object_size: Optional[int] = None, + pid: str | None = None, + data: str | bytes | None = None, + additional_algorithm: str | None = None, + checksum: str | None = None, + checksum_algorithm: str | None = None, + expected_object_size: int | None = None, ) -> "ObjectMetadata": - if pid is None and self._check_arg_data(data): + self._check_arg_data(data) + if pid is None: # If no pid is supplied, store the object only without tagging logging.debug("Request to store data only received.") object_metadata = self._store_data_only(data) self.fhs_logger.info( "Successfully stored object for cid: %s", object_metadata.cid ) - else: - # Else the object will be stored and tagged - self.fhs_logger.debug("Request to store object for pid: %s", pid) - # Validate input parameters - self._check_string(pid, "pid") - # self._check_arg_data(data) - self._check_integer(expected_object_size) - ( - additional_algorithm_checked, - checksum_algorithm_checked, - ) = self._check_arg_algorithms_and_checksum( - additional_algorithm, checksum, checksum_algorithm - ) + return object_metadata + # Else the object will be stored and tagged + self.fhs_logger.debug("Request to store object for pid: %s", pid) + # Validate input parameters + self._check_string(pid, "pid") + self._check_integer(expected_object_size) + ( + additional_algorithm_checked, + checksum_algorithm_checked, + ) = self._check_arg_algorithms_and_checksum( + additional_algorithm, checksum, checksum_algorithm + ) - try: - err_msg = ( - f"Duplicate object request for pid: {pid}. Already in progress." - ) - if self.use_multiprocessing: - with self.object_pid_condition_mp: - # Raise exception immediately if pid is in use - if pid in self.object_locked_pids_mp: - self.fhs_logger.error(err_msg) - raise StoreObjectForPidAlreadyInProgress(err_msg) - else: - with self.object_pid_condition_th: - if pid in self.object_locked_pids_th: - logging.error(err_msg) - raise StoreObjectForPidAlreadyInProgress(err_msg) + try: + err_msg = f"Duplicate object request for pid: {pid}. Already in progress." + if self.use_multiprocessing: + with self.object_pid_condition_mp: + # Raise exception immediately if pid is in use + if pid in self.object_locked_pids_mp: + self.fhs_logger.error(err_msg) + raise StoreObjectForPidAlreadyInProgress(err_msg) + else: + with self.object_pid_condition_th: + if pid in self.object_locked_pids_th: + logging.error(err_msg) + raise StoreObjectForPidAlreadyInProgress(err_msg) - try: - self._synchronize_object_locked_pids(pid) + try: + self._synchronize_object_locked_pids(pid) - self.fhs_logger.debug("Attempting to store object for pid: %s", pid) - object_metadata = self._store_and_validate_data( - pid, - data, - additional_algorithm=additional_algorithm_checked, - checksum=checksum, - checksum_algorithm=checksum_algorithm_checked, - file_size_to_validate=expected_object_size, - ) - self.fhs_logger.debug("Attempting to tag object for pid: %s", pid) - cid = object_metadata.cid - self.tag_object(pid, cid) - self.fhs_logger.info("Successfully stored object for pid: %s", pid) - finally: - # Release pid - self._release_object_locked_pids(pid) - except Exception as err: - err_msg = ( - f"Failed to store object for pid: {pid}. Reference files will not " - f"be created or tagged. Unexpected error: {err})" + self.fhs_logger.debug("Attempting to store object for pid: %s", pid) + object_metadata = self._store_and_validate_data( + pid, + data, + additional_algorithm=additional_algorithm_checked, + checksum=checksum, + checksum_algorithm=checksum_algorithm_checked, + file_size_to_validate=expected_object_size, ) - self.fhs_logger.error(err_msg) - raise err - + self.fhs_logger.debug("Attempting to tag object for pid: %s", pid) + cid = object_metadata.cid + self.tag_object(pid, cid) + self.fhs_logger.info("Successfully stored object for pid: %s", pid) + finally: + # Release pid + self._release_object_locked_pids(pid) + except Exception as err: + err_msg = ( + f"Failed to store object for pid: {pid}. Reference files will not " + f"be created or tagged. Unexpected error: {err})" + ) + self.fhs_logger.error(err_msg) + raise err return object_metadata def tag_object(self, pid: str, cid: str) -> None: @@ -648,7 +646,7 @@ def get_object_status(self, pid: str) -> dict: except KeyError as ke: err_msg = f"No object found for pid: {pid}. Details: {ke}" self.fhs_logger.warning(err_msg) - raise KeyError(err_msg) + raise KeyError(err_msg) from ke return object_status_dict @@ -697,7 +695,7 @@ def delete_if_invalid_object( ) def store_metadata( - self, pid: str, metadata: Union[str, bytes], format_id: Optional[str] = None + self, pid: str, metadata: str | bytes, format_id: str | None = None ) -> str: self.fhs_logger.debug("Request to store metadata for pid: %s", pid) # Validate input parameters @@ -777,7 +775,7 @@ def retrieve_object(self, pid: str) -> IO[bytes]: return obj_stream - def retrieve_metadata(self, pid: str, format_id: Optional[str] = None) -> IO[bytes]: + def retrieve_metadata(self, pid: str, format_id: str | None = None) -> IO[bytes]: self.fhs_logger.debug("Request to retrieve metadata for pid: %s", pid) self._check_string(pid, "pid") checked_format_id = self._check_arg_format_id(format_id, "retrieve_metadata") @@ -799,7 +797,7 @@ def retrieve_metadata(self, pid: str, format_id: Optional[str] = None) -> IO[byt return metadata_stream err_msg = f"No metadata found for pid: {pid}" self.fhs_logger.warning(err_msg) - raise KeyError(err_msg) + raise ValueError(err_msg) def delete_object(self, pid: str) -> None: self.fhs_logger.debug("Request to delete object for id: %s", pid) @@ -938,7 +936,7 @@ def delete_object(self, pid: str) -> None: # Release pid self._release_object_locked_pids(pid) - def delete_metadata(self, pid: str, format_id: Optional[str] = None) -> None: + def delete_metadata(self, pid: str, format_id: str | None = None) -> None: self.fhs_logger.debug("Request to delete metadata for pid: %s", pid) self._check_string(pid, "pid") checked_format_id = self._check_arg_format_id(format_id, "delete_metadata") @@ -1084,9 +1082,9 @@ def store_folder( self, pidpath: list[str], entries: hashstore.folderentry.FolderEntries, - additional_algorithm: Optional[str] = None, - checksum: Optional[str] = None, - checksum_algorithm: Optional[str] = None, + additional_algorithm: str | None = None, + checksum: str | None = None, # noqa: ARG002 + checksum_algorithm: str | None = None, # noqa: ARG002 verify_entry_cids: bool = True, ) -> Optional["ObjectMetadata"]: """Store a folder object. @@ -1101,10 +1099,11 @@ def store_folder( pid + DELIM + path + DELIM + entry.name - The Folder is tagged with an identifier that is "{PID}{DELIM}{path}", that is, the - PID followed by a single delimiter, then the path. If the path portion is an empty - list or a list of length 1 with the first element ".", or DELIM then the Folder is - the root Folder within the context of PID. + The Folder is tagged with an identifier that is "{PID}{DELIM}{path}", + that is, the PID followed by a single delimiter, then the path. If the + path portion is an empty list or a list of length 1 with the first + element ".", or DELIM then the Folder is the root Folder within the + context of PID. Note that since the hash of a Folder is computed from hashes of its content, a Folder hierarchy must be stored starting with the leaves. This method @@ -1136,8 +1135,8 @@ def store_folder( if entry.cid is None or entry.cid == "": # no cid provided, so look it up _entry_pid = hashstore.folderentry.join_pidpath( - pidpath - + [ + [ + *pidpath, entry.name, ] ) @@ -1146,9 +1145,8 @@ def store_folder( else: # verify provided cid is legit if not self._exists("objects", entry.cid): - raise ValueError( - f"object {entry.name} cid {entry.cid} does not exist." - ) + msg = f"object {entry.name} cid {entry.cid} does not exist." + raise ValueError(msg) # Sort the entries by cid for consistent hashing entries.sort(key=lambda entry: entry.cid) @@ -1246,7 +1244,8 @@ def resolve_pidpath(self, pidpath: list[str]) -> dict[str, str]: _name = pidpath[idx] entry = current_folder.entry_by_name(_name) if entry is None: - raise PidRefsDoesNotExist(f"PID not found: {pid}") + msg = f"PID not found: {pid}" + raise PidRefsDoesNotExist(msg) object_info_dict["cid"] = entry.cid object_info_dict["cid_object_path"] = self._get_hashstore_data_object_path( entry.cid @@ -1300,11 +1299,12 @@ def retrieve_folder( # have info, load the folder object from the cid path folder_cid = object_info_dict.get("cid") if folder_cid is None: - raise PidRefsDoesNotExist(f"Entry {pidpath} has no cid?") + msg = f"Entry {pidpath} has no cid?" + raise PidRefsDoesNotExist(msg) cid_path = object_info_dict.get("cid_object_path") return hashstore.folderentry.FolderEntries.from_parquet(cid_path) - def list_pids(self, pattern: Optional[str] = None) -> Generator: + def list_pids(self, pattern: str | None = None) -> Generator: """Yield create_timestamp, CID, PID. Iterates over all CID entries and yields the create timestamp, @@ -1325,25 +1325,27 @@ def list_pids(self, pattern: Optional[str] = None) -> Generator: if cid_entry.is_file() and cid_entry.name not in ignore_names: cid_value = str(cid_entry).replace(cids_path, "").replace("/", "") ctime = cid_entry.stat().st_ctime - for _, entry in enumerate(open(cid_entry, "r", encoding="utf-8")): - pid = entry.strip() - if len(pid) > 0: - if rpattern is not None: - if rpattern.fullmatch(pid): + with open(cid_entry, encoding="utf-8") as cid_entry_f: + for entry in cid_entry_f: + pid = entry.strip() + if len(pid) > 0: + if rpattern is not None: + if rpattern.fullmatch(pid): + yield ctime, cid_value, pid + else: yield ctime, cid_value, pid - else: - yield ctime, cid_value, pid # FileHashStore Core Methods def find_object(self, pid: str) -> dict[str, str]: - """Check if an object referenced by a pid exists and retrieve its content identifier. + """Check if an object referenced by a pid exists and retrieve its + content identifier. - The `find_object` method validates the existence of an object based on the provided - pid and returns the associated content identifier and information about how to - retrieve various accoutrements. Note that the returned dict will contain values - relevant to the type of store, but will always contain a `cid` key if the object - is present. + The `find_object` method validates the existence of an object based on + the provided pid and returns the associated content identifier and + information about how to retrieve various accoutrements. Note that the + returned dict will contain values relevant to the type of store, but + will always contain a `cid` key if the object is present. :param str pid: Authority-based or persistent identifier of the object. @@ -1358,90 +1360,71 @@ def find_object(self, pid: str) -> dict[str, str]: self._check_string(pid, "pid") pid_ref_abs_path = self._get_hashstore_pid_refs_path(pid) - if os.path.isfile(pid_ref_abs_path): - # Read the file to get the cid from the pid reference - pid_refs_cid = self._read_small_file_content(pid_ref_abs_path) - - # Confirm that the cid reference file exists - cid_ref_abs_path = self._get_hashstore_cid_refs_path(pid_refs_cid) - if os.path.isfile(cid_ref_abs_path): - # Check that the pid is actually found in the cid reference file - if self._is_string_in_refs_file(pid, cid_ref_abs_path): - # Object must also exist in order to return the cid retrieved - if not self._exists("objects", pid_refs_cid): - err_msg = ( - f"Reference file found for pid ({pid}) at " - f"{pid_ref_abs_path}, but object referenced does not " - f"exist, cid: {pid_refs_cid}" - ) - self.fhs_logger.error(err_msg) - raise RefsFileExistsButCidObjMissing(err_msg) - else: - sysmeta_doc_name = self._computehash(f"{pid}{self.sysmeta_ns}") - metadata_directory = self._computehash(pid) - metadata_rel_path = Path(*self._shard(metadata_directory)) - sysmeta_full_path = ( - self._get_store_path("metadata") - / metadata_rel_path - / sysmeta_doc_name - ) - return { - "cid": pid_refs_cid, - "cid_object_path": self._get_hashstore_data_object_path( - pid_refs_cid - ), - "cid_refs_path": cid_ref_abs_path, - "pid_refs_path": pid_ref_abs_path, - "sysmeta_path": ( - sysmeta_full_path - if os.path.isfile(sysmeta_full_path) - else "Does not exist." - ), - } - else: - # If not, it is an orphan pid refs file - err_msg = ( - f"Pid reference file exists with cid: {pid_refs_cid} for pid: {pid} but " - f"is missing from cid refs file: {cid_ref_abs_path}" - ) - return { - "cid": pid_refs_cid, - "cid_object_path": self._get_hashstore_data_object_path( - pid_refs_cid - ), - "cid_refs_path": cid_ref_abs_path, - "pid_refs_path": pid_ref_abs_path, - "sysmeta_path": ( - sysmeta_full_path - if os.path.isfile(sysmeta_full_path) - else "Does not exist." - ), - } - # If not, it is an orphan pid refs file - err_msg = ( - f"Pid reference file exists with cid: {pid_refs_cid} for pid: " - f"{pid} but is missing from cid refs file: {cid_ref_abs_path}" - ) - self.fhs_logger.error(err_msg) - raise PidNotFoundInCidRefsFile(err_msg) + if not os.path.isfile(pid_ref_abs_path): + err_msg = ( + f"Pid reference file not found for pid ({pid}): {pid_ref_abs_path}" + ) + self.fhs_logger.error(err_msg) + raise PidRefsDoesNotExist(err_msg) + + # Read the file to get the cid from the pid reference + pid_refs_cid = self._read_small_file_content(pid_ref_abs_path) + + # Confirm that the cid reference file exists + cid_ref_abs_path = self._get_hashstore_cid_refs_path(pid_refs_cid) + if not os.path.isfile(cid_ref_abs_path): err_msg = ( f"Pid reference file exists with cid: {pid_refs_cid} but cid reference " f"file not found: {cid_ref_abs_path} for pid: {pid}" ) self.fhs_logger.error(err_msg) raise OrphanPidRefsFileFound(err_msg) - err_msg = f"Pid reference file not found for pid ({pid}): {pid_ref_abs_path}" - self.fhs_logger.error(err_msg) - raise PidRefsDoesNotExist(err_msg) + + # Check that the pid is actually found in the cid reference file + if not self._is_string_in_refs_file(pid, cid_ref_abs_path): + # If not, it is an orphan pid refs file + err_msg = ( + f"Pid reference file exists with cid: {pid_refs_cid} for pid: " + f"{pid} but is missing from cid refs file: {cid_ref_abs_path}" + ) + self.fhs_logger.error(err_msg) + raise PidNotFoundInCidRefsFile(err_msg) + + # Object must also exist in order to return the cid retrieved + if not self._exists("objects", pid_refs_cid): + err_msg = ( + f"Reference file found for pid ({pid}) at " + f"{pid_ref_abs_path}, but object referenced does not " + f"exist, cid: {pid_refs_cid}" + ) + self.fhs_logger.error(err_msg) + raise RefsFileExistsButCidObjMissing(err_msg) + sysmeta_doc_name = self._computehash(f"{pid}{self.sysmeta_ns}") + metadata_directory = self._computehash(pid) + metadata_rel_path = Path(*self._shard(metadata_directory)) + sysmeta_full_path = ( + self._get_store_path("metadata") / metadata_rel_path / sysmeta_doc_name + ) + return { + "cid": pid_refs_cid, + "cid_object_path": self._get_hashstore_data_object_path(pid_refs_cid), + "cid_refs_path": cid_ref_abs_path, + "pid_refs_path": pid_ref_abs_path, + "sysmeta_path": ( + sysmeta_full_path + if os.path.isfile(sysmeta_full_path) + else "Does not exist." + ), + } def _store_and_validate_data( self, pid: str, - file: Union[str, bytes], - additional_algorithm: Optional[str] = None, - checksum: Optional[str] = None, - checksum_algorithm: Optional[str] = None, - file_size_to_validate: Optional[int] = None, + file: str | bytes, + additional_algorithm: str | None = None, + checksum: str | None = None, + checksum_algorithm: str | None = None, + file_size_to_validate: int | None = None, ) -> "ObjectMetadata": """Store contents of `file` on disk, validate the object's parameters if provided, and tag/reference the object. @@ -1481,7 +1464,7 @@ def _store_and_validate_data( self.fhs_logger.debug("Successfully put object for pid: %s", pid) return object_metadata - def _store_data_only(self, data: Union[str, bytes]) -> "ObjectMetadata": + def _store_data_only(self, data: str | bytes) -> "ObjectMetadata": """Store an object to HashStore and return a metadata object containing the content identifier, object file size and hex digests dictionary of the default algorithms. This method does not validate the object and @@ -1527,12 +1510,12 @@ def _store_data_only(self, data: Union[str, bytes]) -> "ObjectMetadata": def _move_and_get_checksums( self, - pid: Optional[str], + pid: str | None, stream: "Stream", - additional_algorithm: Optional[str] = None, - checksum: Optional[str] = None, - checksum_algorithm: Optional[str] = None, - file_size_to_validate: Optional[int] = None, + additional_algorithm: str | None = None, + checksum: str | None = None, + checksum_algorithm: str | None = None, + file_size_to_validate: int | None = None, ) -> tuple[str, int, dict[str, str]]: """Copy the contents of the `Stream` object onto disk. The copy process uses a temporary file to store the initial contents and returns a @@ -1669,8 +1652,8 @@ def _move_and_get_checksums( def _write_to_tmp_file_and_get_hex_digests( self, stream: "Stream", - additional_algorithm: Optional[str] = None, - checksum_algorithm: Optional[str] = None, + additional_algorithm: str | None = None, + checksum_algorithm: str | None = None, ) -> tuple[dict[str, str], str, int]: """Create a named temporary file from a `Stream` object and return its filename and a dictionary of its algorithms and hex digests. If an @@ -1719,7 +1702,9 @@ def _write_to_tmp_file_and_get_hex_digests( hex_digest_list = [ hash_algorithm.hexdigest() for hash_algorithm in hash_algorithms ] - hex_digest_dict = dict(zip(algorithm_list_to_calculate, hex_digest_list)) + hex_digest_dict = dict( + zip(algorithm_list_to_calculate, hex_digest_list, strict=True) + ) tmp_file_size = os.path.getsize(tmp.name) # Ready for validation and atomic move tmp_file_completion_flag = True @@ -2024,7 +2009,7 @@ def _untag_object(self, pid: str, cid: str) -> None: self.fhs_logger.warning(warn_msg) def _put_metadata( - self, metadata: Union[str, bytes], pid: str, metadata_doc_name: str + self, metadata: str | bytes, pid: str, metadata_doc_name: str ) -> Path: """Store contents of metadata to `[self.root]/metadata` using the hash of the given PID and format ID as the permanent address. @@ -2285,12 +2270,12 @@ def _is_string_in_refs_file(ref_id: str, refs_file_path: Path) -> bool: def _verify_object_information( self, - pid: Optional[str], + pid: str | None, checksum: str, checksum_algorithm: str, entity: str, hex_digests: dict[str, str], - tmp_file_name: Optional[str], + tmp_file_name: str | None, tmp_file_size: int, file_size_to_validate: int, ) -> None: @@ -2373,9 +2358,9 @@ def _verify_hashstore_references( self, pid: str, cid: str, - pid_refs_path: Optional[Path] = None, - cid_refs_path: Optional[Path] = None, - additional_log_string: Optional[str] = None, + pid_refs_path: Path | None = None, + cid_refs_path: Path | None = None, + additional_log_string: str | None = None, ) -> None: """Verifies that the supplied pid and pid reference file and content have been written successfully. @@ -2456,10 +2441,10 @@ def _delete_object_only(self, cid: str) -> None: def _check_arg_algorithms_and_checksum( self, - additional_algorithm: Optional[str], - checksum: Optional[str], - checksum_algorithm: Optional[str], - ) -> tuple[Optional[str], Optional[str]]: + additional_algorithm: str | None, + checksum: str | None, + checksum_algorithm: str | None, + ) -> tuple[str | None, str | None]: """Determines whether the caller has supplied the necessary arguments to validate an object with a checksum value. @@ -2502,7 +2487,7 @@ def _check_arg_format_id(self, format_id: str, method: str) -> str: return self.sysmeta_ns if format_id is None else format_id def _refine_algorithm_list( - self, additional_algorithm: Optional[str], checksum_algorithm: Optional[str] + self, additional_algorithm: str | None, checksum_algorithm: str | None ) -> set[str]: """Create the final list of hash algorithms to calculate. @@ -2561,7 +2546,7 @@ def _clean_algorithm(self, algorithm_string: str) -> str: return cleaned_string def _computehash( - self, stream: Union["Stream", str, IO[bytes]], algorithm: Optional[str] = None + self, stream: Union["Stream", str, IO[bytes]], algorithm: str | None = None ) -> str: """Compute the hash of a file-like object (or string) using the store algorithm by default or with an optional supported algorithm. @@ -2669,9 +2654,7 @@ def _exists(self, entity: str, file: str) -> bool: return False return False - def _open( - self, entity: str, file: str, mode: str = "rb" - ) -> Union[IO[bytes], IO[str]]: + def _open(self, entity: str, file: str, mode: str = "rb") -> IO[bytes] | IO[str]: """Return open buffer object from given id or path. Caller is responsible for closing the stream. @@ -2694,7 +2677,7 @@ def _open( # mode defaults to "rb" return open(realpath, mode) - def _delete(self, entity: str, file: Union[str, Path]) -> None: + def _delete(self, entity: str, file: str | Path) -> None: """Delete file using id or path. Remove any empty directories after deleting. No exception is raised if file doesn't exist. @@ -3049,7 +3032,7 @@ def _read_small_file_content(path_to_file: Path): return opened_path.read() @staticmethod - def _rename_path_for_deletion(path: Union[Path, str]) -> str: + def _rename_path_for_deletion(path: Path | str) -> str: """Rename a given path by appending '_delete' and move it to the renamed path. :param Path path: Path to file to rename @@ -3065,7 +3048,7 @@ def _rename_path_for_deletion(path: Union[Path, str]) -> str: return str(delete_path) @staticmethod - def _get_file_paths(directory: Union[str, Path]) -> Optional[list[Path]]: + def _get_file_paths(directory: str | Path) -> list[Path] | None: """Get the file paths of a given directory if it exists :param mixed directory: String or path to directory. @@ -3083,7 +3066,7 @@ def _get_file_paths(directory: Union[str, Path]) -> Optional[list[Path]]: return None @staticmethod - def _check_arg_data(data: Union[str, os.PathLike, io.BufferedReader]) -> bool: + def _check_arg_data(data: str | os.PathLike | io.BufferedReader) -> bool: """Checks a data argument to ensure that it is either a string, path, or stream object. @@ -3178,7 +3161,7 @@ class Stream: set its position back to ``0``. """ - def __init__(self, obj: Union[IO[bytes], str, Path]): + def __init__(self, obj: IO[bytes] | str | Path): # is it a file like thing if hasattr(obj, "read"): pos = obj.tell() diff --git a/tests/filehashstore/test_filehashstore.py b/tests/filehashstore/test_filehashstore.py index 5a98e0d1..54a4619c 100644 --- a/tests/filehashstore/test_filehashstore.py +++ b/tests/filehashstore/test_filehashstore.py @@ -238,7 +238,7 @@ def test_find_object_no_sysmeta(pids, store): for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(pid, path) - obj_info_dict = store._find_object(pid) + obj_info_dict = store.find_object(pid) retrieved_cid = obj_info_dict["cid"] assert retrieved_cid == object_metadata.hex_digests.get("sha256") @@ -266,7 +266,7 @@ def test_find_object_sysmeta(pids, store): object_metadata = store.store_object(pid, path) stored_metadata_path = store.store_metadata(pid, syspath, format_id) - obj_info_dict = store._find_object(pid) + obj_info_dict = store.find_object(pid) retrieved_cid = obj_info_dict["cid"] assert retrieved_cid == object_metadata.hex_digests.get("sha256") @@ -290,12 +290,12 @@ def test_find_object_refs_exist_but_obj_not_found(pids, store): path = test_dir + pid.replace("/", "_") store.store_object(pid, path) - cid = store._find_object(pid).get("cid") + cid = store.find_object(pid).get("cid") obj_path = store._get_hashstore_data_object_path(cid) os.remove(obj_path) with pytest.raises(RefsFileExistsButCidObjMissing): - store._find_object(pid) + store.find_object(pid) def test_find_object_cid_refs_not_found(pids, store): @@ -314,7 +314,7 @@ def test_find_object_cid_refs_not_found(pids, store): pid_ref_file.truncate() with pytest.raises(OrphanPidRefsFileFound): - store._find_object(pid) + store.find_object(pid) def test_find_object_cid_refs_does_not_contain_pid(pids, store): @@ -332,25 +332,25 @@ def test_find_object_cid_refs_does_not_contain_pid(pids, store): store._update_refs_file(cid_ref_abs_path, pid, "remove") with pytest.raises(PidNotFoundInCidRefsFile): - store._find_object(pid) + store.find_object(pid) def test_find_object_pid_refs_not_found(store): """Test _find_object throws exception when a pid refs file does not exist.""" with pytest.raises(PidRefsDoesNotExist): - store._find_object("dou.test.1") + store.find_object("dou.test.1") def test_find_object_pid_none(store): """Test _find_object throws exception when pid is None.""" with pytest.raises(ValueError, match="empty"): - store._find_object(None) + store.find_object(None) def test_find_object_pid_empty(store): """Test _find_object throws exception when pid is empty.""" with pytest.raises(ValueError, match="empty"): - store._find_object("") + store.find_object("") def test_store_and_validate_data_files_path(pids, store): @@ -895,7 +895,7 @@ def test_untag_object_orphan_pid_refs_file_found(store): os.remove(cid_refs_abs_path) with pytest.raises(OrphanPidRefsFileFound): - store._find_object(pid) + store.find_object(pid) store._synchronize_referenced_locked_pids(pid) store._synchronize_object_locked_cids(cid) @@ -922,7 +922,7 @@ def test_untag_object_orphan_refs_exist_but_data_object_not_found(store): os.remove(data_obj_path) with pytest.raises(RefsFileExistsButCidObjMissing): - store._find_object(pid) + store.find_object(pid) store._synchronize_referenced_locked_pids(pid) store._synchronize_object_locked_cids(cid) @@ -953,7 +953,7 @@ def test_untag_object_refs_found_but_pid_not_in_cid_refs(store): store._update_refs_file(cid_refs_file, pid, "remove") with pytest.raises(PidNotFoundInCidRefsFile): - store._find_object(pid) + store.find_object(pid) store._synchronize_referenced_locked_pids(pid) store._synchronize_object_locked_cids(cid) @@ -984,7 +984,7 @@ def test_untag_object_pid_refs_file_does_not_exist(store): os.remove(pid_refs_file) with pytest.raises(PidRefsDoesNotExist): - store._find_object(pid) + store.find_object(pid) store._synchronize_referenced_locked_pids(pid) store._synchronize_object_locked_cids(cid) @@ -1014,7 +1014,7 @@ def test_untag_object_pid_refs_file_does_not_exist_and_cid_refs_is_empty(store): os.remove(pid_refs_file) with pytest.raises(PidRefsDoesNotExist): - store._find_object(pid) + store.find_object(pid) store._synchronize_referenced_locked_pids(pid) store._synchronize_object_locked_cids(cid) From b194a06d68063a3ffc5a37c31893b05fb5338992 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 2 Jun 2026 09:16:34 -0400 Subject: [PATCH 40/49] update pre-commit to use ruff --- .pre-commit-config.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 656bfca8..506e2b02 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -21,15 +21,15 @@ repos: - id: requirements-txt-fixer - id: trailing-whitespace - # - repo: https://github.com/astral-sh/ruff-pre-commit - # rev: "v0.14.13" - # hooks: - # # first, lint + autofix - # - id: ruff - # types_or: [python, pyi, jupyter] - # args: ["--fix", "--show-fixes"] - # # then, format - # - id: ruff-format + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.14.13" + hooks: + # first, lint + autofix + - id: ruff + types_or: [python, pyi, jupyter] + args: ["--fix", "--show-fixes"] + # then, format + - id: ruff-format # - repo: https://github.com/pre-commit/mirrors-mypy # rev: "v1.19.1" From c860f8e43830bf654ba662d06124fe72bcc8a118 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 2 Jun 2026 09:18:12 -0400 Subject: [PATCH 41/49] Fix formating --- src/hashstore/basehashstore.py | 4 ++-- tests/test_hashstoreproperties.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/hashstore/basehashstore.py b/src/hashstore/basehashstore.py index 5219f7ac..89abfb71 100644 --- a/src/hashstore/basehashstore.py +++ b/src/hashstore/basehashstore.py @@ -9,10 +9,10 @@ from collections.abc import Generator from typing import IO -import hashstore.folderentry - import yaml +import hashstore.folderentry + DATAONE_ALGORITHM_TRANSLATION = { "MD5": "md5", "SHA-1": "sha1", diff --git a/tests/test_hashstoreproperties.py b/tests/test_hashstoreproperties.py index 020fd1d2..a20d1225 100644 --- a/tests/test_hashstoreproperties.py +++ b/tests/test_hashstoreproperties.py @@ -3,7 +3,7 @@ import pytest import hashstore -import hashstore.hashstore +import hashstore.basehashstore def test_defaults(): @@ -29,7 +29,7 @@ def test_from_dict(): p = hashstore.HashStoreProperties.from_dict(props) assert p.store_width == props["store_width"] assert p.store_depth == props["store_depth"] - assert p.store_algorithm == hashstore.hashstore.from_dataone_algorithm_name( + assert p.store_algorithm == hashstore.basehashstore.from_dataone_algorithm_name( props["store_algorithm"] ) assert len(p.store_default_algo_list) == 2 From 36d8735fc3b046794b318abd31d47d5dece5e2b1 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 2 Jun 2026 09:20:14 -0400 Subject: [PATCH 42/49] Dependencies no longer support python 3.9, remove from tests --- .github/workflows/uv-package-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/uv-package-test.yml b/.github/workflows/uv-package-test.yml index 05dcdb89..e5bba0f9 100644 --- a/.github/workflows/uv-package-test.yml +++ b/.github/workflows/uv-package-test.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] + python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v5 From ee889e6c2270720e409cf187690f8dbf3d0654dd Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:14:42 -0400 Subject: [PATCH 43/49] Fleshout baseclass typehints --- src/hashstore/basehashstore.py | 70 +++++++++++++++++++--------------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/src/hashstore/basehashstore.py b/src/hashstore/basehashstore.py index 89abfb71..a474aa59 100644 --- a/src/hashstore/basehashstore.py +++ b/src/hashstore/basehashstore.py @@ -4,14 +4,17 @@ import hashlib import importlib.metadata import importlib.util +import inspect +import io import pathlib from abc import ABC, abstractmethod from collections.abc import Generator -from typing import IO +from typing import Any, cast import yaml import hashstore.folderentry +from hashstore.filehashstore import ObjectMetadata DATAONE_ALGORITHM_TRANSLATION = { "MD5": "md5", @@ -73,14 +76,12 @@ class HashStoreProperties: ] ) - def __post_init__(self): + def __post_init__(self) -> None: """Hash algorthm names are trnaslated from the DataONE names to the names used by the python hashlib. """ - if isinstance(self.store_depth, str): - self.store_depth = int(self.store_depth) - if isinstance(self.store_width, str): - self.store_width = int(self.store_width) + self.store_depth = int(self.store_depth) + self.store_width = int(self.store_width) # Impose reasonable defaults for folder path depth if not (0 < self.store_depth <= 8): msg = "store_depth not between 0 and 8" @@ -89,7 +90,7 @@ def __post_init__(self): msg = "store_width not between 1 and 4" raise ValueError(msg) if ( - self.store_metadata_namespace is None + self.store_metadata_namespace is None # type: ignore[redundant-expr] or len(self.store_metadata_namespace) < 1 ): msg = "Value is required for store_metadata_namespace." @@ -129,7 +130,7 @@ def from_yaml(cls, source: pathlib.Path) -> "HashStoreProperties": return cls(**properties) @classmethod - def from_dict(cls, data: dict) -> "HashStoreProperties": + def from_dict(cls, data: dict[str, Any]) -> "HashStoreProperties": if data["store_algorithm"] not in DATAONE_ALGORITHM_TRANSLATION: msg = ( "store_algorithm must be one of " @@ -201,16 +202,20 @@ def version() -> str: """Return the version number""" return importlib.metadata.version("hashstore") + @abstractmethod + def __init__(self) -> None: + pass + @abstractmethod def store_object( self, pid: str, data: str | pathlib.Path, - additional_algorithm, - checksum, - checksum_algorithm, - expected_object_size, - ): + additional_algorithm: str | None, + checksum: str | None, + checksum_algorithm: str | None, + expected_object_size: int | None, + ) -> ObjectMetadata: """Atomic storage of objects to disk using a given stream. Upon successful storage, it returns an `ObjectMetadata` object containing relevant file information, such as a persistent identifier that @@ -286,7 +291,7 @@ def store_folder( checksum: str | None = None, checksum_algorithm: str | None = None, verify_entry_cids: bool = True, - ): + ) -> ObjectMetadata: """Store a folder object. A Folder is a list of entries that appear in a folder. Each entry @@ -346,7 +351,7 @@ def retrieve_folder( raise NotImplementedError() @abstractmethod - def tag_object(self, pid, cid): + def tag_object(self, pid: str, cid: str) -> None: """Creates references that allow objects stored in HashStore to be discoverable. Retrieving, deleting or calculating a hex digest of an object is based on a pid argument, to proceed, we must be able to find the object @@ -358,7 +363,7 @@ def tag_object(self, pid, cid): raise NotImplementedError() @abstractmethod - def store_metadata(self, pid, metadata, format_id): + def store_metadata(self, pid: str, metadata: str, format_id: str) -> str: """Add or update metadata, such as `sysmeta`, to disk using the given path/stream. The `store_metadata` method uses a persistent identifier `pid` and a metadata `format_id` to determine the permanent address of @@ -382,7 +387,7 @@ def store_metadata(self, pid, metadata, format_id): raise NotImplementedError() @abstractmethod - def retrieve_object(self, pid): + def retrieve_object(self, pid: str) -> io.BufferedReader: """Retrieve an object from disk using a persistent identifier (pid). The `retrieve_object` method opens and returns a buffered object stream ready for reading if the object associated with the provided `pid` @@ -395,7 +400,7 @@ def retrieve_object(self, pid): raise NotImplementedError() @abstractmethod - def retrieve_object_path(self, pidpath: list[str]) -> IO[bytes]: + def retrieve_object_path(self, pidpath: list[str]) -> io.BufferedReader: """Retrieve an object from disk using a persistent identifier (pid). The `retrieve_object` method opens and returns a buffered object stream ready for reading if the object associated with the provided `pid` exists on disk. @@ -407,7 +412,7 @@ def retrieve_object_path(self, pidpath: list[str]) -> IO[bytes]: raise NotImplementedError() @abstractmethod - def retrieve_metadata(self, pid, format_id): + def retrieve_metadata(self, pid: str, format_id: str) -> io.BufferedReader: """Retrieve the metadata object from disk using a persistent identifier (pid) and metadata namespace (format_id). If the metadata document exists, the method opens and returns a buffered metadata stream ready @@ -421,7 +426,7 @@ def retrieve_metadata(self, pid, format_id): raise NotImplementedError() @abstractmethod - def delete_object(self, pid): + def delete_object(self, pid: str) -> None: """Deletes an object and its related data permanently from HashStore using a given persistent identifier. The object associated with the pid will be deleted if it is not referenced by any other pids, along with @@ -434,8 +439,12 @@ def delete_object(self, pid): @abstractmethod def delete_if_invalid_object( - self, object_metadata, checksum, checksum_algorithm, expected_file_size - ): + self, + object_metadata: ObjectMetadata, + checksum: str, + checksum_algorithm: str, + expected_file_size: int, + ) -> None: """Confirm equality of content in an ObjectMetadata. The `delete_invalid_object` method will delete a data object if the object_metadata does not match the specified values. @@ -448,7 +457,7 @@ def delete_if_invalid_object( raise NotImplementedError() @abstractmethod - def delete_metadata(self, pid, format_id): + def delete_metadata(self, pid: str, format_id: str) -> None: """Deletes a metadata document (ex. `sysmeta`) permanently from HashStore using a given persistent identifier (`pid`) and format_id (metadata namespace). If a `format_id` is not supplied, all metadata @@ -460,7 +469,7 @@ def delete_metadata(self, pid, format_id): raise NotImplementedError() @abstractmethod - def get_hex_digest(self, pid, algorithm): + def get_hex_digest(self, pid: str, algorithm: str) -> str: """Calculates the hex digest of an object that exists in HashStore using a given persistent identifier and hash algorithm. @@ -472,7 +481,7 @@ def get_hex_digest(self, pid, algorithm): raise NotImplementedError() @abstractmethod - def list_pids(self, pattern: str | None = None) -> Generator: + def list_pids(self, pattern: str | None = None) -> Generator[float, str, str]: """Yields PIDs from the hashstore. :param str pattern: Optional regexp pattern to match. @@ -480,7 +489,7 @@ def list_pids(self, pattern: str | None = None) -> Generator: raise NotImplementedError() @abstractmethod - def get_object_status(self, pid) -> dict: + def get_object_status(self, pid: str) -> dict[str, Any]: """Returns a dictionary of the object size, modtime, accesstime for the given pid. @@ -491,7 +500,7 @@ def get_object_status(self, pid) -> dict: raise NotImplementedError() @abstractmethod - def find_object(self, pid: str) -> dict[str, str]: + def find_object(self, pid: str) -> dict[str, Any]: """Check if an object referenced by a pid exists and retrieve its content identifier. @@ -526,8 +535,8 @@ class HashStoreFactory: @staticmethod def get_hashstore( - module_name: str, class_name: str, properties: dict | None = None - ): + module_name: str, class_name: str, properties: dict[str, Any] | None = None + ) -> HashStore: """Get a `HashStore`-like object based on the specified `module_name` and `class_name`. @@ -568,6 +577,7 @@ def get_hashstore( # If class is not part of module, raise error if hasattr(imported_module, class_name): hashstore_class = getattr(imported_module, class_name) - return hashstore_class(**properties) + assert inspect.isclass(hashstore_class) + return cast(HashStore, hashstore_class(**properties)) msg = f"Class name '{class_name}' is not an attribute of module '{module_name}'" raise AttributeError(msg) From db3dd6b9c972bac79a9ad2f86766d20529fd8838 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:19:48 -0400 Subject: [PATCH 44/49] Move ObjectProperties to base --- src/hashstore/basehashstore.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/hashstore/basehashstore.py b/src/hashstore/basehashstore.py index a474aa59..ed643521 100644 --- a/src/hashstore/basehashstore.py +++ b/src/hashstore/basehashstore.py @@ -14,7 +14,6 @@ import yaml import hashstore.folderentry -from hashstore.filehashstore import ObjectMetadata DATAONE_ALGORITHM_TRANSLATION = { "MD5": "md5", @@ -192,6 +191,28 @@ def to_yaml(self, dest_path: pathlib.Path) -> None: yaml.dump(properties, dest, default_flow_style=False) +@dataclasses.dataclass +class ObjectMetadata: + """Represents metadata associated with an object. + + The `ObjectMetadata` class represents metadata associated with an object, including + a persistent or authority-based identifier (`pid`), a content identifier (`cid`), + the size of the object in bytes (`obj_size`), and an optional list of hex digests + (`hex_digests`) to assist with validating objects. + + :param str pid: An authority-based or persistent identifier + :param str cid: A unique identifier for the object (Hash ID, hex digest). + :param int obj_size: The size of the object in bytes. + :param dict hex_digests: A list of hex digests to validate objects + (md5, sha1, sha256, sha384, sha512) (optional). + """ + + pid: str + cid: str + obj_size: int + hex_digests: dict + + class HashStore(ABC): """HashStore is a content-addressable file management system that utilizes an object's content identifier (hex digest/checksum) to From 908606ad226d66892df7d0986e921b1d8d3c5d53 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:20:51 -0400 Subject: [PATCH 45/49] Adjust min python version, mypy settings --- pyproject.toml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ebcade13..87169b58 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -55,6 +55,7 @@ dev = [ "pg8000>=1.29.8", "pytest-cov>=7.1.0", "mypy>=1.19.1", + "types-pyyaml>=6.0.12.20260518", ] [build-system] @@ -85,19 +86,15 @@ port.exclude_lines = [ [tool.mypy] files = ["src", "tests"] -python_version = "3.9" +python_version = "3.10" show_error_codes = true warn_unreachable = true -disallow_untyped_defs = false -disallow_incomplete_defs = false +disallow_untyped_defs = true +disallow_incomplete_defs = true check_untyped_defs = true strict = true enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"] -[[tool.mypy.overrides]] -module = "synchronize_member_node.*" -disallow_untyped_defs = true -disallow_incomplete_defs = true [tool.ruff] src = ["src"] From c1bbb8e3d0313fc5e130c587377f48892282ce8a Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:21:35 -0400 Subject: [PATCH 46/49] Make ObjectMetadata top level --- src/hashstore/__init__.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/hashstore/__init__.py b/src/hashstore/__init__.py index 6f2b8c3e..da1ac7d6 100644 --- a/src/hashstore/__init__.py +++ b/src/hashstore/__init__.py @@ -16,7 +16,12 @@ system. """ -from hashstore.basehashstore import HashStore, HashStoreFactory, HashStoreProperties +from hashstore.basehashstore import ( + HashStore, + HashStoreFactory, + HashStoreProperties, + ObjectMetadata, +) -__all__ = ("HashStore", "HashStoreFactory", "HashStoreProperties") +__all__ = ("HashStore", "HashStoreFactory", "HashStoreProperties", "ObjectMetadata") __version__ = "1.1.0" From c6555b81d017b73959fcf1a5682492eb5a3bfe79 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:21:53 -0400 Subject: [PATCH 47/49] updated dependencies --- uv.lock | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/uv.lock b/uv.lock index 0da36378..2eb0ba59 100644 --- a/uv.lock +++ b/uv.lock @@ -256,6 +256,7 @@ dev = [ { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-cov" }, + { name = "types-pyyaml" }, ] [package.metadata] @@ -276,6 +277,7 @@ dev = [ { name = "pre-commit" }, { name = "pytest", specifier = ">=7.2.0" }, { name = "pytest-cov", specifier = ">=7.1.0" }, + { name = "types-pyyaml", specifier = ">=6.0.12.20260518" }, ] [[package]] @@ -828,6 +830,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/61/cceae43728b7de99d9b847560c262873a1f6c98202171fd5ed62640b494b/tomli-2.4.1-py3-none-any.whl", hash = "sha256:0d85819802132122da43cb86656f8d1f8c6587d54ae7dcaf30e90533028b49fe", size = 14583, upload-time = "2026-03-25T20:22:03.012Z" }, ] +[[package]] +name = "types-pyyaml" +version = "6.0.12.20260518" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b8/83/4a1afc3fbfcf5b8d46fc390cd95ed6b0dc9010a265f4e9f46314efffa37a/types_pyyaml-6.0.12.20260518.tar.gz", hash = "sha256:d917f83fb38462550338c1297faedd860b3ec83912b96b1e3d73255f7473e466", size = 17850, upload-time = "2026-05-18T06:01:58.675Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/a2/c01db32be2ae7d6a1689972f3c492b149ee4e164b12fdfd9f64b50888215/types_pyyaml-6.0.12.20260518-py3-none-any.whl", hash = "sha256:d2150f75a231c9fe9c7463bd29487d93e60bac90400287351384bc2284eba7cd", size = 20312, upload-time = "2026-05-18T06:01:57.368Z" }, +] + [[package]] name = "typing-extensions" version = "4.15.0" From c63156b50009461ad6b15ca434d9c67bb57be217 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:22:22 -0400 Subject: [PATCH 48/49] Fix imports --- tests/filehashstore/test_filehashstore.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/filehashstore/test_filehashstore.py b/tests/filehashstore/test_filehashstore.py index 501bd939..6a002437 100644 --- a/tests/filehashstore/test_filehashstore.py +++ b/tests/filehashstore/test_filehashstore.py @@ -8,10 +8,9 @@ import pytest -from hashstore import HashStoreProperties +from hashstore import HashStoreProperties, ObjectMetadata from hashstore.filehashstore import ( FileHashStore, - ObjectMetadata, Stream, ) from hashstore.filehashstore_exceptions import ( From 241554d0568476fc980b5e1bbc59a5b6f55e9c60 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 2 Jun 2026 11:22:46 -0400 Subject: [PATCH 49/49] Move ObjectMetadata to base --- src/hashstore/filehashstore.py | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 2fa61fc2..86490f78 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -13,13 +13,12 @@ import threading from collections.abc import Generator from contextlib import closing -from dataclasses import dataclass from pathlib import Path from tempfile import NamedTemporaryFile from typing import IO, Any, Optional, Union import hashstore.folderentry -from hashstore import HashStore, HashStoreProperties +from hashstore import HashStore, HashStoreProperties, ObjectMetadata from hashstore.filehashstore_exceptions import ( CidRefsContentError, CidRefsFileNotFound, @@ -87,6 +86,7 @@ def __init__( - If path does not exist, then a store is created at that path with the provided properties. """ + super().__init__() self.fhs_logger = logging.getLogger(__name__) store_path = Path(store_path) config_path = FileHashStore.config_path(store_path) @@ -2905,25 +2905,3 @@ def close(self): self._obj.close() else: self._obj.seek(self._pos) - - -@dataclass -class ObjectMetadata: - """Represents metadata associated with an object. - - The `ObjectMetadata` class represents metadata associated with an object, including - a persistent or authority-based identifier (`pid`), a content identifier (`cid`), - the size of the object in bytes (`obj_size`), and an optional list of hex digests - (`hex_digests`) to assist with validating objects. - - :param str pid: An authority-based or persistent identifier - :param str cid: A unique identifier for the object (Hash ID, hex digest). - :param int obj_size: The size of the object in bytes. - :param dict hex_digests: A list of hex digests to validate objects - (md5, sha1, sha256, sha384, sha512) (optional). - """ - - pid: str - cid: str - obj_size: int - hex_digests: dict