From be70bdf4bd593d2998a335e096be57d01532481c Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 4 Dec 2025 07:59:04 -0500 Subject: [PATCH 01/11] Switch dependency manager from poetry to uv --- poetry.lock | 553 ------------------------------------------ pyproject.toml | 77 +++--- uv.lock | 641 +++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 686 insertions(+), 585 deletions(-) delete mode 100644 poetry.lock create mode 100644 uv.lock diff --git a/poetry.lock b/poetry.lock deleted file mode 100644 index 538bc8be..00000000 --- a/poetry.lock +++ /dev/null @@ -1,553 +0,0 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. - -[[package]] -name = "asn1crypto" -version = "1.5.1" -description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" -optional = false -python-versions = "*" -files = [ - {file = "asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67"}, - {file = "asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c"}, -] - -[[package]] -name = "astroid" -version = "2.15.6" -description = "An abstract syntax tree for Python with inference support." -optional = false -python-versions = ">=3.7.2" -files = [ - {file = "astroid-2.15.6-py3-none-any.whl", hash = "sha256:389656ca57b6108f939cf5d2f9a2a825a3be50ba9d589670f393236e0a03b91c"}, - {file = "astroid-2.15.6.tar.gz", hash = "sha256:903f024859b7c7687d7a7f3a3f73b17301f8e42dfd9cc9df9d4418172d3e2dbd"}, -] - -[package.dependencies] -lazy-object-proxy = ">=1.4.0" -typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} -wrapt = [ - {version = ">=1.11,<2", markers = "python_version < \"3.11\""}, - {version = ">=1.14,<2", markers = "python_version >= \"3.11\""}, -] - -[[package]] -name = "black" -version = "22.12.0" -description = "The uncompromising code formatter." -optional = false -python-versions = ">=3.7" -files = [ - {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"}, - {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"}, - {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"}, - {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"}, - {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"}, - {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"}, - {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"}, - {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"}, - {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"}, - {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"}, - {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"}, - {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"}, -] - -[package.dependencies] -click = ">=8.0.0" -mypy-extensions = ">=0.4.3" -pathspec = ">=0.9.0" -platformdirs = ">=2" -tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""} -typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} - -[package.extras] -colorama = ["colorama (>=0.4.3)"] -d = ["aiohttp (>=3.7.4)"] -jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] -uvloop = ["uvloop (>=0.15.2)"] - -[[package]] -name = "click" -version = "8.1.5" -description = "Composable command line interface toolkit" -optional = false -python-versions = ">=3.7" -files = [ - {file = "click-8.1.5-py3-none-any.whl", hash = "sha256:e576aa487d679441d7d30abb87e1b43d24fc53bffb8758443b1a9e1cee504548"}, - {file = "click-8.1.5.tar.gz", hash = "sha256:4be4b1af8d665c6d942909916d31a213a106800c47d0eeba73d34da3cbc11367"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "platform_system == \"Windows\""} - -[[package]] -name = "colorama" -version = "0.4.6" -description = "Cross-platform colored terminal text." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" -files = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] - -[[package]] -name = "dill" -version = "0.3.6" -description = "serialize all of python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "dill-0.3.6-py3-none-any.whl", hash = "sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0"}, - {file = "dill-0.3.6.tar.gz", hash = "sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373"}, -] - -[package.extras] -graph = ["objgraph (>=1.7.2)"] - -[[package]] -name = "exceptiongroup" -version = "1.1.2" -description = "Backport of PEP 654 (exception groups)" -optional = false -python-versions = ">=3.7" -files = [ - {file = "exceptiongroup-1.1.2-py3-none-any.whl", hash = "sha256:e346e69d186172ca7cf029c8c1d16235aa0e04035e5750b4b95039e65204328f"}, - {file = "exceptiongroup-1.1.2.tar.gz", hash = "sha256:12c3e887d6485d16943a309616de20ae5582633e0a2eda17f4e10fd61c1e8af5"}, -] - -[package.extras] -test = ["pytest (>=6)"] - -[[package]] -name = "iniconfig" -version = "2.0.0" -description = "brain-dead simple config-ini parsing" -optional = false -python-versions = ">=3.7" -files = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, -] - -[[package]] -name = "isort" -version = "5.12.0" -description = "A Python utility / library to sort Python imports." -optional = false -python-versions = ">=3.8.0" -files = [ - {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"}, - {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"}, -] - -[package.extras] -colors = ["colorama (>=0.4.3)"] -pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"] -plugins = ["setuptools"] -requirements-deprecated-finder = ["pip-api", "pipreqs"] - -[[package]] -name = "lazy-object-proxy" -version = "1.9.0" -description = "A fast and thorough lazy object proxy." -optional = false -python-versions = ">=3.7" -files = [ - {file = "lazy-object-proxy-1.9.0.tar.gz", hash = "sha256:659fb5809fa4629b8a1ac5106f669cfc7bef26fbb389dda53b3e010d1ac4ebae"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b40387277b0ed2d0602b8293b94d7257e17d1479e257b4de114ea11a8cb7f2d7"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8c6cfb338b133fbdbc5cfaa10fe3c6aeea827db80c978dbd13bc9dd8526b7d4"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:721532711daa7db0d8b779b0bb0318fa87af1c10d7fe5e52ef30f8eff254d0cd"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:66a3de4a3ec06cd8af3f61b8e1ec67614fbb7c995d02fa224813cb7afefee701"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1aa3de4088c89a1b69f8ec0dcc169aa725b0ff017899ac568fe44ddc1396df46"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-win32.whl", hash = "sha256:f0705c376533ed2a9e5e97aacdbfe04cecd71e0aa84c7c0595d02ef93b6e4455"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea806fd4c37bf7e7ad82537b0757999264d5f70c45468447bb2b91afdbe73a6e"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:946d27deaff6cf8452ed0dba83ba38839a87f4f7a9732e8f9fd4107b21e6ff07"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79a31b086e7e68b24b99b23d57723ef7e2c6d81ed21007b6281ebcd1688acb0a"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f699ac1c768270c9e384e4cbd268d6e67aebcfae6cd623b4d7c3bfde5a35db59"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfb38f9ffb53b942f2b5954e0f610f1e721ccebe9cce9025a38c8ccf4a5183a4"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:189bbd5d41ae7a498397287c408617fe5c48633e7755287b21d741f7db2706a9"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-win32.whl", hash = "sha256:81fc4d08b062b535d95c9ea70dbe8a335c45c04029878e62d744bdced5141586"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:f2457189d8257dd41ae9b434ba33298aec198e30adf2dcdaaa3a28b9994f6adb"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d9e25ef10a39e8afe59a5c348a4dbf29b4868ab76269f81ce1674494e2565a6e"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cbf9b082426036e19c6924a9ce90c740a9861e2bdc27a4834fd0a910742ac1e8"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f5fa4a61ce2438267163891961cfd5e32ec97a2c444e5b842d574251ade27d2"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8fa02eaab317b1e9e03f69aab1f91e120e7899b392c4fc19807a8278a07a97e8"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e7c21c95cae3c05c14aafffe2865bbd5e377cfc1348c4f7751d9dc9a48ca4bda"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win32.whl", hash = "sha256:f12ad7126ae0c98d601a7ee504c1122bcef553d1d5e0c3bfa77b16b3968d2734"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:edd20c5a55acb67c7ed471fa2b5fb66cb17f61430b7a6b9c3b4a1e40293b1671"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2d0daa332786cf3bb49e10dc6a17a52f6a8f9601b4cf5c295a4f85854d61de63"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cd077f3d04a58e83d04b20e334f678c2b0ff9879b9375ed107d5d07ff160171"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:660c94ea760b3ce47d1855a30984c78327500493d396eac4dfd8bd82041b22be"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:212774e4dfa851e74d393a2370871e174d7ff0ebc980907723bb67d25c8a7c30"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f0117049dd1d5635bbff65444496c90e0baa48ea405125c088e93d9cf4525b11"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-win32.whl", hash = "sha256:0a891e4e41b54fd5b8313b96399f8b0e173bbbfc03c7631f01efbe29bb0bcf82"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:9990d8e71b9f6488e91ad25f322898c136b008d87bf852ff65391b004da5e17b"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9e7551208b2aded9c1447453ee366f1c4070602b3d932ace044715d89666899b"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f83ac4d83ef0ab017683d715ed356e30dd48a93746309c8f3517e1287523ef4"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7322c3d6f1766d4ef1e51a465f47955f1e8123caee67dd641e67d539a534d006"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:18b78ec83edbbeb69efdc0e9c1cb41a3b1b1ed11ddd8ded602464c3fc6020494"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:09763491ce220c0299688940f8dc2c5d05fd1f45af1e42e636b2e8b2303e4382"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-win32.whl", hash = "sha256:9090d8e53235aa280fc9239a86ae3ea8ac58eff66a705fa6aa2ec4968b95c821"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:db1c1722726f47e10e0b5fdbf15ac3b8adb58c091d12b3ab713965795036985f"}, -] - -[[package]] -name = "mccabe" -version = "0.7.0" -description = "McCabe checker, plugin for flake8" -optional = false -python-versions = ">=3.6" -files = [ - {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, - {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, -] - -[[package]] -name = "mypy-extensions" -version = "1.0.0" -description = "Type system extensions for programs checked with the mypy type checker." -optional = false -python-versions = ">=3.5" -files = [ - {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, - {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, -] - -[[package]] -name = "packaging" -version = "23.1" -description = "Core utilities for Python packages" -optional = false -python-versions = ">=3.7" -files = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, -] - -[[package]] -name = "pathlib" -version = "1.0.1" -description = "Object-oriented filesystem paths" -optional = false -python-versions = "*" -files = [ - {file = "pathlib-1.0.1-py3-none-any.whl", hash = "sha256:f35f95ab8b0f59e6d354090350b44a80a80635d22efdedfa84c7ad1cf0a74147"}, - {file = "pathlib-1.0.1.tar.gz", hash = "sha256:6940718dfc3eff4258203ad5021090933e5c04707d5ca8cc9e73c94a7894ea9f"}, -] - -[[package]] -name = "pathspec" -version = "0.11.1" -description = "Utility library for gitignore style pattern matching of file paths." -optional = false -python-versions = ">=3.7" -files = [ - {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, - {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, -] - -[[package]] -name = "pg8000" -version = "1.29.8" -description = "PostgreSQL interface library" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pg8000-1.29.8-py3-none-any.whl", hash = "sha256:962e9d6687f76057bd6d9c9c0f67f503a503216bf60b3a4d71e4cb8c97f8326d"}, - {file = "pg8000-1.29.8.tar.gz", hash = "sha256:609cfbccea783e15f111cc0cb2f6d4e6b4c349a695c59505a29baba6fc79ffa9"}, -] - -[package.dependencies] -python-dateutil = ">=2.8.2" -scramp = ">=1.4.3" - -[[package]] -name = "platformdirs" -version = "3.8.1" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -optional = false -python-versions = ">=3.7" -files = [ - {file = "platformdirs-3.8.1-py3-none-any.whl", hash = "sha256:cec7b889196b9144d088e4c57d9ceef7374f6c39694ad1577a0aab50d27ea28c"}, - {file = "platformdirs-3.8.1.tar.gz", hash = "sha256:f87ca4fcff7d2b0f81c6a748a77973d7af0f4d526f98f308477c3c436c74d528"}, -] - -[package.extras] -docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)"] - -[[package]] -name = "pluggy" -version = "1.2.0" -description = "plugin and hook calling mechanisms for python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pluggy-1.2.0-py3-none-any.whl", hash = "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849"}, - {file = "pluggy-1.2.0.tar.gz", hash = "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"}, -] - -[package.extras] -dev = ["pre-commit", "tox"] -testing = ["pytest", "pytest-benchmark"] - -[[package]] -name = "pylint" -version = "2.17.4" -description = "python code static checker" -optional = false -python-versions = ">=3.7.2" -files = [ - {file = "pylint-2.17.4-py3-none-any.whl", hash = "sha256:7a1145fb08c251bdb5cca11739722ce64a63db479283d10ce718b2460e54123c"}, - {file = "pylint-2.17.4.tar.gz", hash = "sha256:5dcf1d9e19f41f38e4e85d10f511e5b9c35e1aa74251bf95cdd8cb23584e2db1"}, -] - -[package.dependencies] -astroid = ">=2.15.4,<=2.17.0-dev0" -colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} -dill = [ - {version = ">=0.2", markers = "python_version < \"3.11\""}, - {version = ">=0.3.6", markers = "python_version >= \"3.11\""}, -] -isort = ">=4.2.5,<6" -mccabe = ">=0.6,<0.8" -platformdirs = ">=2.2.0" -tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -tomlkit = ">=0.10.1" -typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} - -[package.extras] -spelling = ["pyenchant (>=3.2,<4.0)"] -testutils = ["gitpython (>3)"] - -[[package]] -name = "pytest" -version = "7.4.0" -description = "pytest: simple powerful testing with Python" -optional = false -python-versions = ">=3.7" -files = [ - {file = "pytest-7.4.0-py3-none-any.whl", hash = "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32"}, - {file = "pytest-7.4.0.tar.gz", hash = "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"}, -] - -[package.dependencies] -colorama = {version = "*", markers = "sys_platform == \"win32\""} -exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -iniconfig = "*" -packaging = "*" -pluggy = ">=0.12,<2.0" -tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} - -[package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] - -[[package]] -name = "python-dateutil" -version = "2.8.2" -description = "Extensions to the standard Python datetime module" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] - -[package.dependencies] -six = ">=1.5" - -[[package]] -name = "pyyaml" -version = "6.0" -description = "YAML parser and emitter for Python" -optional = false -python-versions = ">=3.6" -files = [ - {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, - {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, - {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, - {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, - {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, - {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, - {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, - {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, - {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, - {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, - {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, - {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, - {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, - {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, - {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, - {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, - {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, - {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, - {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, - {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, - {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, - {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, - {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, - {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, - {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, - {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, -] - -[[package]] -name = "scramp" -version = "1.4.4" -description = "An implementation of the SCRAM protocol." -optional = false -python-versions = ">=3.7" -files = [ - {file = "scramp-1.4.4-py3-none-any.whl", hash = "sha256:b142312df7c2977241d951318b7ee923d6b7a4f75ba0f05b621ece1ed616faa3"}, - {file = "scramp-1.4.4.tar.gz", hash = "sha256:b7022a140040f33cf863ab2657917ed05287a807b917950489b89b9f685d59bc"}, -] - -[package.dependencies] -asn1crypto = ">=1.5.1" - -[[package]] -name = "six" -version = "1.16.0" -description = "Python 2 and 3 compatibility utilities" -optional = false -python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" -files = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] - -[[package]] -name = "tomli" -version = "2.0.1" -description = "A lil' TOML parser" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] - -[[package]] -name = "tomlkit" -version = "0.11.8" -description = "Style preserving TOML library" -optional = false -python-versions = ">=3.7" -files = [ - {file = "tomlkit-0.11.8-py3-none-any.whl", hash = "sha256:8c726c4c202bdb148667835f68d68780b9a003a9ec34167b6c673b38eff2a171"}, - {file = "tomlkit-0.11.8.tar.gz", hash = "sha256:9330fc7faa1db67b541b28e62018c17d20be733177d290a13b24c62d1614e0c3"}, -] - -[[package]] -name = "typing-extensions" -version = "4.7.1" -description = "Backported and Experimental Type Hints for Python 3.7+" -optional = false -python-versions = ">=3.7" -files = [ - {file = "typing_extensions-4.7.1-py3-none-any.whl", hash = "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36"}, - {file = "typing_extensions-4.7.1.tar.gz", hash = "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2"}, -] - -[[package]] -name = "wrapt" -version = "1.15.0" -description = "Module for decorators, wrappers and monkey patching." -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" -files = [ - {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, - {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, - {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, - {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, - {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, - {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, - {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, - {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, - {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, - {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, - {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, - {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, - {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, - {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, - {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, - {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, - {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, - {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, - {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, -] - -[metadata] -lock-version = "2.0" -python-versions = ">=3.9" -content-hash = "29d95a36557ed6e054de245ce01f8cc49055e3b478d030a891aa3ee57b981245" diff --git a/pyproject.toml b/pyproject.toml index 13d2e42b..f954c4a4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,42 +1,55 @@ -[tool.poetry] +[project] name = "hashstore" version = "1.1.0" description = "HashStore, an object storage system using content identifiers." -authors = ["Dou Mok ", "Matt Jones ", - "Matthew Brooke", "Jing Tao", "Jeanette Clark", "Ian M. Nesbitt"] +authors = [ + { name = "Dou Mok", email = "douming.mok@gmail.com" }, + { name = "Matt Jones", email = "gitcode@magisa.org" }, + { name = "Matthew Brooke" }, + { name = "Jing Tao" }, + { name = "Jeanette Clark" }, + { name = "Ian M. Nesbitt" }, +] +requires-python = ">=3.9" readme = "README.md" -keywords = ["filesystem", "object storage", "hashstore", "storage"] +keywords = [ + "filesystem", + "object storage", + "hashstore", + "storage", +] classifiers = [ - "Development Status :: 4 - Beta", - "Intended Audience :: Developers", - "Intended Audience :: Science/Research", - "License :: OSI Approved :: Apache Software License", - "Natural Language :: English", - "Operating System :: OS Independent", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.9", - "Programming Language :: Python :: 3.10", - "Programming Language :: Python :: 3.11", - "Programming Language :: Python :: 3.12", - "Topic :: System :: Filesystems" + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "License :: OSI Approved :: Apache Software License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Topic :: System :: Filesystems", +] +dependencies = [ + "pathlib>=1.0.1", + "pyyaml>=6.0", ] -[tool.poetry.dependencies] -python = ">=3.9" -pathlib = ">=1.0.1" -pyyaml = ">=6.0" - -[tool.poetry_bumpversion.file."src/hashstore/__init__.py"] - -[tool.poetry.group.dev.dependencies] -pytest = ">=7.2.0" -black = ">=22.10.0" -pylint = ">=2.17.4" -pg8000 = ">=1.29.8" - -[tool.poetry.scripts] +[project.scripts] hashstore = "hashstore.hashstoreclient:main" +[dependency-groups] +dev = [ + "pytest>=7.2.0", + "black>=22.10.0", + "pylint>=2.17.4", + "pg8000>=1.29.8", +] + [build-system] -requires = ["poetry-core"] -build-backend = "poetry.core.masonry.api" +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.poetry_bumpversion.file."src/hashstore/__init__.py"] diff --git a/uv.lock b/uv.lock new file mode 100644 index 00000000..3bf3e461 --- /dev/null +++ b/uv.lock @@ -0,0 +1,641 @@ +version = 1 +revision = 3 +requires-python = ">=3.9" +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", + "python_full_version < '3.10'", +] + +[[package]] +name = "asn1crypto" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/de/cf/d547feed25b5244fcb9392e288ff9fdc3280b10260362fc45d37a798a6ee/asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c", size = 121080, upload-time = "2022-03-15T14:46:52.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/7f/09065fd9e27da0eda08b4d6897f1c13535066174cc023af248fc2a8d5e5a/asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67", size = 105045, upload-time = "2022-03-15T14:46:51.055Z" }, +] + +[[package]] +name = "astroid" +version = "3.3.11" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/74/dfb75f9ccd592bbedb175d4a32fc643cf569d7c218508bfbd6ea7ef9c091/astroid-3.3.11.tar.gz", hash = "sha256:1e5a5011af2920c7c67a53f65d536d65bfa7116feeaf2354d8b94f29573bb0ce", size = 400439, upload-time = "2025-07-13T18:04:23.177Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/0f/3b8fdc946b4d9cc8cc1e8af42c4e409468c84441b933d037e101b3d72d86/astroid-3.3.11-py3-none-any.whl", hash = "sha256:54c760ae8322ece1abd213057c4b5bba7c49818853fc901ef09719a60dbf9dec", size = 275612, upload-time = "2025-07-13T18:04:21.07Z" }, +] + +[[package]] +name = "astroid" +version = "4.0.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "typing-extensions", marker = "python_full_version == '3.10.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b7/22/97df040e15d964e592d3a180598ace67e91b7c559d8298bdb3c949dc6e42/astroid-4.0.2.tar.gz", hash = "sha256:ac8fb7ca1c08eb9afec91ccc23edbd8ac73bb22cbdd7da1d488d9fb8d6579070", size = 405714, upload-time = "2025-11-09T21:21:18.373Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/ac/a85b4bfb4cf53221513e27f33cc37ad158fce02ac291d18bee6b49ab477d/astroid-4.0.2-py3-none-any.whl", hash = "sha256:d7546c00a12efc32650b19a2bb66a153883185d3179ab0d4868086f807338b9b", size = 276354, upload-time = "2025-11-09T21:21:16.54Z" }, +] + +[[package]] +name = "black" +version = "25.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click", version = "8.1.8", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "click", version = "8.3.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "mypy-extensions" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pytokens" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8c/ad/33adf4708633d047950ff2dfdea2e215d84ac50ef95aff14a614e4b6e9b2/black-25.11.0.tar.gz", hash = "sha256:9a323ac32f5dc75ce7470501b887250be5005a01602e931a15e45593f70f6e08", size = 655669, upload-time = "2025-11-10T01:53:50.558Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/d2/6caccbc96f9311e8ec3378c296d4f4809429c43a6cd2394e3c390e86816d/black-25.11.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ec311e22458eec32a807f029b2646f661e6859c3f61bc6d9ffb67958779f392e", size = 1743501, upload-time = "2025-11-10T01:59:06.202Z" }, + { url = "https://files.pythonhosted.org/packages/69/35/b986d57828b3f3dccbf922e2864223197ba32e74c5004264b1c62bc9f04d/black-25.11.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1032639c90208c15711334d681de2e24821af0575573db2810b0763bcd62e0f0", size = 1597308, upload-time = "2025-11-10T01:57:58.633Z" }, + { url = "https://files.pythonhosted.org/packages/39/8e/8b58ef4b37073f52b64a7b2dd8c9a96c84f45d6f47d878d0aa557e9a2d35/black-25.11.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0c0f7c461df55cf32929b002335883946a4893d759f2df343389c4396f3b6b37", size = 1656194, upload-time = "2025-11-10T01:57:10.909Z" }, + { url = "https://files.pythonhosted.org/packages/8d/30/9c2267a7955ecc545306534ab88923769a979ac20a27cf618d370091e5dd/black-25.11.0-cp310-cp310-win_amd64.whl", hash = "sha256:f9786c24d8e9bd5f20dc7a7f0cdd742644656987f6ea6947629306f937726c03", size = 1347996, upload-time = "2025-11-10T01:57:22.391Z" }, + { url = "https://files.pythonhosted.org/packages/c4/62/d304786b75ab0c530b833a89ce7d997924579fb7484ecd9266394903e394/black-25.11.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:895571922a35434a9d8ca67ef926da6bc9ad464522a5fe0db99b394ef1c0675a", size = 1727891, upload-time = "2025-11-10T02:01:40.507Z" }, + { url = "https://files.pythonhosted.org/packages/82/5d/ffe8a006aa522c9e3f430e7b93568a7b2163f4b3f16e8feb6d8c3552761a/black-25.11.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:cb4f4b65d717062191bdec8e4a442539a8ea065e6af1c4f4d36f0cdb5f71e170", size = 1581875, upload-time = "2025-11-10T01:57:51.192Z" }, + { url = "https://files.pythonhosted.org/packages/cb/c8/7c8bda3108d0bb57387ac41b4abb5c08782b26da9f9c4421ef6694dac01a/black-25.11.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d81a44cbc7e4f73a9d6ae449ec2317ad81512d1e7dce7d57f6333fd6259737bc", size = 1642716, upload-time = "2025-11-10T01:56:51.589Z" }, + { url = "https://files.pythonhosted.org/packages/34/b9/f17dea34eecb7cc2609a89627d480fb6caea7b86190708eaa7eb15ed25e7/black-25.11.0-cp311-cp311-win_amd64.whl", hash = "sha256:7eebd4744dfe92ef1ee349dc532defbf012a88b087bb7ddd688ff59a447b080e", size = 1352904, upload-time = "2025-11-10T01:59:26.252Z" }, + { url = "https://files.pythonhosted.org/packages/7f/12/5c35e600b515f35ffd737da7febdb2ab66bb8c24d88560d5e3ef3d28c3fd/black-25.11.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:80e7486ad3535636657aa180ad32a7d67d7c273a80e12f1b4bfa0823d54e8fac", size = 1772831, upload-time = "2025-11-10T02:03:47Z" }, + { url = "https://files.pythonhosted.org/packages/1a/75/b3896bec5a2bb9ed2f989a970ea40e7062f8936f95425879bbe162746fe5/black-25.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6cced12b747c4c76bc09b4db057c319d8545307266f41aaee665540bc0e04e96", size = 1608520, upload-time = "2025-11-10T01:58:46.895Z" }, + { url = "https://files.pythonhosted.org/packages/f3/b5/2bfc18330eddbcfb5aab8d2d720663cd410f51b2ed01375f5be3751595b0/black-25.11.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6cb2d54a39e0ef021d6c5eef442e10fd71fcb491be6413d083a320ee768329dd", size = 1682719, upload-time = "2025-11-10T01:56:55.24Z" }, + { url = "https://files.pythonhosted.org/packages/96/fb/f7dc2793a22cdf74a72114b5ed77fe3349a2e09ef34565857a2f917abdf2/black-25.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:ae263af2f496940438e5be1a0c1020e13b09154f3af4df0835ea7f9fe7bfa409", size = 1362684, upload-time = "2025-11-10T01:57:07.639Z" }, + { url = "https://files.pythonhosted.org/packages/ad/47/3378d6a2ddefe18553d1115e36aea98f4a90de53b6a3017ed861ba1bd3bc/black-25.11.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0a1d40348b6621cc20d3d7530a5b8d67e9714906dfd7346338249ad9c6cedf2b", size = 1772446, upload-time = "2025-11-10T02:02:16.181Z" }, + { url = "https://files.pythonhosted.org/packages/ba/4b/0f00bfb3d1f7e05e25bfc7c363f54dc523bb6ba502f98f4ad3acf01ab2e4/black-25.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:51c65d7d60bb25429ea2bf0731c32b2a2442eb4bd3b2afcb47830f0b13e58bfd", size = 1607983, upload-time = "2025-11-10T02:02:52.502Z" }, + { url = "https://files.pythonhosted.org/packages/99/fe/49b0768f8c9ae57eb74cc10a1f87b4c70453551d8ad498959721cc345cb7/black-25.11.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:936c4dd07669269f40b497440159a221ee435e3fddcf668e0c05244a9be71993", size = 1682481, upload-time = "2025-11-10T01:57:12.35Z" }, + { url = "https://files.pythonhosted.org/packages/55/17/7e10ff1267bfa950cc16f0a411d457cdff79678fbb77a6c73b73a5317904/black-25.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:f42c0ea7f59994490f4dccd64e6b2dd49ac57c7c84f38b8faab50f8759db245c", size = 1363869, upload-time = "2025-11-10T01:58:24.608Z" }, + { url = "https://files.pythonhosted.org/packages/67/c0/cc865ce594d09e4cd4dfca5e11994ebb51604328489f3ca3ae7bb38a7db5/black-25.11.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:35690a383f22dd3e468c85dc4b915217f87667ad9cce781d7b42678ce63c4170", size = 1771358, upload-time = "2025-11-10T02:03:33.331Z" }, + { url = "https://files.pythonhosted.org/packages/37/77/4297114d9e2fd2fc8ab0ab87192643cd49409eb059e2940391e7d2340e57/black-25.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:dae49ef7369c6caa1a1833fd5efb7c3024bb7e4499bf64833f65ad27791b1545", size = 1612902, upload-time = "2025-11-10T01:59:33.382Z" }, + { url = "https://files.pythonhosted.org/packages/de/63/d45ef97ada84111e330b2b2d45e1dd163e90bd116f00ac55927fb6bf8adb/black-25.11.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bd4a22a0b37401c8e492e994bce79e614f91b14d9ea911f44f36e262195fdda", size = 1680571, upload-time = "2025-11-10T01:57:04.239Z" }, + { url = "https://files.pythonhosted.org/packages/ff/4b/5604710d61cdff613584028b4cb4607e56e148801ed9b38ee7970799dab6/black-25.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:aa211411e94fdf86519996b7f5f05e71ba34835d8f0c0f03c00a26271da02664", size = 1382599, upload-time = "2025-11-10T01:57:57.427Z" }, + { url = "https://files.pythonhosted.org/packages/d5/9a/5b2c0e3215fe748fcf515c2dd34658973a1210bf610e24de5ba887e4f1c8/black-25.11.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:a3bb5ce32daa9ff0605d73b6f19da0b0e6c1f8f2d75594db539fdfed722f2b06", size = 1743063, upload-time = "2025-11-10T02:02:43.175Z" }, + { url = "https://files.pythonhosted.org/packages/a1/20/245164c6efc27333409c62ba54dcbfbe866c6d1957c9a6c0647786e950da/black-25.11.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9815ccee1e55717fe9a4b924cae1646ef7f54e0f990da39a34fc7b264fcf80a2", size = 1596867, upload-time = "2025-11-10T02:00:17.157Z" }, + { url = "https://files.pythonhosted.org/packages/ca/6f/1a3859a7da205f3d50cf3a8bec6bdc551a91c33ae77a045bb24c1f46ab54/black-25.11.0-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:92285c37b93a1698dcbc34581867b480f1ba3a7b92acf1fe0467b04d7a4da0dc", size = 1655678, upload-time = "2025-11-10T01:57:09.028Z" }, + { url = "https://files.pythonhosted.org/packages/56/1a/6dec1aeb7be90753d4fcc273e69bc18bfd34b353223ed191da33f7519410/black-25.11.0-cp39-cp39-win_amd64.whl", hash = "sha256:43945853a31099c7c0ff8dface53b4de56c41294fa6783c0441a8b1d9bf668bc", size = 1347452, upload-time = "2025-11-10T01:57:01.871Z" }, + { url = "https://files.pythonhosted.org/packages/00/5d/aed32636ed30a6e7f9efd6ad14e2a0b0d687ae7c8c7ec4e4a557174b895c/black-25.11.0-py3-none-any.whl", hash = "sha256:e3f562da087791e96cefcd9dda058380a442ab322a02e222add53736451f604b", size = 204918, upload-time = "2025-11-10T01:53:48.917Z" }, +] + +[[package]] +name = "click" +version = "8.1.8" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593, upload-time = "2024-12-21T18:38:44.339Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/d4/7ebdbd03970677812aac39c869717059dbb71a4cfc033ca6e5221787892c/click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2", size = 98188, upload-time = "2024-12-21T18:38:41.666Z" }, +] + +[[package]] +name = "click" +version = "8.3.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "dill" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/12/80/630b4b88364e9a8c8c5797f4602d0f76ef820909ee32f0bacb9f90654042/dill-0.4.0.tar.gz", hash = "sha256:0633f1d2df477324f53a895b02c901fb961bdbf65a17122586ea7019292cbcf0", size = 186976, upload-time = "2025-04-16T00:41:48.867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/3d/9373ad9c56321fdab5b41197068e1d8c25883b3fea29dd361f9b55116869/dill-0.4.0-py3-none-any.whl", hash = "sha256:44f54bf6412c2c8464c14e8243eb163690a9800dbe2c367330883b19c7561049", size = 119668, upload-time = "2025-04-16T00:41:47.671Z" }, +] + +[[package]] +name = "exceptiongroup" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8a/0e/97c33bf5009bdbac74fd2beace167cab3f978feb69cc36f1ef79360d6c4e/exceptiongroup-1.3.1-py3-none-any.whl", hash = "sha256:a7a39a3bd276781e98394987d3a5701d0c4edffb633bb7a5144577f82c773598", size = 16740, upload-time = "2025-11-21T23:01:53.443Z" }, +] + +[[package]] +name = "hashstore" +version = "1.1.0" +source = { editable = "." } +dependencies = [ + { name = "pathlib" }, + { name = "pyyaml" }, +] + +[package.dev-dependencies] +dev = [ + { name = "black" }, + { name = "pg8000" }, + { name = "pylint", version = "3.3.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "pylint", version = "4.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "pytest", version = "8.4.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "pytest", version = "9.0.1", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, +] + +[package.metadata] +requires-dist = [ + { name = "pathlib", specifier = ">=1.0.1" }, + { name = "pyyaml", specifier = ">=6.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "black", specifier = ">=22.10.0" }, + { name = "pg8000", specifier = ">=1.29.8" }, + { name = "pylint", specifier = ">=2.17.4" }, + { name = "pytest", specifier = ">=7.2.0" }, +] + +[[package]] +name = "importlib-metadata" +version = "8.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/66/650a33bd90f786193e4de4b3ad86ea60b53c89b669a5c7be931fac31cdb0/importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000", size = 56641, upload-time = "2025-04-27T15:29:01.736Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/b0/36bd937216ec521246249be3bf9855081de4c5e06a0c9b4219dbeda50373/importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd", size = 27656, upload-time = "2025-04-27T15:29:00.214Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.1.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793, upload-time = "2025-03-19T20:09:59.721Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050, upload-time = "2025-03-19T20:10:01.071Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, +] + +[[package]] +name = "isort" +version = "6.1.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "importlib-metadata", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1e/82/fa43935523efdfcce6abbae9da7f372b627b27142c3419fcf13bf5b0c397/isort-6.1.0.tar.gz", hash = "sha256:9b8f96a14cfee0677e78e941ff62f03769a06d412aabb9e2a90487b3b7e8d481", size = 824325, upload-time = "2025-10-01T16:26:45.027Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/cc/9b681a170efab4868a032631dea1e8446d8ec718a7f657b94d49d1a12643/isort-6.1.0-py3-none-any.whl", hash = "sha256:58d8927ecce74e5087aef019f778d4081a3b6c98f15a80ba35782ca8a2097784", size = 94329, upload-time = "2025-10-01T16:26:43.291Z" }, +] + +[[package]] +name = "isort" +version = "7.0.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/63/53/4f3c058e3bace40282876f9b553343376ee687f3c35a525dc79dbd450f88/isort-7.0.0.tar.gz", hash = "sha256:5513527951aadb3ac4292a41a16cbc50dd1642432f5e8c20057d414bdafb4187", size = 805049, upload-time = "2025-10-11T13:30:59.107Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/ed/e3705d6d02b4f7aea715a353c8ce193efd0b5db13e204df895d38734c244/isort-7.0.0-py3-none-any.whl", hash = "sha256:1bcabac8bc3c36c7fb7b98a76c8abb18e0f841a3ba81decac7691008592499c1", size = 94672, upload-time = "2025-10-11T13:30:57.665Z" }, +] + +[[package]] +name = "mccabe" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e7/ff/0ffefdcac38932a54d2b5eed4e0ba8a408f215002cd178ad1df0f2806ff8/mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325", size = 9658, upload-time = "2022-01-24T01:14:51.113Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/27/1a/1f68f9ba0c207934b35b86a8ca3aad8395a3d6dd7921c0686e23853ff5a9/mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e", size = 7350, upload-time = "2022-01-24T01:14:49.62Z" }, +] + +[[package]] +name = "mypy-extensions" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343, upload-time = "2025-04-22T14:54:24.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963, upload-time = "2025-04-22T14:54:22.983Z" }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727, upload-time = "2025-04-19T11:48:59.673Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, +] + +[[package]] +name = "pathlib" +version = "1.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ac/aa/9b065a76b9af472437a0059f77e8f962fe350438b927cb80184c32f075eb/pathlib-1.0.1.tar.gz", hash = "sha256:6940718dfc3eff4258203ad5021090933e5c04707d5ca8cc9e73c94a7894ea9f", size = 49298, upload-time = "2014-09-03T15:41:57.18Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/f9/690a8600b93c332de3ab4a344a4ac34f00c8f104917061f779db6a918ed6/pathlib-1.0.1-py3-none-any.whl", hash = "sha256:f35f95ab8b0f59e6d354090350b44a80a80635d22efdedfa84c7ad1cf0a74147", size = 14363, upload-time = "2022-05-04T13:37:20.585Z" }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + +[[package]] +name = "pg8000" +version = "1.31.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "scramp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c8/9a/077ab21e700051e03d8c5232b6bcb9a1a4d4b6242c9a0226df2cfa306414/pg8000-1.31.5.tar.gz", hash = "sha256:46ebb03be52b7a77c03c725c79da2ca281d6e8f59577ca66b17c9009618cae78", size = 118933, upload-time = "2025-09-14T09:16:49.748Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/07/5fd183858dff4d24840f07fc845f213cd371a19958558607ba22035dadd7/pg8000-1.31.5-py3-none-any.whl", hash = "sha256:0af2c1926b153307639868d2ee5cef6cd3a7d07448e12736989b10e1d491e201", size = 57816, upload-time = "2025-09-14T09:16:47.798Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.4.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +sdist = { url = "https://files.pythonhosted.org/packages/23/e8/21db9c9987b0e728855bd57bff6984f67952bea55d6f75e055c46b5383e8/platformdirs-4.4.0.tar.gz", hash = "sha256:ca753cf4d81dc309bc67b0ea38fd15dc97bc30ce419a7f58d13eb3bf14c4febf", size = 21634, upload-time = "2025-08-26T14:32:04.268Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/4b/2028861e724d3bd36227adfa20d3fd24c3fc6d52032f4a93c133be5d17ce/platformdirs-4.4.0-py3-none-any.whl", hash = "sha256:abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85", size = 18654, upload-time = "2025-08-26T14:32:02.735Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.5.0" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +sdist = { url = "https://files.pythonhosted.org/packages/61/33/9611380c2bdb1225fdef633e2a9610622310fed35ab11dac9620972ee088/platformdirs-4.5.0.tar.gz", hash = "sha256:70ddccdd7c99fc5942e9fc25636a8b34d04c24b335100223152c2803e4063312", size = 21632, upload-time = "2025-10-08T17:44:48.791Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/73/cb/ac7874b3e5d58441674fb70742e6c374b28b0c7cb988d37d991cde47166c/platformdirs-4.5.0-py3-none-any.whl", hash = "sha256:e578a81bb873cbb89a41fcc904c7ef523cc18284b7e3b3ccf06aca1403b7ebd3", size = 18651, upload-time = "2025-10-08T17:44:47.223Z" }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, +] + +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631, upload-time = "2025-06-21T13:39:12.283Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217, upload-time = "2025-06-21T13:39:07.939Z" }, +] + +[[package]] +name = "pylint" +version = "3.3.9" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "astroid", version = "3.3.11", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, + { name = "dill", marker = "python_full_version < '3.10'" }, + { name = "isort", version = "6.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "mccabe", marker = "python_full_version < '3.10'" }, + { name = "platformdirs", version = "4.4.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "tomli", marker = "python_full_version < '3.10'" }, + { name = "tomlkit", marker = "python_full_version < '3.10'" }, + { name = "typing-extensions", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/04/9d/81c84a312d1fa8133b0db0c76148542a98349298a01747ab122f9314b04e/pylint-3.3.9.tar.gz", hash = "sha256:d312737d7b25ccf6b01cc4ac629b5dcd14a0fcf3ec392735ac70f137a9d5f83a", size = 1525946, upload-time = "2025-10-05T18:41:43.786Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/a7/69460c4a6af7575449e615144aa2205b89408dc2969b87bc3df2f262ad0b/pylint-3.3.9-py3-none-any.whl", hash = "sha256:01f9b0462c7730f94786c283f3e52a1fbdf0494bbe0971a78d7277ef46a751e7", size = 523465, upload-time = "2025-10-05T18:41:41.766Z" }, +] + +[[package]] +name = "pylint" +version = "4.0.4" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "astroid", version = "4.0.2", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, + { name = "dill", marker = "python_full_version >= '3.10'" }, + { name = "isort", version = "7.0.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "mccabe", marker = "python_full_version >= '3.10'" }, + { name = "platformdirs", version = "4.5.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "tomli", marker = "python_full_version == '3.10.*'" }, + { name = "tomlkit", marker = "python_full_version >= '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/d2/b081da1a8930d00e3fc06352a1d449aaf815d4982319fab5d8cdb2e9ab35/pylint-4.0.4.tar.gz", hash = "sha256:d9b71674e19b1c36d79265b5887bf8e55278cbe236c9e95d22dc82cf044fdbd2", size = 1571735, upload-time = "2025-11-30T13:29:04.315Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/92/d40f5d937517cc489ad848fc4414ecccc7592e4686b9071e09e64f5e378e/pylint-4.0.4-py3-none-any.whl", hash = "sha256:63e06a37d5922555ee2c20963eb42559918c20bd2b21244e4ef426e7c43b92e0", size = 536425, upload-time = "2025-11-30T13:29:02.53Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version < '3.10'", +] +dependencies = [ + { name = "colorama", marker = "python_full_version < '3.10' and sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version < '3.10'" }, + { name = "iniconfig", version = "2.1.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, + { name = "packaging", marker = "python_full_version < '3.10'" }, + { name = "pluggy", marker = "python_full_version < '3.10'" }, + { name = "pygments", marker = "python_full_version < '3.10'" }, + { name = "tomli", marker = "python_full_version < '3.10'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "pytest" +version = "9.0.1" +source = { registry = "https://pypi.org/simple" } +resolution-markers = [ + "python_full_version >= '3.12'", + "python_full_version == '3.11.*'", + "python_full_version == '3.10.*'", +] +dependencies = [ + { name = "colorama", marker = "python_full_version >= '3.10' and sys_platform == 'win32'" }, + { name = "exceptiongroup", marker = "python_full_version == '3.10.*'" }, + { name = "iniconfig", version = "2.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, + { name = "packaging", marker = "python_full_version >= '3.10'" }, + { name = "pluggy", marker = "python_full_version >= '3.10'" }, + { name = "pygments", marker = "python_full_version >= '3.10'" }, + { name = "tomli", marker = "python_full_version == '3.10.*'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/07/56/f013048ac4bc4c1d9be45afd4ab209ea62822fb1598f40687e6bf45dcea4/pytest-9.0.1.tar.gz", hash = "sha256:3e9c069ea73583e255c3b21cf46b8d3c56f6e3a1a8f6da94ccb0fcf57b9d73c8", size = 1564125, upload-time = "2025-11-12T13:05:09.333Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/8b/6300fb80f858cda1c51ffa17075df5d846757081d11ab4aa35cef9e6258b/pytest-9.0.1-py3-none-any.whl", hash = "sha256:67be0030d194df2dfa7b556f2e56fb3c3315bd5c8822c6951162b92b32ce7dad", size = 373668, upload-time = "2025-11-12T13:05:07.379Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pytokens" +version = "0.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/8d/a762be14dae1c3bf280202ba3172020b2b0b4c537f94427435f19c413b72/pytokens-0.3.0.tar.gz", hash = "sha256:2f932b14ed08de5fcf0b391ace2642f858f1394c0857202959000b68ed7a458a", size = 17644, upload-time = "2025-11-05T13:36:35.34Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/25/d9db8be44e205a124f6c98bc0324b2bb149b7431c53877fc6d1038dddaf5/pytokens-0.3.0-py3-none-any.whl", hash = "sha256:95b2b5eaf832e469d141a378872480ede3f251a5a5041b8ec6e581d3ac71bbf3", size = 12195, upload-time = "2025-11-05T13:36:33.183Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f4/a0/39350dd17dd6d6c6507025c0e53aef67a9293a6d37d3511f23ea510d5800/pyyaml-6.0.3-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:214ed4befebe12df36bcc8bc2b64b396ca31be9304b8f59e25c11cf94a4c033b", size = 184227, upload-time = "2025-09-25T21:31:46.04Z" }, + { url = "https://files.pythonhosted.org/packages/05/14/52d505b5c59ce73244f59c7a50ecf47093ce4765f116cdb98286a71eeca2/pyyaml-6.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:02ea2dfa234451bbb8772601d7b8e426c2bfa197136796224e50e35a78777956", size = 174019, upload-time = "2025-09-25T21:31:47.706Z" }, + { url = "https://files.pythonhosted.org/packages/43/f7/0e6a5ae5599c838c696adb4e6330a59f463265bfa1e116cfd1fbb0abaaae/pyyaml-6.0.3-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b30236e45cf30d2b8e7b3e85881719e98507abed1011bf463a8fa23e9c3e98a8", size = 740646, upload-time = "2025-09-25T21:31:49.21Z" }, + { url = "https://files.pythonhosted.org/packages/2f/3a/61b9db1d28f00f8fd0ae760459a5c4bf1b941baf714e207b6eb0657d2578/pyyaml-6.0.3-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:66291b10affd76d76f54fad28e22e51719ef9ba22b29e1d7d03d6777a9174198", size = 840793, upload-time = "2025-09-25T21:31:50.735Z" }, + { url = "https://files.pythonhosted.org/packages/7a/1e/7acc4f0e74c4b3d9531e24739e0ab832a5edf40e64fbae1a9c01941cabd7/pyyaml-6.0.3-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9c7708761fccb9397fe64bbc0395abcae8c4bf7b0eac081e12b809bf47700d0b", size = 770293, upload-time = "2025-09-25T21:31:51.828Z" }, + { url = "https://files.pythonhosted.org/packages/8b/ef/abd085f06853af0cd59fa5f913d61a8eab65d7639ff2a658d18a25d6a89d/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:418cf3f2111bc80e0933b2cd8cd04f286338bb88bdc7bc8e6dd775ebde60b5e0", size = 732872, upload-time = "2025-09-25T21:31:53.282Z" }, + { url = "https://files.pythonhosted.org/packages/1f/15/2bc9c8faf6450a8b3c9fc5448ed869c599c0a74ba2669772b1f3a0040180/pyyaml-6.0.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5e0b74767e5f8c593e8c9b5912019159ed0533c70051e9cce3e8b6aa699fcd69", size = 758828, upload-time = "2025-09-25T21:31:54.807Z" }, + { url = "https://files.pythonhosted.org/packages/a3/00/531e92e88c00f4333ce359e50c19b8d1de9fe8d581b1534e35ccfbc5f393/pyyaml-6.0.3-cp310-cp310-win32.whl", hash = "sha256:28c8d926f98f432f88adc23edf2e6d4921ac26fb084b028c733d01868d19007e", size = 142415, upload-time = "2025-09-25T21:31:55.885Z" }, + { url = "https://files.pythonhosted.org/packages/2a/fa/926c003379b19fca39dd4634818b00dec6c62d87faf628d1394e137354d4/pyyaml-6.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:bdb2c67c6c1390b63c6ff89f210c8fd09d9a1217a465701eac7316313c915e4c", size = 158561, upload-time = "2025-09-25T21:31:57.406Z" }, + { url = "https://files.pythonhosted.org/packages/6d/16/a95b6757765b7b031c9374925bb718d55e0a9ba8a1b6a12d25962ea44347/pyyaml-6.0.3-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:44edc647873928551a01e7a563d7452ccdebee747728c1080d881d68af7b997e", size = 185826, upload-time = "2025-09-25T21:31:58.655Z" }, + { url = "https://files.pythonhosted.org/packages/16/19/13de8e4377ed53079ee996e1ab0a9c33ec2faf808a4647b7b4c0d46dd239/pyyaml-6.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:652cb6edd41e718550aad172851962662ff2681490a8a711af6a4d288dd96824", size = 175577, upload-time = "2025-09-25T21:32:00.088Z" }, + { url = "https://files.pythonhosted.org/packages/0c/62/d2eb46264d4b157dae1275b573017abec435397aa59cbcdab6fc978a8af4/pyyaml-6.0.3-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:10892704fc220243f5305762e276552a0395f7beb4dbf9b14ec8fd43b57f126c", size = 775556, upload-time = "2025-09-25T21:32:01.31Z" }, + { url = "https://files.pythonhosted.org/packages/10/cb/16c3f2cf3266edd25aaa00d6c4350381c8b012ed6f5276675b9eba8d9ff4/pyyaml-6.0.3-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:850774a7879607d3a6f50d36d04f00ee69e7fc816450e5f7e58d7f17f1ae5c00", size = 882114, upload-time = "2025-09-25T21:32:03.376Z" }, + { url = "https://files.pythonhosted.org/packages/71/60/917329f640924b18ff085ab889a11c763e0b573da888e8404ff486657602/pyyaml-6.0.3-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b8bb0864c5a28024fac8a632c443c87c5aa6f215c0b126c449ae1a150412f31d", size = 806638, upload-time = "2025-09-25T21:32:04.553Z" }, + { url = "https://files.pythonhosted.org/packages/dd/6f/529b0f316a9fd167281a6c3826b5583e6192dba792dd55e3203d3f8e655a/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:1d37d57ad971609cf3c53ba6a7e365e40660e3be0e5175fa9f2365a379d6095a", size = 767463, upload-time = "2025-09-25T21:32:06.152Z" }, + { url = "https://files.pythonhosted.org/packages/f2/6a/b627b4e0c1dd03718543519ffb2f1deea4a1e6d42fbab8021936a4d22589/pyyaml-6.0.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:37503bfbfc9d2c40b344d06b2199cf0e96e97957ab1c1b546fd4f87e53e5d3e4", size = 794986, upload-time = "2025-09-25T21:32:07.367Z" }, + { url = "https://files.pythonhosted.org/packages/45/91/47a6e1c42d9ee337c4839208f30d9f09caa9f720ec7582917b264defc875/pyyaml-6.0.3-cp311-cp311-win32.whl", hash = "sha256:8098f252adfa6c80ab48096053f512f2321f0b998f98150cea9bd23d83e1467b", size = 142543, upload-time = "2025-09-25T21:32:08.95Z" }, + { url = "https://files.pythonhosted.org/packages/da/e3/ea007450a105ae919a72393cb06f122f288ef60bba2dc64b26e2646fa315/pyyaml-6.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:9f3bfb4965eb874431221a3ff3fdcddc7e74e3b07799e0e84ca4a0f867d449bf", size = 158763, upload-time = "2025-09-25T21:32:09.96Z" }, + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8c/f4bd7f6465179953d3ac9bc44ac1a8a3e6122cf8ada906b4f96c60172d43/pyyaml-6.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:8d1fab6bb153a416f9aeb4b8763bc0f22a5586065f86f7664fc23339fc1c1fac", size = 181814, upload-time = "2025-09-25T21:32:35.712Z" }, + { url = "https://files.pythonhosted.org/packages/bd/9c/4d95bb87eb2063d20db7b60faa3840c1b18025517ae857371c4dd55a6b3a/pyyaml-6.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:34d5fcd24b8445fadc33f9cf348c1047101756fd760b4dacb5c3e99755703310", size = 173809, upload-time = "2025-09-25T21:32:36.789Z" }, + { url = "https://files.pythonhosted.org/packages/92/b5/47e807c2623074914e29dabd16cbbdd4bf5e9b2db9f8090fa64411fc5382/pyyaml-6.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:501a031947e3a9025ed4405a168e6ef5ae3126c59f90ce0cd6f2bfc477be31b7", size = 766454, upload-time = "2025-09-25T21:32:37.966Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/e5e9b168be58564121efb3de6859c452fccde0ab093d8438905899a3a483/pyyaml-6.0.3-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b3bc83488de33889877a0f2543ade9f70c67d66d9ebb4ac959502e12de895788", size = 836355, upload-time = "2025-09-25T21:32:39.178Z" }, + { url = "https://files.pythonhosted.org/packages/88/f9/16491d7ed2a919954993e48aa941b200f38040928474c9e85ea9e64222c3/pyyaml-6.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c458b6d084f9b935061bc36216e8a69a7e293a2f1e68bf956dcd9e6cbcd143f5", size = 794175, upload-time = "2025-09-25T21:32:40.865Z" }, + { url = "https://files.pythonhosted.org/packages/dd/3f/5989debef34dc6397317802b527dbbafb2b4760878a53d4166579111411e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:7c6610def4f163542a622a73fb39f534f8c101d690126992300bf3207eab9764", size = 755228, upload-time = "2025-09-25T21:32:42.084Z" }, + { url = "https://files.pythonhosted.org/packages/d7/ce/af88a49043cd2e265be63d083fc75b27b6ed062f5f9fd6cdc223ad62f03e/pyyaml-6.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5190d403f121660ce8d1d2c1bb2ef1bd05b5f68533fc5c2ea899bd15f4399b35", size = 789194, upload-time = "2025-09-25T21:32:43.362Z" }, + { url = "https://files.pythonhosted.org/packages/23/20/bb6982b26a40bb43951265ba29d4c246ef0ff59c9fdcdf0ed04e0687de4d/pyyaml-6.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:4a2e8cebe2ff6ab7d1050ecd59c25d4c8bd7e6f400f5f82b96557ac0abafd0ac", size = 156429, upload-time = "2025-09-25T21:32:57.844Z" }, + { url = "https://files.pythonhosted.org/packages/f4/f4/a4541072bb9422c8a883ab55255f918fa378ecf083f5b85e87fc2b4eda1b/pyyaml-6.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:93dda82c9c22deb0a405ea4dc5f2d0cda384168e466364dec6255b293923b2f3", size = 143912, upload-time = "2025-09-25T21:32:59.247Z" }, + { url = "https://files.pythonhosted.org/packages/7c/f9/07dd09ae774e4616edf6cda684ee78f97777bdd15847253637a6f052a62f/pyyaml-6.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:02893d100e99e03eda1c8fd5c441d8c60103fd175728e23e431db1b589cf5ab3", size = 189108, upload-time = "2025-09-25T21:32:44.377Z" }, + { url = "https://files.pythonhosted.org/packages/4e/78/8d08c9fb7ce09ad8c38ad533c1191cf27f7ae1effe5bb9400a46d9437fcf/pyyaml-6.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:c1ff362665ae507275af2853520967820d9124984e0f7466736aea23d8611fba", size = 183641, upload-time = "2025-09-25T21:32:45.407Z" }, + { url = "https://files.pythonhosted.org/packages/7b/5b/3babb19104a46945cf816d047db2788bcaf8c94527a805610b0289a01c6b/pyyaml-6.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6adc77889b628398debc7b65c073bcb99c4a0237b248cacaf3fe8a557563ef6c", size = 831901, upload-time = "2025-09-25T21:32:48.83Z" }, + { url = "https://files.pythonhosted.org/packages/8b/cc/dff0684d8dc44da4d22a13f35f073d558c268780ce3c6ba1b87055bb0b87/pyyaml-6.0.3-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a80cb027f6b349846a3bf6d73b5e95e782175e52f22108cfa17876aaeff93702", size = 861132, upload-time = "2025-09-25T21:32:50.149Z" }, + { url = "https://files.pythonhosted.org/packages/b1/5e/f77dc6b9036943e285ba76b49e118d9ea929885becb0a29ba8a7c75e29fe/pyyaml-6.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:00c4bdeba853cc34e7dd471f16b4114f4162dc03e6b7afcc2128711f0eca823c", size = 839261, upload-time = "2025-09-25T21:32:51.808Z" }, + { url = "https://files.pythonhosted.org/packages/ce/88/a9db1376aa2a228197c58b37302f284b5617f56a5d959fd1763fb1675ce6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:66e1674c3ef6f541c35191caae2d429b967b99e02040f5ba928632d9a7f0f065", size = 805272, upload-time = "2025-09-25T21:32:52.941Z" }, + { url = "https://files.pythonhosted.org/packages/da/92/1446574745d74df0c92e6aa4a7b0b3130706a4142b2d1a5869f2eaa423c6/pyyaml-6.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:16249ee61e95f858e83976573de0f5b2893b3677ba71c9dd36b9cf8be9ac6d65", size = 829923, upload-time = "2025-09-25T21:32:54.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/7a/1c7270340330e575b92f397352af856a8c06f230aa3e76f86b39d01b416a/pyyaml-6.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4ad1906908f2f5ae4e5a8ddfce73c320c2a1429ec52eafd27138b7f1cbe341c9", size = 174062, upload-time = "2025-09-25T21:32:55.767Z" }, + { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, + { url = "https://files.pythonhosted.org/packages/9f/62/67fc8e68a75f738c9200422bf65693fb79a4cd0dc5b23310e5202e978090/pyyaml-6.0.3-cp39-cp39-macosx_10_13_x86_64.whl", hash = "sha256:b865addae83924361678b652338317d1bd7e79b1f4596f96b96c77a5a34b34da", size = 184450, upload-time = "2025-09-25T21:33:00.618Z" }, + { url = "https://files.pythonhosted.org/packages/ae/92/861f152ce87c452b11b9d0977952259aa7df792d71c1053365cc7b09cc08/pyyaml-6.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c3355370a2c156cffb25e876646f149d5d68f5e0a3ce86a5084dd0b64a994917", size = 174319, upload-time = "2025-09-25T21:33:02.086Z" }, + { url = "https://files.pythonhosted.org/packages/d0/cd/f0cfc8c74f8a030017a2b9c771b7f47e5dd702c3e28e5b2071374bda2948/pyyaml-6.0.3-cp39-cp39-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3c5677e12444c15717b902a5798264fa7909e41153cdf9ef7ad571b704a63dd9", size = 737631, upload-time = "2025-09-25T21:33:03.25Z" }, + { url = "https://files.pythonhosted.org/packages/ef/b2/18f2bd28cd2055a79a46c9b0895c0b3d987ce40ee471cecf58a1a0199805/pyyaml-6.0.3-cp39-cp39-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5ed875a24292240029e4483f9d4a4b8a1ae08843b9c54f43fcc11e404532a8a5", size = 836795, upload-time = "2025-09-25T21:33:05.014Z" }, + { url = "https://files.pythonhosted.org/packages/73/b9/793686b2d54b531203c160ef12bec60228a0109c79bae6c1277961026770/pyyaml-6.0.3-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0150219816b6a1fa26fb4699fb7daa9caf09eb1999f3b70fb6e786805e80375a", size = 750767, upload-time = "2025-09-25T21:33:06.398Z" }, + { url = "https://files.pythonhosted.org/packages/a9/86/a137b39a611def2ed78b0e66ce2fe13ee701a07c07aebe55c340ed2a050e/pyyaml-6.0.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fa160448684b4e94d80416c0fa4aac48967a969efe22931448d853ada8baf926", size = 727982, upload-time = "2025-09-25T21:33:08.708Z" }, + { url = "https://files.pythonhosted.org/packages/dd/62/71c27c94f457cf4418ef8ccc71735324c549f7e3ea9d34aba50874563561/pyyaml-6.0.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:27c0abcb4a5dac13684a37f76e701e054692a9b2d3064b70f5e4eb54810553d7", size = 755677, upload-time = "2025-09-25T21:33:09.876Z" }, + { url = "https://files.pythonhosted.org/packages/29/3d/6f5e0d58bd924fb0d06c3a6bad00effbdae2de5adb5cda5648006ffbd8d3/pyyaml-6.0.3-cp39-cp39-win32.whl", hash = "sha256:1ebe39cb5fc479422b83de611d14e2c0d3bb2a18bbcb01f229ab3cfbd8fee7a0", size = 142592, upload-time = "2025-09-25T21:33:10.983Z" }, + { url = "https://files.pythonhosted.org/packages/f0/0c/25113e0b5e103d7f1490c0e947e303fe4a696c10b501dea7a9f49d4e876c/pyyaml-6.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:2e71d11abed7344e42a8849600193d15b6def118602c4c176f748e4583246007", size = 158777, upload-time = "2025-09-25T21:33:15.55Z" }, +] + +[[package]] +name = "scramp" +version = "1.4.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asn1crypto" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/58/77/6db18bab446c12cfbee22ca8f65d5b187966bd8f900aeb65db9e60d4be3d/scramp-1.4.6.tar.gz", hash = "sha256:fe055ebbebf4397b9cb323fcc4b299f219cd1b03fd673ca40c97db04ac7d107e", size = 16306, upload-time = "2025-07-05T14:44:03.977Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/bf/54b5d40bea1c1805175ead2d496c267f05eec87561687dd73ab76869d8d9/scramp-1.4.6-py3-none-any.whl", hash = "sha256:a0cf9d2b4624b69bac5432dd69fecfc55a542384fe73c3a23ed9b138cda484e1", size = 12812, upload-time = "2025-07-05T14:44:02.345Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "tomli" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/52/ed/3f73f72945444548f33eba9a87fc7a6e969915e7b1acc8260b30e1f76a2f/tomli-2.3.0.tar.gz", hash = "sha256:64be704a875d2a59753d80ee8a533c3fe183e3f06807ff7dc2232938ccb01549", size = 17392, upload-time = "2025-10-08T22:01:47.119Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/2e/299f62b401438d5fe1624119c723f5d877acc86a4c2492da405626665f12/tomli-2.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:88bd15eb972f3664f5ed4b57c1634a97153b4bac4479dcb6a495f41921eb7f45", size = 153236, upload-time = "2025-10-08T22:01:00.137Z" }, + { url = "https://files.pythonhosted.org/packages/86/7f/d8fffe6a7aefdb61bced88fcb5e280cfd71e08939da5894161bd71bea022/tomli-2.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:883b1c0d6398a6a9d29b508c331fa56adbcdff647f6ace4dfca0f50e90dfd0ba", size = 148084, upload-time = "2025-10-08T22:01:01.63Z" }, + { url = "https://files.pythonhosted.org/packages/47/5c/24935fb6a2ee63e86d80e4d3b58b222dafaf438c416752c8b58537c8b89a/tomli-2.3.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1381caf13ab9f300e30dd8feadb3de072aeb86f1d34a8569453ff32a7dea4bf", size = 234832, upload-time = "2025-10-08T22:01:02.543Z" }, + { url = "https://files.pythonhosted.org/packages/89/da/75dfd804fc11e6612846758a23f13271b76d577e299592b4371a4ca4cd09/tomli-2.3.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a0e285d2649b78c0d9027570d4da3425bdb49830a6156121360b3f8511ea3441", size = 242052, upload-time = "2025-10-08T22:01:03.836Z" }, + { url = "https://files.pythonhosted.org/packages/70/8c/f48ac899f7b3ca7eb13af73bacbc93aec37f9c954df3c08ad96991c8c373/tomli-2.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0a154a9ae14bfcf5d8917a59b51ffd5a3ac1fd149b71b47a3a104ca4edcfa845", size = 239555, upload-time = "2025-10-08T22:01:04.834Z" }, + { url = "https://files.pythonhosted.org/packages/ba/28/72f8afd73f1d0e7829bfc093f4cb98ce0a40ffc0cc997009ee1ed94ba705/tomli-2.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:74bf8464ff93e413514fefd2be591c3b0b23231a77f901db1eb30d6f712fc42c", size = 245128, upload-time = "2025-10-08T22:01:05.84Z" }, + { url = "https://files.pythonhosted.org/packages/b6/eb/a7679c8ac85208706d27436e8d421dfa39d4c914dcf5fa8083a9305f58d9/tomli-2.3.0-cp311-cp311-win32.whl", hash = "sha256:00b5f5d95bbfc7d12f91ad8c593a1659b6387b43f054104cda404be6bda62456", size = 96445, upload-time = "2025-10-08T22:01:06.896Z" }, + { url = "https://files.pythonhosted.org/packages/0a/fe/3d3420c4cb1ad9cb462fb52967080575f15898da97e21cb6f1361d505383/tomli-2.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:4dc4ce8483a5d429ab602f111a93a6ab1ed425eae3122032db7e9acf449451be", size = 107165, upload-time = "2025-10-08T22:01:08.107Z" }, + { url = "https://files.pythonhosted.org/packages/ff/b7/40f36368fcabc518bb11c8f06379a0fd631985046c038aca08c6d6a43c6e/tomli-2.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d7d86942e56ded512a594786a5ba0a5e521d02529b3826e7761a05138341a2ac", size = 154891, upload-time = "2025-10-08T22:01:09.082Z" }, + { url = "https://files.pythonhosted.org/packages/f9/3f/d9dd692199e3b3aab2e4e4dd948abd0f790d9ded8cd10cbaae276a898434/tomli-2.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:73ee0b47d4dad1c5e996e3cd33b8a76a50167ae5f96a2607cbe8cc773506ab22", size = 148796, upload-time = "2025-10-08T22:01:10.266Z" }, + { url = "https://files.pythonhosted.org/packages/60/83/59bff4996c2cf9f9387a0f5a3394629c7efa5ef16142076a23a90f1955fa/tomli-2.3.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:792262b94d5d0a466afb5bc63c7daa9d75520110971ee269152083270998316f", size = 242121, upload-time = "2025-10-08T22:01:11.332Z" }, + { url = "https://files.pythonhosted.org/packages/45/e5/7c5119ff39de8693d6baab6c0b6dcb556d192c165596e9fc231ea1052041/tomli-2.3.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4f195fe57ecceac95a66a75ac24d9d5fbc98ef0962e09b2eddec5d39375aae52", size = 250070, upload-time = "2025-10-08T22:01:12.498Z" }, + { url = "https://files.pythonhosted.org/packages/45/12/ad5126d3a278f27e6701abde51d342aa78d06e27ce2bb596a01f7709a5a2/tomli-2.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e31d432427dcbf4d86958c184b9bfd1e96b5b71f8eb17e6d02531f434fd335b8", size = 245859, upload-time = "2025-10-08T22:01:13.551Z" }, + { url = "https://files.pythonhosted.org/packages/fb/a1/4d6865da6a71c603cfe6ad0e6556c73c76548557a8d658f9e3b142df245f/tomli-2.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7b0882799624980785240ab732537fcfc372601015c00f7fc367c55308c186f6", size = 250296, upload-time = "2025-10-08T22:01:14.614Z" }, + { url = "https://files.pythonhosted.org/packages/a0/b7/a7a7042715d55c9ba6e8b196d65d2cb662578b4d8cd17d882d45322b0d78/tomli-2.3.0-cp312-cp312-win32.whl", hash = "sha256:ff72b71b5d10d22ecb084d345fc26f42b5143c5533db5e2eaba7d2d335358876", size = 97124, upload-time = "2025-10-08T22:01:15.629Z" }, + { url = "https://files.pythonhosted.org/packages/06/1e/f22f100db15a68b520664eb3328fb0ae4e90530887928558112c8d1f4515/tomli-2.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:1cb4ed918939151a03f33d4242ccd0aa5f11b3547d0cf30f7c74a408a5b99878", size = 107698, upload-time = "2025-10-08T22:01:16.51Z" }, + { url = "https://files.pythonhosted.org/packages/89/48/06ee6eabe4fdd9ecd48bf488f4ac783844fd777f547b8d1b61c11939974e/tomli-2.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5192f562738228945d7b13d4930baffda67b69425a7f0da96d360b0a3888136b", size = 154819, upload-time = "2025-10-08T22:01:17.964Z" }, + { url = "https://files.pythonhosted.org/packages/f1/01/88793757d54d8937015c75dcdfb673c65471945f6be98e6a0410fba167ed/tomli-2.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:be71c93a63d738597996be9528f4abe628d1adf5e6eb11607bc8fe1a510b5dae", size = 148766, upload-time = "2025-10-08T22:01:18.959Z" }, + { url = "https://files.pythonhosted.org/packages/42/17/5e2c956f0144b812e7e107f94f1cc54af734eb17b5191c0bbfb72de5e93e/tomli-2.3.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c4665508bcbac83a31ff8ab08f424b665200c0e1e645d2bd9ab3d3e557b6185b", size = 240771, upload-time = "2025-10-08T22:01:20.106Z" }, + { url = "https://files.pythonhosted.org/packages/d5/f4/0fbd014909748706c01d16824eadb0307115f9562a15cbb012cd9b3512c5/tomli-2.3.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4021923f97266babc6ccab9f5068642a0095faa0a51a246a6a02fccbb3514eaf", size = 248586, upload-time = "2025-10-08T22:01:21.164Z" }, + { url = "https://files.pythonhosted.org/packages/30/77/fed85e114bde5e81ecf9bc5da0cc69f2914b38f4708c80ae67d0c10180c5/tomli-2.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4ea38c40145a357d513bffad0ed869f13c1773716cf71ccaa83b0fa0cc4e42f", size = 244792, upload-time = "2025-10-08T22:01:22.417Z" }, + { url = "https://files.pythonhosted.org/packages/55/92/afed3d497f7c186dc71e6ee6d4fcb0acfa5f7d0a1a2878f8beae379ae0cc/tomli-2.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ad805ea85eda330dbad64c7ea7a4556259665bdf9d2672f5dccc740eb9d3ca05", size = 248909, upload-time = "2025-10-08T22:01:23.859Z" }, + { url = "https://files.pythonhosted.org/packages/f8/84/ef50c51b5a9472e7265ce1ffc7f24cd4023d289e109f669bdb1553f6a7c2/tomli-2.3.0-cp313-cp313-win32.whl", hash = "sha256:97d5eec30149fd3294270e889b4234023f2c69747e555a27bd708828353ab606", size = 96946, upload-time = "2025-10-08T22:01:24.893Z" }, + { url = "https://files.pythonhosted.org/packages/b2/b7/718cd1da0884f281f95ccfa3a6cc572d30053cba64603f79d431d3c9b61b/tomli-2.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:0c95ca56fbe89e065c6ead5b593ee64b84a26fca063b5d71a1122bf26e533999", size = 107705, upload-time = "2025-10-08T22:01:26.153Z" }, + { url = "https://files.pythonhosted.org/packages/19/94/aeafa14a52e16163008060506fcb6aa1949d13548d13752171a755c65611/tomli-2.3.0-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:cebc6fe843e0733ee827a282aca4999b596241195f43b4cc371d64fc6639da9e", size = 154244, upload-time = "2025-10-08T22:01:27.06Z" }, + { url = "https://files.pythonhosted.org/packages/db/e4/1e58409aa78eefa47ccd19779fc6f36787edbe7d4cd330eeeedb33a4515b/tomli-2.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:4c2ef0244c75aba9355561272009d934953817c49f47d768070c3c94355c2aa3", size = 148637, upload-time = "2025-10-08T22:01:28.059Z" }, + { url = "https://files.pythonhosted.org/packages/26/b6/d1eccb62f665e44359226811064596dd6a366ea1f985839c566cd61525ae/tomli-2.3.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c22a8bf253bacc0cf11f35ad9808b6cb75ada2631c2d97c971122583b129afbc", size = 241925, upload-time = "2025-10-08T22:01:29.066Z" }, + { url = "https://files.pythonhosted.org/packages/70/91/7cdab9a03e6d3d2bb11beae108da5bdc1c34bdeb06e21163482544ddcc90/tomli-2.3.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0eea8cc5c5e9f89c9b90c4896a8deefc74f518db5927d0e0e8d4a80953d774d0", size = 249045, upload-time = "2025-10-08T22:01:31.98Z" }, + { url = "https://files.pythonhosted.org/packages/15/1b/8c26874ed1f6e4f1fcfeb868db8a794cbe9f227299402db58cfcc858766c/tomli-2.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:b74a0e59ec5d15127acdabd75ea17726ac4c5178ae51b85bfe39c4f8a278e879", size = 245835, upload-time = "2025-10-08T22:01:32.989Z" }, + { url = "https://files.pythonhosted.org/packages/fd/42/8e3c6a9a4b1a1360c1a2a39f0b972cef2cc9ebd56025168c4137192a9321/tomli-2.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:b5870b50c9db823c595983571d1296a6ff3e1b88f734a4c8f6fc6188397de005", size = 253109, upload-time = "2025-10-08T22:01:34.052Z" }, + { url = "https://files.pythonhosted.org/packages/22/0c/b4da635000a71b5f80130937eeac12e686eefb376b8dee113b4a582bba42/tomli-2.3.0-cp314-cp314-win32.whl", hash = "sha256:feb0dacc61170ed7ab602d3d972a58f14ee3ee60494292d384649a3dc38ef463", size = 97930, upload-time = "2025-10-08T22:01:35.082Z" }, + { url = "https://files.pythonhosted.org/packages/b9/74/cb1abc870a418ae99cd5c9547d6bce30701a954e0e721821df483ef7223c/tomli-2.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:b273fcbd7fc64dc3600c098e39136522650c49bca95df2d11cf3b626422392c8", size = 107964, upload-time = "2025-10-08T22:01:36.057Z" }, + { url = "https://files.pythonhosted.org/packages/54/78/5c46fff6432a712af9f792944f4fcd7067d8823157949f4e40c56b8b3c83/tomli-2.3.0-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:940d56ee0410fa17ee1f12b817b37a4d4e4dc4d27340863cc67236c74f582e77", size = 163065, upload-time = "2025-10-08T22:01:37.27Z" }, + { url = "https://files.pythonhosted.org/packages/39/67/f85d9bd23182f45eca8939cd2bc7050e1f90c41f4a2ecbbd5963a1d1c486/tomli-2.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f85209946d1fe94416debbb88d00eb92ce9cd5266775424ff81bc959e001acaf", size = 159088, upload-time = "2025-10-08T22:01:38.235Z" }, + { url = "https://files.pythonhosted.org/packages/26/5a/4b546a0405b9cc0659b399f12b6adb750757baf04250b148d3c5059fc4eb/tomli-2.3.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a56212bdcce682e56b0aaf79e869ba5d15a6163f88d5451cbde388d48b13f530", size = 268193, upload-time = "2025-10-08T22:01:39.712Z" }, + { url = "https://files.pythonhosted.org/packages/42/4f/2c12a72ae22cf7b59a7fe75b3465b7aba40ea9145d026ba41cb382075b0e/tomli-2.3.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c5f3ffd1e098dfc032d4d3af5c0ac64f6d286d98bc148698356847b80fa4de1b", size = 275488, upload-time = "2025-10-08T22:01:40.773Z" }, + { url = "https://files.pythonhosted.org/packages/92/04/a038d65dbe160c3aa5a624e93ad98111090f6804027d474ba9c37c8ae186/tomli-2.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e01decd096b1530d97d5d85cb4dff4af2d8347bd35686654a004f8dea20fc67", size = 272669, upload-time = "2025-10-08T22:01:41.824Z" }, + { url = "https://files.pythonhosted.org/packages/be/2f/8b7c60a9d1612a7cbc39ffcca4f21a73bf368a80fc25bccf8253e2563267/tomli-2.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8a35dd0e643bb2610f156cca8db95d213a90015c11fee76c946aa62b7ae7e02f", size = 279709, upload-time = "2025-10-08T22:01:43.177Z" }, + { url = "https://files.pythonhosted.org/packages/7e/46/cc36c679f09f27ded940281c38607716c86cf8ba4a518d524e349c8b4874/tomli-2.3.0-cp314-cp314t-win32.whl", hash = "sha256:a1f7f282fe248311650081faafa5f4732bdbfef5d45fe3f2e702fbc6f2d496e0", size = 107563, upload-time = "2025-10-08T22:01:44.233Z" }, + { url = "https://files.pythonhosted.org/packages/84/ff/426ca8683cf7b753614480484f6437f568fd2fda2edbdf57a2d3d8b27a0b/tomli-2.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:70a251f8d4ba2d9ac2542eecf008b3c8a9fc5c3f9f02c56a9d7952612be2fdba", size = 119756, upload-time = "2025-10-08T22:01:45.234Z" }, + { url = "https://files.pythonhosted.org/packages/77/b8/0135fadc89e73be292b473cb820b4f5a08197779206b33191e801feeae40/tomli-2.3.0-py3-none-any.whl", hash = "sha256:e95b1af3c5b07d9e643909b5abbec77cd9f1217e6d0bca72b0234736b9fb1f1b", size = 14408, upload-time = "2025-10-08T22:01:46.04Z" }, +] + +[[package]] +name = "tomlkit" +version = "0.13.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cc/18/0bbf3884e9eaa38819ebe46a7bd25dcd56b67434402b66a58c4b8e552575/tomlkit-0.13.3.tar.gz", hash = "sha256:430cf247ee57df2b94ee3fbe588e71d362a941ebb545dec29b53961d61add2a1", size = 185207, upload-time = "2025-06-05T07:13:44.947Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bd/75/8539d011f6be8e29f339c42e633aae3cb73bffa95dd0f9adec09b9c58e85/tomlkit-0.13.3-py3-none-any.whl", hash = "sha256:c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0", size = 38901, upload-time = "2025-06-05T07:13:43.546Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] + +[[package]] +name = "zipp" +version = "3.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50ede074e376733dca2ae7c6eb617489437771209d4180/zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166", size = 25547, upload-time = "2025-06-08T17:06:39.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, +] From f19b491c02d25c91d4e28b847d3d14bbfe8bb655 Mon Sep 17 00:00:00 2001 From: Dou Mok Date: Thu, 4 Dec 2025 11:21:34 -0800 Subject: [PATCH 02/11] Update 'README.md' with instructions on how to install 'uv' for dependency management --- README.md | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 7dd0bd32..0d2a997a 100644 --- a/README.md +++ b/README.md @@ -316,13 +316,23 @@ use_multiprocessing = os.getenv("USE_MULTIPROCESSING", "False") == "True" ## Development build -HashStore is a python package, and built using the [Python Poetry](https://python-poetry.org) -build tool. - -To install `hashstore` locally, create a virtual environment for python 3.9+, -install poetry, and then install or build the package with `poetry install` or `poetry build`, -respectively. Note, installing `hashstore` with poetry will also make the `hashstore` command -available through the command line terminal (see `HashStore Client` section below for details). +HashStore is a python package. We recommend installing it using `uv`. Instructions on how to install and set up `uv` can be found [here](https://gist.github.com/datadavev/3975f244e5db500ba0328ef771ca74dd). + +Friendly Notes: + - You may run into a `command not found: compdef` when adding code to your `.zshrc` file, this can be resolved by adjusting the code to be: + ```sh + # .zshrc + autoload -Uz compinit + compinit + eval "$(uv generate-shell-completion zsh)" + eval "$(uvx --generate-shell-completion zsh)" + ``` + - When downloading the script `uv-python-symlink`, an extension may be added to it, for example: `uv-python-symlink.txt`. It may also not have an executable status. You can execute the following to adjust it: + ```sh + $ mv uv-python-symlink uv-python-symlink.sh + chmod +x uv-python-symlink.sh + ``` + - After following the steps and navigating to the python project, `uv` may not have sufficient permissions to run. Follow the given prompts and execute `direnv allow` To run tests, navigate to the root directory and run `pytest`. The test suite contains tests that take a longer time to run (relating to the storage of large files) - to execute all tests, run From b91e8bb0f6640c368130fa5990ad6ca40653a716 Mon Sep 17 00:00:00 2001 From: Dou Mok Date: Thu, 4 Dec 2025 11:32:03 -0800 Subject: [PATCH 03/11] Add 'exceptiongroup' dependency to 'pyproject.toml' to resolve python 3.10 workflow issue --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index f954c4a4..02e023b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -43,6 +43,7 @@ hashstore = "hashstore.hashstoreclient:main" [dependency-groups] dev = [ "pytest>=7.2.0", + "exceptiongroup>=1.1.0", "black>=22.10.0", "pylint>=2.17.4", "pg8000>=1.29.8", From 16a2598cd5ef794b74e81b65ff968f192caf4cb6 Mon Sep 17 00:00:00 2001 From: Dou Mok Date: Thu, 4 Dec 2025 11:32:33 -0800 Subject: [PATCH 04/11] Add python 3.11 to github workflow test --- .github/workflows/poetry-package-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/poetry-package-test.yml b/.github/workflows/poetry-package-test.yml index dc16d638..9d562d87 100644 --- a/.github/workflows/poetry-package-test.yml +++ b/.github/workflows/poetry-package-test.yml @@ -16,7 +16,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11"] steps: - uses: actions/checkout@v3 From 243953dd16469cab81cd6e3bba3a298acc7dbb80 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 4 Dec 2025 14:51:50 -0500 Subject: [PATCH 05/11] Add workflow for CI with uv, version bump, limit python upper version --- .github/workflows/uv-package-test.yml | 30 +++++++++++++++++++++++++++ pyproject.toml | 4 ++-- 2 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/uv-package-test.yml diff --git a/.github/workflows/uv-package-test.yml b/.github/workflows/uv-package-test.yml new file mode 100644 index 00000000..8e7c27ba --- /dev/null +++ b/.github/workflows/uv-package-test.yml @@ -0,0 +1,30 @@ +name: Python CI with uv and pytest +on: + workflow_dispatch: + push: + branches: [ "main"] + pull_request: + branches: [ "main" ] +jobs: + build: + + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14" ] + steps: + - uses: actions/checkout@v4 + - name: Setup uv + uses: astral-sh/uv-setup-action@v1.2.0 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: 'uv' + - name: Create uv virtual environment + run: uv venv + - name: Install dependencies in uv environment + run: uv pip install -e '.[dev]' + - name: Run tests with pytest + run: uv run pytest diff --git a/pyproject.toml b/pyproject.toml index f954c4a4..95839ac9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "hashstore" -version = "1.1.0" +version = "1.1.1" description = "HashStore, an object storage system using content identifiers." authors = [ { name = "Dou Mok", email = "douming.mok@gmail.com" }, @@ -10,7 +10,7 @@ authors = [ { name = "Jeanette Clark" }, { name = "Ian M. Nesbitt" }, ] -requires-python = ">=3.9" +requires-python = ">=3.9, <4.0" readme = "README.md" keywords = [ "filesystem", From 7ca4b33f4d52c0dc06f1e3e60980d97a94247978 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:03:59 -0500 Subject: [PATCH 06/11] Update uv CI workflow to use current --- .github/workflows/uv-package-test.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/uv-package-test.yml b/.github/workflows/uv-package-test.yml index 8e7c27ba..8333412b 100644 --- a/.github/workflows/uv-package-test.yml +++ b/.github/workflows/uv-package-test.yml @@ -6,25 +6,23 @@ on: pull_request: branches: [ "main" ] jobs: - build: - + build: runs-on: ubuntu-latest strategy: fail-fast: false matrix: python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14" ] steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 + - name: Setup uv - uses: astral-sh/uv-setup-action@v1.2.0 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 + uses: astral-sh/setup-uv@v7 with: + version: '0.9.15' python-version: ${{ matrix.python-version }} - cache: 'uv' - - name: Create uv virtual environment - run: uv venv - - name: Install dependencies in uv environment - run: uv pip install -e '.[dev]' + + - name: Install the project + run: uv sync --locked --all-extras --dev + - name: Run tests with pytest - run: uv run pytest + run: uv run pytest tests From 1d3fdf68ee9dbc22fd13264fa381c139306657ed Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Thu, 4 Dec 2025 15:05:21 -0500 Subject: [PATCH 07/11] uv CI withough lock --- .github/workflows/uv-package-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/uv-package-test.yml b/.github/workflows/uv-package-test.yml index 8333412b..e9ebefe6 100644 --- a/.github/workflows/uv-package-test.yml +++ b/.github/workflows/uv-package-test.yml @@ -22,7 +22,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install the project - run: uv sync --locked --all-extras --dev + run: uv sync --all-extras --dev - name: Run tests with pytest run: uv run pytest tests From 886de819637e685fef071f63f51e479880a15c15 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Tue, 23 Dec 2025 15:36:36 -0500 Subject: [PATCH 08/11] Update lockfile --- uv.lock | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/uv.lock b/uv.lock index 3bf3e461..bcdc3492 100644 --- a/uv.lock +++ b/uv.lock @@ -1,6 +1,6 @@ version = 1 revision = 3 -requires-python = ">=3.9" +requires-python = ">=3.9, <4.0" resolution-markers = [ "python_full_version >= '3.12'", "python_full_version == '3.11.*'", @@ -149,7 +149,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ @@ -158,7 +158,7 @@ wheels = [ [[package]] name = "hashstore" -version = "1.1.0" +version = "1.1.1" source = { editable = "." } dependencies = [ { name = "pathlib" }, @@ -168,6 +168,7 @@ dependencies = [ [package.dev-dependencies] dev = [ { name = "black" }, + { name = "exceptiongroup" }, { name = "pg8000" }, { name = "pylint", version = "3.3.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.10'" }, { name = "pylint", version = "4.0.4", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.10'" }, @@ -184,6 +185,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ { name = "black", specifier = ">=22.10.0" }, + { name = "exceptiongroup", specifier = ">=1.1.0" }, { name = "pg8000", specifier = ">=1.29.8" }, { name = "pylint", specifier = ">=2.17.4" }, { name = "pytest", specifier = ">=7.2.0" }, From 0e2e2bef8fa1994bf05fef5057fa5081d553229c Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Fri, 29 May 2026 17:13:53 -0400 Subject: [PATCH 09/11] Adding ruff formatting --- .pre-commit-config.yaml | 30 ++++++++++++++++++++++++++++++ pyproject.toml | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..aa872ef7 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,30 @@ +ci: + autoupdate_commit_msg: "chore: update pre-commit hooks" + autofix_commit_msg: "style: pre-commit fixes" + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: "v6.0.0" + hooks: + - id: check-added-large-files + - id: check-case-conflict + - id: check-merge-conflict + - id: check-symlinks + - id: check-yaml + - id: debug-statements + - id: end-of-file-fixer + - id: mixed-line-ending + - id: name-tests-test + args: ["--pytest-test-first"] + - id: requirements-txt-fixer + - id: trailing-whitespace + + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: "v0.14.13" + hooks: + # first, lint + autofix + - id: ruff + types_or: [python, pyi, jupyter] + args: ["--fix", "--show-fixes"] + # then, format + - id: ruff-format diff --git a/pyproject.toml b/pyproject.toml index bddee275..9d654796 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,3 +54,44 @@ requires = ["hatchling"] build-backend = "hatchling.build" [tool.poetry_bumpversion.file."src/hashstore/__init__.py"] + +[tool.ruff] +src = ["src"] +extend-exclude = ["tests/testdata/"] +force-exclude = true +line-length = 88 # how long you want lines to be + +[tool.ruff.format] +docstring-code-format = true # code snippets in docstrings will be formatted + +[tool.ruff.lint] +select = [ + "E", "F", "W", # flake8 + "B", # flake8-bugbear + "I", # isort + "ARG", # flake8-unused-arguments + "C4", # flake8-comprehensions + "EM", # flake8-errmsg + "ICN", # flake8-import-conventions + "ISC", # flake8-implicit-str-concat + "G", # flake8-logging-format + "PGH", # pygrep-hooks + "PIE", # flake8-pie + "PL", # pylint + "PT", # flake8-pytest-style + "RET", # flake8-return + "RUF", # Ruff-specific + "SIM", # flake8-simplify + "UP", # pyupgrade + "YTT", # flake8-2020 + "EXE", # flake8-executable +] +ignore = [ + "PLR", # Design related pylint codes + "ISC001", # Conflicts with formatter +] +unfixable = [ + "F401", # Would remove unused imports + "F841", # Would remove unused variables +] +flake8-unused-arguments.ignore-variadic-names = true # allow unused *args/**kwargsisort.required-imports = ["from __future__ import annotations"] From 1636665a0b4440cc1da2046223be2f0b991316d1 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Sat, 30 May 2026 08:13:42 -0400 Subject: [PATCH 10/11] Ruff formatting, PEP cleanup, no functional changes --- .github/workflows/poetry-package-test.yml | 1 - .github/workflows/uv-package-test.yml | 28 +- .pre-commit-config.yaml | 2 + .vscode/settings.json | 2 +- CONTRIBUTING.md | 16 +- README.md | 54 +- hashstore.code-workspace | 2 +- pyproject.toml | 3 + src/hashstore/filehashstore.py | 1017 +++++++++-------- src/hashstore/filehashstore_exceptions.py | 13 +- src/hashstore/hashstore.py | 167 +-- src/hashstore/hashstoreclient.py | 273 ++--- tests/conftest.py | 10 +- tests/filehashstore/test_filehashstore.py | 444 ++++--- .../test_filehashstore_interface.py | 221 ++-- tests/test_hashstore.py | 19 +- tests/test_hashstore_client.py | 23 +- 17 files changed, 1201 insertions(+), 1094 deletions(-) diff --git a/.github/workflows/poetry-package-test.yml b/.github/workflows/poetry-package-test.yml index 9d562d87..9821b50f 100644 --- a/.github/workflows/poetry-package-test.yml +++ b/.github/workflows/poetry-package-test.yml @@ -39,4 +39,3 @@ jobs: - name: Test with pytest run: | poetry run pytest - diff --git a/.github/workflows/uv-package-test.yml b/.github/workflows/uv-package-test.yml index e9ebefe6..05dcdb89 100644 --- a/.github/workflows/uv-package-test.yml +++ b/.github/workflows/uv-package-test.yml @@ -2,27 +2,27 @@ name: Python CI with uv and pytest on: workflow_dispatch: push: - branches: [ "main"] + branches: ["main", "develop"] pull_request: - branches: [ "main" ] + branches: ["main", "develop"] jobs: - build: + build: runs-on: ubuntu-latest strategy: fail-fast: false matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14" ] + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - - uses: actions/checkout@v5 + - uses: actions/checkout@v5 - - name: Setup uv - uses: astral-sh/setup-uv@v7 - with: - version: '0.9.15' - python-version: ${{ matrix.python-version }} + - name: Setup uv + uses: astral-sh/setup-uv@v7 + with: + version: "0.9.15" + python-version: ${{ matrix.python-version }} - - name: Install the project - run: uv sync --all-extras --dev + - name: Install the project + run: uv sync --all-extras --dev - - name: Run tests with pytest - run: uv run pytest tests + - name: Run tests with pytest + run: uv run pytest tests diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index aa872ef7..f4e29488 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,6 +2,8 @@ ci: autoupdate_commit_msg: "chore: update pre-commit hooks" autofix_commit_msg: "style: pre-commit fixes" +exclude: "^(tests/testdata/)" + repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: "v6.0.0" diff --git a/.vscode/settings.json b/.vscode/settings.json index b15ffaa4..11d9542c 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,4 +9,4 @@ "[python]": { "editor.defaultFormatter": "ms-python.black-formatter" } -} \ No newline at end of file +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 59e9c99f..45786319 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,21 +45,21 @@ In short: ## 🔀 Development Workflow -Development is managed through the git repository at https://github.com/DataONEorg/hashstore. The repository is organized into several branches, each with a specific purpose. +Development is managed through the git repository at https://github.com/DataONEorg/hashstore. The repository is organized into several branches, each with a specific purpose. **main**. The `main` branch represents the stable branch that is constantly maintained with the current release. It should generally be safe to install and use the `main` branch the same way as binary releases. The version number in all configuration files and the README on the `main` branch follows [semantic versioning](https://semver.org/) and should always be set to the current stable release, for example `2.8.5`. **develop**. Development takes place on a single branch for integrated development and testing of the set of features targeting the next release. Commits should only be pushed to this branch once they are ready to be deployed to production immediately after being pushed. This keeps the `develop` branch in a state of readiness for the next release. -Any unreleased code changes on the `develop` branch represent changes that have been tested and staged for the next -release. +Any unreleased code changes on the `develop` branch represent changes that have been tested and staged for the next +release. The tip of the `develop` branch always represents the set of features that are awaiting the next release. The develop branch represents the opportunity to integrate changes from multiple features for integrated testing before release. Version numbers on the `develop` branch represent either the planned next release number (e.g., `2.9.0`), or the planned next release number with a `beta` designator or release candidate `rc` designator appended as appropriate. For example, `2.8.6-beta1` or `2.9.0-rc1`. -**feature**. To isolate development on a specific set of capabilities, especially if it may be disruptive to other +**feature**. To isolate development on a specific set of capabilities, especially if it may be disruptive to other developers working on the `develop` branch, feature branches should be created. Feature branches are named as `feature-` + `{issue}` + `-{short-description}`, with `{issue}` being the GitHub issue number related to that new feature. e.g. `feature-23-refactor-storage`. @@ -73,11 +73,11 @@ been tested and are awaiting release. Thus, each `feature-*` branch can be test ### Development flow overview ```mermaid -%%{init: { 'theme': 'base', +%%{init: { 'theme': 'base', 'gitGraph': { 'rotateCommitLabel': false, 'showCommitLabel': false - }, + }, 'themeVariables': { 'commitLabelColor': '#ffffffff', 'commitLabelBackground': '#000000' @@ -110,8 +110,8 @@ gitGraph changes that are desired in a release are merged into the `develop` branch, we run the full set of tests on a clean checkout of the `develop` branch. 2. After testing, the `develop` branch is merged to main, and the `main` branch is tagged with -the new version number (e.g. `2.11.2`). At this point, the tip of the `main` branch will -reflect the new release and the `develop` branch can be fast-forwarded to sync with `main` to +the new version number (e.g. `2.11.2`). At this point, the tip of the `main` branch will +reflect the new release and the `develop` branch can be fast-forwarded to sync with `main` to start work on the next release. 3. Releases can be downloaded from the [GitHub releases page](https://github.com/DataONEorg/hashstore/releases). diff --git a/README.md b/README.md index 0d2a997a..e5f8cba7 100644 --- a/README.md +++ b/README.md @@ -16,18 +16,18 @@ Version: 1.1.0 Cite this software as: -> Dou Mok, Matthew Brooke, Jing Tao, Jeanette Clarke, Ian Nesbitt, Matthew B. Jones. 2024. +> Dou Mok, Matthew Brooke, Jing Tao, Jeanette Clarke, Ian Nesbitt, Matthew B. Jones. 2024. > HashStore: hash-based object storage for DataONE data packages. Arctic Data Center. > [doi:10.18739/A2ZG6G87Q](https://doi.org/10.18739/A2ZG6G87Q) ## Introduction -HashStore is a server-side python package that implements a hash-based object storage file system -for storing and accessing data and metadata for DataONE services. The package is used in DataONE -system components that need direct, filesystem-based access to data objects, their system -metadata, and extended metadata about the objects. This package is a core component of the -[DataONE federation](https://dataone.org), and supports large-scale object storage for a variety -of repositories, including the [KNB Data Repository](http://knb.ecoinformatics.org), the [NSF +HashStore is a server-side python package that implements a hash-based object storage file system +for storing and accessing data and metadata for DataONE services. The package is used in DataONE +system components that need direct, filesystem-based access to data objects, their system +metadata, and extended metadata about the objects. This package is a core component of the +[DataONE federation](https://dataone.org), and supports large-scale object storage for a variety +of repositories, including the [KNB Data Repository](http://knb.ecoinformatics.org), the [NSF Arctic Data Center](https://arcticdata.io/catalog/), the [DataONE search service](https://search.dataone.org), and other repositories. DataONE in general, and HashStore in particular, are open source, community projects. @@ -38,17 +38,17 @@ contributions with us. ## Documentation -The documentation around HashStore's initial design phase can be found here in the [Metacat +The documentation around HashStore's initial design phase can be found here in the [Metacat repository](https://github.com/NCEAS/metacat/blob/feature-1436-storage-and-indexing/docs/user/metacat/source/storage-subsystem.rst#physical-file-layout) as part of the storage re-design planning. Future updates will include documentation here as the package matures. ## HashStore Overview -HashStore is a hash-based object storage system that provides persistent file-based storage using -content hashes to de-duplicate data. The system stores data objects, references (refs) and -metadata in its respective directories and utilizes an identifier-based API for interacting -with the store. HashStore storage classes (like `filehashstore`) must implement the HashStore +HashStore is a hash-based object storage system that provides persistent file-based storage using +content hashes to de-duplicate data. The system stores data objects, references (refs) and +metadata in its respective directories and utilizes an identifier-based API for interacting +with the store. HashStore storage classes (like `filehashstore`) must implement the HashStore interface to ensure the consistent and expected usage of HashStore. ### Public API Methods @@ -160,11 +160,11 @@ metadata_cid_two = hashstore.store_metadata(pid, metadata, format_id) ### Working with objects (store, retrieve, delete) -In HashStore, data objects begin as temporary files while their content identifiers are +In HashStore, data objects begin as temporary files while their content identifiers are calculated. Once the default hash algorithm list and their hashes are generated, objects are stored -in their permanent locations using the hash value of the store's configured algorithm, and -then divided accordingly based on the configured width and depth. Lastly, objects are 'tagged' -with a given identifier (ex. persistent identifier (pid)). This process produces reference +in their permanent locations using the hash value of the store's configured algorithm, and +then divided accordingly based on the configured width and depth. Lastly, objects are 'tagged' +with a given identifier (ex. persistent identifier (pid)). This process produces reference files, which allow objects to be found and retrieved with a given identifier. - Note 1: An identifier can only be used once @@ -176,9 +176,9 @@ files, which allow objects to be found and retrieved with a given identifier. By calling the various interface methods for `store_object`, the calling app/client can validate, store and tag an object simultaneously if the relevant data is available. In the absence of an identifier (ex. persistent identifier (pid)), `store_object` can be called to solely store an -object. The client is then expected to call `delete_if_invalid_object` when the relevant +object. The client is then expected to call `delete_if_invalid_object` when the relevant metadata is available to confirm that the object is what is expected. And to finalize the data-only -storage process (to make the object discoverable), the client calls `tagObject``. In summary, there +storage process (to make the object discoverable), the client calls `tagObject``. In summary, there are two expected paths to store an object: ```py @@ -263,8 +263,8 @@ ex. `store_metadata(stream, pid, format_id)`). ### What are HashStore reference files? -HashStore assumes that every data object is referenced by its a respective identifier. This -identifier is then used when storing, retrieving and deleting an object. In order to facilitate +HashStore assumes that every data object is referenced by its a respective identifier. This +identifier is then used when storing, retrieving and deleting an object. In order to facilitate this process, we create two types of reference files: - pid (persistent identifier) reference files @@ -272,7 +272,7 @@ this process, we create two types of reference files: These reference files are implemented in HashStore underneath the hood with no expectation for modification from the calling app/client. The one and only exception to this process is when the -calling client/app does not have an identifier available (i.e. they receive the stream to store +calling client/app does not have an identifier available (i.e. they receive the stream to store the data object first without any metadata, thus calling `store_object(stream)`). **'pid' Reference Files** @@ -282,7 +282,7 @@ the data object first without any metadata, thus calling `store_object(stream)`) - If an identifier is not available at the time of storing an object, the calling app/client must create this association between a pid and the object it represents by calling `tag_object` separately. -- Each pid reference file contains a single string that represents the content identifier of the +- Each pid reference file contains a single string that represents the content identifier of the object it references - Like how objects are stored once and only once, there is also only one pid reference file for each data object. @@ -297,10 +297,10 @@ the data object first without any metadata, thus calling `store_object(stream)`) ## Concurrency in HashStore -HashStore is both threading and multiprocessing safe, and by default synchronizes calls to store & -delete objects/metadata with Python's threading module. If you wish to use multiprocessing to -parallelize your application, please declare a global environment variable `USE_MULTIPROCESSING` -as `True` before initializing Hashstore. This will direct the relevant Public API calls to +HashStore is both threading and multiprocessing safe, and by default synchronizes calls to store & +delete objects/metadata with Python's threading module. If you wish to use multiprocessing to +parallelize your application, please declare a global environment variable `USE_MULTIPROCESSING` +as `True` before initializing Hashstore. This will direct the relevant Public API calls to synchronize using the Python `multiprocessing` module's locks and conditions. Please see below for example: @@ -414,5 +414,3 @@ California. [![DataONE_footer](https://user-images.githubusercontent.com/6643222/162324180-b5cf0f5f-ae7a-4ca6-87c3-9733a2590634.png)](https://dataone.org) [![nceas_footer](https://www.nceas.ucsb.edu/sites/default/files/2020-03/NCEAS-full%20logo-4C.png)](https://www.nceas.ucsb.edu) - - diff --git a/hashstore.code-workspace b/hashstore.code-workspace index 876a1499..57097327 100644 --- a/hashstore.code-workspace +++ b/hashstore.code-workspace @@ -5,4 +5,4 @@ } ], "settings": {} -} \ No newline at end of file +} diff --git a/pyproject.toml b/pyproject.toml index 9d654796..d6ccb30a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -95,3 +95,6 @@ unfixable = [ "F841", # Would remove unused variables ] flake8-unused-arguments.ignore-variadic-names = true # allow unused *args/**kwargsisort.required-imports = ["from __future__ import annotations"] + +[tool.ruff.lint.per-file-ignores] +"tests/*" = ["E501"] diff --git a/src/hashstore/filehashstore.py b/src/hashstore/filehashstore.py index 74b9c600..a9415517 100644 --- a/src/hashstore/filehashstore.py +++ b/src/hashstore/filehashstore.py @@ -1,46 +1,48 @@ """Core module for FileHashStore""" import atexit +import fcntl +import hashlib +import inspect import io +import logging import multiprocessing +import os import shutil import threading -import hashlib -import os -import logging -import inspect -import fcntl -import yaml -from typing import List, Dict, Union, Optional, IO, Tuple, Set, Any +from contextlib import closing from dataclasses import dataclass from pathlib import Path -from contextlib import closing from tempfile import NamedTemporaryFile +from typing import IO, Any, Optional, Union + +import yaml + from hashstore import HashStore from hashstore.filehashstore_exceptions import ( CidRefsContentError, - OrphanPidRefsFileFound, CidRefsFileNotFound, HashStoreRefsAlreadyExists, + IdentifierNotLocked, NonMatchingChecksum, NonMatchingObjSize, - PidRefsAlreadyExistsError, + OrphanPidRefsFileFound, PidNotFoundInCidRefsFile, + PidRefsAlreadyExistsError, PidRefsContentError, PidRefsDoesNotExist, PidRefsFileNotFound, RefsFileExistsButCidObjMissing, - UnsupportedAlgorithm, StoreObjectForPidAlreadyInProgress, - IdentifierNotLocked, + UnsupportedAlgorithm, ) class FileHashStore(HashStore): - """FileHashStore is an object storage system that was extended from Derrick Gilland's - 'hashfs' library. It supports the storage of objects on disk using a content identifier - to address files (data objects are de-duplicated) and provides a content identifier-based - API to interact with a HashStore. + """FileHashStore is an object storage system that was extended from Derrick + Gilland's 'hashfs' library. It supports the storage of objects on disk using + a content identifier to address files (data objects are de-duplicated) and + provides a content identifier-based API to interact with a HashStore. FileHashStore initializes using a given properties dictionary containing the required keys (see Args). Upon initialization, FileHashStore verifies the provided @@ -52,24 +54,25 @@ class FileHashStore(HashStore): - store_path (str): Path to the HashStore directory. - store_depth (int): Depth when sharding an object's hex digest. - store_width (int): Width of directories when sharding an object's hex digest. - - store_algorithm (str): Hash algorithm used for calculating the object's hex digest. + - store_algorithm (str): Hash algorithm used for calculating the object's hex + digest. - store_metadata_namespace (str): Namespace for the HashStore's system metadata. """ # Property (hashstore configuration) requirements - property_required_keys = [ + property_required_keys = ( "store_path", "store_depth", "store_width", "store_algorithm", "store_metadata_namespace", - ] + ) # Permissions settings for writing files and creating directories f_mode = 0o664 d_mode = 0o755 # The other algorithm list consists of additional algorithms that can be included # for calculating when storing objects, in addition to the default list. - other_algo_list = [ + other_algo_list = ( "sha224", "sha3_224", "sha3_256", @@ -77,7 +80,7 @@ class FileHashStore(HashStore): "sha3_512", "blake2b", "blake2s", - ] + ) def __init__(self, properties=None): self.fhs_logger = logging.getLogger(__name__) @@ -111,12 +114,13 @@ def __init__(self, properties=None): # pylint: disable=W1201 self.fhs_logger.debug( "HashStore does not exist & configuration file not found." - + " Writing configuration file." + " Writing configuration file." ) self._write_properties(properties) # Default algorithm list for FileHashStore based on config file written self._set_default_algorithms() - # Complete initialization/instantiation by setting and creating store directories + # Complete initialization/instantiation by setting and creating store + # directories self.objects = self.root / "objects" self.metadata = self.root / "metadata" self.refs = self.root / "refs" @@ -132,7 +136,8 @@ def __init__(self, properties=None): self._create_path(self.refs / "cids") # Variables to orchestrate parallelization - # Check to see whether a multiprocessing or threading sync lock should be used + # Check to see whether a multiprocessing or threading sync lock should + # be used self.use_multiprocessing = ( os.getenv("USE_MULTIPROCESSING", "False") == "True" ) @@ -200,15 +205,18 @@ def __init__(self, properties=None): @staticmethod def _load_properties( - hashstore_yaml_path: Path, hashstore_required_prop_keys: List[str] - ) -> Dict[str, Union[str, int]]: + hashstore_yaml_path: Path, hashstore_required_prop_keys: list[str] + ) -> dict[str, Union[str, int]]: """Get and return the contents of the current HashStore configuration. :return: HashStore properties with the following keys (and values): - store_depth (int): Depth when sharding an object's hex digest. - - store_width (int): Width of directories when sharding an object's hex digest. - - store_algorithm (str): Hash algo used for calculating the object's hex digest. - - store_metadata_namespace (str): Namespace for the HashStore's system metadata. + - store_width (int): Width of directories when sharding an object's + hex digest. + - store_algorithm (str): Hash algo used for calculating the object's + hex digest. + - store_metadata_namespace (str): Namespace for the HashStore's system + metadata. """ if not os.path.isfile(hashstore_yaml_path): err_msg = "'hashstore.yaml' not found in store root path." @@ -216,7 +224,7 @@ def _load_properties( raise FileNotFoundError(err_msg) # Open file - with open(hashstore_yaml_path, "r", encoding="utf-8") as hs_yaml_file: + with open(hashstore_yaml_path, encoding="utf-8") as hs_yaml_file: yaml_data = yaml.safe_load(hs_yaml_file) # Get hashstore properties @@ -227,17 +235,22 @@ def _load_properties( logging.debug("Successfully retrieved 'hashstore.yaml' properties.") return hashstore_yaml_dict - def _write_properties(self, properties: Dict[str, Union[str, int]]) -> None: + def _write_properties(self, properties: dict[str, Union[str, int]]) -> None: """Writes 'hashstore.yaml' to FileHashStore's root directory with the respective properties object supplied. - :param dict properties: A Python dictionary with the following keys (and values): + :param dict properties: A Python dictionary with the following keys + (and values): - store_depth (int): Depth when sharding an object's hex digest. - - store_width (int): Width of directories when sharding an object's hex digest. - - store_algorithm (str): Hash algo used for calculating the object's hex digest. - - store_metadata_namespace (str): Namespace for the HashStore's system metadata. + - store_width (int): Width of directories when sharding an object's hex + digest. + - store_algorithm (str): Hash algo used for calculating the object's hex + digest. + - store_metadata_namespace (str): Namespace for the HashStore's system + metadata. """ - # If hashstore.yaml already exists, must throw exception and proceed with caution + # If hashstore.yaml already exists, must throw exception and proceed with + # caution if os.path.isfile(self.hashstore_configuration_yaml): err_msg = "Configuration file 'hashstore.yaml' already exists." logging.error(err_msg) @@ -246,13 +259,20 @@ def _write_properties(self, properties: Dict[str, Union[str, int]]) -> None: checked_properties = self._validate_properties(properties) # Collect configuration properties from validated & supplied dictionary - (_, store_depth, store_width, store_algorithm, store_metadata_namespace,) = [ + ( + _, + store_depth, + store_width, + store_algorithm, + store_metadata_namespace, + ) = [ checked_properties[property_name] for property_name in self.property_required_keys ] # Standardize algorithm value for cross-language compatibility - # Note, this must be declared here because HashStore has not yet been initialized + # Note, this must be declared here because HashStore has not yet been + # initialized accepted_store_algorithms = ["MD5", "SHA-1", "SHA-256", "SHA-384", "SHA-512"] if store_algorithm in accepted_store_algorithms: checked_store_algorithm = store_algorithm @@ -297,9 +317,12 @@ def _build_hashstore_yaml_string( """Build a YAML string representing the configuration for a HashStore. :param int store_depth: Depth when sharding an object's hex digest. - :param int store_width: Width of directories when sharding an object's hex digest. - :param str store_algorithm: Hash algorithm used for calculating the object's hex digest. - :param str store_metadata_namespace: Namespace for the HashStore's system metadata. + :param int store_width: Width of directories when sharding an object's hex + digest. + :param str store_algorithm: Hash algorithm used for calculating the object's hex + digest. + :param str store_metadata_namespace: Namespace for the HashStore's system + metadata. :return: A YAML string representing the configuration for a HashStore. """ @@ -317,8 +340,9 @@ def _build_hashstore_yaml_string( ], } - # The tabbing here is intentional otherwise the created .yaml will have extra tabs - hashstore_configuration_comments = f""" + # The tabbing here is intentional otherwise the created .yaml will have + # extra tabs + hashstore_configuration_comments = """ # Default configuration variables for HashStore ############### HashStore Config Notes ############### @@ -355,24 +379,24 @@ def _build_hashstore_yaml_string( """ - hashstore_yaml_with_comments = hashstore_configuration_comments + yaml.dump( + return hashstore_configuration_comments + yaml.dump( hashstore_configuration, sort_keys=False ) - return hashstore_yaml_with_comments - def _verify_hashstore_properties( - self, properties: Dict[str, Union[str, int]], prop_store_path: str + self, properties: dict[str, Union[str, int]], prop_store_path: str ) -> None: - """Determines whether FileHashStore can instantiate by validating a set of arguments - and throwing exceptions. HashStore will not instantiate if an existing configuration - file's properties (`hashstore.yaml`) are different from what is supplied - or if an - object store exists at the given path, but it is missing the `hashstore.yaml` config file. - - If `hashstore.yaml` exists, it will retrieve its properties and compare them with the - given values; and if there is a mismatch, an exception will be thrown. If not, it will - look to see if any directories/files exist in the given store path and throw an exception - if any file or directory is found. + """Determines whether FileHashStore can instantiate by validating a set + of arguments and throwing exceptions. HashStore will not instantiate if + an existing configuration file's properties (`hashstore.yaml`) are + different from what is supplied - or if an object store exists at the + given path, but it is missing the `hashstore.yaml` config file. + + If `hashstore.yaml` exists, it will retrieve its properties and compare + them with the given values; and if there is a mismatch, an exception + will be thrown. If not, it will look to see if any directories/files + exist in the given store path and throw an exception if any file or + directory is found. :param dict properties: HashStore properties. :param str prop_store_path: Store path to check. @@ -387,16 +411,18 @@ def _verify_hashstore_properties( self.hashstore_configuration_yaml, self.property_required_keys ) for key in self.property_required_keys: - # 'store_path' is required to init HashStore but not saved in `hashstore.yaml` + # 'store_path' is required to init HashStore but not saved in + # `hashstore.yaml` if key != "store_path": supplied_key = properties[key] if key == "store_depth" or key == "store_width": supplied_key = int(properties[key]) if hashstore_yaml_dict[key] != supplied_key: err_msg = ( - f"Given properties ({key}: {properties[key]}) does not match." - + f" HashStore configuration ({key}: {hashstore_yaml_dict[key]})" - + f" found at: {self.hashstore_configuration_yaml}" + f"Given properties ({key}: {properties[key]}) does not " + f"match. HashStore configuration " + f"({key}: {hashstore_yaml_dict[key]}) found at: " + f"{self.hashstore_configuration_yaml}" ) self.fhs_logger.critical(err_msg) raise ValueError(err_msg) @@ -409,16 +435,17 @@ def _verify_hashstore_properties( for sub in subfolders ): err_msg = ( - "Unable to initialize HashStore. `hashstore.yaml` is not present but " - "conflicting HashStore directory exists. Please delete '/objects', " - "'/metadata' and/or '/refs' at the store path or supply a new path." + "Unable to initialize HashStore. `hashstore.yaml` is not " + "present but conflicting HashStore directory exists. Please " + "delete '/objects', '/metadata' and/or '/refs' at the store " + "path or supply a new path." ) self.fhs_logger.critical(err_msg) raise RuntimeError(err_msg) def _validate_properties( - self, properties: Dict[str, Union[str, int]] - ) -> Dict[str, Union[str, int]]: + self, properties: dict[str, Union[str, int]] + ) -> dict[str, Union[str, int]]: """Validate a properties dictionary by checking if it contains all the required keys and non-None values. @@ -456,11 +483,11 @@ def _validate_properties( checked_properties[key] = int(value) except Exception as err: err_msg = ( - "Unexpected exception when attempting to ensure store depth and width " - f"are integers. Details: {err}" + "Unexpected exception when attempting to ensure store depth " + f"and width are integers. Details: {err}" ) self.fhs_logger.error(err_msg) - raise ValueError(err_msg) + raise ValueError(err_msg) from err else: checked_properties[key] = value @@ -487,14 +514,13 @@ def lookup_algo(algo_to_translate): self.fhs_logger.critical(err_msg) raise FileNotFoundError(err_msg) - with open( - self.hashstore_configuration_yaml, "r", encoding="utf-8" - ) as hs_yaml_file: + with open(self.hashstore_configuration_yaml, encoding="utf-8") as hs_yaml_file: yaml_data = yaml.safe_load(hs_yaml_file) # Set default store algorithm self.algorithm = lookup_algo(yaml_data["store_algorithm"]) - # Takes DataOne controlled algorithm values and translates to hashlib supported values + # Takes DataOne controlled algorithm values and translates to hashlib supported + # values yaml_store_default_algo_list = yaml_data["store_default_algo_list"] translated_default_algo_list = [] for algo in yaml_store_default_algo_list: @@ -573,8 +599,8 @@ def store_object( self._release_object_locked_pids(pid) except Exception as err: err_msg = ( - f"Failed to store object for pid: {pid}. Reference files will not be " - f"created or tagged. Unexpected error: {err})" + f"Failed to store object for pid: {pid}. Reference files will not " + f"be created or tagged. Unexpected error: {err})" ) self.fhs_logger.error(err_msg) raise err @@ -589,13 +615,16 @@ def tag_object(self, pid: str, cid: str) -> None: try: self._store_hashstore_refs_files(pid, cid) except HashStoreRefsAlreadyExists as hrae: - err_msg = f"Reference files for pid: {pid} and {cid} already exist. Details: {hrae}" + err_msg = ( + f"Reference files for pid: {pid} and {cid} already exist. " + f"Details: {hrae}" + ) self.fhs_logger.error(err_msg) - raise HashStoreRefsAlreadyExists(err_msg) + raise HashStoreRefsAlreadyExists(err_msg) from hrae except PidRefsAlreadyExistsError as praee: err_msg = f"A pid can only reference one cid. Details: {praee}" self.fhs_logger.error(err_msg) - raise PidRefsAlreadyExistsError(err_msg) + raise PidRefsAlreadyExistsError(err_msg) from praee def delete_if_invalid_object( self, @@ -613,36 +642,33 @@ def delete_if_invalid_object( ) self.fhs_logger.error(err_msg) raise ValueError(err_msg) - else: - self.fhs_logger.info( - "Called to verify object with id: %s", object_metadata.cid - ) - object_metadata_hex_digests = object_metadata.hex_digests - object_metadata_file_size = object_metadata.obj_size - checksum_algorithm_checked = self._clean_algorithm(checksum_algorithm) + self.fhs_logger.info("Called to verify object with id: %s", object_metadata.cid) + object_metadata_hex_digests = object_metadata.hex_digests + object_metadata_file_size = object_metadata.obj_size + checksum_algorithm_checked = self._clean_algorithm(checksum_algorithm) - # Throws exceptions if there's an issue - try: - self._verify_object_information( - pid=None, - checksum=checksum, - checksum_algorithm=checksum_algorithm_checked, - entity="objects", - hex_digests=object_metadata_hex_digests, - tmp_file_name=None, - tmp_file_size=object_metadata_file_size, - file_size_to_validate=expected_file_size, - ) - except NonMatchingObjSize as nmose: - self._delete_object_only(object_metadata.cid) - logging.error(nmose) - raise nmose - except NonMatchingChecksum as mmce: - self._delete_object_only(object_metadata.cid) - raise mmce - self.fhs_logger.info( - "Object has been validated for cid: %s", object_metadata.cid + # Throws exceptions if there's an issue + try: + self._verify_object_information( + pid=None, + checksum=checksum, + checksum_algorithm=checksum_algorithm_checked, + entity="objects", + hex_digests=object_metadata_hex_digests, + tmp_file_name=None, + tmp_file_size=object_metadata_file_size, + file_size_to_validate=expected_file_size, ) + except NonMatchingObjSize as nmose: + self._delete_object_only(object_metadata.cid) + logging.error(nmose) + raise nmose + except NonMatchingChecksum as mmce: + self._delete_object_only(object_metadata.cid) + raise mmce + self.fhs_logger.info( + "Object has been validated for cid: %s", object_metadata.cid + ) def store_metadata( self, pid: str, metadata: Union[str, bytes], format_id: Optional[str] = None @@ -655,12 +681,12 @@ def store_metadata( pid_doc = self._computehash(pid + checked_format_id) sync_begin_debug_msg = ( - f" Adding pid: {pid} to locked list, with format_id: {checked_format_id} with doc " - f"name: {pid_doc}" + f" Adding pid: {pid} to locked list, with format_id: {checked_format_id} " + f"with doc name: {pid_doc}" ) sync_wait_msg = ( - f"Pid: {pid} is locked for format_id: {checked_format_id} with doc name: {pid_doc}. " - f"Waiting." + f"Pid: {pid} is locked for format_id: {checked_format_id} with doc name: " + f"{pid_doc}. Waiting." ) if self.use_multiprocessing: with self.metadata_condition_mp: @@ -690,8 +716,8 @@ def store_metadata( finally: # Release pid end_sync_debug_msg = ( - f"Releasing pid doc ({pid_doc}) from locked list for pid: {pid} with format_id: " - + checked_format_id + f"Releasing pid doc ({pid_doc}) from locked list for pid: {pid} with " + f"format_id: {checked_format_id}" ) if self.use_multiprocessing: with self.metadata_condition_mp: @@ -745,10 +771,9 @@ def retrieve_metadata(self, pid: str, format_id: Optional[str] = None) -> IO[byt metadata_stream = self._open(entity, str(metadata_rel_path)) self.fhs_logger.info("Retrieved metadata for pid: %s", pid) return metadata_stream - else: - err_msg = f"No metadata found for pid: {pid}" - self.fhs_logger.error(err_msg) - raise ValueError(err_msg) + err_msg = f"No metadata found for pid: {pid}" + self.fhs_logger.error(err_msg) + raise ValueError(err_msg) def delete_object(self, pid: str) -> None: self.fhs_logger.debug("Request to delete object for id: %s", pid) @@ -758,7 +783,8 @@ def delete_object(self, pid: str) -> None: # Storing and deleting objects are synchronized together # Duplicate store object requests for a pid are rejected, but deleting an object - # will wait for a pid to be released if it's found to be in use before proceeding. + # will wait for a pid to be released if it's found to be in use before + # proceeding. try: # Before we begin deletion process, we look for the `cid` by calling @@ -771,8 +797,8 @@ def delete_object(self, pid: str) -> None: cid = object_info_dict.get("cid") # Proceed with next steps - cid has been retrieved without any issues - # We must synchronize here based on the `cid` because multiple threads may - # try to access the `cid_reference_file` + # We must synchronize here based on the `cid` because multiple threads + # may try to access the `cid_reference_file` self._synchronize_object_locked_cids(cid) try: @@ -784,11 +810,13 @@ def delete_object(self, pid: str) -> None: ) # Remove pid from cid reference file self._update_refs_file(Path(cid_ref_abs_path), pid, "remove") - # Delete cid reference file and object only if the cid refs file is empty + # Delete cid reference file and object only if the cid refs file + # is empty if os.path.getsize(cid_ref_abs_path) == 0: debug_msg = ( - f"Cid reference file is empty (size == 0): {cid_ref_abs_path} - " - + "deleting cid reference file and data object." + "Cid reference file is empty (size == 0): " + "{cid_ref_abs_path} - deleting cid reference file and data " + "object." ) self.fhs_logger.debug(debug_msg) objects_to_delete.append( @@ -805,8 +833,8 @@ def delete_object(self, pid: str) -> None: self.delete_metadata(pid) info_string = ( - f"Successfully deleted references, metadata and object associated" - + f" with pid: {pid}" + "Successfully deleted references, metadata and object " + f"associated with pid: {pid}" ) self.fhs_logger.info(info_string) return @@ -817,8 +845,9 @@ def delete_object(self, pid: str) -> None: except OrphanPidRefsFileFound: warn_msg = ( - f"Orphan pid reference file found for pid: {pid}. Skipping object deletion. " - + "Deleting pid reference file and related metadata documents." + f"Orphan pid reference file found for pid: {pid}. Skipping object " + "deletion. Deleting pid reference file and related metadata " + "documents." ) self.fhs_logger.warning(warn_msg) @@ -834,9 +863,9 @@ def delete_object(self, pid: str) -> None: return except RefsFileExistsButCidObjMissing: warn_msg = ( - f"Reference files exist for pid: {pid}, but the data object is missing. " - + "Deleting pid reference file & related metadata documents. Handling cid " - + "reference file." + f"Reference files exist for pid: {pid}, but the data object is " + "missing. Deleting pid reference file & related metadata " + "documents. Handling cid reference file." ) self.fhs_logger.warning(warn_msg) @@ -864,8 +893,8 @@ def delete_object(self, pid: str) -> None: return except PidNotFoundInCidRefsFile: warn_msg = ( - f"Pid {pid} not found in cid reference file. Deleting pid reference " - + "file and related metadata documents." + f"Pid {pid} not found in cid reference file. Deleting pid " + "reference file and related metadata documents." ) self.fhs_logger.warning(warn_msg) @@ -903,12 +932,12 @@ def delete_metadata(self, pid: str, format_id: Optional[str] = None) -> None: # Synchronize based on doc name # Wait for the pid to release if it's in use sync_begin_debug_msg = ( - f"Adding pid: {pid} to locked list, with format_id: {checked_format_id} " - + f"with doc name: {pid_doc}" + f"Adding pid: {pid} to locked list, with format_id: " + f"{checked_format_id} with doc name: {pid_doc}" ) sync_wait_msg = ( - f"Pid: {pid} is locked for format_id: {checked_format_id} with doc name:" - + f" {pid_doc}. Waiting." + f"Pid: {pid} is locked for format_id: {checked_format_id} with " + f"doc name: {pid_doc}. Waiting." ) if self.use_multiprocessing: with self.metadata_condition_mp: @@ -932,8 +961,8 @@ def delete_metadata(self, pid: str, format_id: Optional[str] = None) -> None: finally: # Release pid end_sync_debug_msg = ( - f"Releasing pid doc ({pid_doc}) from locked list for pid: {pid} with " - + f"format_id: {checked_format_id}" + f"Releasing pid doc ({pid_doc}) from locked list for pid: " + f"{pid} with format_id: {checked_format_id}" ) if self.use_multiprocessing: with self.metadata_condition_mp: @@ -955,12 +984,12 @@ def delete_metadata(self, pid: str, format_id: Optional[str] = None) -> None: pid_doc = self._computehash(pid + checked_format_id) # Wait for the pid to release if it's in use sync_begin_debug_msg = ( - f"Adding pid: {pid} to locked list, with format_id: {checked_format_id} with doc " - + f"name: {pid_doc}" + f"Adding pid: {pid} to locked list, with format_id: " + f"{checked_format_id} with doc name: {pid_doc}" ) sync_wait_msg = ( - f"Pid: {pid} is locked for format_id: {checked_format_id} with doc name:" - + f" {pid_doc}. Waiting." + f"Pid: {pid} is locked for format_id: {checked_format_id} with doc" + f" name: {pid_doc}. Waiting." ) if self.use_multiprocessing: with self.metadata_condition_mp: @@ -989,8 +1018,8 @@ def delete_metadata(self, pid: str, format_id: Optional[str] = None) -> None: finally: # Release pid end_sync_debug_msg = ( - f"Releasing pid doc ({pid_doc}) from locked list for pid: {pid} with " - f"format_id: {checked_format_id}" + f"Releasing pid doc ({pid_doc}) from locked list for pid: {pid} " + f"with format_id: {checked_format_id}" ) if self.use_multiprocessing: with self.metadata_condition_mp: @@ -1018,16 +1047,20 @@ def get_hex_digest(self, pid: str, algorithm: str) -> str: cid_stream = self._open(entity, object_cid) hex_digest = self._computehash(cid_stream, algorithm=algorithm) - info_string = f"Successfully calculated hex digest for pid: {pid}. Hex Digest: {hex_digest}" + info_string = ( + f"Successfully calculated hex digest for pid: {pid}. " + f"Hex Digest: {hex_digest}" + ) logging.info(info_string) return hex_digest # FileHashStore Core Methods - def _find_object(self, pid: str) -> Dict[str, str]: - """Check if an object referenced by a pid exists and retrieve its content identifier. - The `find_object` method validates the existence of an object based on the provided - pid and returns the associated content identifier. + def _find_object(self, pid: str) -> dict[str, str]: + """Check if an object referenced by a pid exists and retrieve its + content identifier. The `find_object` method validates the existence of + an object based on the provided pid and returns the associated content + identifier. :param str pid: Authority-based or persistent identifier of the object. @@ -1054,55 +1087,49 @@ def _find_object(self, pid: str) -> Dict[str, str]: # Object must also exist in order to return the cid retrieved if not self._exists("objects", pid_refs_cid): err_msg = ( - f"Reference file found for pid ({pid}) at {pid_ref_abs_path}" - + f", but object referenced does not exist, cid: {pid_refs_cid}" + f"Reference file found for pid ({pid}) at " + f"{pid_ref_abs_path}, but object referenced does not " + f"exist, cid: {pid_refs_cid}" ) self.fhs_logger.error(err_msg) raise RefsFileExistsButCidObjMissing(err_msg) - else: - sysmeta_doc_name = self._computehash(pid + self.sysmeta_ns) - metadata_directory = self._computehash(pid) - metadata_rel_path = Path(*self._shard(metadata_directory)) - sysmeta_full_path = ( - self._get_store_path("metadata") - / metadata_rel_path - / sysmeta_doc_name - ) - obj_info_dict = { - "cid": pid_refs_cid, - "cid_object_path": self._get_hashstore_data_object_path( - pid_refs_cid - ), - "cid_refs_path": cid_ref_abs_path, - "pid_refs_path": pid_ref_abs_path, - "sysmeta_path": ( - sysmeta_full_path - if os.path.isfile(sysmeta_full_path) - else "Does not exist." - ), - } - return obj_info_dict - else: - # If not, it is an orphan pid refs file - err_msg = ( - f"Pid reference file exists with cid: {pid_refs_cid} for pid: {pid} but " - f"is missing from cid refs file: {cid_ref_abs_path}" + sysmeta_doc_name = self._computehash(pid + self.sysmeta_ns) + metadata_directory = self._computehash(pid) + metadata_rel_path = Path(*self._shard(metadata_directory)) + sysmeta_full_path = ( + self._get_store_path("metadata") + / metadata_rel_path + / sysmeta_doc_name ) - self.fhs_logger.error(err_msg) - raise PidNotFoundInCidRefsFile(err_msg) - else: + return { + "cid": pid_refs_cid, + "cid_object_path": self._get_hashstore_data_object_path( + pid_refs_cid + ), + "cid_refs_path": cid_ref_abs_path, + "pid_refs_path": pid_ref_abs_path, + "sysmeta_path": ( + sysmeta_full_path + if os.path.isfile(sysmeta_full_path) + else "Does not exist." + ), + } + # If not, it is an orphan pid refs file err_msg = ( - f"Pid reference file exists with cid: {pid_refs_cid} but cid reference file " - + f"not found: {cid_ref_abs_path} for pid: {pid}" + f"Pid reference file exists with cid: {pid_refs_cid} for pid: " + f"{pid} but is missing from cid refs file: {cid_ref_abs_path}" ) self.fhs_logger.error(err_msg) - raise OrphanPidRefsFileFound(err_msg) - else: + raise PidNotFoundInCidRefsFile(err_msg) err_msg = ( - f"Pid reference file not found for pid ({pid}): {pid_ref_abs_path}" + f"Pid reference file exists with cid: {pid_refs_cid} but cid reference " + f"file not found: {cid_ref_abs_path} for pid: {pid}" ) self.fhs_logger.error(err_msg) - raise PidRefsDoesNotExist(err_msg) + raise OrphanPidRefsFileFound(err_msg) + err_msg = f"Pid reference file not found for pid ({pid}): {pid_ref_abs_path}" + self.fhs_logger.error(err_msg) + raise PidRefsDoesNotExist(err_msg) def _store_and_validate_data( self, @@ -1113,15 +1140,15 @@ def _store_and_validate_data( checksum_algorithm: Optional[str] = None, file_size_to_validate: Optional[int] = None, ) -> "ObjectMetadata": - """Store contents of `file` on disk, validate the object's parameters if provided, - and tag/reference the object. + """Store contents of `file` on disk, validate the object's parameters if + provided, and tag/reference the object. :param str pid: Authority-based identifier. :param mixed file: Readable object or path to file. - :param str additional_algorithm: Optional algorithm value to include when returning - hex digests. - :param str checksum: Optional checksum to validate object against hex digest before moving - to permanent location. + :param str additional_algorithm: Optional algorithm value to include when + returning hex digests. + :param str checksum: Optional checksum to validate object against hex digest + before moving to permanent location. :param str checksum_algorithm: Algorithm value of the given checksum. :param int file_size_to_validate: Expected size of the object. @@ -1152,10 +1179,10 @@ def _store_and_validate_data( return object_metadata def _store_data_only(self, data: Union[str, bytes]) -> "ObjectMetadata": - """Store an object to HashStore and return a metadata object containing the content - identifier, object file size and hex digests dictionary of the default algorithms. This - method does not validate the object and writes directly to `/objects` after the hex - digests are calculated. + """Store an object to HashStore and return a metadata object containing + the content identifier, object file size and hex digests dictionary of + the default algorithms. This method does not validate the object and + writes directly to `/objects` after the hex digests are calculated. :param mixed data: String or path to object. @@ -1203,20 +1230,21 @@ def _move_and_get_checksums( checksum: Optional[str] = None, checksum_algorithm: Optional[str] = None, file_size_to_validate: Optional[int] = None, - ) -> Tuple[str, int, Dict[str, str]]: - """Copy the contents of the `Stream` object onto disk. The copy process uses a temporary - file to store the initial contents and returns a dictionary of algorithms and their - hex digest values. If the file already exists, the method will immediately - raise an exception. If an algorithm and checksum are provided, it will proceed to - validate the object (and delete the temporary file created if the hex digest stored does - not match what is provided). + ) -> tuple[str, int, dict[str, str]]: + """Copy the contents of the `Stream` object onto disk. The copy process + uses a temporary file to store the initial contents and returns a + dictionary of algorithms and their hex digest values. If the file + already exists, the method will immediately raise an exception. If an + algorithm and checksum are provided, it will proceed to validate the + object (and delete the temporary file created if the hex digest stored + does not match what is provided). :param Optional[str] pid: Authority-based identifier. :param Stream stream: Object stream when saving. - :param str additional_algorithm: Optional algorithm value to include when returning hex - digests. - :param str checksum: Optional checksum to validate the object against hex digest before - moving to the permanent location. + :param str additional_algorithm: Optional algorithm value to include + when returning hex digests. + :param str checksum: Optional checksum to validate the object against + hex digest before moving to the permanent location. :param str checksum_algorithm: Algorithm value of the given checksum. :param int file_size_to_validate: Expected size of the object. @@ -1238,7 +1266,8 @@ def _move_and_get_checksums( abs_file_path = self._build_hashstore_data_object_path(object_cid) # Only move file if it doesn't exist. We do not check before we create the tmp - # file and calculate the hex digests because the given checksum could be incorrect. + # file and calculate the hex digests because the given checksum could be + # incorrect. if not os.path.isfile(abs_file_path): # Files are stored once and only once self._verify_object_information( @@ -1270,32 +1299,32 @@ def _move_and_get_checksums( if pid_checksum == hex_digests.get(self.algorithm): # If the checksums match, return and log warning err_msg = ( - f"Object exists at: {abs_file_path} but an unexpected issue has been " - + "encountered. Reference files will not be created and/or tagged." + f"Object exists at: {abs_file_path} but an unexpected " + "issue has been encountered. Reference files will not be " + "created and/or tagged." ) self.fhs_logger.warning(err_msg) raise err - else: - debug_msg = ( - f"Object exists at {abs_file_path} but the pid object checksum " - + "provided does not match what has been calculated. Deleting object. " - + "References will not be created and/or tagged.", - ) - self.fhs_logger.debug(debug_msg) - self._delete("objects", abs_file_path) - raise err - else: - self.fhs_logger.debug("Deleting temporary file: %s", tmp_file_name) - self._delete("tmp", tmp_file_name) - err_msg = ( - f"Object has not been stored for pid: {pid} - an unexpected error has " - + f"occurred when moving tmp file to: {object_cid}. Reference files will " - + f"not be created and/or tagged. Error: {err}" + debug_msg = ( + f"Object exists at {abs_file_path} but the pid object checksum " + "provided does not match what has been calculated. Deleting " + "object. References will not be created and/or tagged.", ) - self.fhs_logger.warning(err_msg) - raise + self.fhs_logger.debug(debug_msg) + self._delete("objects", abs_file_path) + raise err + self.fhs_logger.debug("Deleting temporary file: %s", tmp_file_name) + self._delete("tmp", tmp_file_name) + err_msg = ( + f"Object has not been stored for pid: {pid} - an unexpected error " + f"has occurred when moving tmp file to: {object_cid}. Reference " + f"files will not be created and/or tagged. Error: {err}" + ) + self.fhs_logger.warning(err_msg) + raise else: - # If the data object already exists, do not move the file but attempt to verify it + # If the data object already exists, do not move the file but attempt to + # verify it try: self._verify_object_information( pid, @@ -1310,24 +1339,25 @@ def _move_and_get_checksums( except NonMatchingObjSize as nmose: # If any exception is thrown during validation, we do not tag. err_msg = ( - f"Object already exists for pid: {pid}, deleting temp file. Reference files " - + "will not be created and/or tagged due to an issue with the supplied pid " - + f"object metadata. {str(nmose)}" + f"Object already exists for pid: {pid}, deleting temp file. " + "Reference files will not be created and/or tagged due to an issue " + f"with the supplied pid object metadata. {nmose!s}" ) self.fhs_logger.debug(err_msg) raise NonMatchingObjSize(err_msg) from nmose except NonMatchingChecksum as nmce: # If any exception is thrown during validation, we do not tag. err_msg = ( - f"Object already exists for pid: {pid}, deleting temp file. Reference files " - + "will not be created and/or tagged due to an issue with the supplied pid " - + f"object metadata. {str(nmce)}" + f"Object already exists for pid: {pid}, deleting temp file. " + "Reference files will not be created and/or tagged due to an issue" + f" with the supplied pid object metadata. {nmce!s}" ) self.fhs_logger.debug(err_msg) raise NonMatchingChecksum(err_msg) from nmce finally: - # Ensure that the tmp file has been removed, the data object already exists, so it - # is redundant. No exception is thrown so 'store_object' can proceed to tag object + # Ensure that the tmp file has been removed, the data object already + # exists, so it is redundant. No exception is thrown so 'store_object' + # can proceed to tag object if os.path.isfile(tmp_file_name): self._delete("tmp", tmp_file_name) @@ -1338,15 +1368,17 @@ def _write_to_tmp_file_and_get_hex_digests( stream: "Stream", additional_algorithm: Optional[str] = None, checksum_algorithm: Optional[str] = None, - ) -> Tuple[Dict[str, str], str, int]: - """Create a named temporary file from a `Stream` object and return its filename - and a dictionary of its algorithms and hex digests. If an additional and/or checksum - algorithm is provided, it will add the respective hex digest to the dictionary if - it is supported. + ) -> tuple[dict[str, str], str, int]: + """Create a named temporary file from a `Stream` object and return its + filename and a dictionary of its algorithms and hex digests. If an + additional and/or checksum algorithm is provided, it will add the + respective hex digest to the dictionary if it is supported. :param Stream stream: Object stream. - :param str additional_algorithm: Algorithm of additional hex digest to generate. - :param str checksum_algorithm: Algorithm of additional checksum algo to generate. + :param str additional_algorithm: Algorithm of additional hex digest to + generate. + :param str checksum_algorithm: Algorithm of additional checksum algo to + generate. :return: tuple - hex_digest_dict, tmp.name - hex_digest_dict (dict): Algorithms and their hex digests. @@ -1396,7 +1428,7 @@ def _write_to_tmp_file_and_get_hex_digests( err_msg = f"Unexpected {err=}, {type(err)=}" self.fhs_logger.error(err_msg) # pylint: disable=W0707,W0719 - raise Exception(err_msg) + raise Exception(err_msg) from err except KeyboardInterrupt: err_msg = "Keyboard interruption by user." self.fhs_logger.error(err_msg) @@ -1411,7 +1443,7 @@ def _write_to_tmp_file_and_get_hex_digests( except Exception as err: err_msg = ( f"Unexpected {err=} while attempting to delete tmp file: " - + f"{tmp.name}, {type(err)=}" + f"{tmp.name}, {type(err)=}" ) self.fhs_logger.error(err_msg) @@ -1426,7 +1458,7 @@ def _mktmpfile(self, path: Path) -> IO[bytes]: if os.path.exists(path) is False: self._create_path(path) - tmp = NamedTemporaryFile(dir=path, delete=False) + tmp = NamedTemporaryFile(dir=path, delete=False) # noqa: SIM115 # Delete tmp file if python interpreter crashes or thread is interrupted def delete_tmp_file(): @@ -1445,8 +1477,8 @@ def delete_tmp_file(): return tmp def _store_hashstore_refs_files(self, pid: str, cid: str) -> None: - """Create the pid refs file and create/update cid refs files in HashStore to establish - the relationship between a 'pid' and a 'cid'. + """Create the pid refs file and create/update cid refs files in + HashStore to establish the relationship between a 'pid' and a 'cid'. :param str pid: Persistent or authority-based identifier. :param str cid: Content identifier @@ -1460,14 +1492,16 @@ def _store_hashstore_refs_files(self, pid: str, cid: str) -> None: tmp_root_path = self._get_store_path("refs") / "tmp" pid_refs_path = self._get_hashstore_pid_refs_path(pid) cid_refs_path = self._get_hashstore_cid_refs_path(cid) - # Create paths for pid ref file in '.../refs/pid' and cid ref file in '.../refs/cid' + # Create paths for pid ref file in '.../refs/pid' and cid ref file + # in '.../refs/cid' self._create_path(Path(os.path.dirname(pid_refs_path))) self._create_path(Path(os.path.dirname(cid_refs_path))) if os.path.isfile(pid_refs_path) and os.path.isfile(cid_refs_path): - # If both reference files exist, we confirm that reference files are where they - # are expected to be and throw an exception to inform the client that everything - # is in place - and include other issues for context + # If both reference files exist, we confirm that reference files are + # where they are expected to be and throw an exception to inform the + # client that everything is in place - and include other issues + # for context err_msg = ( f"Object with cid: {cid} exists and is tagged with pid: {pid}." ) @@ -1484,13 +1518,13 @@ def _store_hashstore_refs_files(self, pid: str, cid: str) -> None: except Exception as e: rev_msg = err_msg + " " + str(e) self.fhs_logger.error(rev_msg) - raise HashStoreRefsAlreadyExists(err_msg) + raise HashStoreRefsAlreadyExists(err_msg) from e elif os.path.isfile(pid_refs_path) and not os.path.isfile( cid_refs_path ): - # If pid refs exists, the pid has already been claimed and cannot be tagged we - # throw an exception immediately + # If pid refs exists, the pid has already been claimed and cannot + # be tagged we throw an exception immediately error_msg = f"Pid refs file already exists for pid: {pid}." self.fhs_logger.error(error_msg) raise PidRefsAlreadyExistsError(error_msg) @@ -1499,8 +1533,8 @@ def _store_hashstore_refs_files(self, pid: str, cid: str) -> None: cid_refs_path ): debug_msg = ( - f"Pid reference file does not exist for pid {pid} but cid refs file " - + f"found at: {cid_refs_path} for cid: {cid}" + f"Pid reference file does not exist for pid {pid} but cid refs " + f"file found at: {cid_refs_path} for cid: {cid}" ) self.fhs_logger.debug(debug_msg) # Move the pid refs file @@ -1514,7 +1548,10 @@ def _store_hashstore_refs_files(self, pid: str, cid: str) -> None: cid, pid_refs_path, cid_refs_path, - f"Updated existing cid refs file: {cid_refs_path} with pid: {pid}", + ( + f"Updated existing cid refs file: {cid_refs_path} " + f"with pid: {pid}" + ), ) info_msg = f"Successfully updated cid: {cid} with pid: {pid}" self.fhs_logger.info(info_msg) @@ -1525,7 +1562,10 @@ def _store_hashstore_refs_files(self, pid: str, cid: str) -> None: cid_tmp_file_path = self._write_refs_file(tmp_root_path, pid, "cid") shutil.move(pid_tmp_file_path, pid_refs_path) shutil.move(cid_tmp_file_path, cid_refs_path) - log_msg = "Refs files have been moved to their permanent location. Verifying refs." + log_msg = ( + "Refs files have been moved to their permanent location. " + "Verifying refs." + ) self._verify_hashstore_references( pid, cid, pid_refs_path, cid_refs_path, log_msg ) @@ -1539,9 +1579,13 @@ def _store_hashstore_refs_files(self, pid: str, cid: str) -> None: raise expected_exceptions except Exception as ue: - # For all other unexpected exceptions, we are to revert the tagging process as - # much as possible. No exceptions from the reverting process will be thrown. - err_msg = f"Unexpected exception: {ue}, reverting tagging process (untag obj)." + # For all other unexpected exceptions, we are to revert the tagging + # process as much as possible. No exceptions from the reverting process + # will be thrown. + err_msg = ( + f"Unexpected exception: {ue}, reverting tagging " + "process (untag obj)." + ) self.fhs_logger.error(err_msg) self._untag_object(pid, cid) raise ue @@ -1552,10 +1596,11 @@ def _store_hashstore_refs_files(self, pid: str, cid: str) -> None: self._release_reference_locked_pids(pid) def _untag_object(self, pid: str, cid: str) -> None: - """Untags a data object in HashStore by deleting the 'pid reference file' and removing - the 'pid' from the 'cid reference file'. This method will never delete a data - object. `_untag_object` will attempt to proceed with as much of the untagging process as - possible and swallow relevant exceptions. + """Untags a data object in HashStore by deleting the 'pid reference + file' and removing the 'pid' from the 'cid reference file'. This method + will never delete a data object. `_untag_object` will attempt to proceed + with as much of the untagging process as possible and swallow relevant + exceptions. :param str cid: Content identifier :param str pid: Persistent or authority-based identifier. @@ -1569,9 +1614,9 @@ def _untag_object(self, pid: str, cid: str) -> None: # The pid will not be released until this process is over self._check_reference_locked_pids(pid) - # Before we begin the untagging process, we look for the `cid` by calling `find_object` - # which will throw custom exceptions if there is an issue with the reference files, - # which help us determine the path to proceed with. + # Before we begin the untagging process, we look for the `cid` by calling + # `find_object`which will throw custom exceptions if there is an issue with the + # reference files, which help us determine the path to proceed with. try: obj_info_dict = self._find_object(pid) cid_to_check = obj_info_dict["cid"] @@ -1606,14 +1651,14 @@ def _untag_object(self, pid: str, cid: str) -> None: self._delete_marked_files(untag_obj_delete_list) warn_msg = ( - f"Cid refs file does not exist for pid: {pid}. Deleted orphan pid refs file. " - f"Additional info: {oprff}" + f"Cid refs file does not exist for pid: {pid}. Deleted orphan pid refs " + f"file. Additional info: {oprff}" ) self.fhs_logger.warning(warn_msg) except RefsFileExistsButCidObjMissing as rfebcom: - # `find_object` throws this exception when both pid/cid refs files exist but the - # actual data object does not. + # `find_object` throws this exception when both pid/cid refs files exist + # but the actual data object does not. pid_refs_path = self._get_hashstore_pid_refs_path(pid) cid_read = self._read_small_file_content(pid_refs_path) self._validate_and_check_cid_lock(pid, cid, cid_read) @@ -1631,15 +1676,15 @@ def _untag_object(self, pid: str, cid: str) -> None: self._delete_marked_files(untag_obj_delete_list) warn_msg = ( - f"data object for cid: {cid_read}. does not exist, but pid and cid references " - + f"files found for pid: {pid}, Deleted pid and cid refs files. " - + f"Additional info: {rfebcom}" + f"data object for cid: {cid_read}. does not exist, but pid and cid " + f"references files found for pid: {pid}, Deleted pid and cid refs " + f"files. Additional info: {rfebcom}" ) self.fhs_logger.warning(warn_msg) except PidNotFoundInCidRefsFile as pnficrf: - # `find_object` throws this exception when both the pid and cid refs file exists - # but the pid is not found in the cid refs file + # `find_object` throws this exception when both the pid and cid refs file + # exists but the pid is not found in the cid refs file pid_refs_path = self._get_hashstore_pid_refs_path(pid) cid_read = self._read_small_file_content(pid_refs_path) self._validate_and_check_cid_lock(pid, cid, cid_read) @@ -1651,8 +1696,8 @@ def _untag_object(self, pid: str, cid: str) -> None: self._delete_marked_files(untag_obj_delete_list) warn_msg = ( - f"Pid not found in expected cid refs file for pid: {pid}. Deleted orphan pid refs " - f"file. Additional info: {pnficrf}" + f"Pid not found in expected cid refs file for pid: {pid}. Deleted " + f"orphan pid refs file. Additional info: {pnficrf}" ) self.fhs_logger.warning(warn_msg) @@ -1671,7 +1716,7 @@ def _untag_object(self, pid: str, cid: str) -> None: warn_msg = ( "Pid refs file not found, removed pid from cid reference file for cid:" - + f" {cid}. Additional info: {prdne}" + f" {cid}. Additional info: {prdne}" ) self.fhs_logger.warning(warn_msg) @@ -1718,8 +1763,8 @@ def _put_metadata( raise else: err_msg = ( - f"Attempted to move metadata for pid: {pid}, but metadata temp file not found:" - + f" {metadata_tmp}" + f"Attempted to move metadata for pid: {pid}, but metadata temp file " + f"not found: {metadata_tmp}" ) self.fhs_logger.error(err_msg) raise FileNotFoundError(err_msg) @@ -1760,15 +1805,18 @@ def _delete_marked_files(delete_list: list[str]) -> None: warn_msg = f"Unable to remove {obj} in given delete_list. " + str(e) logging.warning(warn_msg) else: - raise ValueError("list cannot be None") + msg = "list cannot be None" + raise ValueError(msg) def _mark_pid_refs_file_for_deletion( - self, pid: str, delete_list: List[str], pid_refs_path: Path + self, pid: str, delete_list: list[str], pid_refs_path: Path ) -> None: - """Attempt to rename a pid refs file and add the renamed file to a provided list. + """Attempt to rename a pid refs file and add the renamed file to a provided + list. :param str pid: Persistent or authority-based identifier. - :param list delete_list: List to add the renamed pid refs file marked for deletion to + :param list delete_list: List to add the renamed pid refs file marked for + deletion to :param path pid_refs_path: Path to the pid reference file """ try: @@ -1781,13 +1829,14 @@ def _mark_pid_refs_file_for_deletion( self.fhs_logger.error(err_msg) def _remove_pid_and_handle_cid_refs_deletion( - self, pid: str, delete_list: List[str], cid_refs_path: Path + self, pid: str, delete_list: list[str], cid_refs_path: Path ) -> None: - """Attempt to remove a pid from a 'cid refs file' and add the 'cid refs file' to the - delete list if it is empty. + """Attempt to remove a pid from a 'cid refs file' and add the 'cid refs + file' to the delete list if it is empty. :param str pid: Persistent or authority-based identifier. - :param list delete_list: List to add the renamed pid refs file marked for deletion to + :param list delete_list: List to add the renamed pid refs file marked for + deletion to :param path cid_refs_path: Path to the pid reference file """ try: @@ -1799,16 +1848,16 @@ def _remove_pid_and_handle_cid_refs_deletion( except Exception as e: err_msg = ( - f"Unable to delete remove pid from cid refs file: {cid_refs_path} for pid:" - f" {pid}. " + str(e) + f"Unable to delete remove pid from cid refs file: {cid_refs_path} for " + f"pid: {pid}. " + str(e) ) self.fhs_logger.error(err_msg) def _validate_and_check_cid_lock( self, pid: str, cid: str, cid_to_check: str ) -> None: - """Confirm that the two content identifiers provided are equal and is locked to ensure - thread safety. + """Confirm that the two content identifiers provided are equal and is locked + to ensure thread safety. :param str pid: Persistent identifier :param str cid: Content identifier @@ -1852,7 +1901,7 @@ def _write_refs_file(self, path: Path, ref_id: str, ref_type: str) -> str: except Exception as err: err_msg = ( f"Failed to write cid refs file for pid: {ref_id} into path: {path}. " - + f"Unexpected error: {err=}, {type(err)=}" + f"Unexpected error: {err=}, {type(err)=}" ) self.fhs_logger.error(err_msg) raise err @@ -1866,12 +1915,15 @@ def _update_refs_file( :param str ref_id: Authority-based or persistent identifier of the object. :param str update_type: 'add' or 'remove' """ - debug_msg = f"Updating ({update_type}) for ref_id: {ref_id} at refs file: {refs_file_path}." + debug_msg = ( + f"Updating ({update_type}) for ref_id: {ref_id} at " + "refs file: {refs_file_path}." + ) self.fhs_logger.debug(debug_msg) if not os.path.isfile(refs_file_path): err_msg = ( f"Refs file: {refs_file_path} does not exist." - + f"Cannot {update_type} ref_id: {ref_id}" + f"Cannot {update_type} ref_id: {ref_id}" ) self.fhs_logger.error(err_msg) raise FileNotFoundError(err_msg) @@ -1900,13 +1952,13 @@ def _update_refs_file( ref_file.truncate() debug_msg = ( f"Update ({update_type}) for ref_id: {ref_id} " - + f"completed on refs file: {refs_file_path}." + f"completed on refs file: {refs_file_path}." ) self.fhs_logger.debug(debug_msg) except Exception as err: err_msg = ( f"Failed to {update_type} for ref_id: {ref_id}" - + f" at refs file: {refs_file_path}. Unexpected {err=}, {type(err)=}" + f" at refs file: {refs_file_path}. Unexpected {err=}, {type(err)=}" ) self.fhs_logger.error(err_msg) raise err @@ -1920,7 +1972,7 @@ def _is_string_in_refs_file(ref_id: str, refs_file_path: Path) -> bool: :return: pid_found """ - with open(refs_file_path, "r", encoding="utf8") as ref_file: + with open(refs_file_path, encoding="utf8") as ref_file: # Confirm that pid is not currently already tagged for line in ref_file: value = line.strip() @@ -1934,7 +1986,7 @@ def _verify_object_information( checksum: str, checksum_algorithm: str, entity: str, - hex_digests: Dict[str, str], + hex_digests: dict[str, str], tmp_file_name: Optional[str], tmp_file_size: int, file_size_to_validate: int, @@ -1951,22 +2003,24 @@ def _verify_object_information( :param int tmp_file_size: Size of the temporary file. :param int file_size_to_validate: Expected size of the object. """ - if file_size_to_validate is not None and file_size_to_validate > 0: - if file_size_to_validate != tmp_file_size: - err_msg = ( - f"Object file size calculated: {tmp_file_size} does not match with expected " - f"size: {file_size_to_validate}." + if ( + file_size_to_validate is not None + and file_size_to_validate > 0 + and file_size_to_validate != tmp_file_size + ): + err_msg = ( + f"Object file size calculated: {tmp_file_size} does not match with " + f"expected size: {file_size_to_validate}." + ) + if pid is not None: + self._delete(entity, tmp_file_name) + err_msg_for_pid = ( + f"{err_msg} Tmp file deleted and file not stored for pid: {pid}" ) - if pid is not None: - self._delete(entity, tmp_file_name) - err_msg_for_pid = ( - f"{err_msg} Tmp file deleted and file not stored for pid: {pid}" - ) - self.fhs_logger.debug(err_msg_for_pid) - raise NonMatchingObjSize(err_msg_for_pid) - else: - self.fhs_logger.debug(err_msg) - raise NonMatchingObjSize(err_msg) + self.fhs_logger.debug(err_msg_for_pid) + raise NonMatchingObjSize(err_msg_for_pid) + self.fhs_logger.debug(err_msg) + raise NonMatchingObjSize(err_msg) if checksum_algorithm is not None and checksum is not None: if checksum_algorithm not in hex_digests: # Check to see if it is a supported algorithm @@ -1986,10 +2040,10 @@ def _verify_object_information( ) if hex_digest_calculated != checksum: err_msg = ( - f"Checksum_algorithm ({checksum_algorithm}) cannot be found in the " - + "default hex digests dict, but is supported. New checksum calculated: " - + f"{hex_digest_calculated}, does not match what has been provided: " - + checksum + f"Checksum_algorithm ({checksum_algorithm}) cannot be found " + "in the default hex digests dict, but is supported. New " + f"checksum calculated: {hex_digest_calculated}, does not match " + f"what has been provided: {checksum}" ) self.fhs_logger.debug(err_msg) raise NonMatchingChecksum(err_msg) @@ -1997,9 +2051,9 @@ def _verify_object_information( hex_digest_stored = hex_digests[checksum_algorithm] if hex_digest_stored != checksum.lower(): err_msg = ( - f"Hex digest and checksum do not match - file not stored for pid: {pid}. " - + f"Algorithm: {checksum_algorithm}. Checksum provided: {checksum} !=" - + f" HexDigest: {hex_digest_stored}." + f"Hex digest and checksum do not match - file not stored for " + f"Algorithm: {checksum_algorithm}. Checksum provided: " + f"{checksum} != HexDigest: {hex_digest_stored}." ) if pid is not None: # Delete the tmp file @@ -2009,9 +2063,8 @@ def _verify_object_information( ) self.fhs_logger.error(err_msg_for_pid) raise NonMatchingChecksum(err_msg_for_pid) - else: - self.fhs_logger.error(err_msg) - raise NonMatchingChecksum(err_msg) + self.fhs_logger.error(err_msg) + raise NonMatchingChecksum(err_msg) def _verify_hashstore_references( self, @@ -2041,7 +2094,10 @@ def _verify_hashstore_references( # Check that reference files were created if not os.path.isfile(pid_refs_path): - err_msg = f" Pid refs file missing: {pid_refs_path}. Note: {additional_log_string}" + err_msg = ( + f" Pid refs file missing: {pid_refs_path}. " + f"Note: {additional_log_string}" + ) self.fhs_logger.error(err_msg) raise PidRefsFileNotFound(err_msg) if not os.path.isfile(cid_refs_path): @@ -2055,8 +2111,8 @@ def _verify_hashstore_references( retrieved_cid = self._read_small_file_content(pid_refs_path) if retrieved_cid != cid: err_msg = ( - f"Pid refs file exists ({pid_refs_path}) but cid ({cid}) does not match." - + f" Note: {additional_log_string}" + f"Pid refs file exists ({pid_refs_path}) but cid ({cid}) does not " + f"match. Note: {additional_log_string}" ) self.fhs_logger.error(err_msg) raise PidRefsContentError(err_msg) @@ -2065,14 +2121,15 @@ def _verify_hashstore_references( if not pid_found: err_msg = ( f"Cid refs file exists ({cid_refs_path}) but pid ({pid}) not found." - + f" Note: {additional_log_string}" + f" Note: {additional_log_string}" ) self.fhs_logger.error(err_msg) raise CidRefsContentError(err_msg) def _delete_object_only(self, cid: str) -> None: - """Attempt to delete an object based on the given content identifier (cid). If the object - has any pids references and/or a cid refs file exists, the object will not be deleted. + """Attempt to delete an object based on the given content identifier + (cid). If the object has any pids references and/or a cid refs file + exists, the object will not be deleted. :param str cid: Content identifier """ @@ -2099,9 +2156,9 @@ def _check_arg_algorithms_and_checksum( additional_algorithm: Optional[str], checksum: Optional[str], checksum_algorithm: Optional[str], - ) -> Tuple[Optional[str], Optional[str]]: - """Determines whether the caller has supplied the necessary arguments to validate - an object with a checksum value. + ) -> tuple[Optional[str], Optional[str]]: + """Determines whether the caller has supplied the necessary arguments to + validate an object with a checksum value. :param additional_algorithm: Value of the additional algorithm to calculate. :type additional_algorithm: str or None @@ -2139,16 +2196,11 @@ def _check_arg_format_id(self, format_id: str, method: str) -> str: err_msg = f"FileHashStore - {method}: Format_id cannot be empty." self.fhs_logger.error(err_msg) raise ValueError(err_msg) - elif format_id is None: - # Use default value set by hashstore config - checked_format_id = self.sysmeta_ns - else: - checked_format_id = format_id - return checked_format_id + return self.sysmeta_ns if format_id is None else format_id def _refine_algorithm_list( self, additional_algorithm: Optional[str], checksum_algorithm: Optional[str] - ) -> Set[str]: + ) -> set[str]: """Create the final list of hash algorithms to calculate. :param str additional_algorithm: Additional algorithm. @@ -2161,8 +2213,8 @@ def _refine_algorithm_list( self._clean_algorithm(checksum_algorithm) if checksum_algorithm in self.other_algo_list: debug_additional_other_algo_str = ( - f"Checksum algo: {checksum_algorithm} found in other_algo_lists, adding to " - + f"list of algorithms to calculate." + f"Checksum algo: {checksum_algorithm} found in other_algo_lists, " + f"adding to list of algorithms to calculate." ) self.fhs_logger.debug(debug_additional_other_algo_str) algorithm_list_to_calculate.append(checksum_algorithm) @@ -2170,15 +2222,14 @@ def _refine_algorithm_list( self._clean_algorithm(additional_algorithm) if additional_algorithm in self.other_algo_list: debug_additional_other_algo_str = ( - f"Additional algo: {additional_algorithm} found in other_algo_lists, " - + f"adding to list of algorithms to calculate." + f"Additional algo: {additional_algorithm} found in " + "other_algo_lists, adding to list of algorithms to calculate." ) self.fhs_logger.debug(debug_additional_other_algo_str) algorithm_list_to_calculate.append(additional_algorithm) # Remove duplicates - algorithm_list_to_calculate = set(algorithm_list_to_calculate) - return algorithm_list_to_calculate + return set(algorithm_list_to_calculate) def _clean_algorithm(self, algorithm_string: str) -> str: """Format a string and ensure that it is supported and compatible with @@ -2209,11 +2260,12 @@ def _clean_algorithm(self, algorithm_string: str) -> str: def _computehash( self, stream: Union["Stream", str, IO[bytes]], algorithm: Optional[str] = None ) -> str: - """Compute the hash of a file-like object (or string) using the store algorithm by - default or with an optional supported algorithm. + """Compute the hash of a file-like object (or string) using the store + algorithm by default or with an optional supported algorithm. - :param mixed stream: A buffered stream (`io.BufferedReader`) of an object. A string is - also acceptable as they are a sequence of characters (Python only). + :param mixed stream: A buffered stream (`io.BufferedReader`) of an object. A + string is also acceptable as they are a sequence of characters + (Python only). :param str algorithm: Algorithm of hex digest to generate. :return: Hex digest. @@ -2225,21 +2277,23 @@ def _computehash( hash_obj = hashlib.new(check_algorithm) for data in stream: hash_obj.update(self._cast_to_bytes(data)) - hex_digest = hash_obj.hexdigest() - return hex_digest + return hash_obj.hexdigest() - def _shard(self, checksum: str) -> List[str]: - """Splits the given checksum into a list of tokens of length `self.width`, followed by - the remainder. + def _shard(self, checksum: str) -> list[str]: + """Splits the given checksum into a list of tokens of length + `self.width`, followed by the remainder. - This method divides the checksum into `self.depth` number of tokens, each with a fixed - width of `self.width`, taken from the beginning of the checksum. Any leftover characters - are added as the final element in the list. + This method divides the checksum into `self.depth` number of tokens, + each with a fixed width of `self.width`, taken from the beginning of the + checksum. Any leftover characters are added as the final element in the + list. Example: - For a checksum of '0d555ed77052d7e166017f779cbc193357c3a5006ee8b8457230bcf7abcef65e', + For a checksum of + '0d555ed77052d7e166017f779cbc193357c3a5006ee8b8457230bcf7abcef65e', the result may be: - ['0d', '55', '5e', 'd77052d7e166017f779cbc193357c3a5006ee8b8457230bcf7abcef65e'] + ['0d', '55', '5e', + 'd77052d7e166017f779cbc193357c3a5006ee8b8457230bcf7abcef65e'] :param str checksum: The checksum string to be split into tokens. @@ -2247,7 +2301,7 @@ def _shard(self, checksum: str) -> List[str]: characters as the last element. """ - def compact(items: List[Any]) -> List[Any]: + def compact(items: list[Any]) -> list[Any]: """Return only truthy elements of `items`.""" # truthy_items = [] # for item in items: @@ -2258,13 +2312,11 @@ def compact(items: List[Any]) -> List[Any]: # This creates a list of `depth` number of tokens with width # `width` from the first part of the id plus the remainder. - hierarchical_list = compact( + return compact( [checksum[i * self.width : self.width * (i + 1)] for i in range(self.depth)] + [checksum[self.depth * self.width :]] ) - return hierarchical_list - def _count(self, entity: str) -> int: """Return the count of the number of files in the `root` directory. @@ -2284,9 +2336,10 @@ def _count(self, entity: str) -> int: elif entity == "tmp": directory_to_count = self.objects / "tmp" else: - raise ValueError( + msg = ( f"entity: {entity} does not exist. Do you mean 'objects' or 'metadata'?" ) + raise ValueError(msg) for _, _, files in os.walk(directory_to_count): for _ in files: @@ -2311,6 +2364,7 @@ def _exists(self, entity: str, file: str) -> bool: return bool(self._get_hashstore_metadata_path(file)) except FileNotFoundError: return False + return False def _open( self, entity: str, file: str, mode: str = "rb" @@ -2330,12 +2384,12 @@ def _open( if entity == "metadata": realpath = self._get_hashstore_metadata_path(file) if realpath is None: - raise IOError(f"Could not locate file: {file}") + msg = f"Could not locate file: {file}" + raise OSError(msg) # pylint: disable=W1514 # mode defaults to "rb" - buffer = io.open(realpath, mode) - return buffer + return open(realpath, mode) def _delete(self, entity: str, file: Union[str, Path]) -> None: """Delete file using id or path. Remove any empty directories after @@ -2359,9 +2413,8 @@ def _delete(self, entity: str, file: Union[str, Path]) -> None: # Check if the given path is an absolute path realpath = file else: - raise IOError( - f"FileHashStore - delete(): Could not locate file: {file}" - ) + msg = f"FileHashStore - delete(): Could not locate file: {file}" + raise OSError(msg) if realpath is not None: os.remove(realpath) @@ -2382,27 +2435,31 @@ def _create_path(self, path: Path) -> None: assert os.path.isdir(path), f"expected {path} to be a directory" def _get_store_path(self, entity: str) -> Path: - """Return a path object to the root directory of the requested hashstore directory type + """Return a path object to the root directory of the requested hashstore + directory type. + + :param str entity: Desired entity type: "objects", "metadata", "refs", + "cid" and "pid". - :param str entity: Desired entity type: "objects", "metadata", "refs", "cid" and "pid". Note, "cid" and "pid" are refs specific directories. :return: Path to requested store entity type """ if entity == "objects": return Path(self.objects) - elif entity == "metadata": + if entity == "metadata": return Path(self.metadata) - elif entity == "refs": + if entity == "refs": return Path(self.refs) - elif entity == "cid": + if entity == "cid": return Path(self.cids) - elif entity == "pid": + if entity == "pid": return Path(self.pids) - else: - raise ValueError( - f"entity: {entity} does not exist. Do you mean 'objects', 'metadata' or 'refs'?" - ) + msg = ( + f"entity: {entity} does not exist. Do you mean 'objects', " + "'metadata' or 'refs'?" + ) + raise ValueError(msg) def _build_hashstore_data_object_path(self, hash_id: str) -> str: """Build the absolute file path for a given content identifier @@ -2413,13 +2470,14 @@ def _build_hashstore_data_object_path(self, hash_id: str) -> str: """ paths = self._shard(hash_id) root_dir = self._get_store_path("objects") - absolute_path = os.path.join(root_dir, *paths) - return absolute_path + return os.path.join(root_dir, *paths) def _get_hashstore_data_object_path(self, cid_or_relative_path: str) -> Path: - """Get the expected path to a hashstore data object that exists using a content identifier. + """Get the expected path to a hashstore data object that exists using a + content identifier. - :param str cid_or_relative_path: Content identifier or relative path in '/objects' to check + :param str cid_or_relative_path: Content identifier or relative path in + '/objects' to check :return: Path to the data object referenced by the pid """ @@ -2428,26 +2486,25 @@ def _get_hashstore_data_object_path(self, cid_or_relative_path: str) -> Path: ) if os.path.isfile(expected_abs_data_obj_path): return Path(expected_abs_data_obj_path) - else: - if os.path.isfile(cid_or_relative_path): - # Check whether the supplied arg is an abs path that exists or not for convenience - return Path(cid_or_relative_path) - else: - # Check the relative path - relpath = os.path.join(self.objects, cid_or_relative_path) - if os.path.isfile(relpath): - return Path(relpath) - else: - raise FileNotFoundError( - "Could not locate a data object in '/objects' for the supplied " - + f"cid_or_relative_path: {cid_or_relative_path}" - ) + if os.path.isfile(cid_or_relative_path): + # Check whether the supplied arg is an abs path that exists or not for + # convenience + return Path(cid_or_relative_path) + # Check the relative path + relpath = os.path.join(self.objects, cid_or_relative_path) + if os.path.isfile(relpath): + return Path(relpath) + msg = ( + "Could not locate a data object in '/objects' for the supplied " + f"cid_or_relative_path: {cid_or_relative_path}" + ) + raise FileNotFoundError(msg) def _get_hashstore_metadata_path(self, metadata_relative_path: str) -> Path: """Return the expected metadata path to a hashstore metadata object that exists. - :param str metadata_relative_path: Metadata path to check or relative path in '/metadata' - to check + :param str metadata_relative_path: Metadata path to check or relative path in + '/metadata' to check :return: Path to the data object referenced by the pid """ @@ -2455,24 +2512,26 @@ def _get_hashstore_metadata_path(self, metadata_relative_path: str) -> Path: expected_abs_metadata_path = os.path.join(self.metadata, metadata_relative_path) if os.path.isfile(expected_abs_metadata_path): return Path(expected_abs_metadata_path) - else: - if os.path.isfile(metadata_relative_path): - # Check whether the supplied arg is an abs path that exists or not for convenience - return Path(metadata_relative_path) - else: - raise FileNotFoundError( - "Could not locate a metadata object in '/metadata' for the supplied " - + f"metadata_relative_path: {metadata_relative_path}" - ) + if os.path.isfile(metadata_relative_path): + # Check whether the supplied arg is an abs path that exists or not for + # convenience + return Path(metadata_relative_path) + msg = ( + "Could not locate a metadata object in '/metadata' for the supplied " + f"metadata_relative_path: {metadata_relative_path}" + ) + raise FileNotFoundError(msg) def _get_hashstore_pid_refs_path(self, pid: str) -> Path: - """Return the expected path to a pid reference file. The path may or may not exist. + """Return the expected path to a pid reference file. The path may or may not + exist. :param str pid: Persistent or authority-based identifier :return: Path to pid reference file """ - # The pid refs file is named after the hash of the pid using the store's algorithm + # The pid refs file is named after the hash of the pid using the + # store's algorithm hash_id = self._computehash(pid, self.algorithm) root_dir = self._get_store_path("pid") directories_and_path = self._shard(hash_id) @@ -2480,7 +2539,8 @@ def _get_hashstore_pid_refs_path(self, pid: str) -> Path: return Path(pid_ref_file_abs_path) def _get_hashstore_cid_refs_path(self, cid: str) -> Path: - """Return the expected path to a cid reference file. The path may or may not exist. + """Return the expected path to a cid reference file. The path may or may + not exist. :param str cid: Content identifier @@ -2495,8 +2555,8 @@ def _get_hashstore_cid_refs_path(self, cid: str) -> Path: # Synchronization Methods def _synchronize_object_locked_pids(self, pid: str) -> None: - """Threads must work with 'pid's one identifier at a time to ensure thread safety when - handling requests to store, delete or tag pids. + """Threads must work with 'pid's one identifier at a time to ensure + thread safety when handling requests to store, delete or tag pids. :param str pid: Persistent or authority-based identifier """ @@ -2504,21 +2564,25 @@ def _synchronize_object_locked_pids(self, pid: str) -> None: with self.object_pid_condition_mp: # Wait for the cid to release if it's being tagged while pid in self.object_locked_pids_mp: - self.fhs_logger.debug(f"Pid ({pid}) is locked. Waiting.") + self.fhs_logger.debug("Pid (%s) is locked. Waiting.", pid) self.object_pid_condition_mp.wait() self.object_locked_pids_mp.append(pid) - self.fhs_logger.debug(f"Synchronizing object_locked_pids_mp for pid: {pid}") + self.fhs_logger.debug( + "Synchronizing object_locked_pids_mp for pid: %s", pid + ) else: with self.object_pid_condition_th: while pid in self.object_locked_pids_th: - self.fhs_logger.debug(f"Pid ({pid}) is locked. Waiting.") + self.fhs_logger.debug("Pid (%s) is locked. Waiting.", pid) self.object_pid_condition_th.wait() self.object_locked_pids_th.append(pid) - self.fhs_logger.debug(f"Synchronizing object_locked_pids_th for pid: {pid}") + self.fhs_logger.debug( + "Synchronizing object_locked_pids_th for pid: %s", pid + ) def _release_object_locked_pids(self, pid: str) -> None: - """Remove the given persistent identifier from 'object_locked_pids' and notify other - waiting threads or processes. + """Remove the given persistent identifier from 'object_locked_pids' and + notify other waiting threads or processes. :param str pid: Persistent or authority-based identifier """ @@ -2526,18 +2590,18 @@ def _release_object_locked_pids(self, pid: str) -> None: with self.object_pid_condition_mp: self.object_locked_pids_mp.remove(pid) self.object_pid_condition_mp.notify() - self.fhs_logger.debug(f"Releasing pid ({pid}) from object_locked_pids_mp.") + self.fhs_logger.debug("Releasing pid (%s) from object_locked_pids_mp.", pid) else: # Release pid with self.object_pid_condition_th: self.object_locked_pids_th.remove(pid) self.object_pid_condition_th.notify() - self.fhs_logger.debug(f"Releasing pid ({pid}) from object_locked_pids_th.") + self.fhs_logger.debug("Releasing pid (%s) from object_locked_pids_th.", pid) def _synchronize_object_locked_cids(self, cid: str) -> None: - """Multiple threads may access a data object via its 'cid' or the respective 'cid - reference file' (which contains a list of 'pid's that reference a 'cid') and this needs - to be coordinated. + """Multiple threads may access a data object via its 'cid' or the + respective 'cid reference file' (which contains a list of 'pid's that + reference a 'cid') and this needs to be coordinated. :param str cid: Content identifier """ @@ -2545,18 +2609,22 @@ def _synchronize_object_locked_cids(self, cid: str) -> None: with self.object_cid_condition_mp: # Wait for the cid to release if it's being tagged while cid in self.object_locked_cids_mp: - self.fhs_logger.debug(f"Cid ({cid}) is locked. Waiting.") + self.fhs_logger.debug("Cid (%s) is locked. Waiting.", cid) self.object_cid_condition_mp.wait() # Modify reference_locked_cids consecutively self.object_locked_cids_mp.append(cid) - self.fhs_logger.debug(f"Synchronizing object_locked_cids_mp for cid: {cid}") + self.fhs_logger.debug( + "Synchronizing object_locked_cids_mp for cid: %s", cid + ) else: with self.object_cid_condition_th: while cid in self.object_locked_cids_th: - self.fhs_logger.debug(f"Cid ({cid}) is locked. Waiting.") + self.fhs_logger.debug("Cid (%s) is locked. Waiting.", cid) self.object_cid_condition_th.wait() self.object_locked_cids_th.append(cid) - self.fhs_logger.debug(f"Synchronizing object_locked_cids_th for cid: {cid}") + self.fhs_logger.debug( + "Synchronizing object_locked_cids_th for cid: %s", cid + ) def _check_object_locked_cids(self, cid: str) -> None: """Check that a given content identifier is currently locked (found in the @@ -2576,8 +2644,8 @@ def _check_object_locked_cids(self, cid: str) -> None: raise IdentifierNotLocked(err_msg) def _release_object_locked_cids(self, cid: str) -> None: - """Remove the given content identifier from 'object_locked_cids' and notify other - waiting threads or processes. + """Remove the given content identifier from 'object_locked_cids' and + notify other waiting threads or processes. :param str cid: Content identifier """ @@ -2586,19 +2654,20 @@ def _release_object_locked_cids(self, cid: str) -> None: self.object_locked_cids_mp.remove(cid) self.object_cid_condition_mp.notify() self.fhs_logger.debug( - f"Releasing cid ({cid}) from object_cid_condition_mp." + "Releasing cid (%s) from object_cid_condition_mp.", cid ) else: with self.object_cid_condition_th: self.object_locked_cids_th.remove(cid) self.object_cid_condition_th.notify() self.fhs_logger.debug( - f"Releasing cid ({cid}) from object_cid_condition_th." + "Releasing cid (%s) from object_cid_condition_th.", cid ) def _synchronize_referenced_locked_pids(self, pid: str) -> None: - """Multiple threads may interact with a pid (to tag, untag, delete) and these actions - must be coordinated to prevent unexpected behaviour/race conditions that cause chaos. + """Multiple threads may interact with a pid (to tag, untag, delete) and + these actions must be coordinated to prevent unexpected behaviour/race + conditions that cause chaos. :param str pid: Persistent or authority-based identifier """ @@ -2606,21 +2675,21 @@ def _synchronize_referenced_locked_pids(self, pid: str) -> None: with self.reference_pid_condition_mp: # Wait for the pid to release if it's in use while pid in self.reference_locked_pids_mp: - self.fhs_logger.debug(f"Pid ({pid}) is locked. Waiting.") + self.fhs_logger.debug("Pid (%s) is locked. Waiting.", pid) self.reference_pid_condition_mp.wait() # Modify reference_locked_pids consecutively self.reference_locked_pids_mp.append(pid) self.fhs_logger.debug( - f"Synchronizing reference_locked_pids_mp for pid: {pid}" + "Synchronizing reference_locked_pids_mp for pid: %s", pid ) else: with self.reference_pid_condition_th: while pid in self.reference_locked_pids_th: - logging.debug(f"Pid ({pid}) is locked. Waiting.") + logging.debug("Pid (%s) is locked. Waiting.", pid) self.reference_pid_condition_th.wait() self.reference_locked_pids_th.append(pid) self.fhs_logger.debug( - f"Synchronizing reference_locked_pids_th for pid: {pid}" + "Synchronizing reference_locked_pids_th for pid: %s", pid ) def _check_reference_locked_pids(self, pid: str) -> None: @@ -2641,8 +2710,8 @@ def _check_reference_locked_pids(self, pid: str) -> None: raise IdentifierNotLocked(err_msg) def _release_reference_locked_pids(self, pid: str) -> None: - """Remove the given persistent identifier from 'reference_locked_pids' and notify other - waiting threads or processes. + """Remove the given persistent identifier from 'reference_locked_pids' + and notify other waiting threads or processes. :param str pid: Persistent or authority-based identifier """ @@ -2651,7 +2720,7 @@ def _release_reference_locked_pids(self, pid: str) -> None: self.reference_locked_pids_mp.remove(pid) self.reference_pid_condition_mp.notify() self.fhs_logger.debug( - f"Releasing pid ({pid}) from reference_locked_pids_mp." + "Releasing pid (%s) from reference_locked_pids_mp.", pid ) else: # Release pid @@ -2659,22 +2728,22 @@ def _release_reference_locked_pids(self, pid: str) -> None: self.reference_locked_pids_th.remove(pid) self.reference_pid_condition_th.notify() self.fhs_logger.debug( - f"Releasing pid ({pid}) from reference_locked_pids_th." + "Releasing pid (%s) from reference_locked_pids_th.", pid ) # Other Static Methods @staticmethod def _read_small_file_content(path_to_file: Path): - """Read the contents of a file with the given path. This method is not optimized for - large files - so it should only be used for small files (like reference files). + """Read the contents of a file with the given path. This method is not + optimized for large files - so it should only be used for small files + (like reference files). :param path path_to_file: Path to the file to read :return: Content of the given file """ - with open(path_to_file, "r", encoding="utf8") as opened_path: - content = opened_path.read() - return content + with open(path_to_file, encoding="utf8") as opened_path: + return opened_path.read() @staticmethod def _rename_path_for_deletion(path: Union[Path, str]) -> str: @@ -2688,27 +2757,27 @@ def _rename_path_for_deletion(path: Union[Path, str]) -> str: path = Path(path) delete_path = path.with_name(path.stem + "_delete" + path.suffix) shutil.move(path, delete_path) - # TODO: Adjust all code for constructing paths to use path and revise accordingly + # TODO: Adjust all code for constructing paths to use path and revise + # accordingly return str(delete_path) @staticmethod - def _get_file_paths(directory: Union[str, Path]) -> Optional[List[Path]]: + def _get_file_paths(directory: Union[str, Path]) -> Optional[list[Path]]: """Get the file paths of a given directory if it exists :param mixed directory: String or path to directory. :raises FileNotFoundError: If the directory doesn't exist - :return: file_paths - File paths of the given directory or None if directory doesn't exist + :return: file_paths - File paths of the given directory or None if directory + doesn't exist """ if os.path.exists(directory): files = os.listdir(directory) - file_paths = [ + return [ directory / file for file in files if os.path.isfile(directory / file) ] - return file_paths - else: - return None + return None @staticmethod def _check_arg_data(data: Union[str, os.PathLike, io.BufferedReader]) -> bool: @@ -2726,18 +2795,15 @@ def _check_arg_data(data: Union[str, os.PathLike, io.BufferedReader]) -> bool: and not isinstance(data, io.BufferedIOBase) ): err_msg = ( - "FileHashStore - _validate_arg_data: Data must be a path, string or buffered" - + f" stream type. Data type supplied: {type(data)}" + "FileHashStore - _validate_arg_data: Data must be a path, string or " + f"buffered stream type. Data type supplied: {type(data)}" ) logging.error(err_msg) raise TypeError(err_msg) - if isinstance(data, str): - if data.strip() == "": - err_msg = ( - "FileHashStore - _validate_arg_data: Data string cannot be empty." - ) - logging.error(err_msg) - raise TypeError(err_msg) + if isinstance(data, str) and data.strip() == "": + err_msg = "FileHashStore - _validate_arg_data: Data string cannot be empty." + logging.error(err_msg) + raise TypeError(err_msg) return True @staticmethod @@ -2751,7 +2817,7 @@ def _check_integer(file_size: int) -> None: if not isinstance(file_size, int): err_msg = ( "FileHashStore - _check_integer: size given must be an integer." - + f" File size: {file_size}. Arg Type: {type(file_size)}." + f" File size: {file_size}. Arg Type: {type(file_size)}." ) logging.error(err_msg) raise TypeError(err_msg) @@ -2762,8 +2828,8 @@ def _check_integer(file_size: int) -> None: @staticmethod def _check_string(string: str, arg: str) -> None: - """Check whether a string is None or empty - or if it contains an illegal character; - throws an exception if so. + """Check whether a string is None or empty - or if it contains an + illegal character; throws an exception if so. :param str string: Value to check. :param str arg: Name of the argument to check. @@ -2772,7 +2838,7 @@ def _check_string(string: str, arg: str) -> None: method = inspect.stack()[1].function err_msg = ( f"FileHashStore - {method}: {arg} cannot be None" - + f" or empty, {arg}: {string}." + f" or empty, {arg}: {string}." ) logging.error(err_msg) raise ValueError(err_msg) @@ -2807,10 +2873,11 @@ def __init__(self, obj: Union[IO[bytes], str, Path]): if hasattr(obj, "read"): pos = obj.tell() elif os.path.isfile(obj): - obj = io.open(obj, "rb") + obj = open(obj, "rb") # noqa: SIM115 pos = None else: - raise ValueError("Object must be a valid file path or a readable object") + msg = "Object must be a valid file path or a readable object" + raise ValueError(msg) try: file_stat = os.stat(obj.name) diff --git a/src/hashstore/filehashstore_exceptions.py b/src/hashstore/filehashstore_exceptions.py index 7acb77f8..e48b39fc 100644 --- a/src/hashstore/filehashstore_exceptions.py +++ b/src/hashstore/filehashstore_exceptions.py @@ -2,9 +2,9 @@ class StoreObjectForPidAlreadyInProgress(Exception): - """Custom exception thrown when called to store a data object for a pid that is already - progress. A pid can only ever reference one data object/content identifier so duplicate - requests are rejected immediately.""" + """Custom exception thrown when called to store a data object for a pid that + is already progress. A pid can only ever reference one data object/content + identifier so duplicate requests are rejected immediately.""" def __init__(self, message, errors=None): super().__init__(message) @@ -12,8 +12,8 @@ def __init__(self, message, errors=None): class IdentifierNotLocked(Exception): - """Custom exception thrown when an identifier (ex. 'pid' or 'cid') is not locked, which is - required to ensure thread safety.""" + """Custom exception thrown when an identifier (ex. 'pid' or 'cid') is not + locked, which is required to ensure thread safety.""" def __init__(self, message, errors=None): super().__init__(message) @@ -119,7 +119,8 @@ def __init__(self, message, errors=None): class HashStoreRefsAlreadyExists(Exception): - """Custom exception thrown when called to tag an object that is already tagged appropriately.""" + """Custom exception thrown when called to tag an object that is already + tagged appropriately.""" def __init__(self, message, errors=None): super().__init__(message) diff --git a/src/hashstore/hashstore.py b/src/hashstore/hashstore.py index 20a93fd8..d741e433 100644 --- a/src/hashstore/hashstore.py +++ b/src/hashstore/hashstore.py @@ -1,19 +1,19 @@ """Hashstore Interface""" -from abc import ABC, abstractmethod import importlib.metadata import importlib.util +from abc import ABC, abstractmethod class HashStore(ABC): - """HashStore is a content-addressable file management system that utilizes - an object's content identifier (hex digest/checksum) to address files.""" + """HashStore is a content-addressable file management + system that utilizes an object's content identifier (hex digest/checksum) to + address files.""" @staticmethod def version(): """Return the version number""" - __version__ = importlib.metadata.version("hashstore") - return __version__ + return importlib.metadata.version("hashstore") @abstractmethod def store_object( @@ -25,32 +25,36 @@ def store_object( checksum_algorithm, expected_object_size, ): - """Atomic storage of objects to disk using a given stream. Upon successful storage, - it returns an `ObjectMetadata` object containing relevant file information, such as - a persistent identifier that references the data file, the file's size, and a hex digest - dictionary of algorithms and checksums. The method also tags the object, creating + """Atomic storage of objects to disk using a given stream. Upon + successful storage, it returns an `ObjectMetadata` object containing + relevant file information, such as a persistent identifier that + references the data file, the file's size, and a hex digest dictionary + of algorithms and checksums. The method also tags the object, creating references for discoverability. - `store_object` ensures that an object is stored only once by synchronizing multiple calls - and rejecting attempts to store duplicate objects. If called without a pid, it stores the - object without tagging, and it becomes the caller's responsibility to finalize the process - by calling `tag_object` after verifying the correct object is stored. - - The file's permanent address is determined by calculating the object's content identifier - based on the store's default algorithm, which is also the permanent address of the file. - The content identifier is then sharded using the store's configured depth and width, - delimited by '/', and concatenated to produce the final permanent address. This address - is stored in the `/store_directory/objects/` directory. - - By default, the hex digest map includes common hash algorithms (md5, sha1, sha256, sha384, - sha512). If an additional algorithm is provided, the method checks if it is supported and - adds it to the hex digests dictionary along with its corresponding hex digest. An algorithm - is considered "supported" if it is recognized as a valid hash algorithm in the `hashlib` - library. - - If file size and/or checksum & checksum_algorithm values are provided, `store_object` - validates the object to ensure it matches the given arguments before moving the file to - its permanent address. + `store_object` ensures that an object is stored only once by + synchronizing multiple calls and rejecting attempts to store duplicate + objects. If called without a pid, it stores the object without tagging, + and it becomes the caller's responsibility to finalize the process by + calling `tag_object` after verifying the correct object is stored. + + The file's permanent address is determined by calculating the object's + content identifier based on the store's default algorithm, which is also + the permanent address of the file. The content identifier is then + sharded using the store's configured depth and width, delimited by '/', + and concatenated to produce the final permanent address. This address is + stored in the `/store_directory/objects/` directory. + + By default, the hex digest map includes common hash algorithms (md5, + sha1, sha256, sha384, sha512). If an additional algorithm is provided, + the method checks if it is supported and adds it to the hex digests + dictionary along with its corresponding hex digest. An algorithm is + considered "supported" if it is recognized as a valid hash algorithm in + the `hashlib` library. + + If file size and/or checksum & checksum_algorithm values are provided, + `store_object` validates the object to ensure it matches the given + arguments before moving the file to its permanent address. :param str pid: Authority-based identifier. :param mixed data: String or path to the object. @@ -67,8 +71,9 @@ def store_object( @abstractmethod def tag_object(self, pid, cid): """Creates references that allow objects stored in HashStore to be discoverable. - Retrieving, deleting or calculating a hex digest of an object is based on a pid - argument, to proceed, we must be able to find the object associated with the pid. + Retrieving, deleting or calculating a hex digest of an object is based + on a pid argument, to proceed, we must be able to find the object + associated with the pid. :param str pid: Authority-based or persistent identifier of the object. :param str cid: Content identifier of the object. @@ -77,17 +82,19 @@ def tag_object(self, pid, cid): @abstractmethod def store_metadata(self, pid, metadata, format_id): - """Add or update metadata, such as `sysmeta`, to disk using the given path/stream. The - `store_metadata` method uses a persistent identifier `pid` and a metadata `format_id` - to determine the permanent address of the metadata object. All metadata documents for a - given `pid` will be stored in a directory that follows the HashStore configuration - settings (under ../metadata) that is determined by calculating the hash of the given pid. - Metadata documents are stored in this directory, and is each named using the hash of the pid - and metadata format (`pid` + `format_id`). - - Upon successful storage of metadata, the method returns a string representing the file's - permanent address. Metadata objects are stored in parallel to objects in the - `/store_directory/metadata/` directory. + """Add or update metadata, such as `sysmeta`, to disk using the given + path/stream. The `store_metadata` method uses a persistent identifier + `pid` and a metadata `format_id` to determine the permanent address of + the metadata object. All metadata documents for a given `pid` will be + stored in a directory that follows the HashStore configuration settings + (under ../metadata) that is determined by calculating the hash of the + given pid. Metadata documents are stored in this directory, and is each + named using the hash of the pid and metadata format (`pid` + + `format_id`). + + Upon successful storage of metadata, the method returns a string + representing the file's permanent address. Metadata objects are stored + in parallel to objects in the `/store_directory/metadata/` directory. :param str pid: Authority-based identifier. :param mixed metadata: String or path to the metadata document. @@ -99,9 +106,10 @@ def store_metadata(self, pid, metadata, format_id): @abstractmethod def retrieve_object(self, pid): - """Retrieve an object from disk using a persistent identifier (pid). The `retrieve_object` - method opens and returns a buffered object stream ready for reading if the object - associated with the provided `pid` exists on disk. + """Retrieve an object from disk using a persistent identifier (pid). The + `retrieve_object` method opens and returns a buffered object stream + ready for reading if the object associated with the provided `pid` + exists on disk. :param str pid: Authority-based identifier. @@ -111,9 +119,10 @@ def retrieve_object(self, pid): @abstractmethod def retrieve_metadata(self, pid, format_id): - """Retrieve the metadata object from disk using a persistent identifier (pid) - and metadata namespace (format_id). If the metadata document exists, the method opens - and returns a buffered metadata stream ready for reading. + """Retrieve the metadata object from disk using a persistent identifier + (pid) and metadata namespace (format_id). If the metadata document + exists, the method opens and returns a buffered metadata stream ready + for reading. :param str pid: Authority-based identifier. :param str format_id: Metadata format. @@ -124,10 +133,11 @@ def retrieve_metadata(self, pid, format_id): @abstractmethod def delete_object(self, pid): - """Deletes an object and its related data permanently from HashStore using a given - persistent identifier. The object associated with the pid will be deleted if it is not - referenced by any other pids, along with its reference files and all metadata documents - found in its respective metadata directory. + """Deletes an object and its related data permanently from HashStore + using a given persistent identifier. The object associated with the pid + will be deleted if it is not referenced by any other pids, along with + its reference files and all metadata documents found in its respective + metadata directory. :param str pid: Persistent or Authority-based identifier. """ @@ -137,8 +147,9 @@ def delete_object(self, pid): def delete_if_invalid_object( self, object_metadata, checksum, checksum_algorithm, expected_file_size ): - """Confirm equality of content in an ObjectMetadata. The `delete_invalid_object` method - will delete a data object if the object_metadata does not match the specified values. + """Confirm equality of content in an ObjectMetadata. The + `delete_invalid_object` method will delete a data object if the + object_metadata does not match the specified values. :param ObjectMetadata object_metadata: ObjectMetadata object. :param str checksum: Value of the checksum. @@ -149,9 +160,10 @@ def delete_if_invalid_object( @abstractmethod def delete_metadata(self, pid, format_id): - """Deletes a metadata document (ex. `sysmeta`) permanently from HashStore using a given - persistent identifier (`pid`) and format_id (metadata namespace). If a `format_id` is - not supplied, all metadata documents associated with the given `pid` will be deleted. + """Deletes a metadata document (ex. `sysmeta`) permanently from + HashStore using a given persistent identifier (`pid`) and format_id + (metadata namespace). If a `format_id` is not supplied, all metadata + documents associated with the given `pid` will be deleted. :param str pid: Authority-based identifier. :param str format_id: Metadata format. @@ -160,8 +172,8 @@ def delete_metadata(self, pid, format_id): @abstractmethod def get_hex_digest(self, pid, algorithm): - """Calculates the hex digest of an object that exists in HashStore using a given persistent - identifier and hash algorithm. + """Calculates the hex digest of an object that exists in HashStore using + a given persistent identifier and hash algorithm. :param str pid: Authority-based identifier. :param str algorithm: Algorithm of hex digest to generate. @@ -174,24 +186,28 @@ def get_hex_digest(self, pid, algorithm): class HashStoreFactory: """A factory class for creating `HashStore`-like objects. - The `HashStoreFactory` class serves as a factory for creating `HashStore`-like objects, - which are classes that implement the 'HashStore' abstract methods. + The `HashStoreFactory` class serves as a factory for creating + `HashStore`-like objects, which are classes that implement the 'HashStore' + abstract methods. - This factory class provides a method to retrieve a `HashStore` object based on a given module - (e.g., "hashstore.filehashstore.filehashstore") and class name (e.g., "FileHashStore"). - """ + This factory class provides a method to retrieve a `HashStore` object based + on a given module (e.g., "hashstore.filehashstore.filehashstore") and class + name (e.g., "FileHashStore").""" @staticmethod def get_hashstore(module_name, class_name, properties=None): - """Get a `HashStore`-like object based on the specified `module_name` and `class_name`. + """Get a `HashStore`-like object based on the specified `module_name` + and `class_name`. - The `get_hashstore` method retrieves a `HashStore`-like object based on the provided - `module_name` and `class_name`, with optional custom properties. + The `get_hashstore` method retrieves a `HashStore`-like object based on + the provided `module_name` and `class_name`, with optional custom + properties. :param str module_name: Name of the package (e.g., "hashstore.filehashstore"). - :param str class_name: Name of the class in the given module (e.g., "FileHashStore"). - :param dict properties: Desired HashStore properties (optional). If `None`, default values - will be used. Example Properties Dictionary: + :param str class_name: Name of the class in the given module + (e.g., "FileHashStore"). + :param dict properties: Desired HashStore properties (optional). If `None`, + default values will be used. Example Properties Dictionary: { "store_path": "var/metacat", "store_depth": 3, @@ -200,14 +216,16 @@ def get_hashstore(module_name, class_name, properties=None): "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata" } - :return: HashStore - A hash store object based on the given `module_name` and `class_name`. + :return: HashStore - A hash store object based on the given `module_name` + and `class_name`. :raises ModuleNotFoundError: If the module is not found. :raises AttributeError: If the class does not exist within the module. """ # Validate module if importlib.util.find_spec(module_name) is None: - raise ModuleNotFoundError(f"No module found for '{module_name}'") + msg = f"No module found for '{module_name}'" + raise ModuleNotFoundError(msg) # Get HashStore imported_module = importlib.import_module(module_name) @@ -216,6 +234,5 @@ def get_hashstore(module_name, class_name, properties=None): if hasattr(imported_module, class_name): hashstore_class = getattr(imported_module, class_name) return hashstore_class(properties=properties) - raise AttributeError( - f"Class name '{class_name}' is not an attribute of module '{module_name}'" - ) + msg = f"Class name '{class_name}' is not an attribute of module '{module_name}'" + raise AttributeError(msg) diff --git a/src/hashstore/hashstoreclient.py b/src/hashstore/hashstoreclient.py index b1c1cc67..a6f989d8 100644 --- a/src/hashstore/hashstoreclient.py +++ b/src/hashstore/hashstoreclient.py @@ -1,14 +1,15 @@ -#!/usr/bin/env python """HashStore Command Line App""" import logging +import multiprocessing import os from argparse import ArgumentParser from datetime import datetime -import multiprocessing from pathlib import Path -import yaml + import pg8000 +import yaml + from hashstore import HashStoreFactory @@ -21,8 +22,8 @@ def __init__(self): program_name = "HashStore Command Line Client" description = ( "Command line tool to call store, retrieve and delete with a HashStore." - + " Additionally, methods are available to test functionality with a" - + " metacat postgres db." + " Additionally, methods are available to test functionality with a" + " metacat postgres db." ) epilog = "Created for DataONE (NCEAS)" @@ -199,9 +200,12 @@ def load_store_properties(hashstore_yaml): :return: HashStore properties with the following keys (and values): - store_depth (int): Depth when sharding an object's hex digest. - - store_width (int): Width of directories when sharding an object's hex digest. - - store_algorithm (str): Hash algorithm used for calculating the object's hex digest. - - store_metadata_namespace (str): Namespace for the HashStore's system metadata. + - store_width (int): Width of directories when sharding an object's hex + digest. + - store_algorithm (str): Hash algorithm used for calculating the object's + hex digest. + - store_metadata_namespace (str): Namespace for the HashStore's system + metadata. :rtype: dict """ property_required_keys = [ @@ -214,11 +218,11 @@ def load_store_properties(hashstore_yaml): if not os.path.exists(hashstore_yaml): exception_string = ( "HashStoreParser - load_store_properties: hashstore.yaml not found" - + " in store root path." + " in store root path." ) raise FileNotFoundError(exception_string) # Open file - with open(hashstore_yaml, "r", encoding="utf-8") as file: + with open(hashstore_yaml, encoding="utf-8") as file: yaml_data = yaml.safe_load(file) # Get hashstore properties @@ -324,8 +328,8 @@ def store_to_hashstore_from_list(self, origin_dir, obj_type, num, skip_obj_size) content = ( f"HashStoreClient (store_to_hashstore_from_list):\n" f"Start Time: {start_time}\nEnd Time: {end_time}\n" - + f"Total Time to Store {len(checked_obj_list)} {obj_type}" - + f" Objects: {end_time - start_time}\n" + f"Total Time to Store {len(checked_obj_list)} {obj_type}" + f" Objects: {end_time - start_time}\n" ) logging.info(content) @@ -337,7 +341,6 @@ def try_store_object(self, obj_tuple): try: self.hashstore.store_object(*obj_tuple) return - # pylint: disable=W0718 except Exception as so_exception: print(so_exception) @@ -402,8 +405,8 @@ def retrieve_and_validate_from_hashstore( content = ( f"retrieve_and_validate_from_hashstore:\n" f"Start Time: {start_time}\nEnd Time: {end_time}\n" - + f"Total Time to retrieve and validate {len(checked_obj_list)} {obj_type}" - + f" Objects: {end_time - start_time}\n" + f"Total Time to retrieve and validate {len(checked_obj_list)} {obj_type}" + f" Objects: {end_time - start_time}\n" ) logging.info(content) @@ -423,8 +426,8 @@ def validate_object(self, obj_tuple): if computed_digest != obj_db_checksum: err_msg = ( f"Assertion Error for pid/guid: {pid_guid} -" - + f" Digest calculated from stream ({computed_digest}) does not match" - + f" checksum from metacat db: {obj_db_checksum}" + f" Digest calculated from stream ({computed_digest}) does not match" + f" checksum from metacat db: {obj_db_checksum}" ) logging.error(err_msg) print(err_msg) @@ -434,7 +437,8 @@ def validate_object(self, obj_tuple): def validate_metadata(self, obj_tuple): """Retrieves a metadata from HashStore and validates its checksum. - :param obj_tuple: Tuple containing pid_guid, format_id, obj_checksum, obj_algorithm. + :param obj_tuple: Tuple containing pid_guid, format_id, obj_checksum, + obj_algorithm. """ pid_guid = obj_tuple[0] namespace = obj_tuple[1] @@ -448,8 +452,8 @@ def validate_metadata(self, obj_tuple): if computed_digest != metadata_db_checksum: err_msg = ( f"Assertion Error for pid/guid: {pid_guid} -" - + f" Digest calculated from stream ({computed_digest}) does not match" - + f" checksum from metacat db: {metadata_db_checksum}" + f" Digest calculated from stream ({computed_digest}) does not match" + f" checksum from metacat db: {metadata_db_checksum}" ) logging.error(err_msg) print(err_msg) @@ -504,8 +508,8 @@ def delete_objects_from_list(self, origin_dir, obj_type, num, skip_obj_size): content = ( f"HashStoreClient (delete_objects_from_list):\n" f"Start Time: {start_time}\nEnd Time: {end_time}\n" - + f"Total Time to Delete {len(checked_obj_list)} {obj_type}" - + f" Objects: {end_time - start_time}\n" + f"Total Time to Delete {len(checked_obj_list)} {obj_type}" + f" Objects: {end_time - start_time}\n" ) logging.info(content) @@ -517,7 +521,6 @@ def try_delete_object(self, obj_pid): try: self.hashstore.delete_object(obj_pid) return - # pylint: disable=W0718 except Exception as do_exception: print(do_exception) @@ -531,7 +534,6 @@ def try_delete_metadata(self, obj_tuple): try: self.hashstore.delete_metadata(pid_guid, namespace) return - # pylint: disable=W0718 except Exception as do_exception: print(do_exception) @@ -553,13 +555,13 @@ def __init__(self, hashstore_path, hashstore): pgyaml_path = hashstore_path + "/pgdb.yaml" if not os.path.exists(pgyaml_path): exception_string = ( - "HashStore CLI Client - _load_metacat_db_properties: pgdb.yaml not found" - + " in store root path. Must be manually created with the following keys:" - + " db_user, db_password, db_host, db_port, db_name" + "HashStore CLI Client - _load_metacat_db_properties: pgdb.yaml not " + "found in store root path. Must be manually created with the " + "following keys: db_user, db_password, db_host, db_port, db_name" ) raise FileNotFoundError(exception_string) # Open file - with open(pgyaml_path, "r", encoding="utf-8") as file: + with open(pgyaml_path, encoding="utf-8") as file: yaml_data = yaml.safe_load(file) # Get database values @@ -570,11 +572,13 @@ def __init__(self, hashstore_path, hashstore): self.db_yaml_dict[key] = checked_property def get_object_metadata_list(self, origin_directory, num, skip_obj_size=None): - """Query the Metacat database for the full object and metadata list, ordered by GUID. + """Query the Metacat database for the full object and metadata list, ordered + by GUID. :param str origin_directory: 'var/metacat/data' or 'var/metacat/documents'. :param int num: Number of rows to retrieve from the Metacat database. - :param int skip_obj_size: Size of obj in GB to skip (ex. 4 = 4GB), defaults to 'None' + :param int skip_obj_size: Size of obj in GB to skip (ex. 4 = 4GB), defaults + to 'None' """ # Create a connection to the database db_user = self.db_yaml_dict["db_user"] @@ -595,15 +599,12 @@ def get_object_metadata_list(self, origin_directory, num, skip_obj_size=None): cursor = conn.cursor() # Query to refine rows between `identifier` and `systemmetadata`` table - if num is None: - limit_query = "" - else: - limit_query = f" LIMIT {num}" + limit_query = "" if num is None else f" LIMIT {num}" query = f"""SELECT identifier.guid, identifier.docid, identifier.rev, systemmetadata.object_format, systemmetadata.checksum, - systemmetadata.checksum_algorithm, systemmetadata.size FROM identifier INNER JOIN - systemmetadata ON identifier.guid = systemmetadata.guid ORDER BY - identifier.guid{limit_query};""" + systemmetadata.checksum_algorithm, systemmetadata.size FROM identifier + INNER JOIN systemmetadata ON identifier.guid = systemmetadata.guid + ORDER BY identifier.guid{limit_query};""" cursor.execute(query) # Fetch all rows from the result set @@ -620,23 +621,20 @@ def get_object_metadata_list(self, origin_directory, num, skip_obj_size=None): size = int(row[6]) if gb_files_to_skip is not None and size > gb_files_to_skip: continue - else: - # Get pid, filepath and formatId - pid_guid = row[0] - metadatapath_docid_rev = ( - origin_directory + "/" + row[1] + "." + str(row[2]) - ) - metadata_namespace = row[3] - row_checksum = row[4] - row_checksum_algorithm = row[5] - tuple_item = ( - pid_guid, - metadatapath_docid_rev, - metadata_namespace, - row_checksum, - row_checksum_algorithm, - ) - object_metadata_list.append(tuple_item) + # Get pid, filepath and formatId + pid_guid = row[0] + metadatapath_docid_rev = origin_directory + "/" + row[1] + "." + str(row[2]) + metadata_namespace = row[3] + row_checksum = row[4] + row_checksum_algorithm = row[5] + tuple_item = ( + pid_guid, + metadatapath_docid_rev, + metadata_namespace, + row_checksum, + row_checksum_algorithm, + ) + object_metadata_list.append(tuple_item) # Close the cursor and connection when done cursor.close() @@ -646,9 +644,11 @@ def get_object_metadata_list(self, origin_directory, num, skip_obj_size=None): @staticmethod def refine_list_for_objects(metacat_obj_list, action): - """Refine a list of objects by checking for file existence and removing duplicates. + """Refine a list of objects by checking for file existence and removing + duplicates. - :param List metacat_obj_list: List of tuple objects representing rows from Metacat database. + :param List metacat_obj_list: List of tuple objects representing rows from + Metacat database. :param str action: Action to perform. Options: "store", "retrieve", or "delete". - "store": Create a list of objects to store that do not exist in HashStore. - "retrieve": Create a list of objects that exist in HashStore. @@ -689,11 +689,14 @@ def refine_list_for_objects(metacat_obj_list, action): @staticmethod def refine_list_for_metadata(metacat_obj_list, action): - """Refine a list of metadata by checking for file existence and removing duplicates. + """Refine a list of metadata by checking for file existence and removing + duplicates. - :param List metacat_obj_list: List of tuple objects representing rows from metacat db. + :param List metacat_obj_list: List of tuple objects representing rows from + metacat db. :param str action: Action to perform - "store", "retrieve", or "delete". - - "store": Create a list of metadata to store that do not exist in HashStore. + - "store": Create a list of metadata to store that do not exist in + HashStore. - "retrieve": Create a list of metadata that exist in HashStore. - "delete": Create a list of metadata pids with their format_ids. @@ -709,22 +712,25 @@ def refine_list_for_metadata(metacat_obj_list, action): item_checksum_algorithm = tuple_item[4] if os.path.exists(filepath_docid_rev): if action == "store": - tuple_item = (pid_guid, filepath_docid_rev, metadata_namespace) - refined_metadata_list.append(tuple_item) + refined_metadata_list.append( + (pid_guid, filepath_docid_rev, metadata_namespace) + ) if action == "retrieve": - tuple_item = ( - pid_guid, - metadata_namespace, - item_checksum, - item_checksum_algorithm, + refined_metadata_list.append( + ( + pid_guid, + metadata_namespace, + item_checksum, + item_checksum_algorithm, + ) ) - refined_metadata_list.append(tuple_item) if action == "delete": - tuple_item = ( - pid_guid, - metadata_namespace, + refined_metadata_list.append( + ( + pid_guid, + metadata_namespace, + ) ) - refined_metadata_list.append(tuple_item) return refined_metadata_list @@ -735,31 +741,31 @@ def main(): args = parser.get_parser_args() # Client setup process - if getattr(args, "create_hashstore"): + if args.create_hashstore: # Create HashStore if -chs flag is true in a given directory # Get store attributes, HashStore will validate properties props = { - "store_path": getattr(args, "store_path"), - "store_depth": int(getattr(args, "depth")), - "store_width": int(getattr(args, "width")), - "store_algorithm": getattr(args, "algorithm"), - "store_metadata_namespace": getattr(args, "formatid"), + "store_path": args.store_path, + "store_depth": int(args.depth), + "store_width": int(args.width), + "store_algorithm": args.algorithm, + "store_metadata_namespace": args.formatid, } HashStoreClient(props) # Can't use client app without first initializing HashStore - store_path = getattr(args, "store_path") + store_path = args.store_path store_path_config_yaml = store_path + "/hashstore.yaml" if not os.path.exists(store_path_config_yaml): - raise FileNotFoundError( + msg = ( f"Missing config file (hashstore.yaml) at store path: {store_path}." - + " HashStore must first be initialized, use `--help` for more information." + " HashStore must first be initialized, use `--help` for more information." ) - else: - # Get the default format_id for sysmeta - with open(store_path_config_yaml, "r", encoding="utf-8") as hs_yaml_file: - yaml_data = yaml.safe_load(hs_yaml_file) + raise FileNotFoundError(msg) + # Get the default format_id for sysmeta + with open(store_path_config_yaml, encoding="utf-8") as hs_yaml_file: + yaml_data = yaml.safe_load(hs_yaml_file) - default_formatid = yaml_data["store_metadata_namespace"] + default_formatid = yaml_data["store_metadata_namespace"] # Setup logging, create log file if it doesn't already exist hashstore_py_log = store_path + "/python_client.log" @@ -768,11 +774,8 @@ def main(): python_log_file_path.parent.mkdir(parents=True, exist_ok=True) open(python_log_file_path, "w", encoding="utf-8").close() # Check for logging level - logging_level_arg = getattr(args, "logging_level") - if logging_level_arg is None: - logging_level = "INFO" - else: - logging_level = logging_level_arg + logging_level_arg = args.logging_level + logging_level = "INFO" if logging_level_arg is None else logging_level_arg logging.basicConfig( filename=python_log_file_path, level=logging_level, @@ -781,51 +784,52 @@ def main(): ) # Collect arguments to process - pid = getattr(args, "object_pid") - path = getattr(args, "object_path") - algorithm = getattr(args, "object_algorithm") - checksum = getattr(args, "object_checksum") - checksum_algorithm = getattr(args, "object_checksum_algorithm") - size = getattr(args, "object_size") - formatid = getattr(args, "object_formatid") + pid = args.object_pid + path = args.object_path + algorithm = args.object_algorithm + checksum = args.object_checksum + checksum_algorithm = args.object_checksum_algorithm + size = args.object_size + formatid = args.object_formatid if formatid is None: formatid = default_formatid - knbvm_test = getattr(args, "knbvm_flag") + knbvm_test = args.knbvm_flag # Instantiate HashStore Client props = parser.load_store_properties(store_path_config_yaml) # Reminder: 'hashstore.yaml' only contains 4 of the required 5 properties props["store_path"] = store_path hashstore_c = HashStoreClient(props, knbvm_test) if knbvm_test: - directory_to_convert = getattr(args, "source_directory") + directory_to_convert = args.source_directory # Check if the directory to convert exists if os.path.exists(directory_to_convert): # If -nobj is supplied, limit the objects we work with - number_of_objects_to_convert = getattr(args, "num_obj_to_convert") + number_of_objects_to_convert = args.num_obj_to_convert # Determine if we are working with objects or metadata - directory_type = getattr(args, "source_directory_type") - size_of_obj_to_skip = getattr(args, "gb_file_size_to_skip") + directory_type = args.source_directory_type + size_of_obj_to_skip = args.gb_file_size_to_skip accepted_directory_types = ["object", "metadata"] if directory_type not in accepted_directory_types: - raise ValueError( - "Directory `-stype` cannot be empty, must be 'object' or 'metadata'." - + f" source_directory_type: {directory_type}" + msg = ( + "Directory `-stype` cannot be empty, must be 'object' or " + f"'metadata'. source_directory_type: {directory_type}" ) - if getattr(args, "store_to_hashstore"): + raise ValueError(msg) + if args.store_to_hashstore: hashstore_c.store_to_hashstore_from_list( directory_to_convert, directory_type, number_of_objects_to_convert, size_of_obj_to_skip, ) - if getattr(args, "retrieve_and_validate"): + if args.retrieve_and_validate: hashstore_c.retrieve_and_validate_from_hashstore( directory_to_convert, directory_type, number_of_objects_to_convert, size_of_obj_to_skip, ) - if getattr(args, "delete_from_hashstore"): + if args.delete_from_hashstore: hashstore_c.delete_objects_from_list( directory_to_convert, directory_type, @@ -833,43 +837,52 @@ def main(): size_of_obj_to_skip, ) else: - raise FileNotFoundError( - f"Directory to convert is None or does not exist: {directory_to_convert}." + msg = ( + "Directory to convert is None or does not " + f"exist: {directory_to_convert}." ) - elif getattr(args, "client_getchecksum"): + raise FileNotFoundError(msg) + elif args.client_getchecksum: if pid is None: - raise ValueError("'-pid' option is required") + msg = "'-pid' option is required" + raise ValueError(msg) if algorithm is None: - raise ValueError("'-algo' option is required") + msg = "'-algo' option is required" + raise ValueError(msg) # Calculate the hex digest of a given pid with algorithm supplied digest = hashstore_c.hashstore.get_hex_digest(pid, algorithm) print(f"guid/pid: {pid}") print(f"algorithm: {algorithm}") print(f"Checksum/Hex Digest: {digest}") - elif getattr(args, "client_storeobject"): + elif args.client_storeobject: if pid is None: - raise ValueError("'-pid' option is required") + msg = "'-pid' option is required" + raise ValueError(msg) if path is None: - raise ValueError("'-path' option is required") + msg = "'-path' option is required" + raise ValueError(msg) # Store object to HashStore object_metadata = hashstore_c.hashstore.store_object( pid, path, algorithm, checksum, checksum_algorithm, size ) print(f"Object Metadata:\n{object_metadata}") - elif getattr(args, "client_storemetadata"): + elif args.client_storemetadata: if pid is None: - raise ValueError("'-pid' option is required") + msg = "'-pid' option is required" + raise ValueError(msg) if path is None: - raise ValueError("'-path' option is required") + msg = "'-path' option is required" + raise ValueError(msg) # Store metadata to HashStore metadata_cid = hashstore_c.hashstore.store_metadata(pid, path, formatid) print(f"Metadata Path: {metadata_cid}") - elif getattr(args, "client_retrieveobject"): + elif args.client_retrieveobject: if pid is None: - raise ValueError("'-pid' option is required") + msg = "'-pid' option is required" + raise ValueError(msg) # Retrieve object from HashStore and display the first 1000 bytes object_stream = hashstore_c.hashstore.retrieve_object(pid) object_content = object_stream.read(1000).decode("utf-8") @@ -877,9 +890,10 @@ def main(): print(object_content) print("...\n<-- Truncated for Display Purposes -->") - elif getattr(args, "client_retrievemetadata"): + elif args.client_retrievemetadata: if pid is None: - raise ValueError("'-pid' option is required") + msg = "'-pid' option is required" + raise ValueError(msg) # Retrieve metadata from HashStore and display the first 1000 bytes metadata_stream = hashstore_c.hashstore.retrieve_metadata(pid, formatid) metadata_content = metadata_stream.read(1000).decode("utf-8") @@ -887,20 +901,23 @@ def main(): print(metadata_content) print("...\n<-- Truncated for Display Purposes -->") - elif getattr(args, "client_deleteobject"): + elif args.client_deleteobject: if pid is None: - raise ValueError("'-pid' option is required") + msg = "'-pid' option is required" + raise ValueError(msg) # Delete object from HashStore delete_status = hashstore_c.hashstore.delete_object(pid) print(f"Object Deleted (T/F): {delete_status}") - elif getattr(args, "client_deletemetadata"): + elif args.client_deletemetadata: if pid is None: - raise ValueError("'-pid' option is required") + msg = "'-pid' option is required" + raise ValueError(msg) # Delete metadata from HashStore delete_status = hashstore_c.hashstore.delete_metadata(pid, formatid) print( - f"Metadata for pid: {pid} & formatid: {formatid}\nDeleted (T/F): {delete_status}" + f"Metadata for pid: {pid} & formatid: {formatid}\n" + f"Deleted (T/F): {delete_status}" ) diff --git a/tests/conftest.py b/tests/conftest.py index e10a83e0..e6efba3f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ """Pytest overall configuration file for fixtures""" import pytest + from hashstore.filehashstore import FileHashStore @@ -22,21 +23,19 @@ def init_props(tmp_path): hashstore_path = directory.as_posix() # Note, objects generated via tests are placed in a temporary folder # with the 'directory' parameter above appended - properties = { + return { "store_path": hashstore_path, "store_depth": 3, "store_width": 2, "store_algorithm": "SHA-256", "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", } - return properties @pytest.fixture(name="store") def init_store(props): """Create FileHashStore instance for all tests.""" - store = FileHashStore(props) - return store + return FileHashStore(props) @pytest.fixture(name="pids") @@ -45,7 +44,7 @@ def init_pids(): - object_cid: hex digest of the pid - metadata_cid: hex digest of the pid + store_metadata_namespace """ - test_pids = { + return { "doi:10.18739/A2901ZH2M": { "file_size_bytes": 39993, "metadata_cid": "323e0799524cec4c7e14d31289cefd884b563b5c052f154a066de5ec1e477da7", @@ -80,4 +79,3 @@ def init_pids(): "blake2s": "c8c9aea2f7ddcfaf8db93ce95f18e467b6293660d1a0b08137636a3c92896765", }, } - return test_pids diff --git a/tests/filehashstore/test_filehashstore.py b/tests/filehashstore/test_filehashstore.py index 0b0e94c3..8613fe7b 100644 --- a/tests/filehashstore/test_filehashstore.py +++ b/tests/filehashstore/test_filehashstore.py @@ -1,30 +1,31 @@ """Test module for FileHashStore init, core, utility and supporting methods.""" +import hashlib import io import os -import hashlib import shutil from pathlib import Path + import pytest + from hashstore.filehashstore import FileHashStore, ObjectMetadata, Stream from hashstore.filehashstore_exceptions import ( - OrphanPidRefsFileFound, + CidRefsContentError, + CidRefsFileNotFound, + HashStoreRefsAlreadyExists, + IdentifierNotLocked, NonMatchingChecksum, NonMatchingObjSize, + OrphanPidRefsFileFound, PidNotFoundInCidRefsFile, - PidRefsDoesNotExist, - RefsFileExistsButCidObjMissing, - UnsupportedAlgorithm, - HashStoreRefsAlreadyExists, PidRefsAlreadyExistsError, - CidRefsContentError, - CidRefsFileNotFound, PidRefsContentError, + PidRefsDoesNotExist, PidRefsFileNotFound, - IdentifierNotLocked, + RefsFileExistsButCidObjMissing, + UnsupportedAlgorithm, ) - # pylint: disable=W0212 @@ -52,7 +53,7 @@ def test_init_existing_store_incorrect_algorithm_format(store): "store_algorithm": "sha256", "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", } - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Must be one of"): FileHashStore(properties) @@ -84,7 +85,7 @@ def test_init_with_existing_hashstore_mismatched_config_depth(store): "store_algorithm": "SHA-256", "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", } - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="depth"): FileHashStore(properties) @@ -98,7 +99,7 @@ def test_init_with_existing_hashstore_mismatched_config_width(store): "store_algorithm": "SHA-256", "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", } - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="width"): FileHashStore(properties) @@ -112,7 +113,7 @@ def test_init_with_existing_hashstore_mismatched_config_algo(store): "store_algorithm": "SHA-512", "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", } - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="configuration"): FileHashStore(properties) @@ -126,7 +127,7 @@ def test_init_with_existing_hashstore_mismatched_config_metadata_ns(store): "store_algorithm": "SHA-512", "store_metadata_namespace": "http://ns.dataone.org/service/types/v5.0", } - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="configuration"): FileHashStore(properties) @@ -134,7 +135,7 @@ def test_init_with_existing_hashstore_missing_yaml(store, pids): """Test init with existing store raises RuntimeError when hashstore.yaml not found but objects exist.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") store._store_and_validate_data(pid, path) os.remove(store.hashstore_configuration_yaml) @@ -205,14 +206,14 @@ def test_validate_properties_key_value_is_none(store): "store_algorithm": "SHA-256", "store_metadata_namespace": None, } - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Value for key"): store._validate_properties(properties) def test_validate_properties_incorrect_type(store): """Confirm exception raised when a bad properties value is given.""" properties = "etc/filehashstore/hashstore.yaml" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Invalid"): store._validate_properties(properties) @@ -220,7 +221,7 @@ def test_set_default_algorithms_missing_yaml(store, pids): """Confirm set_default_algorithms raises FileNotFoundError when hashstore.yaml not found.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") store._store_and_validate_data(pid, path) os.remove(store.hashstore_configuration_yaml) @@ -234,7 +235,7 @@ def test_set_default_algorithms_missing_yaml(store, pids): def test_find_object_no_sysmeta(pids, store): """Test _find_object returns the correct content and expected value for non-existent sysmeta.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(pid, path) obj_info_dict = store._find_object(pid) @@ -258,7 +259,7 @@ def test_find_object_sysmeta(pids, store): """Test _find_object returns the correct content along with the sysmeta path""" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename @@ -285,7 +286,7 @@ def test_find_object_sysmeta(pids, store): def test_find_object_refs_exist_but_obj_not_found(pids, store): """Test _find_object throws exception when refs file exist but the object does not.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") store.store_object(pid, path) @@ -301,7 +302,7 @@ def test_find_object_cid_refs_not_found(pids, store): """Test _find_object throws exception when pid refs file is found (and contains a cid) but the cid refs file does not exist.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") _object_metadata = store.store_object(pid, path) @@ -320,7 +321,7 @@ def test_find_object_cid_refs_does_not_contain_pid(pids, store): """Test _find_object throws exception when pid refs file is found (and contains a cid) but the cid refs file does not contain the pid.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(pid, path) @@ -342,20 +343,20 @@ def test_find_object_pid_refs_not_found(store): def test_find_object_pid_none(store): """Test _find_object throws exception when pid is None.""" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store._find_object(None) def test_find_object_pid_empty(store): """Test _find_object throws exception when pid is empty.""" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store._find_object("") def test_store_and_validate_data_files_path(pids, store): """Test _store_and_validate_data accepts path object for the path arg.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = Path(test_dir) / pid.replace("/", "_") object_metadata = store._store_and_validate_data(pid, path) assert store._exists("objects", object_metadata.cid) @@ -364,7 +365,7 @@ def test_store_and_validate_data_files_path(pids, store): def test_store_and_validate_data_files_string(pids, store): """Test _store_and_validate_data accepts string for the path arg.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store._store_and_validate_data(pid, path) assert store._exists("objects", object_metadata.cid) @@ -373,11 +374,10 @@ def test_store_and_validate_data_files_string(pids, store): def test_store_and_validate_data_files_stream(pids, store): """Test _store_and_validate_data accepts stream for the path arg.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") - input_stream = io.open(path, "rb") - object_metadata = store._store_and_validate_data(pid, input_stream) - input_stream.close() + with open(path, "rb") as input_stream: + object_metadata = store._store_and_validate_data(pid, input_stream) assert store._exists("objects", object_metadata.cid) assert store._count("objects") == 3 @@ -385,7 +385,7 @@ def test_store_and_validate_data_files_stream(pids, store): def test_store_and_validate_data_cid(pids, store): """Check _store_and_validate_data returns the expected content identifier""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store._store_and_validate_data(pid, path) assert object_metadata.cid == pids[pid][store.algorithm] @@ -394,7 +394,7 @@ def test_store_and_validate_data_cid(pids, store): def test_store_and_validate_data_file_size(pids, store): """Check _store_and_validate_data returns correct file size.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store._store_and_validate_data(pid, path) assert object_metadata.obj_size == pids[pid]["file_size_bytes"] @@ -403,7 +403,7 @@ def test_store_and_validate_data_file_size(pids, store): def test_store_and_validate_data_hex_digests(pids, store): """Check _store_and_validate_data successfully generates hex digests dictionary.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store._store_and_validate_data(pid, path) assert object_metadata.hex_digests.get("md5") == pids[pid]["md5"] @@ -417,7 +417,7 @@ def test_store_and_validate_data_additional_algorithm(pids, store): """Check _store_and_validate_data returns an additional algorithm in hex digests when provided with an additional algo value.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: algo = "sha224" path = test_dir + pid.replace("/", "_") object_metadata = store._store_and_validate_data( @@ -431,7 +431,7 @@ def test_store_and_validate_data_with_correct_checksums(pids, store): """Check _store_and_validate_data stores a data object when a valid checksum and checksum algorithm is supplied.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: algo = "sha224" algo_checksum = pids[pid][algo] path = test_dir + pid.replace("/", "_") @@ -445,7 +445,7 @@ def test_store_and_validate_data_with_incorrect_checksum(pids, store): """Check _store_and_validate_data does not store data objects when a bad checksum supplied.""" test_dir = "tests/testdata/" entity = "objects" - for pid in pids.keys(): + for pid in pids: algo = "sha224" algo_checksum = "badChecksumValue" path = test_dir + pid.replace("/", "_") @@ -459,7 +459,7 @@ def test_store_and_validate_data_with_incorrect_checksum(pids, store): def test_store_data_only_cid(pids, store): """Check _store_data_only returns correct id.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store._store_data_only(path) assert object_metadata.cid == pids[pid][store.algorithm] @@ -468,7 +468,7 @@ def test_store_data_only_cid(pids, store): def test_store_data_only_file_size(pids, store): """Check _store_data_only returns correct file size.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store._store_data_only(path) assert object_metadata.obj_size == pids[pid]["file_size_bytes"] @@ -477,7 +477,7 @@ def test_store_data_only_file_size(pids, store): def test_store_data_only_hex_digests(pids, store): """Check _store_data_only generates a hex digests dictionary.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store._store_data_only(path) assert object_metadata.hex_digests.get("md5") == pids[pid]["md5"] @@ -490,45 +490,42 @@ def test_store_data_only_hex_digests(pids, store): def test_move_and_get_checksums_id(pids, store): """Test _move_and_get_checksums returns correct id.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") - input_stream = io.open(path, "rb") - ( - move_id, - _, - _, - ) = store._move_and_get_checksums(pid, input_stream) - input_stream.close() + with open(path, "rb") as input_stream: + ( + move_id, + _, + _, + ) = store._move_and_get_checksums(pid, input_stream) assert move_id == pids[pid][store.algorithm] def test_move_and_get_checksums_file_size(pids, store): """Test _move_and_get_checksums returns correct file size.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") - input_stream = io.open(path, "rb") - ( - _, - tmp_file_size, - _, - ) = store._move_and_get_checksums(pid, input_stream) - input_stream.close() + with open(path, "rb") as input_stream: + ( + _, + tmp_file_size, + _, + ) = store._move_and_get_checksums(pid, input_stream) assert tmp_file_size == pids[pid]["file_size_bytes"] def test_move_and_get_checksums_hex_digests(pids, store): """Test _move_and_get_checksums returns correct hex digests.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") - input_stream = io.open(path, "rb") - ( - _, - _, - hex_digests, - ) = store._move_and_get_checksums(pid, input_stream) - input_stream.close() + with open(path, "rb") as input_stream: + ( + _, + _, + hex_digests, + ) = store._move_and_get_checksums(pid, input_stream) assert hex_digests.get("md5") == pids[pid]["md5"] assert hex_digests.get("sha1") == pids[pid]["sha1"] assert hex_digests.get("sha256") == pids[pid]["sha256"] @@ -539,26 +536,23 @@ def test_move_and_get_checksums_hex_digests(pids, store): def test_move_and_get_checksums_does_not_store_duplicate(pids, store): """Test _move_and_get_checksums does not store duplicate objects.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") - input_stream = io.open(path, "rb") - store._move_and_get_checksums(pid, input_stream) - input_stream.close() - for pid in pids.keys(): + with open(path, "rb") as input_stream: + store._move_and_get_checksums(pid, input_stream) + for pid in pids: path = test_dir + pid.replace("/", "_") - input_stream = io.open(path, "rb") - store._move_and_get_checksums(pid, input_stream) - input_stream.close() + with open(path, "rb") as input_stream: + store._move_and_get_checksums(pid, input_stream) assert store._count("objects") == 3 def test_move_and_get_checksums_raises_error_with_nonmatching_checksum(pids, store): """Test _move_and_get_checksums raises error when incorrect checksum supplied.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") - input_stream = io.open(path, "rb") - with pytest.raises(NonMatchingChecksum): + with open(path, "rb") as input_stream, pytest.raises(NonMatchingChecksum): # pylint: disable=W0212 store._move_and_get_checksums( pid, @@ -566,22 +560,24 @@ def test_move_and_get_checksums_raises_error_with_nonmatching_checksum(pids, sto checksum="nonmatchingchecksum", checksum_algorithm="sha256", ) - input_stream.close() assert store._count("objects") == 0 def test_move_and_get_checksums_incorrect_file_size(pids, store): """Test _move_and_get_checksums raises error with an incorrect file size.""" test_dir = "tests/testdata/" - for pid in pids.keys(): - with pytest.raises(NonMatchingObjSize): - path = test_dir + pid.replace("/", "_") - input_stream = io.open(path, "rb") - incorrect_file_size = 1000 - (_, _, _, _,) = store._move_and_get_checksums( + for pid in pids: + path = test_dir + pid.replace("/", "_") + incorrect_file_size = 1000 + with open(path, "rb") as input_stream, pytest.raises(NonMatchingObjSize): + ( + _, + _, + _, + _, + ) = store._move_and_get_checksums( pid, input_stream, file_size_to_validate=incorrect_file_size ) - input_stream.close() def test_write_to_tmp_file_and_get_hex_digests_additional_algo(store): @@ -589,15 +585,14 @@ def test_write_to_tmp_file_and_get_hex_digests_additional_algo(store): test_dir = "tests/testdata/" pid = "jtao.1700.1" path = test_dir + pid - input_stream = io.open(path, "rb") checksum_algo = "sha3_256" checksum_correct = ( "b748069cd0116ba59638e5f3500bbff79b41d6184bc242bd71f5cbbb8cf484cf" ) - hex_digests, _, _ = store._write_to_tmp_file_and_get_hex_digests( - input_stream, additional_algorithm=checksum_algo - ) - input_stream.close() + with open(path, "rb") as input_stream: + hex_digests, _, _ = store._write_to_tmp_file_and_get_hex_digests( + input_stream, additional_algorithm=checksum_algo + ) assert hex_digests.get("sha3_256") == checksum_correct @@ -607,15 +602,14 @@ def test_write_to_tmp_file_and_get_hex_digests_checksum_algo(store): test_dir = "tests/testdata/" pid = "jtao.1700.1" path = test_dir + pid - input_stream = io.open(path, "rb") checksum_algo = "sha3_256" checksum_correct = ( "b748069cd0116ba59638e5f3500bbff79b41d6184bc242bd71f5cbbb8cf484cf" ) - hex_digests, _, _ = store._write_to_tmp_file_and_get_hex_digests( - input_stream, checksum_algorithm=checksum_algo - ) - input_stream.close() + with open(path, "rb") as input_stream: + hex_digests, _, _ = store._write_to_tmp_file_and_get_hex_digests( + input_stream, checksum_algorithm=checksum_algo + ) assert hex_digests.get("sha3_256") == checksum_correct @@ -625,7 +619,6 @@ def test_write_to_tmp_file_and_get_hex_digests_checksum_and_additional_algo(stor test_dir = "tests/testdata/" pid = "jtao.1700.1" path = test_dir + pid - input_stream = io.open(path, "rb") additional_algo = "sha224" additional_algo_checksum_correct = ( "9b3a96f434f3c894359193a63437ef86fbd5a1a1a6cc37f1d5013ac1" @@ -634,12 +627,12 @@ def test_write_to_tmp_file_and_get_hex_digests_checksum_and_additional_algo(stor checksum_correct = ( "b748069cd0116ba59638e5f3500bbff79b41d6184bc242bd71f5cbbb8cf484cf" ) - hex_digests, _, _ = store._write_to_tmp_file_and_get_hex_digests( - input_stream, - additional_algorithm=additional_algo, - checksum_algorithm=checksum_algo, - ) - input_stream.close() + with open(path, "rb") as input_stream: + hex_digests, _, _ = store._write_to_tmp_file_and_get_hex_digests( + input_stream, + additional_algorithm=additional_algo, + checksum_algorithm=checksum_algo, + ) assert hex_digests.get("sha3_256") == checksum_correct assert hex_digests.get("sha224") == additional_algo_checksum_correct @@ -651,38 +644,39 @@ def test_write_to_tmp_file_and_get_hex_digests_checksum_and_additional_algo_dupl test_dir = "tests/testdata/" pid = "jtao.1700.1" path = test_dir + pid - input_stream = io.open(path, "rb") additional_algo = "sha224" checksum_algo = "sha224" checksum_correct = "9b3a96f434f3c894359193a63437ef86fbd5a1a1a6cc37f1d5013ac1" - hex_digests, _, _ = store._write_to_tmp_file_and_get_hex_digests( - input_stream, - additional_algorithm=additional_algo, - checksum_algorithm=checksum_algo, - ) - input_stream.close() + with open(path, "rb") as input_stream: + hex_digests, _, _ = store._write_to_tmp_file_and_get_hex_digests( + input_stream, + additional_algorithm=additional_algo, + checksum_algorithm=checksum_algo, + ) assert hex_digests.get("sha224") == checksum_correct def test_write_to_tmp_file_and_get_hex_digests_file_size(pids, store): """Test _write...hex_digests returns correct file size.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") - input_stream = io.open(path, "rb") - _, _, tmp_file_size = store._write_to_tmp_file_and_get_hex_digests(input_stream) - input_stream.close() + with open(path, "rb") as input_stream: + _, _, tmp_file_size = store._write_to_tmp_file_and_get_hex_digests( + input_stream + ) assert tmp_file_size == pids[pid]["file_size_bytes"] def test_write_to_tmp_file_and_get_hex_digests_hex_digests(pids, store): """Test _write...hex_digests returns correct hex digests.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") - input_stream = io.open(path, "rb") - hex_digests, _, _ = store._write_to_tmp_file_and_get_hex_digests(input_stream) - input_stream.close() + with open(path, "rb") as input_stream: + hex_digests, _, _ = store._write_to_tmp_file_and_get_hex_digests( + input_stream + ) assert hex_digests.get("md5") == pids[pid]["md5"] assert hex_digests.get("sha1") == pids[pid]["sha1"] assert hex_digests.get("sha256") == pids[pid]["sha256"] @@ -693,30 +687,30 @@ def test_write_to_tmp_file_and_get_hex_digests_hex_digests(pids, store): def test_write_to_tmp_file_and_get_hex_digests_tmpfile_object(pids, store): """Test _write...hex_digests returns a tmp file successfully.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") - input_stream = io.open(path, "rb") - _, tmp_file_name, _ = store._write_to_tmp_file_and_get_hex_digests(input_stream) - input_stream.close() + with open(path, "rb") as input_stream: + _, tmp_file_name, _ = store._write_to_tmp_file_and_get_hex_digests( + input_stream + ) assert os.path.isfile(tmp_file_name) is True def test_write_to_tmp_file_and_get_hex_digests_with_unsupported_algorithm(pids, store): """Test _write...hex_digests raises an exception when an unsupported algorithm supplied.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") - input_stream = io.open(path, "rb") algo = "md2" - with pytest.raises(UnsupportedAlgorithm): - _, _, _ = store._write_to_tmp_file_and_get_hex_digests( - input_stream, additional_algorithm=algo - ) - with pytest.raises(UnsupportedAlgorithm): - _, _, _ = store._write_to_tmp_file_and_get_hex_digests( - input_stream, checksum_algorithm=algo - ) - input_stream.close() + with open(path, "rb") as input_stream: + with pytest.raises(UnsupportedAlgorithm): + _, _, _ = store._write_to_tmp_file_and_get_hex_digests( + input_stream, additional_algorithm=algo + ) + with pytest.raises(UnsupportedAlgorithm): + _, _, _ = store._write_to_tmp_file_and_get_hex_digests( + input_stream, checksum_algorithm=algo + ) def test_mktmpfile(store): @@ -729,7 +723,7 @@ def test_mktmpfile(store): def test_store_hashstore_refs_files_(pids, store): """Test _store_hashstore_refs_files does not throw exception when successful.""" - for pid in pids.keys(): + for pid in pids: cid = pids[pid][store.algorithm] store._store_hashstore_refs_files(pid, cid) assert store._count("pid") == 3 @@ -738,7 +732,7 @@ def test_store_hashstore_refs_files_(pids, store): def test_store_hashstore_refs_files_pid_refs_file_exists(pids, store): """Test _store_hashstore_refs_file creates the expected pid reference file.""" - for pid in pids.keys(): + for pid in pids: cid = pids[pid][store.algorithm] store._store_hashstore_refs_files(pid, cid) pid_refs_file_path = store._get_hashstore_pid_refs_path(pid) @@ -747,7 +741,7 @@ def test_store_hashstore_refs_files_pid_refs_file_exists(pids, store): def test_store_hashstore_refs_file_cid_refs_file_exists(pids, store): """Test _store_hashstore_refs_file creates the cid reference file.""" - for pid in pids.keys(): + for pid in pids: cid = pids[pid][store.algorithm] store._store_hashstore_refs_files(pid, cid) cid_refs_file_path = store._get_hashstore_cid_refs_path(cid) @@ -756,11 +750,11 @@ def test_store_hashstore_refs_file_cid_refs_file_exists(pids, store): def test_store_hashstore_refs_file_pid_refs_file_content(pids, store): """Test _store_hashstore_refs_file created the pid reference file with the expected cid.""" - for pid in pids.keys(): + for pid in pids: cid = pids[pid][store.algorithm] store._store_hashstore_refs_files(pid, cid) pid_refs_file_path = store._get_hashstore_pid_refs_path(pid) - with open(pid_refs_file_path, "r", encoding="utf8") as f: + with open(pid_refs_file_path, encoding="utf8") as f: pid_refs_cid = f.read() assert pid_refs_cid == cid @@ -768,11 +762,11 @@ def test_store_hashstore_refs_file_pid_refs_file_content(pids, store): def test_store_hashstore_refs_file_cid_refs_file_content(pids, store): """Test _store_hashstore_refs_file creates the cid reference file successfully with pid tagged.""" - for pid in pids.keys(): + for pid in pids: cid = pids[pid][store.algorithm] store._store_hashstore_refs_files(pid, cid) cid_refs_file_path = store._get_hashstore_cid_refs_path(cid) - with open(cid_refs_file_path, "r", encoding="utf8") as f: + with open(cid_refs_file_path, encoding="utf8") as f: pid_refs_cid = f.read().strip() assert pid_refs_cid == pid @@ -780,7 +774,7 @@ def test_store_hashstore_refs_file_cid_refs_file_content(pids, store): def test_store_hashstore_refs_file_pid_refs_found_cid_refs_found(pids, store): """Test _store_hashstore_refs_file does not throw an exception when any refs file already exists and verifies the content, and does not double tag the cid refs file.""" - for pid in pids.keys(): + for pid in pids: cid = pids[pid][store.algorithm] store._store_hashstore_refs_files(pid, cid) @@ -789,7 +783,7 @@ def test_store_hashstore_refs_file_pid_refs_found_cid_refs_found(pids, store): cid_refs_file_path = store._get_hashstore_cid_refs_path(cid) line_count = 0 - with open(cid_refs_file_path, "r", encoding="utf8") as ref_file: + with open(cid_refs_file_path, encoding="utf8") as ref_file: for _line in ref_file: line_count += 1 assert line_count == 1 @@ -798,7 +792,7 @@ def test_store_hashstore_refs_file_pid_refs_found_cid_refs_found(pids, store): def test_store_hashstore_refs_files_pid_refs_found_cid_refs_not_found(store, pids): """Test that _store_hashstore_refs_files throws an exception when pid refs file exists, contains a different cid, and is correctly referenced in the associated cid refs file""" - for pid in pids.keys(): + for pid in pids: cid = pids[pid][store.algorithm] store._store_hashstore_refs_files(pid, cid) @@ -821,11 +815,11 @@ def test_store_hashstore_refs_files_refs_not_found_cid_refs_found(store): # Read cid file to confirm cid refs file contains the additional pid line_count = 0 cid_ref_abs_path = store._get_hashstore_cid_refs_path(cid) - with open(cid_ref_abs_path, "r", encoding="utf8") as f: + with open(cid_ref_abs_path, encoding="utf8") as f: for _, line in enumerate(f, start=1): value = line.strip() line_count += 1 - assert value == pid or value == additional_pid + assert value in (pid, additional_pid) assert line_count == 2 assert store._count("pid") == 2 assert store._count("cid") == 1 @@ -834,7 +828,7 @@ def test_store_hashstore_refs_files_refs_not_found_cid_refs_found(store): def test_untag_object(pids, store): """Test _untag_object untags successfully.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = Path(test_dir + pid.replace("/", "_")) object_metadata = store.store_object(pid, path) cid = object_metadata.cid @@ -853,7 +847,7 @@ def test_untag_object(pids, store): def test_untag_object_pid_not_locked(pids, store): """Test _untag_object throws exception when pid is not locked""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = Path(test_dir + pid.replace("/", "_")) object_metadata = store.store_object(pid, path) cid = object_metadata.cid @@ -865,15 +859,15 @@ def test_untag_object_pid_not_locked(pids, store): def test_untag_object_cid_not_locked(pids, store): """Test _untag_object throws exception with cid is not locked""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = Path(test_dir + pid.replace("/", "_")) object_metadata = store.store_object(pid, path) cid = object_metadata.cid + store._synchronize_referenced_locked_pids(pid) with pytest.raises(IdentifierNotLocked): - store._synchronize_referenced_locked_pids(pid) store._untag_object(pid, cid) - store._release_reference_locked_pids(pid) + store._release_reference_locked_pids(pid) def test_untag_object_orphan_pid_refs_file_found(store): @@ -1025,7 +1019,7 @@ def test_put_metadata_with_path(pids, store): entity = "metadata" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename metadata_stored_path = store._put_metadata(syspath, pid, format_id) @@ -1038,7 +1032,7 @@ def test_put_metadata_with_string(pids, store): entity = "metadata" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath = str(Path(test_dir) / filename) metadata_stored_path = store._put_metadata(syspath, pid, format_id) @@ -1050,7 +1044,7 @@ def test_put_metadata_stored_path(pids, store): """Test put metadata returns correct path to the metadata stored.""" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: metadata_document_name = store._computehash(pid + format_id) filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename @@ -1068,13 +1062,11 @@ def test_put_metadata_stored_path(pids, store): def test_mktmpmetadata(pids, store): """Test mktmpmetadata creates tmpFile.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename - sys_stream = io.open(syspath, "rb") - # pylint: disable=W0212 - tmp_name = store._mktmpmetadata(sys_stream) - sys_stream.close() + with open(syspath, "rb") as sys_stream: + tmp_name = store._mktmpmetadata(sys_stream) assert os.path.exists(tmp_name) @@ -1111,7 +1103,7 @@ def test_delete_marked_files_empty_list_or_none(store): list_to_check = [] store._delete_marked_files(list_to_check) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="None"): store._delete_marked_files(None) @@ -1172,7 +1164,7 @@ def test_validate_and_check_cid_lock_non_matching_cid(store): cid = "thegoodcid" cid_to_check = "thebadcid" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="match"): store._validate_and_check_cid_lock(pid, cid, cid_to_check) @@ -1194,10 +1186,10 @@ def test_write_refs_file_ref_type_cid(store): def test_write_refs_file_ref_type_content_cid(pids, store): """Test that write_refs_file writes the expected content when given a 'cid' update_type.""" - for pid in pids.keys(): + for pid in pids: tmp_root_path = store._get_store_path("refs") / "tmp" tmp_cid_refs_file = store._write_refs_file(tmp_root_path, pid, "cid") - with open(tmp_cid_refs_file, "r", encoding="utf8") as f: + with open(tmp_cid_refs_file, encoding="utf8") as f: cid_ref_file_pid = f.read() assert pid == cid_ref_file_pid.strip() @@ -1205,7 +1197,7 @@ def test_write_refs_file_ref_type_content_cid(pids, store): def test_write_refs_file_ref_type_pid(pids, store): """Test that write_pid_refs_file writes a reference file when given a 'pid' update_type.""" - for pid in pids.keys(): + for pid in pids: cid = pids[pid]["sha256"] tmp_root_path = store._get_store_path("refs") / "tmp" tmp_pid_refs_file = store._write_refs_file(tmp_root_path, cid, "pid") @@ -1214,11 +1206,11 @@ def test_write_refs_file_ref_type_pid(pids, store): def test_write_refs_file_ref_type_content_pid(pids, store): """Test that write_refs_file writes the expected content when given a 'pid' update_type""" - for pid in pids.keys(): + for pid in pids: cid = pids[pid]["sha256"] tmp_root_path = store._get_store_path("refs") / "tmp" tmp_pid_refs_file = store._write_refs_file(tmp_root_path, cid, "pid") - with open(tmp_pid_refs_file, "r", encoding="utf8") as f: + with open(tmp_pid_refs_file, encoding="utf8") as f: pid_refs_cid = f.read() assert cid == pid_refs_cid @@ -1226,31 +1218,31 @@ def test_write_refs_file_ref_type_content_pid(pids, store): def test_update_refs_file_content(pids, store): """Test that update_refs_file updates the ref file as expected.""" - for pid in pids.keys(): + for pid in pids: tmp_root_path = store._get_store_path("refs") / "tmp" tmp_cid_refs_file = store._write_refs_file(tmp_root_path, pid, "cid") pid_other = "dou.test.1" store._update_refs_file(tmp_cid_refs_file, pid_other, "add") - with open(tmp_cid_refs_file, "r", encoding="utf8") as f: + with open(tmp_cid_refs_file, encoding="utf8") as f: for _, line in enumerate(f, start=1): value = line.strip() - assert value == pid or value == pid_other + assert value in (pid, pid_other) def test_update_refs_file_content_multiple(pids, store): """Test that _update_refs_file adds multiple references successfully.""" - for pid in pids.keys(): + for pid in pids: tmp_root_path = store._get_store_path("refs") / "tmp" tmp_cid_refs_file = store._write_refs_file(tmp_root_path, pid, "cid") cid_reference_list = [pid] - for i in range(0, 5): + for i in range(5): store._update_refs_file(tmp_cid_refs_file, f"dou.test.{i}", "add") cid_reference_list.append(f"dou.test.{i}") line_count = 0 - with open(tmp_cid_refs_file, "r", encoding="utf8") as f: + with open(tmp_cid_refs_file, encoding="utf8") as f: for _, line in enumerate(f, start=1): line_count += 1 value = line.strip() @@ -1262,14 +1254,14 @@ def test_update_refs_file_content_multiple(pids, store): def test_update_refs_file_deduplicates_pid_already_found(pids, store): """Test that _update_refs_file does not add a pid to a refs file that already contains the pid.""" - for pid in pids.keys(): + for pid in pids: tmp_root_path = store._get_store_path("refs") / "tmp" tmp_cid_refs_file = store._write_refs_file(tmp_root_path, pid, "cid") # Exception should not be thrown store._update_refs_file(tmp_cid_refs_file, pid, "add") line_count = 0 - with open(tmp_cid_refs_file, "r", encoding="utf8") as ref_file: + with open(tmp_cid_refs_file, encoding="utf8") as ref_file: for _line in ref_file: line_count += 1 assert line_count == 1 @@ -1277,7 +1269,7 @@ def test_update_refs_file_deduplicates_pid_already_found(pids, store): def test_update_refs_file_content_cid_refs_does_not_exist(pids, store): """Test that _update_refs_file throws exception if refs file doesn't exist.""" - for pid in pids.keys(): + for pid in pids: cid = pids[pid]["sha256"] cid_ref_abs_path = store._get_hashstore_cid_refs_path(cid) with pytest.raises(FileNotFoundError): @@ -1286,7 +1278,7 @@ def test_update_refs_file_content_cid_refs_does_not_exist(pids, store): def test_update_refs_file_remove(pids, store): """Test that _update_refs_file deletes the given pid from the ref file.""" - for pid in pids.keys(): + for pid in pids: tmp_root_path = store._get_store_path("refs") / "tmp" tmp_cid_refs_file = store._write_refs_file(tmp_root_path, pid, "cid") @@ -1294,7 +1286,7 @@ def test_update_refs_file_remove(pids, store): store._update_refs_file(tmp_cid_refs_file, pid_other, "add") store._update_refs_file(tmp_cid_refs_file, pid, "remove") - with open(tmp_cid_refs_file, "r", encoding="utf8") as f: + with open(tmp_cid_refs_file, encoding="utf8") as f: for _, line in enumerate(f, start=1): value = line.strip() assert value == pid_other @@ -1302,7 +1294,7 @@ def test_update_refs_file_remove(pids, store): def test_update_refs_file_empty_file(pids, store): """Test that _update_refs_file leaves a file empty when removing the last pid.""" - for pid in pids.keys(): + for pid in pids: tmp_root_path = store._get_store_path("refs") / "tmp" tmp_cid_refs_file = store._write_refs_file(tmp_root_path, pid, "cid") # First remove the pid @@ -1314,12 +1306,12 @@ def test_update_refs_file_empty_file(pids, store): def test_is_string_in_refs_file(pids, store): """Test that _update_refs_file leaves a file empty when removing the last pid.""" - for pid in pids.keys(): + for pid in pids: tmp_root_path = store._get_store_path("refs") / "tmp" tmp_cid_refs_file = store._write_refs_file(tmp_root_path, pid, "cid") cid_reference_list = [pid] - for i in range(0, 5): + for i in range(5): store._update_refs_file(tmp_cid_refs_file, f"dou.test.{i}", "add") cid_reference_list.append(f"dou.test.{i}") @@ -1329,7 +1321,7 @@ def test_is_string_in_refs_file(pids, store): def test_verify_object_information(pids, store): """Test _verify_object_information succeeds given good arguments.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(data=path) hex_digests = object_metadata.hex_digests @@ -1351,7 +1343,7 @@ def test_verify_object_information(pids, store): def test_verify_object_information_incorrect_size(pids, store): """Test _verify_object_information throws exception when size is incorrect.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(data=path) hex_digests = object_metadata.hex_digests @@ -1374,7 +1366,7 @@ def test_verify_object_information_incorrect_size_with_pid(pids, store): """Test _verify_object_information deletes the expected tmp file if obj size does not match and raises an exception.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(data=path) hex_digests = object_metadata.hex_digests @@ -1396,7 +1388,7 @@ def test_verify_object_information_incorrect_size_with_pid(pids, store): 1000, expected_file_size, ) - assert not os.path.isfile(tmp_file.name) + assert not os.path.isfile(tmp_file.name) def test_verify_object_information_missing_key_in_hex_digests_unsupported_algo( @@ -1405,7 +1397,7 @@ def test_verify_object_information_missing_key_in_hex_digests_unsupported_algo( """Test _verify_object_information throws exception when algorithm is not found in hex digests and is not supported.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(data=path) checksum = object_metadata.hex_digests.get(store.algorithm) @@ -1430,7 +1422,7 @@ def test_verify_object_information_missing_key_in_hex_digests_supported_algo( """Test _verify_object_information throws exception when algorithm is not found in hex digests but is supported, and the checksum calculated does not match.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(data=path) checksum = object_metadata.hex_digests.get(store.algorithm) @@ -1455,7 +1447,7 @@ def test_verify_object_information_missing_key_in_hex_digests_matching_checksum( """Test _verify_object_information does not throw exception when algorithm is not found in hex digests but is supported, and the checksum calculated matches.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(data=path) checksum_algorithm = "blake2s" @@ -1475,7 +1467,7 @@ def test_verify_object_information_missing_key_in_hex_digests_matching_checksum( def test_verify_hashstore_references_pid_refs_file_missing(pids, store): """Test _verify_hashstore_references throws exception when pid refs file is missing.""" - for pid in pids.keys(): + for pid in pids: cid = pids[pid]["sha256"] with pytest.raises(PidRefsFileNotFound): store._verify_hashstore_references(pid, cid) @@ -1483,7 +1475,7 @@ def test_verify_hashstore_references_pid_refs_file_missing(pids, store): def test_verify_hashstore_references_pid_refs_incorrect_cid(pids, store): """Test _verify_hashstore_references throws exception when pid refs file cid is incorrect.""" - for pid in pids.keys(): + for pid in pids: cid = pids[pid]["sha256"] # Write the cid refs file and move it where it needs to be tmp_root_path = store._get_store_path("refs") / "tmp" @@ -1506,7 +1498,7 @@ def test_verify_hashstore_references_pid_refs_incorrect_cid(pids, store): def test_verify_hashstore_references_cid_refs_file_missing(pids, store): """Test _verify_hashstore_references throws exception when cid refs file is missing.""" - for pid in pids.keys(): + for pid in pids: cid = pids[pid]["sha256"] pid_ref_abs_path = store._get_hashstore_pid_refs_path(pid) store._create_path(os.path.dirname(pid_ref_abs_path)) @@ -1521,7 +1513,7 @@ def test_verify_hashstore_references_cid_refs_file_missing(pids, store): def test_verify_hashstore_references_cid_refs_file_missing_pid(pids, store): """Test _verify_hashstore_references throws exception when cid refs file does not contain the expected pid.""" - for pid in pids.keys(): + for pid in pids: cid = pids[pid]["sha256"] # Get a tmp cid refs file and write the wrong pid into it tmp_root_path = store._get_store_path("refs") / "tmp" @@ -1545,7 +1537,7 @@ def test_verify_hashstore_references_cid_refs_file_with_multiple_refs_missing_pi ): """Test _verify_hashstore_references throws exception when cid refs file with multiple references does not contain the expected pid.""" - for pid in pids.keys(): + for pid in pids: cid = pids[pid]["sha256"] # Write the wrong pid into a cid refs file and move it where it needs to be tmp_root_path = store._get_store_path("refs") / "tmp" @@ -1560,7 +1552,7 @@ def test_verify_hashstore_references_cid_refs_file_with_multiple_refs_missing_pi tmp_pid_refs_file = store._write_refs_file(tmp_root_path, cid, "pid") shutil.move(tmp_pid_refs_file, pid_ref_abs_path) - for i in range(0, 5): + for i in range(5): store._update_refs_file(cid_ref_abs_path, f"dou.test.{i}", "add") with pytest.raises(CidRefsContentError): @@ -1571,7 +1563,7 @@ def test_delete_object_only(pids, store): """Test _delete_object successfully deletes only object.""" test_dir = "tests/testdata/" entity = "objects" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(pid=None, data=path) store._delete_object_only(object_metadata.cid) @@ -1583,7 +1575,7 @@ def test_delete_object_only_cid_refs_file_exists(pids, store): test_dir = "tests/testdata/" entity = "objects" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename @@ -1618,11 +1610,10 @@ def test_clean_algorithm_unsupported_algo(store): def test_computehash(pids, store): """Test to check computehash method.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") - obj_stream = io.open(path, "rb") - obj_sha256_hash = store._computehash(obj_stream, "sha256") - obj_stream.close() + with open(path, "rb") as obj_stream: + obj_sha256_hash = store._computehash(obj_stream, "sha256") assert pids[pid]["sha256"] == obj_sha256_hash @@ -1643,7 +1634,7 @@ def test_count(pids, store): """Check that count returns expected number of objects.""" test_dir = "tests/testdata/" entity = "objects" - for pid in pids.keys(): + for pid in pids: path_string = test_dir + pid.replace("/", "_") store._store_and_validate_data(pid, path_string) assert store._count(entity) == 3 @@ -1653,7 +1644,7 @@ def test_exists_object_with_object_metadata_id(pids, store): """Test exists method with an absolute file path.""" test_dir = "tests/testdata/" entity = "objects" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store._store_and_validate_data(pid, path) assert store._exists(entity, object_metadata.cid) @@ -1663,7 +1654,7 @@ def test_exists_object_with_sharded_path(pids, store): """Test exists method with an absolute file path.""" test_dir = "tests/testdata/" entity = "objects" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store._store_and_validate_data(pid, path) object_metadata_shard_path = os.path.join(*store._shard(object_metadata.cid)) @@ -1675,7 +1666,7 @@ def test_exists_metadata_files_path(pids, store): test_dir = "tests/testdata/" entity = "metadata" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename metadata_stored_path = store.store_metadata(pid, syspath, format_id) @@ -1694,7 +1685,7 @@ def test_open_objects(pids, store): """Test open returns a stream.""" test_dir = "tests/testdata/" entity = "objects" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store._store_and_validate_data(pid, path) object_metadata_id = object_metadata.cid @@ -1706,7 +1697,7 @@ def test_open_objects(pids, store): def test_private_delete_objects(pids, store): """Confirm _delete deletes for entity type 'objects'""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = Path(test_dir + pid.replace("/", "_")) object_metadata = store.store_object(pid, path) @@ -1718,7 +1709,7 @@ def test_private_delete_metadata(pids, store): """Confirm _delete deletes for entity type 'metadata'""" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename store.store_metadata(pid, syspath, format_id) @@ -1736,7 +1727,7 @@ def test_private_delete_metadata(pids, store): def test_private_delete_absolute_path(pids, store): """Confirm _delete deletes for absolute paths'""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = Path(test_dir + pid.replace("/", "_")) object_metadata = store.store_object(pid, path) @@ -1793,7 +1784,7 @@ def test_get_hashstore_data_object_path_file_does_not_exist(store): def test_get_hashstore_data_object_path_with_object_id(store, pids): """Test _get_hashstore_data_object_path returns absolute path given an object id.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store._store_and_validate_data(pid, path) obj_abs_path = store._get_hashstore_data_object_path(object_metadata.cid) @@ -1804,7 +1795,7 @@ def test_get_hashstore_metadata_path_absolute_path(store, pids): """Test _get_hashstore_metadata_path returns absolute path given a metadata id.""" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename metadata_stored_path = store.store_metadata(pid, syspath, format_id) @@ -1816,7 +1807,7 @@ def test_get_hashstore_metadata_path_relative_path(pids, store): """Confirm resolve path returns correct metadata path.""" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename _metadata_stored_path = store.store_metadata(pid, syspath, format_id) @@ -1837,7 +1828,7 @@ def test_get_hashstore_metadata_path_relative_path(pids, store): def test_get_hashstore_pid_refs_path(pids, store): """Confirm resolve path returns correct object pid refs path""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = Path(test_dir + pid.replace("/", "_")) _object_metadata = store.store_object(pid, path) @@ -1853,7 +1844,7 @@ def test_get_hashstore_pid_refs_path(pids, store): def test_get_hashstore_cid_refs_path(pids, store): """Confirm resolve path returns correct object pid refs path""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = Path(test_dir + pid.replace("/", "_")) object_metadata = store.store_object(pid, path) cid = object_metadata.cid @@ -1868,23 +1859,23 @@ def test_check_string(store): """Confirm that an exception is raised when a string is None, empty or contains an illegal character (ex. tabs or new lines)""" empty_pid_with_spaces = " " - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store._check_string(empty_pid_with_spaces, "empty_pid_with_spaces") none_value = None - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store._check_string(none_value, "none_value") new_line = "\n" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store._check_string(new_line, "new_line") new_line_with_other_chars = "hello \n" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store._check_string(new_line_with_other_chars, "new_line_with_other_chars") tab_line = "\t" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store._check_string(tab_line, "tab_line") @@ -1899,7 +1890,7 @@ def test_cast_to_bytes(store): def test_stream_reads_file(pids): """Test that a stream can read a file and yield its contents.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path_string = test_dir + pid.replace("/", "_") obj_stream = Stream(path_string) hashobj = hashlib.new("sha256") @@ -1913,7 +1904,7 @@ def test_stream_reads_file(pids): def test_stream_reads_path_object(pids): """Test that a stream can read a file-like object and yield its contents.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = Path(test_dir + pid.replace("/", "_")) obj_stream = Stream(path) hash_obj = hashlib.new("sha256") @@ -1927,23 +1918,22 @@ def test_stream_reads_path_object(pids): def test_stream_returns_to_original_position_on_close(pids): """Test that a stream returns to its original position after closing the file.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path_string = test_dir + pid.replace("/", "_") - input_stream = io.open(path_string, "rb") - input_stream.seek(5) - hashobj = hashlib.new("sha256") - obj_stream = Stream(input_stream) - for data in obj_stream: - hashobj.update(data) - obj_stream.close() - assert input_stream.tell() == 5 - input_stream.close() + with open(path_string, "rb") as input_stream: + input_stream.seek(5) + hashobj = hashlib.new("sha256") + obj_stream = Stream(input_stream) + for data in obj_stream: + hashobj.update(data) + obj_stream.close() + assert input_stream.tell() == 5 # noinspection PyTypeChecker def test_stream_raises_error_for_invalid_object(): """Test that a stream raises ValueError for an invalid input object.""" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="readable"): Stream(1234) diff --git a/tests/filehashstore/test_filehashstore_interface.py b/tests/filehashstore/test_filehashstore_interface.py index 381e1035..44bc50ba 100644 --- a/tests/filehashstore/test_filehashstore_interface.py +++ b/tests/filehashstore/test_filehashstore_interface.py @@ -1,23 +1,42 @@ """Test module for FileHashStore HashStore interface methods.""" -import io import os -from pathlib import Path -from threading import Thread import random import threading import time +from pathlib import Path +from threading import Thread + import pytest from hashstore.filehashstore_exceptions import ( + HashStoreRefsAlreadyExists, NonMatchingChecksum, NonMatchingObjSize, + PidRefsAlreadyExistsError, PidRefsDoesNotExist, + StoreObjectForPidAlreadyInProgress, UnsupportedAlgorithm, - HashStoreRefsAlreadyExists, - PidRefsAlreadyExistsError, ) + +class ExceptionThread(Thread): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.exception = None + + def run(self): + try: + super().run() + except BaseException as e: + self.exception = e + + def join(self, timeout=None): + super().join(timeout) + if self.exception: + raise self.exception + + # pylint: disable=W0212 @@ -31,7 +50,7 @@ def test_store_object_refs_files_and_object(pids, store): """Test store object stores objects and creates reference files.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = Path(test_dir + pid.replace("/", "_")) object_metadata = store.store_object(pid, path) assert object_metadata.cid == pids[pid][store.algorithm] @@ -43,7 +62,7 @@ def test_store_object_refs_files_and_object(pids, store): def test_store_object_only_object(pids, store): """Test store object stores an object only (no reference files will be created)""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = Path(test_dir + pid.replace("/", "_")) object_metadata = store.store_object(data=path) assert object_metadata.cid == pids[pid][store.algorithm] @@ -55,7 +74,7 @@ def test_store_object_only_object(pids, store): def test_store_object_files_path(pids, store): """Test store object when given a path object.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = Path(test_dir + pid.replace("/", "_")) _object_metadata = store.store_object(pid, path) assert store._exists("objects", pids[pid][store.algorithm]) @@ -65,7 +84,7 @@ def test_store_object_files_path(pids, store): def test_store_object_files_string(pids, store): """Test store object when given a string object.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path_string = test_dir + pid.replace("/", "_") _object_metadata = store.store_object(pid, path_string) assert store._exists("objects", pids[pid][store.algorithm]) @@ -75,11 +94,10 @@ def test_store_object_files_string(pids, store): def test_store_object_files_input_stream(pids, store): """Test store object when given a stream object.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") - input_stream = io.open(path, "rb") - _object_metadata = store.store_object(pid, input_stream) - input_stream.close() + with open(path, "rb") as input_stream: + _object_metadata = store.store_object(pid, input_stream) assert store._exists("objects", pids[pid][store.algorithm]) assert store._count("objects") == 3 @@ -87,7 +105,7 @@ def test_store_object_files_input_stream(pids, store): def test_store_object_cid(pids, store): """Test store object returns expected content identifier.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(pid, path) assert object_metadata.cid == pids[pid][store.algorithm] @@ -96,7 +114,7 @@ def test_store_object_cid(pids, store): def test_store_object_pid(pids, store): """Test store object returns expected persistent identifier.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(pid, path) assert object_metadata.pid == pid @@ -105,7 +123,7 @@ def test_store_object_pid(pids, store): def test_store_object_obj_size(pids, store): """Test store object returns expected file size.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(pid, path) object_size = object_metadata.obj_size @@ -115,7 +133,7 @@ def test_store_object_obj_size(pids, store): def test_store_object_hex_digests(pids, store): """Test store object returns expected hex digests dictionary.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(pid, path) assert object_metadata.hex_digests.get("md5") == pids[pid]["md5"] @@ -130,7 +148,7 @@ def test_store_object_pid_empty(store): test_dir = "tests/testdata/" pid = "jtao.1700.1" path = test_dir + pid - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="cannot be None or empty"): store.store_object("", path) @@ -139,7 +157,7 @@ def test_store_object_pid_empty_spaces(store): test_dir = "tests/testdata/" pid = "jtao.1700.1" path = test_dir + pid - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="cannot be None or empty"): store.store_object(" ", path) @@ -180,7 +198,7 @@ def test_store_object_data_incorrect_type_path_with_special_character(store): test_dir = "tests/testdata/" pid = "jtao.1700.1" path = test_dir + pid + "\n" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="cannot be None or empty"): store.store_object("", path) @@ -302,7 +320,7 @@ def test_store_object_checksum_empty(store): pid = "jtao.1700.1" path = test_dir + pid checksum_algorithm = "sha3_256" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="checksum"): store.store_object( pid, path, checksum="", checksum_algorithm=checksum_algorithm ) @@ -315,7 +333,7 @@ def test_store_object_checksum_empty_spaces(store): pid = "jtao.1700.1" path = test_dir + pid checksum_algorithm = "sha3_256" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="checksum"): store.store_object( pid, path, checksum=" ", checksum_algorithm=checksum_algorithm ) @@ -359,7 +377,7 @@ def test_store_object_checksum_algorithm_empty(store): checksum_correct = ( "b748069cd0116ba59638e5f3500bbff79b41d6184bc242bd71f5cbbb8cf484cf" ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="checksum"): store.store_object(pid, path, checksum=checksum_correct, checksum_algorithm="") @@ -372,7 +390,7 @@ def test_store_object_checksum_algorithm_empty_spaces(store): checksum_correct = ( "b748069cd0116ba59638e5f3500bbff79b41d6184bc242bd71f5cbbb8cf484cf" ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="checksum"): store.store_object( pid, path, checksum=checksum_correct, checksum_algorithm=" " ) @@ -387,7 +405,7 @@ def test_store_object_checksum_algorithm_special_character(store): checksum_correct = ( "b748069cd0116ba59638e5f3500bbff79b41d6184bc242bd71f5cbbb8cf484cf" ) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="checksum"): store.store_object( pid, path, checksum=checksum_correct, checksum_algorithm="\n" ) @@ -445,11 +463,11 @@ def test_store_object_duplicate_object_references_file_content(pids, store): # Confirm the content of the cid reference files cid_ref_abs_path = store._get_hashstore_cid_refs_path(pids[pid][store.algorithm]) cid_count = 0 - with open(cid_ref_abs_path, "r", encoding="utf8") as f: + with open(cid_ref_abs_path, encoding="utf8") as f: for _, line in enumerate(f, start=1): cid_count += 1 value = line.strip() - assert value == pid or value == pid_two or value == pid_three + assert value in (pid, pid_two, pid_three) assert cid_count == 3 @@ -476,7 +494,7 @@ def test_store_object_duplicate_raises_error_with_bad_validation_data(pids, stor def test_store_object_with_obj_file_size(store, pids): """Test store object stores object with correct file sizes.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: obj_file_size = pids[pid]["file_size_bytes"] path = test_dir + pid.replace("/", "_") object_metadata = store.store_object( @@ -489,7 +507,7 @@ def test_store_object_with_obj_file_size(store, pids): def test_store_object_with_obj_file_size_incorrect(store, pids): """Test store object throws exception with incorrect file size.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: obj_file_size = 1234 path = test_dir + pid.replace("/", "_") with pytest.raises(NonMatchingObjSize): @@ -501,7 +519,7 @@ def test_store_object_with_obj_file_size_non_integer(store, pids): """Test store object throws exception with a non integer value (ex. a string) as the file size.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: obj_file_size = "Bob" path = test_dir + pid.replace("/", "_") with pytest.raises(TypeError): @@ -511,10 +529,10 @@ def test_store_object_with_obj_file_size_non_integer(store, pids): def test_store_object_with_obj_file_size_zero(store, pids): """Test store object throws exception with zero as the file size.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: obj_file_size = 0 path = test_dir + pid.replace("/", "_") - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="size"): store.store_object(pid, path, expected_object_size=obj_file_size) @@ -525,25 +543,21 @@ def test_store_object_duplicates_threads(pids, store): path = test_dir + pid entity = "objects" - def store_object_wrapper(obj_pid, obj_path): - try: - store.store_object(obj_pid, obj_path) # Call store_object inside the thread - # pylint: disable=W0718 - except Exception as e: - assert ( - type(e).__name__ == "HashStoreRefsAlreadyExists" - or type(e).__name__ == "StoreObjectForPidAlreadyInProgress" - ) - - thread1 = Thread(target=store_object_wrapper, args=(pid, path)) - thread2 = Thread(target=store_object_wrapper, args=(pid, path)) - thread3 = Thread(target=store_object_wrapper, args=(pid, path)) + thread1 = ExceptionThread(target=store.store_object, args=(pid, path)) + thread2 = ExceptionThread(target=store.store_object, args=(pid, path)) + thread3 = ExceptionThread(target=store.store_object, args=(pid, path)) thread1.start() thread2.start() thread3.start() + with pytest.raises( + (StoreObjectForPidAlreadyInProgress, HashStoreRefsAlreadyExists) + ): + thread2.join() + with pytest.raises( + (StoreObjectForPidAlreadyInProgress, HashStoreRefsAlreadyExists) + ): + thread3.join() thread1.join() - thread2.join() - thread3.join() # One thread will succeed, file count must still be 1 assert store._count(entity) == 1 assert store._exists(entity, pids[pid][store.algorithm]) @@ -561,7 +575,7 @@ def test_store_object_threads_multiple_pids_one_cid_content(pids, store): test_dir = "tests/testdata/" path = test_dir + "jtao.1700.1" pid_list = ["jtao.1700.1"] - for n in range(0, 5): + for n in range(5): pid_list.append(f"dou.test.{n}") def store_object_wrapper(obj_pid, obj_path): @@ -593,7 +607,7 @@ def store_object_wrapper(obj_pid, obj_path): "94f9b6c88f1f458e410c30c351c6384ea42ac1b5ee1f8430d3e365e43b78a38a" ) number_of_pids_reffed = 0 - with open(cid_refs_path, "r", encoding="utf8") as ref_file: + with open(cid_refs_path, encoding="utf8") as ref_file: # Confirm that pid is not currently already tagged for pid in ref_file: if pid.strip() in pid_list: @@ -607,7 +621,7 @@ def test_store_object_threads_multiple_pids_one_cid_files(store): test_dir = "tests/testdata/" path = test_dir + "jtao.1700.1" pid_list = ["jtao.1700.1"] - for n in range(0, 5): + for n in range(5): pid_list.append(f"dou.test.{n}") def store_object_wrapper(obj_pid, obj_path): @@ -750,7 +764,7 @@ def test_store_object_sparse_large_file(store): def test_tag_object(pids, store): """Test tag_object does not throw exception when successful.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(None, path) store.tag_object(pid, object_metadata.cid) @@ -775,11 +789,11 @@ def test_tag_object_pid_refs_not_found_cid_refs_found(store): # Read cid file to confirm cid refs file contains the additional pid line_count = 0 cid_ref_abs_path = store._get_hashstore_cid_refs_path(cid) - with open(cid_ref_abs_path, "r", encoding="utf8") as f: + with open(cid_ref_abs_path, encoding="utf8") as f: for _, line in enumerate(f, start=1): value = line.strip() line_count += 1 - assert value == pid or value == additional_pid + assert value in (pid, additional_pid) assert line_count == 2 assert store._count("pid") == 2 assert store._count("cid") == 1 @@ -788,7 +802,7 @@ def test_tag_object_pid_refs_not_found_cid_refs_found(store): def test_tag_object_hashstore_refs_already_exist(pids, store): """Confirm that tag throws HashStoreRefsAlreadyExists when refs already exist""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(pid, path) @@ -799,7 +813,7 @@ def test_tag_object_hashstore_refs_already_exist(pids, store): def test_tag_object_pid_refs_already_exist(pids, store): """Confirm that tag throws PidRefsAlreadyExistsError when a pid refs already exists""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(pid, path) cid_refs_file_path = store._get_hashstore_cid_refs_path(object_metadata.cid) @@ -812,7 +826,7 @@ def test_tag_object_pid_refs_already_exist(pids, store): def test_delete_if_invalid_object(pids, store): """Test delete_if_invalid_object does not throw exception given good arguments.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(data=path) checksum = object_metadata.hex_digests.get(store.algorithm) @@ -828,7 +842,7 @@ def test_delete_if_invalid_object_supported_other_algo_not_in_default(pids, stor """Test delete_if_invalid_object does not throw exception when supported add algo is supplied.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: supported_algo = "sha224" path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(data=path) @@ -844,13 +858,13 @@ def test_delete_if_invalid_object_exception_incorrect_object_metadata_type(pids, """Test delete_if_invalid_object throws exception when incorrect obj type is given to object_metadata arg.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(data=path) checksum = object_metadata.hex_digests.get(store.algorithm) checksum_algorithm = store.algorithm expected_file_size = object_metadata.obj_size - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="ObjectMetadata"): store.delete_if_invalid_object( "not_object_metadata", checksum, checksum_algorithm, expected_file_size ) @@ -860,7 +874,7 @@ def test_delete_if_invalid_object_exception_incorrect_size(pids, store): """Test delete_if_invalid_object throws exception when incorrect size is supplied and that data object is deleted as we are storing without a pid.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(data=path) checksum = object_metadata.hex_digests.get(store.algorithm) @@ -878,11 +892,11 @@ def test_delete_if_invalid_object_exception_incorrect_size_object_exists(pids, s """Test delete_if_invalid_object throws exception when incorrect size is supplied and that data object is not deleted since it already exists (a cid refs file is present).""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") store.store_object(pid, data=path) # Store again without pid and wrong object size - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(data=path) checksum = object_metadata.hex_digests.get(store.algorithm) @@ -900,7 +914,7 @@ def test_delete_if_invalid_object_exception_incorrect_size_object_exists(pids, s def test_delete_if_invalid_object_exception_incorrect_checksum(pids, store): """Test delete_if_invalid_object throws exception when incorrect checksum is supplied.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(data=path) checksum_algorithm = store.algorithm @@ -917,7 +931,7 @@ def test_delete_if_invalid_object_exception_incorrect_checksum(pids, store): def test_delete_if_invalid_object_exception_incorrect_checksum_algo(pids, store): """Test delete_if_invalid_object throws exception when unsupported algorithm is supplied.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(data=path) checksum = object_metadata.hex_digests.get(store.algorithm) @@ -936,7 +950,7 @@ def test_delete_if_invalid_object_exception_supported_other_algo_bad_checksum( ): """Test delete_if_invalid_object throws exception when incorrect checksum is supplied.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(data=path) checksum = object_metadata.hex_digests.get(store.algorithm) @@ -953,7 +967,7 @@ def test_store_metadata(pids, store): """Test store_metadata.""" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename stored_metadata_path = store.store_metadata(pid, syspath, format_id) @@ -1001,7 +1015,7 @@ def test_store_metadata_default_format_id(pids, store): """Test store_metadata returns expected id when storing with default format_id.""" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename stored_metadata_path = store.store_metadata(pid, syspath) @@ -1020,7 +1034,7 @@ def test_store_metadata_files_string(pids, store): test_dir = "tests/testdata/" entity = "metadata" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath_string = str(Path(test_dir) / filename) stored_metadata_path = store.store_metadata(pid, syspath_string, format_id) @@ -1033,12 +1047,11 @@ def test_store_metadata_files_input_stream(pids, store): test_dir = "tests/testdata/" entity = "metadata" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath_string = str(Path(test_dir) / filename) - syspath_stream = io.open(syspath_string, "rb") - _stored_metadata_path = store.store_metadata(pid, syspath_stream, format_id) - syspath_stream.close() + with open(syspath_string, "rb") as syspath_stream: + _stored_metadata_path = store.store_metadata(pid, syspath_stream, format_id) assert store._count(entity) == 3 @@ -1049,7 +1062,7 @@ def test_store_metadata_pid_empty(store): pid = "" filename = pid.replace("/", "_") + ".xml" syspath_string = str(Path(test_dir) / filename) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store.store_metadata(pid, syspath_string, format_id) @@ -1060,7 +1073,7 @@ def test_store_metadata_pid_empty_spaces(store): pid = " " filename = pid.replace("/", "_") + ".xml" syspath_string = str(Path(test_dir) / filename) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store.store_metadata(pid, syspath_string, format_id) @@ -1071,7 +1084,7 @@ def test_store_metadata_pid_format_id_spaces(store): pid = "jtao.1700.1" filename = pid.replace("/", "_") + ".xml" syspath_string = str(Path(test_dir) / filename) - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store.store_metadata(pid, syspath_string, format_id) @@ -1097,7 +1110,7 @@ def test_store_metadata_metadata_path(pids, store): """Test store_metadata returns expected path to metadata document.""" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename @@ -1137,7 +1150,7 @@ def test_store_metadata_thread_lock(store): def test_retrieve_object(pids, store): """Test retrieve_object returns a stream to the correct object data.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(pid, path) obj_stream = store.retrieve_object(pid) @@ -1149,7 +1162,7 @@ def test_retrieve_object(pids, store): def test_retrieve_object_pid_empty(store): """Test retrieve_object raises error when supplied with empty pid.""" pid = " " - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store.retrieve_object(pid) @@ -1194,7 +1207,7 @@ def test_retrieve_metadata_bytes_pid_invalid(store): """Test retrieve_metadata raises exception when supplied with pid with no system metadata.""" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" pid_does_not_exist = "jtao.1700.1.metadata.does.not.exist" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="No metadata"): store.retrieve_metadata(pid_does_not_exist, format_id) @@ -1202,7 +1215,7 @@ def test_retrieve_metadata_bytes_pid_empty(store): """Test retrieve_metadata raises exception when supplied with empty pid.""" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" pid = " " - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store.retrieve_metadata(pid, format_id) @@ -1210,7 +1223,7 @@ def test_retrieve_metadata_format_id_empty(store): """Test retrieve_metadata raises error when supplied with an empty format_id.""" format_id = "" pid = "jtao.1700.1" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="No metadata"): store.retrieve_metadata(pid, format_id) @@ -1218,7 +1231,7 @@ def test_retrieve_metadata_format_id_empty_spaces(store): """Test retrieve_metadata raises exception when supplied with empty spaces as the format_id.""" format_id = " " pid = "jtao.1700.1" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store.retrieve_metadata(pid, format_id) @@ -1226,7 +1239,7 @@ def test_delete_object_object_deleted(pids, store): """Test delete_object successfully deletes object.""" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename @@ -1240,7 +1253,7 @@ def test_delete_object_metadata_deleted(pids, store): """Test delete_object successfully deletes associated metadata files.""" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename @@ -1254,7 +1267,7 @@ def test_delete_object_refs_files_deleted(pids, store): """Test delete_object successfully deletes refs files.""" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename @@ -1269,7 +1282,7 @@ def test_delete_object_pid_refs_file_deleted(pids, store): """Test delete_object deletes the associated pid refs file for the object.""" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename @@ -1284,7 +1297,7 @@ def test_delete_object_cid_refs_file_deleted(pids, store): """Test delete_object deletes the associated cid refs file for the object.""" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename @@ -1299,7 +1312,7 @@ def test_delete_object_cid_refs_file_deleted(pids, store): def test_delete_object_cid_refs_file_with_pid_refs_remaining(pids, store): """Test delete_object does not delete the cid refs file that still contains refs.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") object_metadata = store.store_object(pid, path) cid = object_metadata.cid @@ -1314,14 +1327,14 @@ def test_delete_object_cid_refs_file_with_pid_refs_remaining(pids, store): def test_delete_object_pid_empty(store): """Test delete_object raises error when empty pid supplied.""" pid = " " - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store.delete_object(pid) def test_delete_object_pid_none(store): """Test delete_object raises error when pid is 'None'.""" pid = None - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store.delete_object(pid) @@ -1329,7 +1342,7 @@ def test_delete_metadata(pids, store): """Test delete_metadata successfully deletes metadata.""" test_dir = "tests/testdata/" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename _stored_metadata_path = store.store_metadata(pid, syspath, format_id) @@ -1379,14 +1392,14 @@ def test_delete_metadata_does_not_exist(pids, store): """Test delete_metadata does not throw exception when called to delete metadata that does not exist.""" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: store.delete_metadata(pid, format_id) def test_delete_metadata_default_format_id(store, pids): """Test delete_metadata deletes successfully with default format_id.""" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename _stored_metadata_path = store.store_metadata(pid, syspath) @@ -1398,7 +1411,7 @@ def test_delete_metadata_pid_empty(store): """Test delete_metadata raises error when empty pid supplied.""" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" pid = " " - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store.delete_metadata(pid, format_id) @@ -1406,7 +1419,7 @@ def test_delete_metadata_pid_none(store): """Test delete_metadata raises error when pid is 'None'.""" format_id = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" pid = None - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store.delete_metadata(pid, format_id) @@ -1414,7 +1427,7 @@ def test_delete_metadata_format_id_empty(store): """Test delete_metadata raises error when empty format_id supplied.""" format_id = " " pid = "jtao.1700.1" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store.delete_metadata(pid, format_id) @@ -1458,7 +1471,7 @@ def test_get_hex_digest_pid_empty(store): """Test get_hex_digest raises error when supplied pid is empty.""" pid = " " algorithm = "sm3" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store.get_hex_digest(pid, algorithm) @@ -1466,7 +1479,7 @@ def test_get_hex_digest_pid_none(store): """Test get_hex_digest raises error when supplied pid is 'None'.""" pid = None algorithm = "sm3" - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store.get_hex_digest(pid, algorithm) @@ -1474,7 +1487,7 @@ def test_get_hex_digest_algorithm_empty(store): """Test get_hex_digest raises error when supplied algorithm is empty.""" pid = "jtao.1700.1" algorithm = " " - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store.get_hex_digest(pid, algorithm) @@ -1482,7 +1495,7 @@ def test_get_hex_digest_algorithm_none(store): """Test get_hex_digest raises error when supplied algorithm is 'None'.""" pid = "jtao.1700.1" algorithm = None - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="empty"): store.get_hex_digest(pid, algorithm) @@ -1496,14 +1509,14 @@ def test_store_and_delete_objects_100_pids_1_cid(store): # Store upper_limit = 101 for i in range(1, upper_limit): - pid_modified = f"dou.test.{str(i)}" + pid_modified = f"dou.test.{i!s}" store.store_object(pid_modified, path) assert sum([len(files) for _, _, files in os.walk(refs_pids_path)]) == 100 assert sum([len(files) for _, _, files in os.walk(refs_cids_path)]) == 1 assert store._count("objects") == 1 # Delete for i in range(1, upper_limit): - pid_modified = f"dou.test.{str(i)}" + pid_modified = f"dou.test.{i!s}" store.delete_object(pid_modified) assert sum([len(files) for _, _, files in os.walk(refs_pids_path)]) == 0 assert sum([len(files) for _, _, files in os.walk(refs_cids_path)]) == 0 @@ -1519,7 +1532,7 @@ def store_object_wrapper(pid_var): path = test_dir + "jtao.1700.1" upper_limit = 101 for i in range(1, upper_limit): - pid_modified = f"dou.test.{pid_var}.{str(i)}" + pid_modified = f"dou.test.{pid_var}.{i!s}" store.store_object(pid_modified, path) # pylint: disable=W0718 except Exception as e: @@ -1540,7 +1553,7 @@ def delete_object_wrapper(pid_var): try: upper_limit = 101 for i in range(1, upper_limit): - pid_modified = f"dou.test.{pid_var}.{str(i)}" + pid_modified = f"dou.test.{pid_var}.{i!s}" store.delete_object(pid_modified) # pylint: disable=W0718 except Exception as e: diff --git a/tests/test_hashstore.py b/tests/test_hashstore.py index 02d83e3b..a2a28569 100644 --- a/tests/test_hashstore.py +++ b/tests/test_hashstore.py @@ -1,16 +1,17 @@ """Test module for HashStore's HashStoreFactory and ObjectMetadata class.""" import os + import pytest -from hashstore.hashstore import HashStoreFactory + from hashstore.filehashstore import FileHashStore +from hashstore.hashstore import HashStoreFactory @pytest.fixture(name="factory") def init_factory(): """Create factory for all tests.""" - factory = HashStoreFactory() - return factory + return HashStoreFactory() def test_init(factory): @@ -29,17 +30,17 @@ def test_factory_get_hashstore_filehashstore(factory, props): def test_factory_get_hashstore_unsupported_class(factory): """Check that AttributeError is raised when provided with unsupported class.""" + module_name = "hashstore.filehashstore" + class_name = "S3HashStore" with pytest.raises(AttributeError): - module_name = "hashstore.filehashstore" - class_name = "S3HashStore" factory.get_hashstore(module_name, class_name) def test_factory_get_hashstore_unsupported_module(factory): """Check that ModuleNotFoundError is raised when provided with unsupported module.""" + module_name = "hashstore.s3filestore" + class_name = "FileHashStore" with pytest.raises(ModuleNotFoundError): - module_name = "hashstore.s3filestore" - class_name = "FileHashStore" factory.get_hashstore(module_name, class_name) @@ -56,7 +57,7 @@ def test_factory_get_hashstore_filehashstore_unsupported_algorithm(factory): "store_algorithm": "MD2", "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", } - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Must be one of"): factory.get_hashstore(module_name, class_name, properties) @@ -72,7 +73,7 @@ def test_factory_get_hashstore_filehashstore_incorrect_algorithm_format(factory) "store_algorithm": "dou_algo", "store_metadata_namespace": "https://ns.dataone.org/service/types/v2.0#SystemMetadata", } - with pytest.raises(ValueError): + with pytest.raises(ValueError, match="Must be one of"): factory.get_hashstore(module_name, class_name, properties) diff --git a/tests/test_hashstore_client.py b/tests/test_hashstore_client.py index ba0bd566..7ba12e09 100644 --- a/tests/test_hashstore_client.py +++ b/tests/test_hashstore_client.py @@ -1,8 +1,9 @@ """Test module for the Python client (Public API calls only).""" -import sys import os +import sys from pathlib import Path + from hashstore import hashstoreclient # pylint: disable=W0212 @@ -48,7 +49,7 @@ def test_get_checksum(capsys, store, pids): """Test calculating a hash via HashStore through client.""" client_directory = os.getcwd() + "/src/hashstore" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") store.store_object(pid, path) @@ -74,8 +75,8 @@ def test_get_checksum(capsys, store, pids): capsystext = capsys.readouterr().out expected_output = ( f"guid/pid: {pid}\n" - + f"algorithm: {store.algorithm}\n" - + f"Checksum/Hex Digest: {pids[pid][store.algorithm]}\n" + f"algorithm: {store.algorithm}\n" + f"Checksum/Hex Digest: {pids[pid][store.algorithm]}\n" ) assert capsystext == expected_output @@ -84,12 +85,12 @@ def test_store_object(store, pids): """Test storing objects to HashStore through client.""" client_directory = os.getcwd() + "/src/hashstore" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: client_module_path = f"{client_directory}/client.py" test_store = str(store.root) store_object_opt = "-storeobject" client_pid_arg = f"-pid={pid}" - path = f'-path={test_dir + pid.replace("/", "_")}' + path = f"-path={test_dir + pid.replace('/', '_')}" chs_args = [ client_module_path, test_store, @@ -113,7 +114,7 @@ def test_store_metadata(capsys, store, pids): test_dir = "tests/testdata/" namespace = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" entity = "metadata" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename client_module_path = f"{client_directory}/client.py" @@ -154,7 +155,7 @@ def test_retrieve_objects(capsys, pids, store): """Test retrieving objects from a HashStore through client.""" client_directory = os.getcwd() + "/src/hashstore" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") store.store_object(pid, path) @@ -193,7 +194,7 @@ def test_retrieve_metadata(capsys, pids, store): client_directory = os.getcwd() + "/src/hashstore" test_dir = "tests/testdata/" namespace = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename _metadata_cid = store.store_metadata(pid, syspath, namespace) @@ -234,7 +235,7 @@ def test_delete_objects(pids, store): """Test deleting objects from a HashStore through client.""" client_directory = os.getcwd() + "/src/hashstore" test_dir = "tests/testdata/" - for pid in pids.keys(): + for pid in pids: path = test_dir + pid.replace("/", "_") store.store_object(pid, path) @@ -263,7 +264,7 @@ def test_delete_metadata(pids, store): client_directory = os.getcwd() + "/src/hashstore" test_dir = "tests/testdata/" namespace = "https://ns.dataone.org/service/types/v2.0#SystemMetadata" - for pid in pids.keys(): + for pid in pids: filename = pid.replace("/", "_") + ".xml" syspath = Path(test_dir) / filename _metadata_cid = store.store_metadata(pid, syspath, namespace) From 099473e7a573bb3ac13916a0bd37328684ada600 Mon Sep 17 00:00:00 2001 From: datadavev <605409+datadavev@users.noreply.github.com> Date: Sat, 30 May 2026 08:18:11 -0400 Subject: [PATCH 11/11] remove poetry package test workflow --- .github/workflows/poetry-package-test.yml | 41 ----------------------- 1 file changed, 41 deletions(-) delete mode 100644 .github/workflows/poetry-package-test.yml diff --git a/.github/workflows/poetry-package-test.yml b/.github/workflows/poetry-package-test.yml deleted file mode 100644 index 9821b50f..00000000 --- a/.github/workflows/poetry-package-test.yml +++ /dev/null @@ -1,41 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python - -name: Python poetry package - -on: - push: - branches: [ "main", "develop" ] - pull_request: - branches: [ "main", "develop" ] - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11"] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip poetry - poetry install --no-interaction - #python -m pip install flake8 pytest - #if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - #- name: Lint with flake8 - #run: | - # stop the build if there are Python syntax errors or undefined names - # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - poetry run pytest