Merge branch 'main' into qa

Author: fatimarahman

.github/workflows/deploy-production.yml

@@ -4,6 +4,10 @@ on:
   release:
     types: [ released ]
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   check_production_tag:
     name: Check if the release is tagged as production
@@ -26,10 +30,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.9
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
           cache: 'pip'
@@ -52,6 +56,4 @@ jobs:
           .
 
       - name: Publish distribution package to PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          password: ${{ secrets.PYPI_API_TOKEN }}
+        uses: pypa/gh-action-pypi-publish@release/v1

.github/workflows/deploy-qa.yml

@@ -4,6 +4,10 @@ on:
   release:
     types: [ released ]
 
+permissions:
+  id-token: write
+  contents: read
+
 jobs:
   check_qa_tag:
     name: Check if the release is tagged as QA
@@ -26,10 +30,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.9
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
           cache: 'pip'
@@ -54,5 +58,4 @@ jobs:
       - name: Publish distribution package to Test PyPI
         uses: pypa/gh-action-pypi-publish@release/v1
         with:
-          password: ${{ secrets.TEST_PYPI_API_TOKEN }}
-          repository_url: https://test.pypi.org/legacy/
+          repository-url: https://test.pypi.org/legacy/

.github/workflows/run-unit-tests.yml

@@ -1,24 +1,24 @@
 name: Run Python unit tests
 
-on: 
+on:
   pull_request:
-    actions: [ opened ]
+    types: [ labeled, unlabeled, opened, reopened, synchronize ]
 
 jobs:
   changelog:
     name: Updates changelog
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - uses: dangoslen/changelog-enforcer@v3
   test:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.9
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.9'
           cache: 'pip'

CHANGELOG.md

@@ -1,6 +1,24 @@
 # Changelog
+## v1.5.0 11/19/24
+- Added cloudLibrary client
+
+## v1.4.0 9/23/24
+- Added SFTP client
+
+## v1.3.2 8/1/24
+- Replaced info statements with debug for security purposes
+
+## v1.3.1 7/31/24
+- Replaced log statement in Avro client with debug
+
+## v1.3.0 7/30/24
+- Added SecretsManager client
+
+## v1.2.1 7/25/24
+- Add retry for fetching Avro schemas
+
 ## v1.2.0 7/17/24
-- Generalized Avro functions and separated encoding/decoding behavior.
+- Generalized Avro functions and separated encoding/decoding behavior
 
 ## v1.1.6 7/12/24
 - Add put functionality to Oauth2 Client

README.md

@@ -7,11 +7,14 @@ This package contains common Python utility classes and functions.
 * Setting and retrieving a resource in S3
 * Decrypting values with KMS
 * Encoding and decoding records using a given Avro schema
+* Retrieving secrets from AWS Secrets Manager
+* Downloading files from a remote SSH SFTP server
 * Connecting to and querying a MySQL database
 * Connecting to and querying a PostgreSQL database
 * Connecting to and querying a PostgreSQL database using a connection pool
 * Connecting to and querying Redshift
 * Making requests to the Oauth2 authenticated APIs such as NYPL Platform API and Sierra
+* Interacting with vendor APIs such as cloudLibrary
 
 ## Functions
 * Reading a YAML config file and putting the contents in os.environ -- see `config/sample.yaml` for an example of how the config file should be formatted
@@ -35,7 +38,7 @@ kinesis_client = KinesisClient(...)
 # Do not use any version below 1.0.0
 # All available optional dependencies can be found in pyproject.toml.
 # See the "Managing dependencies" section below for more details.
-nypl-py-utils[kinesis-client,config-helper]==1.1.2
+nypl-py-utils[kinesis-client,config-helper]==1.5.0
 ```
 
 ## Developing locally
@@ -61,7 +64,7 @@ The optional dependency sets also give the developer the option to manually list
 ### Using PostgreSQLClient in an AWS Lambda
 Because `psycopg` requires a statically linked version of the `libpq` library, the `PostgreSQLClient` cannot be installed as-is in an AWS Lambda function. Instead, it must be packaged as follows:
 ```bash
-pip install --target ./package nypl-py-utils[postgresql-client]==1.1.2
+pip install --target ./package nypl-py-utils[postgresql-client]==1.5.0
 
 pip install \
     --platform manylinux2014_x86_64 \

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "nypl_py_utils"
-version = "1.2.0"
+version = "1.5.0"
 authors = [
   { name="Aaron Friedman", email="aaronfriedman@nypl.org" },
 ]
@@ -27,6 +27,9 @@ avro-client = [
     "avro>=1.11.1",
     "requests>=2.28.1"
 ]
+cloudlibrary-client = [
+    "requests>=2.28.1"
+]
 kinesis-client = [
     "boto3>=1.26.5",
     "botocore>=1.29.5"
@@ -56,6 +59,13 @@ s3-client = [
     "boto3>=1.26.5",
     "botocore>=1.29.5"
 ]
+secrets-manager-client = [
+    "boto3>=1.26.5",
+    "botocore>=1.29.5"
+]
+sftp-client = [
+    "paramiko>=3.4.1"
+]
 config-helper = [
     "nypl_py_utils[kms-client]",
     "PyYAML>=6.0"
@@ -67,7 +77,7 @@ research-catalog-identifier-helper = [
     "requests>=2.28.1"
 ]
 development = [
-    "nypl_py_utils[avro-client,kinesis-client,kms-client,mysql-client,oauth2-api-client,postgresql-client,postgresql-pool-client,redshift-client,s3-client,config-helper,obfuscation-helper,research-catalog-identifier-helper]",
+    "nypl_py_utils[avro-client,kinesis-client,kms-client,mysql-client,oauth2-api-client,postgresql-client,postgresql-pool-client,redshift-client,s3-client,secrets-manager-client,sftp-client,config-helper,obfuscation-helper,research-catalog-identifier-helper]",
     "flake8>=6.0.0",
     "freezegun>=1.2.2",
     "mock>=4.0.3",

src/nypl_py_utils/classes/avro_client.py

@@ -5,7 +5,8 @@
 from avro.io import BinaryDecoder, BinaryEncoder, DatumReader, DatumWriter
 from io import BytesIO
 from nypl_py_utils.functions.log_helper import create_log
-from requests.exceptions import JSONDecodeError, RequestException
+from requests.adapters import HTTPAdapter, Retry
+from requests.exceptions import JSONDecodeError
 
 
 class AvroClient:
@@ -15,7 +16,13 @@ class AvroClient:
     """
 
     def __init__(self, platform_schema_url):
-        self.logger = create_log("avro_encoder")
+        self.logger = create_log("avro_client")
+        retry_policy = Retry(total=3, backoff_factor=45,
+                             status_forcelist=[500, 502, 503, 504],
+                             allowed_methods=frozenset(['GET']))
+        self.session = requests.Session()
+        self.session.mount("https://",
+                           HTTPAdapter(max_retries=retry_policy))
         self.schema = avro.schema.parse(
             self.get_json_schema(platform_schema_url))
 
@@ -27,9 +34,11 @@ def get_json_schema(self, platform_schema_url):
         self.logger.info(
             "Fetching Avro schema from {}".format(platform_schema_url))
         try:
-            response = requests.get(platform_schema_url)
+
+            response = self.session.get(url=platform_schema_url,
+                                        timeout=60)
             response.raise_for_status()
-        except RequestException as e:
+        except Exception as e:
             self.logger.error(
                 "Failed to retrieve schema from {url}: {error}".format(
                     url=platform_schema_url, error=e
@@ -39,7 +48,7 @@ def get_json_schema(self, platform_schema_url):
                 "Failed to retrieve schema from {url}: {error}".format(
                     url=platform_schema_url, error=e
                 )
-            ) from None
+            )
 
         try:
             json_response = response.json()
@@ -124,7 +133,7 @@ def decode_record(self, record):
 
         Returns a dictionary where each key is a field in the schema.
         """
-        self.logger.info(
+        self.logger.debug(
             "Decoding {rec} using {schema} schema".format(
                 rec=record, schema=self.schema.name
             )