Merge branch 'main' into qa

Author: aaronfriedman6

CHANGELOG.md

@@ -1,6 +1,12 @@
 # Changelog
+## v1.3.0 7/30/24
+- Added SecretsManager client
+
+## v1.2.1 7/25/24
+- Add retry for fetching Avro schemas
+
 ## v1.2.0 7/17/24
-- Generalized Avro functions and separated encoding/decoding behavior.
+- Generalized Avro functions and separated encoding/decoding behavior
 
 ## v1.1.6 7/12/24
 - Add put functionality to Oauth2 Client

README.md

@@ -7,6 +7,7 @@ This package contains common Python utility classes and functions.
 * Setting and retrieving a resource in S3
 * Decrypting values with KMS
 * Encoding and decoding records using a given Avro schema
+* Retrieving secrets from AWS Secrets Manager
 * Connecting to and querying a MySQL database
 * Connecting to and querying a PostgreSQL database
 * Connecting to and querying a PostgreSQL database using a connection pool
@@ -35,7 +36,7 @@ kinesis_client = KinesisClient(...)
 # Do not use any version below 1.0.0
 # All available optional dependencies can be found in pyproject.toml.
 # See the "Managing dependencies" section below for more details.
-nypl-py-utils[kinesis-client,config-helper]==1.1.2
+nypl-py-utils[kinesis-client,config-helper]==1.3.0
 ```
 
 ## Developing locally

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "nypl_py_utils"
-version = "1.2.0"
+version = "1.3.0"
 authors = [
   { name="Aaron Friedman", email="aaronfriedman@nypl.org" },
 ]
@@ -56,6 +56,10 @@ s3-client = [
     "boto3>=1.26.5",
     "botocore>=1.29.5"
 ]
+secrets-manager-client = [
+    "boto3>=1.26.5",
+    "botocore>=1.29.5"
+]
 config-helper = [
     "nypl_py_utils[kms-client]",
     "PyYAML>=6.0"
@@ -67,7 +71,7 @@ research-catalog-identifier-helper = [
     "requests>=2.28.1"
 ]
 development = [
-    "nypl_py_utils[avro-client,kinesis-client,kms-client,mysql-client,oauth2-api-client,postgresql-client,postgresql-pool-client,redshift-client,s3-client,config-helper,obfuscation-helper,research-catalog-identifier-helper]",
+    "nypl_py_utils[avro-client,kinesis-client,kms-client,mysql-client,oauth2-api-client,postgresql-client,postgresql-pool-client,redshift-client,s3-client,secrets-manager-client,config-helper,obfuscation-helper,research-catalog-identifier-helper]",
     "flake8>=6.0.0",
     "freezegun>=1.2.2",
     "mock>=4.0.3",

src/nypl_py_utils/classes/avro_client.py

@@ -5,7 +5,8 @@
 from avro.io import BinaryDecoder, BinaryEncoder, DatumReader, DatumWriter
 from io import BytesIO
 from nypl_py_utils.functions.log_helper import create_log
-from requests.exceptions import JSONDecodeError, RequestException
+from requests.adapters import HTTPAdapter, Retry
+from requests.exceptions import JSONDecodeError
 
 
 class AvroClient:
@@ -15,7 +16,13 @@ class AvroClient:
     """
 
     def __init__(self, platform_schema_url):
-        self.logger = create_log("avro_encoder")
+        self.logger = create_log("avro_client")
+        retry_policy = Retry(total=3, backoff_factor=45,
+                             status_forcelist=[500, 502, 503, 504],
+                             allowed_methods=frozenset(['GET']))
+        self.session = requests.Session()
+        self.session.mount("https://",
+                           HTTPAdapter(max_retries=retry_policy))
         self.schema = avro.schema.parse(
             self.get_json_schema(platform_schema_url))
 
@@ -27,9 +34,11 @@ def get_json_schema(self, platform_schema_url):
         self.logger.info(
             "Fetching Avro schema from {}".format(platform_schema_url))
         try:
-            response = requests.get(platform_schema_url)
+
+            response = self.session.get(url=platform_schema_url,
+                                        timeout=60)
             response.raise_for_status()
-        except RequestException as e:
+        except Exception as e:
             self.logger.error(
                 "Failed to retrieve schema from {url}: {error}".format(
                     url=platform_schema_url, error=e
@@ -39,7 +48,7 @@ def get_json_schema(self, platform_schema_url):
                 "Failed to retrieve schema from {url}: {error}".format(
                     url=platform_schema_url, error=e
                 )
-            ) from None
+            )
 
         try:
             json_response = response.json()

src/nypl_py_utils/classes/secrets_manager_client.py

@@ -0,0 +1,67 @@
+import boto3
+import json
+import os
+
+from botocore.exceptions import ClientError
+from nypl_py_utils.functions.log_helper import create_log
+
+
+class SecretsManagerClient:
+    """Client for interacting with AWS Secrets Manager"""
+
+    def __init__(self):
+        self.logger = create_log('secrets_manager_client')
+
+        try:
+            self.secrets_manager_client = boto3.client(
+                'secretsmanager', region_name=os.environ.get('AWS_REGION',
+                                                             'us-east-1'))
+        except ClientError as e:
+            self.logger.error(
+                'Could not create Secrets Manager client: {err}'.format(
+                    err=e))
+            raise SecretsManagerClientError(
+                'Could not create Secrets Manager client: {err}'.format(
+                    err=e)) from None
+
+    def close(self):
+        self.secrets_manager_client.close()
+
+    def get_secret(self, secret_name, is_json=True):
+        """
+        Retrieves secret with the given name from the Secrets Manager.
+
+        Parameters
+        ----------
+        secret_name: str
+            The name of the secret to retrieve
+        is_json: bool, optional
+            Whether the value of the secret is a JSON string that should be
+            returned as a dictionary
+
+        Returns
+        -------
+        dict or str
+            Dictionary if `is_json` is True; string if `is_json` is False
+        """
+        self.logger.debug('Retrieving \'{}\' from Secrets Manager'.format(
+            secret_name))
+        try:
+            response = self.secrets_manager_client.get_secret_value(
+                SecretId=secret_name)
+            if is_json:
+                return json.loads(response['SecretString'])
+            else:
+                return response['SecretString']
+        except ClientError as e:
+            self.logger.error(
+                ('Could not retrieve \'{secret}\' from Secrets Manager: {err}')
+                .format(secret=secret_name, err=e))
+            raise SecretsManagerClientError(
+                ('Could not retrieve \'{secret}\' from Secrets Manager: {err}')
+                .format(secret=secret_name, err=e)) from None
+
+
+class SecretsManagerClientError(Exception):
+    def __init__(self, message=None):
+        self.message = message

tests/test_avro_client.py

@@ -25,26 +25,26 @@ class TestAvroClient:
     @pytest.fixture
     def test_avro_encoder_instance(self, requests_mock):
         requests_mock.get(
-            'https://test_schema_url', text=json.dumps(_TEST_SCHEMA))
-        return AvroEncoder('https://test_schema_url')
+            "https://test_schema_url", text=json.dumps(_TEST_SCHEMA))
+        return AvroEncoder("https://test_schema_url")
 
     @pytest.fixture
     def test_avro_decoder_instance(self, requests_mock):
         requests_mock.get(
-            'https://test_schema_url', text=json.dumps(_TEST_SCHEMA))
-        return AvroDecoder('https://test_schema_url')
-
-    def test_get_json_schema(self, test_avro_encoder_instance,
-                             test_avro_decoder_instance):
-        assert test_avro_encoder_instance.schema == _TEST_SCHEMA['data'][
-            'schema']
-        assert test_avro_decoder_instance.schema == _TEST_SCHEMA['data'][
-            'schema']
-
-    def test_request_error(self, requests_mock):
-        requests_mock.get('https://test_schema_url', exc=ConnectTimeout)
+            "https://test_schema_url", text=json.dumps(_TEST_SCHEMA))
+        return AvroDecoder("https://test_schema_url")
+
+    def test_get_json_schema_success(self, test_avro_encoder_instance,
+                                     test_avro_decoder_instance):
+        assert test_avro_encoder_instance.schema == _TEST_SCHEMA["data"][
+            "schema"]
+        assert test_avro_decoder_instance.schema == _TEST_SCHEMA["data"][
+            "schema"]
+
+    def test_get_json_schema_error(self, requests_mock):
+        requests_mock.get("https://test_schema_url", exc=ConnectTimeout)
         with pytest.raises(AvroClientError):
-            AvroEncoder('https://test_schema_url')
+            AvroEncoder("https://test_schema_url")
 
     def test_bad_json_error(self, requests_mock):
         requests_mock.get(

tests/test_kms_client.py

@@ -21,10 +21,10 @@ def test_instance(self, mocker):
 
     def test_decrypt(self, test_instance):
         test_instance.kms_client.decrypt.return_value = _TEST_DECRYPTION
-        assert test_instance.kms_client.decrypt.called_once_with(
-            CiphertextBlob=b'test-encrypted-value')
         assert test_instance.decrypt(
             _TEST_ENCRYPTED_VALUE) == 'test-decrypted-value'
+        test_instance.kms_client.decrypt.assert_called_once_with(
+            CiphertextBlob=b'test-encrypted-value')
 
     def test_base64_error(self, test_instance):
         with pytest.raises(KmsClientError):

tests/test_mysql_client.py

@@ -60,7 +60,7 @@ def test_execute_write_query_with_params(self, mock_mysql_conn,
             'test query %s %s', query_params=('a', 1)) is None
         mock_cursor.execute.assert_called_once_with('test query %s %s',
                                                     ('a', 1))
-        test_instance.conn.commit.called_once()
+        test_instance.conn.commit.assert_called_once()
         mock_cursor.close.assert_called_once()
 
     def test_execute_query_with_exception(

tests/test_secrets_manager_client.py

@@ -0,0 +1,55 @@
+import pytest
+
+from botocore.exceptions import ClientError
+from datetime import datetime
+from nypl_py_utils.classes.secrets_manager_client import (
+    SecretsManagerClient, SecretsManagerClientError)
+
+_TEST_RESPONSE = {
+    'ARN': 'test_arn',
+    'Name': 'test_secret',
+    'VersionId': 'test_version',
+    'SecretString': '{\n  "key1": "value1",\n  "key2": "value2"\n}',
+    'VersionStages': ['AWSCURRENT'],
+    'CreatedDate': datetime(2024, 1, 1, 1, 1, 1, 1),
+    'ResponseMetadata': {
+        'RequestId': 'test-request-id',
+        'HTTPStatusCode': 200,
+        'HTTPHeaders': {
+            'x-amzn-requestid': 'test-request-id',
+            'content-type': 'application/x-amz-json-1.1',
+            'content-length': '155',
+            'date': 'Mon, 1 Jan 2024 07:01:01 GMT'
+        },
+        'RetryAttempts': 0}
+}
+
+
+class TestSecretsManagerClient:
+
+    @pytest.fixture
+    def test_instance(self, mocker):
+        mocker.patch('boto3.client')
+        return SecretsManagerClient()
+
+    def test_get_secret(self, test_instance):
+        test_instance.secrets_manager_client.get_secret_value.return_value = \
+            _TEST_RESPONSE
+        assert test_instance.get_secret('test_secret') == {
+            'key1': 'value1', 'key2': 'value2'}
+        test_instance.secrets_manager_client.get_secret_value\
+            .assert_called_once_with(SecretId='test_secret')
+
+    def test_get_secret_non_json(self, test_instance):
+        test_instance.secrets_manager_client.get_secret_value.return_value = \
+            _TEST_RESPONSE
+        assert test_instance.get_secret('test_secret', is_json=False) == (
+            '{\n  "key1": "value1",\n  "key2": "value2"\n}')
+        test_instance.secrets_manager_client.get_secret_value\
+            .assert_called_once_with(SecretId='test_secret')
+
+    def test_get_secret_error(self, test_instance):
+        test_instance.secrets_manager_client.get_secret_value.side_effect = \
+            ClientError({}, 'GetSecretValue')
+        with pytest.raises(SecretsManagerClientError):
+            test_instance.get_secret('test_secret')