Merge branch 'main' into qa

Author: fatimarahman

CHANGELOG.md

@@ -1,4 +1,20 @@
 # Changelog
+## v1.6.3 1/27/25
+- Add capability to pull cloudLibrary events by the millisecond
+
+## v1.6.2 12/2/24
+- Add record_num capability to patron_data_helper
+
+## v1.6.1 11/26/24
+- Accidental duplicate of v1.6.0
+
+## v1.6.0 11/20/24
+- Added patron_data_helper functions
+- Use executemany instead of execute when appropriate in PostgreSQLClient
+- Add capability to retry connecting to a database to the MySQL, PostgreSQL, and Redshift clients
+- Automatically close database connection upon error in the MySQL, PostgreSQL, and Redshift clients
+- Delete old PostgreSQLPoolClient, which was not production ready
+
 ## v1.5.0 11/19/24
 - Added cloudLibrary client
 

Makefile

@@ -9,7 +9,7 @@ help:
 	@echo "    lint project files using the flake8 linter"
 
 test:
-	pytest
+	pytest -W ignore::FutureWarning
 
 lint:
 	flake8 --exclude *env

README.md

@@ -11,7 +11,6 @@ This package contains common Python utility classes and functions.
 * Downloading files from a remote SSH SFTP server
 * Connecting to and querying a MySQL database
 * Connecting to and querying a PostgreSQL database
-* Connecting to and querying a PostgreSQL database using a connection pool
 * Connecting to and querying Redshift
 * Making requests to the Oauth2 authenticated APIs such as NYPL Platform API and Sierra
 * Interacting with vendor APIs such as cloudLibrary
@@ -21,6 +20,7 @@ This package contains common Python utility classes and functions.
 * Creating a logger in the appropriate format
 * Obfuscating a value using bcrypt
 * Parsing/building Research Catalog identifiers
+* Mapping between barcodes and Sierra patron ids plus getting patron data from Sierra and Redshift using those ids or record_nums
 
 ## Usage
 ```python
@@ -38,7 +38,7 @@ kinesis_client = KinesisClient(...)
 # Do not use any version below 1.0.0
 # All available optional dependencies can be found in pyproject.toml.
 # See the "Managing dependencies" section below for more details.
-nypl-py-utils[kinesis-client,config-helper]==1.5.0
+nypl-py-utils[kinesis-client,config-helper]==1.x.y
 ```
 
 ## Developing locally
@@ -64,7 +64,7 @@ The optional dependency sets also give the developer the option to manually list
 ### Using PostgreSQLClient in an AWS Lambda
 Because `psycopg` requires a statically linked version of the `libpq` library, the `PostgreSQLClient` cannot be installed as-is in an AWS Lambda function. Instead, it must be packaged as follows:
 ```bash
-pip install --target ./package nypl-py-utils[postgresql-client]==1.5.0
+pip install --target ./package nypl-py-utils[postgresql-client]==1.x.y
 
 pip install \
     --platform manylinux2014_x86_64 \

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "nypl_py_utils"
-version = "1.5.0"
+version = "1.6.2"
 authors = [
   { name="Aaron Friedman", email="aaronfriedman@nypl.org" },
 ]
@@ -48,9 +48,6 @@ oauth2-api-client = [
 postgresql-client = [
     "psycopg[binary]>=3.1.6"
 ]
-postgresql-pool-client = [
-    "psycopg[binary,pool]>=3.1.6"
-]
 redshift-client = [
     "botocore>=1.29.5",
     "redshift-connector>=2.0.909"
@@ -73,11 +70,15 @@ config-helper = [
 obfuscation-helper = [
     "bcrypt>=4.0.1"
 ]
+patron-data-helper = [
+    "nypl_py_utils[postgresql-client,redshift-client]>=1.1.5",
+    "pandas>=2.2.2"
+]
 research-catalog-identifier-helper = [
     "requests>=2.28.1"
 ]
 development = [
-    "nypl_py_utils[avro-client,kinesis-client,kms-client,mysql-client,oauth2-api-client,postgresql-client,postgresql-pool-client,redshift-client,s3-client,secrets-manager-client,sftp-client,config-helper,obfuscation-helper,research-catalog-identifier-helper]",
+    "nypl_py_utils[avro-client,kinesis-client,kms-client,mysql-client,oauth2-api-client,postgresql-client,redshift-client,s3-client,secrets-manager-client,sftp-client,config-helper,obfuscation-helper,patron-data-helper,research-catalog-identifier-helper]",
     "flake8>=6.0.0",
     "freezegun>=1.2.2",
     "mock>=4.0.3",

src/nypl_py_utils/classes/cloudlibrary_client.py

@@ -3,7 +3,7 @@
 import hmac
 import requests
 
-from datetime import datetime, timedelta, timezone
+from datetime import datetime, timezone
 from nypl_py_utils.functions.log_helper import create_log
 from requests.adapters import HTTPAdapter, Retry
 
@@ -35,28 +35,25 @@ def get_library_events(self, start_date=None,
         optional timeframe. Pulls past 24 hours of events by default.
 
         start_date and end_date are optional parameters, and must be
-        formatted either YYYY-MM-DD or YYYY-MM-DDTHH:MM:SS
+        formatted either YYYY-MM-DD, YYYY-MM-DD:SS, or YYYY-MM-DDTHH:MM:SS.fff
         """
-        date_format = "%Y-%m-%dT%H:%M:%S"
-        today = datetime.now(timezone.utc)
-        yesterday = today - timedelta(1)
-        start_date = datetime.strftime(
-            yesterday, date_format) if start_date is None else start_date
-        end_date = datetime.strftime(
-            today, date_format) if end_date is None else end_date
-
-        if (datetime.strptime(start_date, date_format) >
-                datetime.strptime(end_date, date_format)):
-            error_message = (f"Start date {start_date} greater than end date "
-                             f"{end_date}, cannot retrieve library events")
-            self.logger.error(error_message)
-            raise CloudLibraryClientError(error_message)
-
-        self.logger.info(
-            (f"Fetching all library events in "
-             f"time frame {start_date} to {end_date}..."))
+        path = "data/cloudevents"
+        if None not in (start_date, end_date):
+            if (self._parse_event_date(start_date) >
+                    self._parse_event_date(end_date)):
+                error_message = (f"Start date {start_date} greater than end "
+                                 f"date {end_date}, cannot retrieve library "
+                                 f"events")
+                self.logger.error(error_message)
+                raise CloudLibraryClientError(error_message)
+
+            path += f"?startdate={start_date}&enddate={end_date}"
+            self.logger.info(f"Fetching all library events in "
+                             f"time frame {start_date} to {end_date}...")
+        else:
+            self.logger.info("Fetching all library events "
+                             "from the past day...")
 
-        path = f"data/cloudevents?startdate={start_date}&enddate={end_date}"
         response = self.request(path=path, method_type="GET")
         return response
 
@@ -66,7 +63,7 @@ def create_request_body(self, request_type,
         Helper function to generate request body when performing item
         and/or patron-specific functions (ex. checking out a title).
         """
-        request_template = "<%(request_type)s><ItemId>%(item_id)s</ItemId><PatronId>%(patron_id)s</PatronId></%(request_type)s>" # noqa
+        request_template = "<%(request_type)s><ItemId>%(item_id)s</ItemId><PatronId>%(patron_id)s</PatronId></%(request_type)s>"  # noqa
         return request_template % {
             "request_type": request_type,
             "item_id": item_id,
@@ -113,6 +110,16 @@ def request(self, path, method_type="GET",
 
         return response
 
+    def _parse_event_date(self, event_date) -> datetime:
+        for fmt in ("%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%S.%f"):
+            try:
+                return datetime.strptime(event_date, fmt)
+            except ValueError:
+                pass
+        error_message = (f"Invalid date format found: {event_date}")
+        self.logger.error(error_message)
+        raise CloudLibraryClientError(error_message)
+
     def _build_headers(self, method_type, path) -> dict:
         time, authorization = self._build_authorization(
             method_type, path)

src/nypl_py_utils/classes/mysql_client.py

@@ -1,4 +1,5 @@
 import mysql.connector
+import time
 
 from nypl_py_utils.functions.log_helper import create_log
 
@@ -15,35 +16,49 @@ def __init__(self, host, port, database, user, password):
         self.user = user
         self.password = password
 
-    def connect(self, **kwargs):
+    def connect(self, retry_count=0, backoff_factor=5, **kwargs):
         """
         Connects to a MySQL database using the given credentials.
 
-        Keyword args can be passed into the connection to set certain options.
-        All possible arguments can be found here:
-        https://dev.mysql.com/doc/connector-python/en/connector-python-connectargs.html.
-
-        Common arguments include:
-            autocommit: bool
-                Whether to automatically commit each query rather than running
-                them as part of a transaction. By default False.
+        Parameters
+        ----------
+        retry_count: int, optional
+            The number of times to retry connecting before throwing an error.
+            By default no retry occurs.
+        backoff_factor: int, optional
+            The backoff factor when retrying. The amount of time to wait before
+            retrying is backoff_factor ** number_of_retries_made.
+        kwargs:
+            All possible arguments can be found here:
+            https://dev.mysql.com/doc/connector-python/en/connector-python-connectargs.html
         """
         self.logger.info('Connecting to {} database'.format(self.database))
-        try:
-            self.conn = mysql.connector.connect(
-                host=self.host,
-                port=self.port,
-                database=self.database,
-                user=self.user,
-                password=self.password,
-                **kwargs)
-        except mysql.connector.Error as e:
-            self.logger.error(
-                'Error connecting to {name} database: {error}'.format(
-                    name=self.database, error=e))
-            raise MySQLClientError(
-                'Error connecting to {name} database: {error}'.format(
-                    name=self.database, error=e)) from None
+        attempt_count = 0
+        while attempt_count <= retry_count:
+            try:
+                try:
+                    self.conn = mysql.connector.connect(
+                        host=self.host,
+                        port=self.port,
+                        database=self.database,
+                        user=self.user,
+                        password=self.password,
+                        **kwargs)
+                    break
+                except (mysql.connector.Error):
+                    if attempt_count < retry_count:
+                        self.logger.info('Failed to connect -- retrying')
+                        time.sleep(backoff_factor ** attempt_count)
+                        attempt_count += 1
+                    else:
+                        raise
+            except Exception as e:
+                self.logger.error(
+                    'Error connecting to {name} database: {error}'.format(
+                        name=self.database, error=e))
+                raise MySQLClientError(
+                    'Error connecting to {name} database: {error}'.format(
+                        name=self.database, error=e)) from None
 
     def execute_query(self, query, query_params=None, **kwargs):
         """
@@ -83,6 +98,8 @@ def execute_query(self, query, query_params=None, **kwargs):
                 return cursor.fetchall()
         except Exception as e:
             self.conn.rollback()
+            cursor.close()
+            self.close_connection()
             self.logger.error(
                 ('Error executing {name} database query \'{query}\': {error}')
                 .format(name=self.database, query=query, error=e))