Merge pull request #4 from TaskarCenterAtUW/develop

v0.0.7 version to Master

Author: susrisha

.github/workflows/deploy_to_test.yml

@@ -20,8 +20,8 @@ jobs:
         with:
           python-version: '3.10'
 
-      - name: Install Requirements
-        run: pip install -r requirements.txt
+      - name: Installing git
+        run: pip install gitpython
 
       - name: Generating version file
         run: python freeze_version.py
@@ -32,9 +32,9 @@ jobs:
       - name: Building Package
         run: python setup.py bdist_wheel
 
-#      - name: Publish package
-#        uses: pypa/gh-action-pypi-publish@release/v1
-#        with:
-#          skip_existing: true
-#          password: ${{ secrets.PYPI_API_TOKEN }}
-#          repository_url: https://test.pypi.org/legacy/
+      - name: Publish package
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          skip_existing: true
+          password: ${{ secrets.PYPI_API_TOKEN }}
+          repository_url: https://test.pypi.org/legacy/

.github/workflows/publish_to_pypi.yml

@@ -41,8 +41,6 @@ jobs:
       - name: Compare local and remote version with dpkg
         run: dpkg --compare-versions ${{env.localVersion}} "gt" ${{env.remoteVersion}}
 
-      - name: Install Requirements
-        run: pip install -r requirements.txt
 
       - name: Setup python wheel
         run: pip install wheel

CHANGELOG.md

@@ -1,5 +1,30 @@
 # Change log
 
+### 0.0.7
+- Fixed configuration issues
+- Fixed all the review comments
+- Refactor code
+- Removed unused code
+- Added local dev env using TDEI-local-server Package
+- Added local env support to -
+  - Logger
+  - Queue
+  - Topic
+  - Storage
+
+### 0.0.6
+- Added get_remote_url function in FileEntity class which will return the uploaded file url
+
+### 0.0.5
+- Fixed Storage Blob stream
+
+### 0.0.4
+- Fixed threading
+
+### 0.0.3
+- Removed blocking call from topic subscription
+- Added TODOs and FIXMEs
+
 ### 0.0.2
 - Added filtered exception handling
 - Removed unused packages

README.md

@@ -24,7 +24,7 @@ All the cloud connections are initialized with `initialize` function of core whi
 Eg.
 ```python
 from python_ms_core import Core
-Core.initialize()
+core = Core() or Core(config='Local')
 ```
 The method analyzes the `.env` variables and does a health check on what components are available
 
@@ -56,12 +56,13 @@ This file will have to be generated or shared offline as per the developer requi
 Offers helper classes to help log the information. It is also used to record the audit messages
 as well as the analytics information required.
 
-Use `Core.get_logger()`
+Use `core.get_logger()`
 Eg.
 ```python
 from python_ms_core import Core
 
-logger = Core.get_logger()
+core = Core()
+logger = core.get_logger()
 
 # Record a metric
 logger.record_metric(name='test', value='test') # Metric and value
@@ -74,7 +75,8 @@ Eg.
 ```python
 from python_ms_core import Core
 
-logger = Core.get_logger()
+core = Core()
+logger = core.get_logger()
 
 # Record a metric
 logger.debug('Debug Message')
@@ -102,15 +104,12 @@ The configuration required by Queue and Topic is similar and will be handled via
 
 Topic can be accessed by the core method `get_topic`. This method takes two parameters
 1. topic name (required)
-2. callback function where messages will be received
 
 ```python
 from python_ms_core import Core
 
-def callback(instance):
-    print(instance.get_messages())
-
-topic = Core.get_topic(topic_name='topicName', callback=callback)
+core = Core()
+topic = core.get_topic(topic_name='topicName')
 
 ```
 
@@ -121,7 +120,8 @@ Once the topic object is got, use `publish` method to publish the message to top
 from python_ms_core import Core
 from python_ms_core.core.queue.models.queue_message import QueueMessage
 
-topic = Core.get_topic(topic_name='topicName')
+core = Core()
+topic = core.get_topic(topic_name='topicName')
 topic.publish(QueueMessage.data_from(
     {
         'message': 'Test message'
@@ -139,18 +139,26 @@ It takes one parameter
 ```python
 from python_ms_core import Core
 
-topic = Core.get_topic(topic_name='topicName')
-msg = topic.subscribe(subscription='subscriptionName')
-print(msg)
+core = Core()
+topic_object = topic = Core.get_topic(topic_name='topicName')
+
+def process(message):
+    print(f'Message Received: {message}')
+
+try:
+    topic_object.subscribe(subscription='subscriptionName')
+except Exception as e:
+    print(e)
 ```
 
 ### Storage
 For all the azure blobs and other storages, storage components will offer simple ways to upload/download and read the existing data.
 ```python
 from python_ms_core import Core
 
+core = Core()
 # Create storage client
-azure_client = Core.get_storage_client()
+azure_client = core.get_storage_client()
 
 # Get a container in the storage client
 container = azure_client.get_container(container_name='tdei-storage-test')
@@ -168,7 +176,8 @@ File upload is done only through read stream.
 from python_ms_core import Core
 from io import BytesIO, StringIO
 
-azure_client = Core.get_storage_client()
+core = Core()
+azure_client = core.get_storage_client()
 container = azure_client.get_container(container_name='tdei-storage-test')
 # Create an instance of `AzureFileEntity` with name and mime-type
 txt = 'foo\nbar\nbaz'

freeze_version.py

@@ -11,7 +11,7 @@
 
 build_date = date.today().strftime('%Y-%m-%d')
 
-version = '0.0.2'
+version = '0.0.7'
 
 with open(version_file_path, 'w+') as version_file:
     version_file.write("version = '{}'\n".format(version))

requirements.txt

@@ -2,6 +2,20 @@ azure-common==1.1.28
 azure-core==1.26.1
 azure-servicebus==7.0.0
 azure-storage-blob==12.14.1
+certifi==2022.12.7
+cffi==1.15.1
+charset-normalizer==2.1.1
+cryptography==39.0.0
+idna==3.4
+isodate==0.6.1
+msrest==0.7.1
+oauthlib==3.2.2
+pycparser==2.21
 python-dotenv==0.21.0
 requests==2.28.1
-GitPython==3.1.29
+requests-oauthlib==1.3.1
+six==1.16.0
+typing_extensions==4.4.0
+uamqp==1.6.3
+urllib3==1.26.14
+pika==1.3.1

src/example.py

@@ -1,71 +1,79 @@
 # Testing code
-
 import sys
+import os
+import time
 import uuid
+import random
 import datetime
 from io import BytesIO, StringIO
-import time
 
 from python_ms_core import Core
-from python_ms_core.core.queue.providers import azure_queue_config
 from python_ms_core.core.queue.models.queue_message import QueueMessage
 
-Core.initialize()
+core = Core(config='Local')
 print('Hello')
 
-topic = 'gtfs-flex-upload'
+topic = 'gtfspathways'
 subscription = 'upload-validation-processor-test'
 some_other_sub = 'usdufs'
 
-topic_config = azure_queue_config.AzureQueueConfig()
 
-azure_client = Core.get_storage_client()
+def publish_messages(topic_name):
+    topic_object = core.get_topic(topic_name=topic_name)
+    queue_message = QueueMessage.data_from({
+        'message': str(uuid.uuid4().hex),
+        'data': {'a': random.randint(0, 1000)}
+    })
+    topic_object.publish(data=queue_message)
+    print('Message Published')
+
+
+def subscribe(topic_name, subscription_name):
+    def process(message):
+        print(f'Message Received: {message}')
+
+    topic_object = core.get_topic(topic_name=topic_name)
+    try:
+        topic_object.subscribe(subscription=subscription_name, callback=process)
+    except Exception as e:
+        print(e)
 
-container = azure_client.get_container(container_name='tdei-storage-test')
+
+subscribe(topic, subscription)
+
+azure_client = core.get_storage_client()
+
+container = azure_client.get_container(container_name='gtfspathways')
 
 list_of_files = container.list_files()
-for single in list_of_files:
-    print(single.name)
-firstFile = list_of_files[2]
-# print(firstFile.name+'<><>')
-file_content = firstFile.get_body_text()
+if len(list_of_files) > 0:
+    for single in list_of_files:
+        print(single.path)
+    firstFile = list_of_files[0]
+    # print(firstFile.name+'<><>')
+    file_content = firstFile.get_body_text()
 
 # Creating a text stream
 txt = 'foo\nbar\nbaz'
 file_like_io = StringIO(txt)
 basename = 'sample-file'
 suffix = datetime.datetime.now().strftime("%y%m%d_%H%M%S")
 filename = '_'.join([basename, suffix])
-test_file = container.create_file(f'{filename}.txt', 'text/plain')
+try:
+    test_file = container.create_file(f'{filename}.txt')
+except Exception as e:
+    print(e)
 print('Start uploading...')
 test_file.upload(file_like_io.read())
+print(test_file.get_remote_url())
 print('Uploaded Successfully')
 
-logger = Core.get_logger()
-logger.record_metric(name='test', value='test')
-
-logger = Core.get_logger(provider='Local')
+logger = core.get_logger()
 logger.record_metric(name='test', value='test')
 
+publish_messages(topic)
+time.sleep(2)
 
-def on_connect_callback(instance):
-    print('Connected with result code {}'.format(instance.get_messages()))
-    time.sleep(30)
-    sys.exit()
-
-
-topicObject1 = Core.get_topic(topic_name=topic, callback=on_connect_callback)
-topicObject2 = Core.get_topic(topic_name='tipic')
-try:
-    topicObject2.subscribe(subscription=subscription)
-except Exception as e:
-    print(e)
-
-queue_message = QueueMessage.data_from({
-    'message': str(uuid.uuid4().hex),
-    'data': {'a': 1}
-})
-
-topicObject1.publish(data=queue_message)
-
-topicObject1.subscribe(subscription=subscription)
+# logger = core.get_logger()
+# logger.record_metric(name='test', value='test')
+os._exit(os.EX_OK)