Merge branch 'develop'

Author: onekiloparsec

arcsecond/__init__.py

@@ -7,4 +7,4 @@
            "ArcsecondConnectionError",
            "ArcsecondInvalidEndpointError"]
 
-__version__ = '0.7.3'
+__version__ = '0.7.4'

arcsecond/api/endpoints/_base.py

@@ -3,7 +3,10 @@
 
 import click
 import requests
+from requests_toolbelt.multipart import encoder
+
 from progress.spinner import Spinner
+from progress.bar import Bar
 
 from arcsecond.api.constants import (
     ARCSECOND_API_URL_DEV,
@@ -14,12 +17,18 @@
     API_AUTH_PATH_REGISTER)
 
 from arcsecond.api.error import ArcsecondConnectionError, ArcsecondError
+from arcsecond.api.helpers import transform_payload_for_multipart_encoder_fields
 from arcsecond.config import config_file_read_api_key, config_file_read_organisation_memberships
 from arcsecond.options import State
 
 SAFE_METHODS = ['GET', 'OPTIONS']
 WRITABLE_MEMBERSHIPS = ['superadmin', 'admin', 'member']
 
+EVENT_METHOD_WILL_START = 'EVENT_METHOD_WILL_START'
+EVENT_METHOD_DID_FINISH = 'EVENT_METHOD_DID_FINISH'
+EVENT_METHOD_DID_FAIL = 'EVENT_METHOD_DID_FAIL'
+EVENT_METHOD_PROGRESS_PERCENT = 'EVENT_METHOD_PROGRESS_PERCENT'
+
 
 class APIEndPoint(object):
     name = None
@@ -84,22 +93,23 @@ def _check_organisation_membership_and_permission(self, method_name, organisatio
         memberships = config_file_read_organisation_memberships(self.state.config_section())
         if self.state.organisation not in memberships.keys():
             raise ArcsecondError('No membership found for organisation {}'.format(organisation))
+
         membership = memberships[self.state.organisation]
         if method_name not in SAFE_METHODS and membership not in WRITABLE_MEMBERSHIPS:
             raise ArcsecondError('Membership for organisation {} has no write permission'.format(organisation))
 
-    def _async_perform_request(self, url, method, payload=None, files=None, **headers):
-        def _async_perform_request_store_response(storage, method, url, payload, files, headers):
+    def _async_perform_request(self, url, method, payload=None, **headers):
+        def _async_perform_request_store_response(storage, method, url, payload, headers):
             try:
-                storage['response'] = method(url, json=payload, files=files, headers=headers)
+                storage['response'] = method(url, json=payload, headers=headers)
             except requests.exceptions.ConnectionError:
                 storage['error'] = ArcsecondConnectionError(self._get_base_url())
             except Exception as e:
                 storage['error'] = ArcsecondError(str(e))
 
         storage = {}
         thread = threading.Thread(target=_async_perform_request_store_response,
-                                  args=(storage, method, url, payload, files, headers))
+                                  args=(storage, method, url, payload, headers))
         thread.start()
 
         spinner = Spinner()
@@ -115,31 +125,59 @@ def _async_perform_request_store_response(storage, method, url, payload, files,
 
         return storage.get('response', None)
 
-    def _perform_request(self, url, method, payload, **headers):
+    def _prepare_request(self, url, method, payload, **headers):
         assert (url and method)
 
         if not isinstance(method, str) or callable(method):
             raise ArcsecondError('Invalid HTTP request method {}. '.format(str(method)))
 
-        # Check API key, hence login state. Must do before check for org.
-        headers = self._check_and_set_api_key(headers, url)
-
         # Put method name aside in its own var.
         method_name = method.upper() if isinstance(method, str) else ''
-        method = getattr(requests, method.lower()) if isinstance(method, str) else method
-        files = payload.pop('files', None) if payload else None
 
         if self.state and self.state.organisation:
             self._check_organisation_membership_and_permission(method_name, self.state.organisation)
 
+        # Check API key, hence login state. Must do before check for org.
+        headers = self._check_and_set_api_key(headers, url)
+        method = getattr(requests, method.lower()) if isinstance(method, str) else method
+
         if payload:
+            # Filtering None values out of payload.
             payload = {k: v for k, v in payload.items() if v is not None}
 
+        return url, method_name, method, payload, headers
+
+    def _perform_request(self, url, method, payload, callback=None, **headers):
+        if self.state.verbose:
+            click.echo('Preparing request...')
+
+        url, method_name, method, payload, headers = self._prepare_request(url, method, payload, **headers)
+
         if self.state.verbose:
             click.echo('Sending {} request to {}'.format(method_name, url))
-            click.echo('Payload: {}'.format(payload))
 
-        response = self._async_perform_request(url, method, payload, files, **headers)
+        payload, fields = transform_payload_for_multipart_encoder_fields(payload)
+        if fields:
+            encoded_data = encoder.MultipartEncoder(fields=fields)
+            bar, upload_callback = None, None
+
+            if self.state.is_using_cli is False and callback:
+                upload_callback = lambda m: callback(EVENT_METHOD_PROGRESS_PERCENT, m.bytes_read / m.len * 100)
+            elif self.state.verbose:
+                bar = Bar('Uploading ' + fields['file'][0], suffix='%(percent)d%%')
+                upload_callback = lambda m: bar.goto(m.bytes_read / m.len * 100)
+
+            upload_monitor = encoder.MultipartEncoderMonitor(encoded_data, upload_callback)
+            headers.update(**{'Content-Type': upload_monitor.content_type})
+            response = method(url, data=upload_monitor, headers=headers)
+
+            if self.state.verbose:
+                bar.finish()
+        else:
+            if self.state.verbose:
+                click.echo('Payload: {}'.format(payload))
+
+            response = self._async_perform_request(url, method, payload, **headers)
 
         if response is None:
             raise ArcsecondConnectionError(url)
@@ -153,16 +191,16 @@ def _perform_request(self, url, method, payload, **headers):
             return None, response.text
 
     def list(self, name='', **headers):
-        return self._perform_request(self._list_url(name), 'get', None, **headers)
+        return self._perform_request(self._list_url(name), 'get', None, None, **headers)
 
-    def create(self, payload, **headers):
-        return self._perform_request(self._list_url(), 'post', payload, **headers)
+    def create(self, payload, callback=None, **headers):
+        return self._perform_request(self._list_url(), 'post', payload, callback, **headers)
 
     def read(self, id_name_uuid, **headers):
-        return self._perform_request(self._detail_url(id_name_uuid), 'get', None, **headers)
+        return self._perform_request(self._detail_url(id_name_uuid), 'get', None, None, **headers)
 
     def update(self, id_name_uuid, payload, **headers):
-        return self._perform_request(self._detail_url(id_name_uuid), 'put', payload, **headers)
+        return self._perform_request(self._detail_url(id_name_uuid), 'put', payload, None, **headers)
 
     def delete(self, id_name_uuid, **headers):
-        return self._perform_request(self._detail_url(id_name_uuid), 'delete', None, **headers)
+        return self._perform_request(self._detail_url(id_name_uuid), 'delete', None, None, **headers)

arcsecond/api/helpers.py

@@ -3,8 +3,23 @@
 from .error import ArcsecondInputValueError
 
 
-def make_file_upload_payload(filepath):
-    return {'files': {'file': open(os.path.abspath(filepath), 'rb')}}
+def make_file_upload_multipart_dict(filepath):
+    return {'fields': {'file': (os.path.basename(filepath), open(os.path.abspath(filepath), 'rb'))}}
+
+
+def transform_payload_for_multipart_encoder_fields(payload):
+    if isinstance(payload, str) and os.path.exists(payload) and os.path.isfile(payload):
+        payload = make_file_upload_multipart_dict(payload)  # transform a str into a dict
+
+    elif isinstance(payload, dict) and 'file' in payload.keys():
+        file_value = payload.pop('file')  # .pop() not .get()
+        if file_value and os.path.exists(file_value) and os.path.isfile(file_value):
+            payload.update(**make_file_upload_multipart_dict(file_value))  # unpack the resulting dict of make_file...()
+        else:
+            payload.update(file=file_value)  # do nothing, it's not a file...
+
+    fields = payload.pop('fields', None) if payload else None
+    return payload, fields
 
 
 def make_coords_dict(kwargs):

arcsecond/api/main.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 
 import json
-import os
 import pprint
 import types
 import webbrowser
@@ -21,7 +20,6 @@
 from arcsecond.options import State
 from .auth import AuthAPIEndPoint
 from .error import ArcsecondInvalidEndpointError, ArcsecondNotLoggedInError, ArcsecondTooManyPrefixesError
-from .helpers import make_file_upload_payload
 
 from .endpoints import (ActivitiesAPIEndPoint, CataloguesAPIEndPoint, DatasetsAPIEndPoint, ExoplanetsAPIEndPoint,
                         DataFilesAPIEndPoint, FindingChartsAPIEndPoint, InstrumentsAPIEndPoint, NightLogAPIEndPoint,
@@ -31,6 +29,7 @@
 
 pp = pprint.PrettyPrinter(indent=4, depth=5)
 ECHO_PREFIX = u' • '
+ECHO_ERROR_PREFIX = u' • [error] '
 
 __all__ = ["ArcsecondAPI"]
 
@@ -70,7 +69,7 @@ def get_api_state(state=None, **kwargs):
 
 
 def set_api_factory(cls):
-    def factory(endpoint_class, state, **kwargs):
+    def factory(endpoint_class, state=None, **kwargs):
         return ArcsecondAPI(endpoint_class, state, **kwargs)
 
     for endpoint_class in ENDPOINTS:
@@ -144,13 +143,15 @@ def _echo_error(cls, state, error):
         else:
             json_obj = json.loads(error)
             if 'detail' in json_obj.keys():
-                click.echo(ECHO_PREFIX + json_obj['detail'])
+                detail_msg = ', '.join(json_obj['detail']) if isinstance(json_obj['detail'], list) else json_obj['detail']
+                click.echo(ECHO_ERROR_PREFIX + detail_msg)
             elif 'error' in json_obj.keys():
-                click.echo(ECHO_PREFIX + json_obj['error'])
+                error_msg = ', '.join(json_obj['error']) if isinstance(json_obj['error'], list) else json_obj['error']
+                click.echo(ECHO_ERROR_PREFIX + error_msg)
             elif 'non_field_errors' in json_obj.keys():
                 errors = json_obj['non_field_errors']
                 message = ', '.join(errors) if isinstance(error, list) else str(errors)
-                click.echo(ECHO_PREFIX + message)
+                click.echo(ECHO_ERROR_PREFIX + message)
             else:
                 click.echo(ECHO_PREFIX + str(error))
 
@@ -166,22 +167,10 @@ def _check_endpoint_class(self, endpoint):
             raise ArcsecondInvalidEndpointError(endpoint, ENDPOINTS)
         return endpoint
 
-    def _check_for_file_in_payload(self, payload):
-        if isinstance(payload, str) and os.path.exists(payload) and os.path.isfile(payload):
-            return make_file_upload_payload(payload)  # transform a str into a dict
-        elif isinstance(payload, dict) and 'file' in payload.keys():
-            file_value = payload.pop('file')  # .pop() not .get()
-            if file_value and os.path.exists(file_value) and os.path.isfile(file_value):
-                payload.update(**make_file_upload_payload(file_value))  # unpack the resulting dict of make_file...()
-            else:
-                payload.update(file=file_value)  # do nothing, it's not a file...
-        return payload
-
     def list(self, name=None, **headers):
         return self._echo_response(self.endpoint.list(name, **headers))
 
     def create(self, payload, **headers):
-        payload = self._check_for_file_in_payload(payload)
         return self._echo_response(self.endpoint.create(payload, **headers))
 
     def read(self, id_name_uuid, **headers):

arcsecond/cli.py

@@ -149,7 +149,7 @@ def activities(state, method, pk, **kwargs):
     api = Arcsecond.create_activities_api(state)
     if method == 'create':
         kwargs.update(coordinates=make_coords_dict(kwargs))
-        api.create(kwargs)
+        api.create(kwargs)  # the kwargs dict is the payload!
     elif method == 'read':
         api.read(pk)  # will handle list if pk is None
     elif method == 'update':
@@ -169,7 +169,7 @@ def activities(state, method, pk, **kwargs):
 def datasets(state, method, uuid, **kwargs):
     api = Arcsecond.create_datasets_api(state)
     if method == 'create':
-        api.create(kwargs)
+        api.create(kwargs)  # the kwargs dict is the payload!
     elif method == 'read':
         api.read(uuid)  # will handle list if pk is None
     elif method == 'update':
@@ -184,7 +184,8 @@ def datasets(state, method, uuid, **kwargs):
 @click.argument('dataset', required=True, nargs=1)
 @click.argument('method', required=False, nargs=1, type=MethodChoiceParamType(), default='read')
 @click.argument('pk', required=False, nargs=1)
-@click.option('--file', required=False, nargs=1, help="The path to the data file to upload. Can be zipped with gzip or bzip2.")
+@click.option('--file', required=False, nargs=1,
+              help="The path to the data file to upload. Can be zipped with gzip or bzip2.")
 @click.option('--instrument', required=False, nargs=1, help="The UUID of the instrument.")
 @organisation_options
 @pass_state
@@ -200,7 +201,7 @@ def datafiles(state, dataset, method, pk, **kwargs):
         kwargs.pop('organisation')
     api = Arcsecond.create_datafiles_api(state=state, dataset=dataset)
     if method == 'create':
-        api.create(kwargs)
+        api.create(kwargs)  # the kwargs dict is the payload!
     elif method == 'read':
         api.read(pk)  # will handle list if pk is None
     elif method == 'update':

setup.py

@@ -28,6 +28,7 @@
         'six',
         'click',
         'requests',
+        'requests_toolbelt',
         'pygments',
         'configparser',
         'progress'