Merge pull request #61 from mattmanley/batch_add_tasks

Adding batch functionality for posting new tasks

Author: Daniel B

examples/challenge_examples.py

@@ -58,4 +58,4 @@
     data = json.loads(data_file.read())
 
 # Printing response:
-print(api.add_tasks_to_challenge(data, challenge_id))
+print(json.dumps(api.add_tasks_to_challenge(data, challenge_id)))

maproulette/api/challenge.py

@@ -250,7 +250,7 @@ def reset_task_instructions(self, challenge_id):
     def add_tasks_to_challenge(self, data, challenge_id):
         """Method to add tasks to an existing challenge
 
-        :param data: a geojson containing geometry of tasks to be added to a challenge
+        :param data: a GeoJSON containing geometry of tasks to be added to a challenge
         :param challenge_id: the ID corresponding to the challenge that tasks will be added to
         :returns: the API response from the PUT request
         """

maproulette/api/maproulette_server.py

@@ -43,6 +43,7 @@ def __check_health(self, retries=3, delay=5):
                     return True
             except requests.exceptions.ConnectionError:
                 print(f"Connection not available. Attempt {str(i+1)} out of {str(retries)}")
+                time.sleep(delay)
 
         raise ConnectionUnavailableError(
             message='Specified server unavailable'

maproulette/api/task.py

@@ -30,15 +30,20 @@ def get_task_history(self, task_id):
             endpoint=f"/task/{task_id}/history")
         return response
 
-    def create_tasks(self, data):
-        """Method to create a batch of tasks
+    def create_tasks(self, data, batch_size=5000):
+        """Method to create a batch of tasks using the specified batch_size.
 
         :param data: a JSON input containing task details
+        :param batch_size: the number of tasks to post per API call. The default is 5000.
+        :type batch_size: int
         :returns: the API response from the POST request
         """
-        response = self.post(
-            endpoint="/tasks",
-            body=data)
+        response = []
+        for batch in self.batch_generator(input_list=data, chunk_size=batch_size):
+            response.append(self.post(
+                endpoint="/tasks",
+                body=batch)
+            )
         return response
 
     def update_tasks(self, data):
@@ -157,3 +162,15 @@ def is_task_model(input_object):
         :returns: True if instance of model
         """
         return bool(isinstance(input_object, TaskModel))
+
+    @staticmethod
+    def batch_generator(input_list, chunk_size):
+        """Method to yield successive n-sized chunks from input_list
+
+        :param input_list: the list to break into chunks
+        :param chunk_size: the number of list items to include per chunk
+        :type chunk_size: int
+        :returns: an iterator for the n-sized chunks of the input_list
+        """
+        for i in range(0, len(input_list), chunk_size):
+            yield input_list[i:i + chunk_size]

maproulette/models/task.py

@@ -190,6 +190,7 @@ def to_dict(self):
             "id": self._id,
             "name": self._name,
             "parent": self._parent,
+            "geometries": self._geometries,
             "instruction": self._instruction,
             "location": self._location,
             "suggestedFix": self._suggested_fix,

tests/test_task_api.py

@@ -26,13 +26,16 @@ def test_get_task_history(self, mock_request, api_instance=api):
 
     @patch('maproulette.api.maproulette_server.requests.Session.post')
     def test_create_tasks(self, mock_request, api_instance=api):
+        test_tasks = []
         geometries = test_geojson['features'][0]['geometry']
         test_task_model = maproulette.TaskModel(name='test_task',
                                                 parent='12345',
                                                 geometries=geometries)
+        test_tasks.append(test_task_model.to_dict())
         mock_request.return_value.status_code = '200'
-        response = api_instance.create_tasks(test_task_model)
-        self.assertEqual(response['status'], '200')
+        responses = api_instance.create_tasks(test_tasks)
+        for response in responses:
+            self.assertEqual(response['status'], '200')
 
     @patch('maproulette.api.maproulette_server.requests.Session.put')
     def test_update_tasks(self, mock_request, api_instance=api):
@@ -88,3 +91,15 @@ def test_get_task_comments(self, mock_request, api_instance=api):
         mock_request.return_value.status_code = '200'
         response = api_instance.get_task_comments(task_id)
         self.assertEqual(response['status'], '200')
+
+    def test_batch_generator(self, api_instance=api):
+
+        batch_size = 10
+        test_length = 1234
+        test_list = [i for i in range(test_length)]
+        running_total = 0
+        for chunk in api_instance.batch_generator(test_list, batch_size):
+            running_total += len(chunk)
+            self.assertIsInstance(chunk, list)
+            self.assertLessEqual(len(chunk), batch_size)
+        self.assertEqual(test_length, running_total)