New query method and tests.

Author: victor73

osdf.py

@@ -228,6 +228,31 @@ def validate_node(self, json_data):
 
         return (valid, error_msg)
 
+    def oql_query(self, namespace, query, page=1):
+        """
+        Issue an OSDF Query Language (OQL) query against OSDF.
+
+        Returns the specified page of results.
+        """
+        url = "/nodes/oql/%s/page/%s" % (namespace, str(page))
+
+        osdf_response = self._request.post(url, query)
+
+        if osdf_response["code"] != 200 and osdf_response["code"] != 206:
+            headers = osdf_response["headers"]
+
+            if 'x-osdf-error' in headers:
+                msg = "Unable to query namespace %s. Reason: %s" % (namespace, headers['x-osdf-error'])
+            else:
+                msg = "Unable to query namespace."
+
+            raise Exception(msg)
+
+        data = json.loads( osdf_response['content'] )
+
+        data = self._byteify(data)
+
+        return data
 
     def query(self, namespace, query, page=1):
         """
@@ -256,6 +281,33 @@ def query(self, namespace, query, page=1):
 
         return data
 
+    def oql_query_all_pages(self, namespace, query):
+        """
+        Issue an OSDF Query Language (OQL) query against OSDF, as in the
+        oql_query() method, but retrieves ALL results by aggregating all
+        the available pages of results. Use with caution, as this may
+        consume a lot of memory with large result sets.
+        """
+        more_results = True
+        page = 1
+        cumulative_results = []
+
+        while more_results:
+           results = self.oql_query(namespace, query, page)
+
+           cumulative_results.extend(results['results'])
+
+           if results['result_count'] > 0:
+               page += 1
+           else:
+               more_results = False
+
+        results['results'] = cumulative_results
+        results['result_count'] = len(results['results'])
+        del results['page']
+
+        return results;
+
     def query_all_pages(self, namespace, query):
         """
         Issue a query against OSDF, as in the query() method, but retrieves
@@ -291,7 +343,7 @@ def create_osdf_node(self, namespace, node_type, domain_json, linkage={}, read="
 
         return node_json
 
-    def header_error(self, heades=[], method_type='retrieve',
+    def header_error(self, headers=[], method_type='retrieve',
             document_type=None):
         if 'x-osdf-error' in headers:
             msg = "Unable to %s %s document. Reason: %s" \

test/test_osdf.py

@@ -3,13 +3,28 @@
 import unittest
 from osdf import OSDF
 
+def _get_osdf():
+    osdf = OSDF(OsdfTest.server, OsdfTest.username, OsdfTest.password)
+    return osdf
+
 class OsdfTest(unittest.TestCase):
     server = "osdf-devel.igs.umaryland.edu"
     username = "test"
     password = "test"
 
+    test_node = {
+                  "ns": "test",
+                  "acl": { "read": [ "all" ], "write": [ "all" ] },
+                  "linkage": {},
+                  "node_type": "example",
+                  "meta": {
+                      "description": "something",
+                      "color": "blue"
+                  }
+              }
+
     def testGetInfo(self):
-        osdf = OSDF(OsdfTest.server, OsdfTest.username, OsdfTest.password)
+        osdf = _get_osdf()
 
         info = None
 
@@ -33,36 +48,151 @@ def testGetInfo(self):
 
 
     def testOqlQuery(self):
-        osdf = OSDF(OsdfTest.server, OsdfTest.username, OsdfTest.password)
+        osdf = _get_osdf()
 
         query = '"project"[node_type]'
         namespace = "test"
 
         results = osdf.oql_query(namespace, query)
 
-        self.assertIsNotNone(results, "OQL query result is not None.")
+        self._examine_paged_results(results, "OQL")
+
+    def testOqlQueryAllPages(self):
+        osdf = _get_osdf()
+
+        query = '"project"[node_type]'
+        namespace = "test"
+
+        results = osdf.oql_query_all_pages(namespace, query)
+        self._examine_all_results(results, "OQL")
+
+    def _examine_all_results(self, results, search_type):
+        self.assertIsNotNone(results, "%s query result is not None." % search_type)
+
+        self.assertTrue(type(results) == dict, "%s query result is a list." % search_type)
+
+        self.assertTrue('result_count' in results,
+                        "%s result contains 'result_count' key." % search_type)
 
-        self.assertTrue(type(results) == dict, "OQL query result is a list.")
+        self.assertTrue('results' in results,
+                        "%s result contains 'results' key." % search_type)
 
-        self.assertTrue('result_count' in results, "Result contains 'result_count' key.")
-        self.assertTrue('results' in results, "Result contains 'results' key.")
-        self.assertTrue('page' in results, "Result contains 'page' key.")
+    def _examine_paged_results(self, results, search_type):
+        self.assertIsNotNone(results,
+                        "%s query result is not None." % search_type)
+
+        self.assertTrue(type(results) == dict,
+                        "%s query result is a list." % search_type)
+
+        self.assertTrue('result_count' in results,
+                        "%s result contains 'result_count' key." % search_type)
+        self.assertTrue('results' in results,
+                        "%s result contains 'results' key." % search_type)
+        self.assertTrue('page' in results,
+                        "%s result contains 'page' key." % search_type)
 
     def testQuery(self):
-        osdf = OSDF(OsdfTest.server, OsdfTest.username, OsdfTest.password)
+        osdf = _get_osdf()
 
         query = '{ "term" : { "node_type" : "project" }}'
         namespace = "test"
 
         results = osdf.query(namespace, query)
 
-        self.assertIsNotNone(results, "Query result is not None.")
+        self._examine_paged_results(results, "ES QueryDSL")
+
+    def testInsertNode(self):
+
+        osdf = _get_osdf()
+
+        exception_thrown = False;
+        node_id = None
+
+        try:
+            node_id = osdf.insert_node(OsdfTest.test_node)
+        except Exception as e:
+            print(e)
+            exception_thrown = True
+
+        # Check that the insertion did not raise an exception
+        self.assertFalse(exception_thrown);
+        self.assertIsNotNone(node_id,
+                        "Node ID for inserted data is not None.")
+        self.assertTrue(type(node_id) == str,
+                        "Node ID for inserted data is a string.")
+
+    def testGetNode(self):
+        osdf = _get_osdf()
+
+        exception_thrown = False;
+        node_id = osdf.insert_node(OsdfTest.test_node)
+        retrieved = None
+
+        try:
+            retrieved = osdf.get_node(node_id)
+        except Exception as e:
+            exception_thrown = True
+
+        # Check that there were no issues
+        self.assertFalse(exception_thrown);
+
+        # Check that the retrieval yeilded some data
+        self.assertIsNotNone(retrieved);
+        self.assertTrue(type(retrieved) == dict,
+                        "Node retrieved has correct type.")
+
+        # Check that the data has the right structure
+        self.assertTrue('ns' in retrieved)
+        self.assertTrue('meta' in retrieved)
+        self.assertTrue('linkage' in retrieved)
+        self.assertTrue('ver' in retrieved)
+        self.assertTrue('acl' in retrieved)
+        self.assertTrue('id' in retrieved)
+
+        self.assertEqual(retrieved['id'], node_id,
+                         "Retrieved node has the right ID.")
+
+        # Do some cleanup after ourselves
+        try:
+            osdf.delete_node(node_id)
+        except:
+            # ignore any problems as they are not relevant to this
+            # particular test
+            pass
+
+    def testDeleteNode(self):
+        osdf = _get_osdf()
+
+        exception_thrown = False;
+        node_id = osdf.insert_node(OsdfTest.test_node)
+
+        try:
+            osdf.delete_node(node_id)
+        except Exception as e:
+            exception_thrown = True
+
+        # Check that the deletion did not raise an exception
+        self.assertFalse(exception_thrown);
+
+        # Now verify that the deletion actually work. A node
+        # retrieval on that node ID should faile...
+        get_success = True
+        try:
+            osdf.get_node(node_id)
+        except Exception as e:
+            get_success = False
+
+        self.assertFalse(get_success, "Deletion persisted in OSDF.")
+
+    def testQueryAllPages(self):
+        osdf = _get_osdf()
+
+        query = '{ "term" : { "node_type" : "project" }}'
+        namespace = "test"
 
-        self.assertTrue(type(results) == dict, "Query result is a list.")
+        results = osdf.query_all_pages(namespace, query)
 
-        self.assertTrue('result_count' in results, "Result contains 'result_count' key.")
-        self.assertTrue('results' in results, "Result contains 'results' key.")
-        self.assertTrue('page' in results, "Result contains 'page' key.")
+        self._examine_all_results(results, "ES QueryDSL")
 
 
 if __name__ == "__main__":