Merge pull request #204 from wltrimbl/master

Maintenance update

Author: wilke

examples/python/abundance_matrix.py

@@ -4,18 +4,20 @@
 
 from __future__ import print_function
 import sys
+import json
 from optparse import OptionParser
 from mglib import async_rest_api, sparse_to_dense, get_auth_token, API_URL
 
 DEBUG = 0
 
 if __name__ == '__main__':
-    usage = "usage: %prog -i <input sequence file> -o <output file>"
+    usage = "Usage: %prog [options]\nFunction: retrieves data from MG-RAST matrix API, unwraps into csv format"
     parser = OptionParser(usage)
     parser.add_option("-s", "--source", dest="source", default="RefSeq", help="Annotation source: RefSeq, GenBank, IMG, SEED, TrEMBL, SwissProt, PATRIC, KEG, RDP, Greengenes, LSU, SSU")
     parser.add_option("-g", "--grouplevel", dest="grouplevel", default="domain", help="Grouping level: strain, species, genus, family, order, class, phylum, domain / function, level1, level2, level3")
     parser.add_option("-i", "--hittype", dest="hittype", default="single", help="Hit type: all, single, lca")
     parser.add_option("-c", "--call", dest="call", default="organism", help="organism or function")
+    parser.add_option("-b", "--biom", dest="biom", action="store_true", help="biom output (csv defaut)")
     parser.add_option("-d", "--identity", dest="identity", default=1, help="% identity threshold")
     parser.add_option("-e", "--evalue", dest="evalue", default="1", help="organism or function")
     parser.add_option("-t", "--type", dest="resulttype", default="abundance", help="Result type: abundnaance, evalue, identity, or length")
@@ -36,17 +38,22 @@
     length = opts.length
     identity = opts.identity
     hittype = opts.hittype
+    biom = opts.biom
 # construct API call
     base_url = API_URL + "/matrix/organism"
     if opts.call == "function" or opts.source == "SubSystems":
         base_url = API_URL + "/matrix/function"
-    base_url = base_url + "?asynchronous=1&group_level=%s&result_type=%s&auth=%s&source=%s&evalue=%s&length=%s&identity=%s&hittype=%s&" % (group_level, result_type, key, source, evalue, length, identity, hittype)
+    base_url = base_url + "?asynchronous=1&group_level=%s&result_type=%s&source=%s&evalue=%s&length=%s&identity=%s&hittype=%s&" % (group_level, result_type, source, evalue, length, identity, hittype)
     URI = base_url + "&".join(["id=%s" % m for m in metagenomes.split(",")])
     print(URI, file=sys.stderr)
-    print("#"+ URI, file=sys.stdout)
 # retrieve the data by sending at HTTP GET request to the MG-RAST API
 
     jsonstructure = async_rest_api(URI, auth=key)
+    if biom:
+        print(json.dumps(jsonstructure))
+        sys.exit()
+    else:
+        print("#"+ URI, file=sys.stdout)
 
 # unpack and display the data table
     cols = [x["id"] for x in jsonstructure["columns"]]

examples/python/annotation_table.py

@@ -9,9 +9,8 @@
 
 DEBUG = 0
 
-
 if __name__ == '__main__':
-    usage = "usage: %prog -i <input sequence file> -o <output file>"
+    usage = "usage: %prog [options]\nFunction: retrieves and presents table of sequence IDs and annotation table results"
     parser = OptionParser(usage)
     parser.add_option("-s", "--source", dest="source", default="RefSeq", help="Annotation source: RefSeq, GenBank, IMG, SEED, TrEMBL, SwissProt, PATRIC, KEG, RDP, Greengenes, LSU, SSU")
     parser.add_option("-g", "--grouplevel", dest="grouplevel", default="domain", help="Grouping level: strain, species, genus, family, order, class, phylum, domain / function, level1, level2, level3")
@@ -36,7 +35,7 @@
 
 # construct API call
     base_url = API_URL + "/profile/{}".format(metagenomes)
-    base_url = base_url + "?asynchronous=1&group_level=%s&result_type=%s&auth=%s&source=%s&evalue=%s&" % (group_level, result_type, key, source, evalue)
+    base_url = base_url + "?asynchronous=1&group_level=%s&result_type=%s&source=%s&evalue=%s&" % (group_level, result_type, source, evalue)
     URI = base_url + "&".join(["id=%s" % m for m in metagenomes.split(",")])
     URI = base_url 
     print(URI, file=sys.stderr)

examples/python/list_all_mg.py

@@ -12,7 +12,7 @@ def printlist(js):
         if "public" in item.keys():
             public = item["public"]
         else:
-            public = False
+            public = "False"
         try:
             mg_name= item["name"]
             project_id = item["project_id"]

mglib/mglib.py

@@ -73,7 +73,10 @@ def body_from_url(url, accept, auth=None, data=None, debug=False, method=None):
 # return python struct from JSON output of MG-RAST or Shock API
 def obj_from_url(url, auth=None, data=None, debug=False, method=None):
     result = body_from_url(url, 'application/json', auth=auth, data=data, debug=debug, method=method)
-    obj = json.loads(result.read().decode("utf8"))
+    if result.headers["content-type"] == "application/x-download":
+        return(result.read()) 
+    else:
+        obj = json.loads(result.read().decode("utf8"))
     if obj is None:
         sys.stderr.write("ERROR: return structure not valid json format\n")
         sys.exit(1)
@@ -120,26 +123,31 @@ def async_rest_api(url, auth=None, data=None, debug=False, delay=60):
     submit = obj_from_url(url, auth=auth, data=data, debug=debug)
 # If "status" is nor present, or if "status" is somehow not "submitted" 
 # assume this is not an asynchronous call and it's done.
-    if ('status' in submit) and (submit['status'] == 'done') and ('url' in submit):
+    if ('status' in submit) and (submit['status'] != 'submitted') and ('data' in submit):
         return submit['data']
-    if not (('status' in submit) and (submit['status'] == 'submitted') and ('url' in submit)):
-        return submit
-    result = obj_from_url(submit['url'], debug=debug)
+#    if not (('status' in submit) and (submit['status'] == 'submitted') and ('url' in submit)):
+#        return submit  # No status, no url and no submitted
+    result = obj_from_url(submit['url'], auth=auth, debug=debug)
     try:
-        while result['status'] != 'done':
+        while result['status'] == 'submitted':
             if debug:
                 print("waiting %d seconds ..."%delay)
             time.sleep(delay)
-            result = obj_from_url(submit['url'], debug=debug)
+            result = obj_from_url(submit['url'], auth=auth, debug=debug)
     except KeyError:
         print("Error in response to "+url, file=sys.stderr)
         print("Does not contain 'status' field, likely API syntax error", file=sys.stderr)
         print(json.dumps(result), file=sys.stderr)
         sys.exit(1)
-    if 'ERROR' in result['data']:
-        sys.stderr.write("ERROR: %s\n" %result['data']['ERROR'])
-        print(json.dumps(result), file=sys.stderr)
-        sys.exit(1)
+    except TypeError:  # result isn't json, return it anyway
+        return(result.decode("utf8"))
+    try: 
+        if 'ERROR' in result['data']:
+            sys.stderr.write("ERROR: %s\n" %result['data']['ERROR'])
+            print(json.dumps(result), file=sys.stderr)
+            sys.exit(1)
+    except KeyError:  # result doesn't have "data"
+        return result    
     return result['data']
 
 # POST file to MG-RAST or Shock
@@ -407,7 +415,7 @@ def kbids_to_mgids(kbids):
 def kbid_lookup(ids, reverse=False):
     request = 'mg2kb' if reverse else 'kb2mg'
     post = json.dumps({'ids': ids}, separators=(',',':'))
-    data = obj_from_url(API_URL+'/job/'+request, data=post)
+    data = obj_from_url(API_URL+'/job/'+request, auth=auth, data=post)
     return data['data']
 
 def get_auth_token(opts=None):

scripts/mg-biom2metadata

@@ -13,14 +13,15 @@
 if __name__ == '__main__':
     usage = "usage: %prog [options]  URI"
     parser = ArgumentParser(usage)
-#    parser.add_argument("-v", "--verbose", dest="verbose", action="store_true")
+    parser.add_argument("-v", "--verbose", dest="verbose", action="store_true")
     parser.add_argument("-k", "--token", dest="token", type=str,
                       help="Auth token")
     parser.add_argument("URI", type=str, help="URI to query")
 
     opts = parser.parse_args()
     key = get_auth_token(opts)
-
+    if opts.verbose:
+        print("KEY = {}".format(key), file=sys.stderr)   
 # assign parameters
     URI = opts.URI
 
@@ -31,4 +32,7 @@
     jsonstructure = async_rest_api(URI, auth=key)
 
 # unpack and display the data table
-    print(json.dumps(jsonstructure))
+    if type(jsonstructure) == str:    # If we have data, not json structure
+        sys.stdout.write(jsonstructure)
+    else:
+        print(json.dumps(jsonstructure), file=sys.stdout)

scripts/mg-biom2taxa

@@ -4,5 +4,5 @@ RUN apt-get update && apt-get install -y git make python python-setuptools  gcc
 
 # setproctile  needs Python.h from python-dev
 # requests needs gcc to build
-RUN git clone http://github.com/wltrimbl/MG-RAST-Tools
+RUN git clone http://github.com/MG-RAST/MG-RAST-Tools
 RUN cd MG-RAST-Tools && make install