Merge pull request #206 from wltrimbl/master

Client maintenence

Author: wltrimbl

examples/python/abundance_matrix.py

@@ -55,23 +55,23 @@
     else:
         print("#"+ URI, file=sys.stdout)
 
-# unpack and display the data table
-    cols = [x["id"] for x in jsonstructure["columns"]]
-    rows = [x["id"] for x in jsonstructure["rows"]]
-    matrixtype = jsonstructure["type"]
-
+    matrixtype = jsonstructure["data"]["matrix_type"]
     if matrixtype == "sparse":
         data = sparse_to_dense(jsonstructure["data"], len(rows), len(cols))
     else:
         data = jsonstructure["data"]
+# unpack and display the data table
+    cols = [x["id"] for x in data["columns"]]
+    rows = [x["id"] for x in data["rows"]]
+
 
     if DEBUG:
         print(jsonstructure)
         print("COLS", cols)
         print("ROWS", rows)
         print("TYPE", matrixtype)
         print("DATA", data)
-    h = data
+    h = data["data"]
 
     sys.stdout.write("Taxon\t")
     for j in range(0, len(cols)):

examples/python/annotation_table.py

@@ -47,6 +47,7 @@
 #    rows = [x["id"] for x in jsondata["rows"]]
 
     data = jsondata # ["data"]
+    data = jsondata["data"]
 
     if DEBUG:
         print(jsonstructure)

examples/python/list_all_mg.py

@@ -3,6 +3,7 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 import sys
+import time
 
 from mglib import get_auth_token, obj_from_url, API_URL, urlencode
 
@@ -18,7 +19,7 @@ def printlist(js):
             project_id = item["project_id"]
             project_name = item["project_name"]
         except KeyError:
-            sys.stderr.write(repr(item))
+            sys.stderr.write(repr(item) + "\n")
         sys.stdout.write(("\t".join([item["metagenome_id"],
 #                        str(len(item.keys())),
                          repr(public), item["created_on"],
@@ -32,7 +33,7 @@ def printlist(js):
 limit = 1000 # initial call
 
 # construct API call
-
+# public = 0 means "don't show public metagenomes"
 parameters = {"limit": limit, "order":"created_on", "direction": "asc", "public": "1"}
 API_URL= "https://api.mg-rast.org/"
 
@@ -45,12 +46,17 @@ def printlist(js):
 total_count = int(jsonstructure["total_count"])
 sys.stderr.write("Total number of records: {:d}\n".format(total_count))
 
-for i in range(0, int(total_count / limit) +1):
-    sys.stderr.write("Page {:d}\t".format(i))
+for i in range(0, int(total_count / limit) +2):
+#    sys.stderr.write("Page {:d}\t".format(i))
+    sys.stderr.write("Page {:d}\t{}\n".format(i, base_url))
     jsonstructure = obj_from_url(base_url, auth=key)
     printlist(jsonstructure)
-    try:
+    time.sleep(1)
+    if "next" in jsonstructure.keys():
         next_url = jsonstructure["next"]
         base_url = next_url
-    except KeyError:
+        continue
+    else:
+        sys.stderr.write("No next, page {} url {} \n".format(i, base_url))
+        sys.stderr.write(repr(jsonstructure)) 
         break

mglib/mglib.py

@@ -72,13 +72,24 @@ def body_from_url(url, accept, auth=None, data=None, debug=False, method=None):
 
 # return python struct from JSON output of MG-RAST or Shock API
 def obj_from_url(url, auth=None, data=None, debug=False, method=None):
-    result = body_from_url(url, 'application/json', auth=auth, data=data, debug=debug, method=method)
-    if result.headers["content-type"] == "application/x-download":
-        return(result.read()) 
+    try:
+        result = body_from_url(url, 'application/json', auth=auth, data=data, debug=debug, method=method)
+        read = result.read()
+    except:  # try one more time  ConnectionResetError is incompatible with python2
+        result = body_from_url(url, 'application/json', auth=auth, data=data, debug=debug, method=method)
+        read = result.read()
+    if result.headers["content-type"] == "application/x-download" or result.headers["content-type"] == "application/octet-stream":
+        return(read)   # Watch out!
+    if result.headers["content-type"][0:9] == "text/html":  # json decoder won't work
+        return(read)   # Watch out!
+    if result.headers["content-type"] == "application/json":  # If header is set, this should work 
+        data = read.decode("utf8")
+        obj = json.loads(data)
     else:
-        obj = json.loads(result.read().decode("utf8"))
+        data = read.decode("utf8")
+        obj = json.loads(data)
     if obj is None:
-        sys.stderr.write("ERROR: return structure not valid json format\n")
+        sys.stderr.write("ERROR: return structure not valid json format\n" + repr(data))
         sys.exit(1)
     if len(list(obj.keys())) == 0:
         if debug:
@@ -121,33 +132,44 @@ def async_rest_api(url, auth=None, data=None, debug=False, delay=60):
     except:
         parameters = {"asynchronous": 1}
     submit = obj_from_url(url, auth=auth, data=data, debug=debug)
-# If "status" is nor present, or if "status" is somehow not "submitted" 
+# If "status" is nor present, or if "status" is somehow not "submitted"
 # assume this is not an asynchronous call and it's done.
-    if ('status' in submit) and (submit['status'] != 'submitted') and ('data' in submit):
-        return submit['data']
+    if type(submit) == bytes:   # can't decode
+        try: 
+            return decode("utf-8", submit)
+        except:
+            return submit
+    if ('status' in submit) and (submit['status'] != 'submitted') and (submit['status'] != "processing") and ('data' in submit):
+        return submit
+    if not ('url' in submit.keys()):
+        return submit
 #    if not (('status' in submit) and (submit['status'] == 'submitted') and ('url' in submit)):
 #        return submit  # No status, no url and no submitted
     result = obj_from_url(submit['url'], auth=auth, debug=debug)
-    try:
-        while result['status'] == 'submitted':
+    if type(result) is bytes:
+        return(result)
+    if 'status' in result.keys():
+        while result['status'] == 'submitted' or result['status'] == "processing":
             if debug:
                 print("waiting %d seconds ..."%delay)
             time.sleep(delay)
             result = obj_from_url(submit['url'], auth=auth, debug=debug)
-    except KeyError:
+    if 'url' in result.keys() or 'next' in result.keys(): # does not need to wait
+        return(result)
+    try:
         print("Error in response to "+url, file=sys.stderr)
-        print("Does not contain 'status' field, likely API syntax error", file=sys.stderr)
+        print("Does not contain 'status' or 'next' field, likely API syntax error", file=sys.stderr)
         print(json.dumps(result), file=sys.stderr)
         sys.exit(1)
     except TypeError:  # result isn't json, return it anyway
         return(result.decode("utf8"))
-    try: 
+    try:
         if 'ERROR' in result['data']:
             sys.stderr.write("ERROR: %s\n" %result['data']['ERROR'])
             print(json.dumps(result), file=sys.stderr)
             sys.exit(1)
     except KeyError:  # result doesn't have "data"
-        return result    
+        return result
     return result['data']
 
 # POST file to MG-RAST or Shock
@@ -214,14 +236,15 @@ def sparse_to_dense(sMatrix, rmax, cmax):
 # transform BIOM format to tabbed table
 # returns max value of matrix
 def biom_to_tab(biom, hdl, rows=None, use_id=True, col_name=False):
+    assert 'matrix_type' in biom.keys(), repr(biom)
     if biom['matrix_type'] == 'sparse':
         matrix = sparse_to_dense(biom['data'], biom['shape'][0], biom['shape'][1])
     else:
         matrix = biom['data']
     if col_name:
-        hdl.write( "\t%s\n" %"\t".join([c['name'] for c in biom['columns']]) )
+        hdl.write("\t%s\n" %"\t".join([c['name'] for c in biom['columns']]))
     else:
-        hdl.write( "\t%s\n" %"\t".join([c['id'] for c in biom['columns']]) )
+        hdl.write("\t%s\n" %"\t".join([c['id'] for c in biom['columns']]))
     rowmax = []
     for i, row in enumerate(matrix):
         name = biom['rows'][i]['id']
@@ -231,7 +254,7 @@ def biom_to_tab(biom, hdl, rows=None, use_id=True, col_name=False):
             continue
         try:
             rowmax.append(max(row))
-            hdl.write( "%s\t%s\n" %(name, "\t".join(map(str, row))) )
+            hdl.write("%s\t%s\n" %(name, "\t".join(map(str, row))))
         except:
             try:
                 hdl.close()
@@ -262,6 +285,7 @@ def profile_to_matrix(p):
     p['matrix_element_type'] = 'int'
     p['matrix_element_value'] = 'abundance'
     p['date'] = time.strftime("%Y-%m-%d %H:%M:%S")
+    assert 'matrix_type' in p.keys(), repr(p)
     if p['matrix_type'] == 'sparse':
         p['data'] = sparse_to_dense(p['data'], p['shape'][0], p['shape'][1])
     if trim:
@@ -301,6 +325,7 @@ def merge_biom(b1, b2):
                "id": b1['id']+'_'+b2['id'],
                "type": b1['type'] }
     # make sure we are dense
+    assert 'matrix_type' in b2.keys(), repr(b2)
     if b2['matrix_type'] == 'sparse':
         b2['data'] = sparse_to_dense(b2['data'], b2['shape'][0], b2['shape'][1])
     # get lists of ids
@@ -352,14 +377,15 @@ def biom_to_matrix(biom, col_name=False, sig_stats=False):
     except KeyError:
         rows = [r['id'] for r in biom['rows']]
 #        rows = [";".join(r['metadata']['hierarchy']) for r in biom['rows']]
+    assert "matrix_type" in biom.keys(), repr(biom)
     if biom['matrix_type'] == 'sparse':
         data = sparse_to_dense(biom['data'], len(rows), len(cols))
     else:
         data = biom['data']
     if sig_stats and ('significance' in biom['rows'][0]['metadata']) and (len(biom['rows'][0]['metadata']['significance']) > 0):
-        cols.extend( [s[0] for s in biom['rows'][0]['metadata']['significance']] )
+        cols.extend([s[0] for s in biom['rows'][0]['metadata']['significance']] )
         for i, r in enumerate(biom['rows']):
-            data[i].extend( [s[1] for s in r['metadata']['significance']] )
+            data[i].extend([s[1] for s in r['metadata']['significance']] )
     return rows, cols, data
 
 # transform tabbed table to matrix in json format
@@ -382,7 +408,7 @@ def sub_matrix(matrix, ncols):
         return matrix
     sub = list()
     for row in matrix:
-        sub.append( row[:ncols] )
+        sub.append(row[:ncols] )
     return sub
 
 # return KBase id for MG-RAST id

scripts/mg-abundant-functions.py

@@ -94,8 +94,10 @@ def main(args):
         data = obj_from_url(url)
         level = 'level4' if opts.level == 'function' else opts.level
         sub_ann = set(map(lambda x: x[level], data['data']))
-    
+    biomorig = biom
+    biom = biomorig["data"] 
     # sort data
+    assert "matrix_type" in biom.keys(), repr(biom)
     if biom["matrix_type"] == "sparse":
         for d in sorted(biom['data'], key=itemgetter(2), reverse=True):
             name = biom['rows'][d[0]]['id']  # if opts.source != 'Subsystems' else biom['rows'][d[0]]['metadata']['ontology'][-1]

scripts/mg-abundant-taxa.py

@@ -92,6 +92,8 @@ def main(args):
         url = opts.url+'/m5nr/taxonomy?'+urlencode(params, True)
         data = obj_from_url(url)
         sub_ann = set(map(lambda x: x[opts.level], data['data']))
+    biomorig = biom
+    biom = biomorig["data"]
     if biom['matrix_type'] == "dense":
         data = biom['data']
     else:

scripts/mg-compare-functions.py

@@ -166,7 +166,7 @@ def main(args):
     if opts.format == 'biom':
         out_hdl.write(json.dumps(biom)+"\n")
     else:
-        biom_to_tab(biom, out_hdl, rows=sub_ann)
+        biom_to_tab(biom["data"], out_hdl, rows=sub_ann)
 
     out_hdl.close()
     return 0

scripts/mg-download.py

@@ -38,10 +38,13 @@
 
 # download a file
 def file_download(auth, info, dirpath="."):
-    fhandle = open(os.path.join(dirpath, info['file_name']), 'w')
     sys.stdout.write("Downloading %s for %s ... "%(info['file_name'], info['id']))
-    file_from_url(info['url'], fhandle, auth=auth)
-    fhandle.close()
+    if "url" in info.keys():  # all is well
+        fhandle = open(os.path.join(dirpath, info['file_name']), 'w')
+        file_from_url(info['url'], fhandle, auth=auth)
+        fhandle.close()
+    else:   # Don't open empty file if download doesn't have url
+        sys.stderr.write("WARNING Download info does not contain url.  Possibly datasets pre- human screening?\n" + repr(info)+"\n")
     sys.stdout.write("Done\n")
 
 def main(args):
@@ -54,42 +57,47 @@ def main(args):
     parser.add_argument("--token", dest="token", default=None, help="OAuth token")
     parser.add_argument("--project", dest="project", default=None, help="project ID")
     parser.add_argument("--metagenome", dest="metagenome", default=None, help="metagenome ID")
-    parser.add_argument("--file", dest="file", default=None, help="file ID for given project or metagenome")
+    parser.add_argument("--file", dest="file", default="299.1", help="file ID for given project or metagenome")
     parser.add_argument("--dir", dest="dir", default=".", help="directory to do downloads")
     parser.add_argument("--list", dest="list", action="store_true", default=False, help="list files and their info for given ID")
-    
+
     # get inputs
     opts = parser.parse_args()
-    if not (opts.project or opts.metagenome):
+    PROJECT = opts.project
+    DOWNDIR = opts.dir
+    METAGENOME = opts.metagenome
+    LIST = opts.list
+    FILE = opts.file
+    URL = opts.url
+
+    if not (PROJECT or METAGENOME):
         sys.stderr.write("ERROR: a project or metagenome id is required\n")
         return 1
-    if not os.path.isdir(opts.dir):
-        sys.stderr.write("ERROR: dir '%s' does not exist\n"%opts.dir)
+    if not os.path.isdir(DOWNDIR):
+        sys.stderr.write("ERROR: dir '%s' does not exist\n"%DOWNDIR)
         return 1
-    downdir = opts.dir
-    
+
     # get auth
     token = get_auth_token(opts)
-    
+
     # get metagenome list
     mgs = []
-    if opts.project:
-        url  = opts.url+'/project/'+opts.project+'?verbosity=full'
+    if PROJECT:
+        url = URL +'/project/'+PROJECT+'?verbosity=full'
         data = obj_from_url(url, auth=token)
         for mg in data['metagenomes']:
             mgs.append(mg["metagenome_id"])
-    elif opts.metagenome:
-        mgs.append(opts.metagenome)
-    
+    elif METAGENOME:
+        mgs.append(METAGENOME)
     # get file lists
     all_files = {}
     for mg in mgs:
-        url  = opts.url+'/download/'+mg
+        url = URL + '/download/' + mg
         data = obj_from_url(url, auth=token)
         all_files[mg] = data['data']
-    
+
     # just list
-    if opts.list:
+    if LIST:
         pt = PrettyTable(["Metagenome", "File Name", "File ID", "Checksum", "Byte Size"])
         for mg, files in all_files.items():
             for f in files:
@@ -99,27 +107,31 @@ def main(args):
         pt.align['Byte Size'] = "r"
         print(pt)
         return 0
-    
+
     # download all in dirs by ID
-    if opts.project:
-        downdir = os.path.join(downdir, opts.project)
-        if not os.path.isdir(downdir):
-            os.mkdir(downdir)
+    if PROJECT:
+        DOWNDIR = os.path.join(DOWNDIR, PROJECT)
+        if not os.path.isdir(DOWNDIR):
+            os.mkdir(DOWNDIR)
     for mg, files in all_files.items():
-        mgdir = os.path.join(downdir, mg)
+        mgdir = os.path.join(DOWNDIR, mg)
         if not os.path.isdir(mgdir):
             os.mkdir(mgdir)
+        filecount = 0
         for f in files:
-            if opts.file:
-                if f['file_id'] == opts.file:
+            if FILE:
+                if f['file_id'] == FILE:
+                    filecount += 1
                     file_download(token, f, dirpath=mgdir)
-                elif f['file_name'] == opts.file:
+                elif f['file_name'] == FILE:
+                    filecount += 1
                     file_download(token, f, dirpath=mgdir)
             else:
                 file_download(token, f, dirpath=mgdir)
-    
+                filecount += 1
+        if filecount == 0:
+            sys.exit("Didn't find file number " + FILE)
     return 0
 
-
 if __name__ == "__main__":
     sys.exit(main(sys.argv))