Merge pull request #199 from wltrimbl/master

Maintenance, removed dependency on legacy poster package

Author: wgerlach

.gitignore

@@ -7,6 +7,12 @@
 /dist/*
 /mglib.egg-info/*
 
+# testing files
+/.cache/*
+/.pytest_cache/*
+/htmlcov/*
+.coverage 
+
 # ignore files with # and ~
 *\#*
 *\~*

mglib/mglib.py

@@ -1,5 +1,4 @@
 from __future__ import print_function
-import io
 import os
 import sys
 import time
@@ -10,12 +9,6 @@
 import random
 import hashlib
 import subprocess
-try:
-    from StringIO import StringIO
-except ImportError:
-    from io import StringIO
-from poster.encode import multipart_encode
-from poster.streaminghttp import register_openers
 import requests
 from requests_toolbelt import MultipartEncoder
 
@@ -151,19 +144,19 @@ def async_rest_api(url, auth=None, data=None, debug=False, delay=60):
 
 # POST file to MG-RAST or Shock
 def post_file(url, keyname, filename, data={}, auth=None, debug=False):
-    register_openers()
     if debug:
-        print("data:\t"+json.dumps(data))
-    data[keyname] = open(filename)
-    datagen, header = multipart_encode(data)
+        print("post_file", url)
+    data[keyname] = (filename, open(filename, 'rb'))
+    datagen = MultipartEncoder(data)
+    header = {"Content-Type": datagen.content_type}
     if auth:
         header['Authorization'] = 'mgrast '+auth
     if debug:
-        print("header:\t"+json.dumps(header))
+        print("data:\t"+repr(data))
+        print("header:\t"+repr(header))
         print("url:\t"+url)
     try:
-        req = Request(url, datagen, header)
-        res = urlopen(req)
+        res = requests.post(url, data=datagen, headers=header, stream=True)
     except HTTPError as error:
         try:
             eobj = json.loads(error.read())
@@ -178,14 +171,11 @@ def post_file(url, keyname, filename, data={}, auth=None, debug=False):
     if not res:
         sys.stderr.write("ERROR: no results returned\n")
         sys.exit(1)
-    obj = json.loads(res.read().decode("utf8"))
+    obj = json.loads(res.content.decode("utf8"))
+    if debug:
+        print(json.dumps(obj))
     if obj is None:
         sys.stderr.write("ERROR: return structure not valid json format\n")
-    while True:
-        chunk = res.read(8192)
-        if not chunk:
-            break
-        handle.write(chunk.decode('utf8'))
     return(obj)
 
 # safe handling of stdout for piping

scripts/mg-export-research-object.py

@@ -6,19 +6,19 @@
 import yaml
 import shutil
 import hashlib
-from optparse import OptionParser
+from argparse import ArgumentParser
 from prettytable import PrettyTable
 from mglib import VERSION, get_auth_token, AUTH_LIST, API_URL, obj_from_url, file_from_url, random_str
 
-VERSION = 'alpha'
+RO_VERSION = 'alpha'
 
 prehelp = """
 NAME
     mg-export-research-object
 
 VERSION
-    %s
-
+    MGRAST-Tools %s
+    mg-export-research-object %s
 SYNOPSIS
     mg-export-research-object [ --help, --user <user>, --passwd <password>, --token <oAuth token>, --metagenome <metagenome id>, --dir <directory name> --list <list manifest>]
 
@@ -73,19 +73,19 @@ def edit_input(text, mg):
     return yaml.dump(info, allow_unicode=True, default_flow_style=False)
 
 def main(args):
-    OptionParser.format_description = lambda self, formatter: self.description
-    OptionParser.format_epilog = lambda self, formatter: self.epilog
-    parser = OptionParser(usage='', description=prehelp%VERSION, epilog=posthelp%AUTH_LIST)
-    parser.add_option("", "--url", dest="url", default=API_URL, help="MG-RAST API url")
-    parser.add_option("", "--user", dest="user", default=None, help="OAuth username")
-    parser.add_option("", "--passwd", dest="passwd", default=None, help="OAuth password")
-    parser.add_option("", "--token", dest="token", default=None, help="OAuth token")
-    parser.add_option("", "--metagenome", dest="metagenome", default=None, help="metagenome ID")
-    parser.add_option("", "--dir", dest="dir", default=".", help="directory to export to")
-    parser.add_option("", "--list", dest="list", action="store_true", default=False, help="list files in manifest")
+    ArgumentParser.format_description = lambda self, formatter: self.description
+    ArgumentParser.format_epilog = lambda self, formatter: self.epilog
+    parser = ArgumentParser(usage='', description=prehelp.format(VERSION, RO_VERSION), epilog=posthelp%AUTH_LIST)
+    parser.add_argument("--url", dest="url", default=API_URL, help="MG-RAST API url")
+    parser.add_argument("--user", dest="user", default=None, help="OAuth username")
+    parser.add_argument("--passwd", dest="passwd", default=None, help="OAuth password")
+    parser.add_argument("--token", dest="token", default=None, help="OAuth token")
+    parser.add_argument("--metagenome", dest="metagenome", default=None, help="metagenome ID")
+    parser.add_argument("--dir", dest="dir", default=".", help="directory to export to")
+    parser.add_argument("--list", dest="list", action="store_true", default=False, help="list files in manifest")
 
     # get inputs
-    (opts, args) = parser.parse_args()
+    opts = parser.parse_args()
     if not opts.metagenome:
         sys.stderr.write("ERROR: a metagenome id is required\n")
         return 1

scripts/mg-inbox.py

@@ -179,7 +179,7 @@ def upload(files):
         data = {
             "attributes_str": attr
         }
-        result = post_file(SHOCK_URL+"/node", fformat, f, data=data, auth=mgrast_auth['token'])
+        result = post_file(SHOCK_URL+"/node", fformat, f, data=data, auth=mgrast_auth['token'], debug=DEBUG)
         # compute file info
         info = obj_from_url(API_URL+"/inbox/info/"+result['data']['id'], auth=mgrast_auth['token'], debug=DEBUG)
         print(info['status'])
@@ -212,7 +212,7 @@ def upload_archive(afile):
         "file_name": os.path.basename(afile),
         "attributes_str": attr
     }
-    result = post_file(SHOCK_URL+"/node", "upload", afile, data=data, auth=mgrast_auth['token'])
+    result = post_file(SHOCK_URL+"/node", "upload", afile, data=data, auth=mgrast_auth['token'], debug=DEBUG)
     data = {
         "unpack_node": result['data']['id'],
         "archive_format": aformat,
@@ -286,7 +286,7 @@ def compute(action, files, retain, joinfile, rc_index):
             "rc_index": 1 if rc_index else 0
         }
     else:
-        sys.stderr.write("ERROR: invalid compute ion. use one of: %s\n"%", ".join(compute_options))
+        sys.stderr.write("ERROR: invalid compute action. use one of: %s\n"%", ".join(compute_actions))
     info = obj_from_url(API_URL+"/inbox/"+action, data=json.dumps(data), auth=mgrast_auth['token'], debug=DEBUG)
     print(info['status'])
 
@@ -408,8 +408,8 @@ def main(args):
             sys.stderr.write("ERROR: validate missing file\n")
             return 1
     elif action == "compute":
-        if (len(args) < 2) or (args[1] not in compute_options):
-            sys.stderr.write("ERROR: invalid compute ion. use one of: %s\n"%", ".join(compute_options))
+        if (len(args) < 2) or (args[1] not in compute_actions):
+            sys.stderr.write("ERROR: invalid compute action. use one of: %s\n"%", ".join(compute_actions))
             return 1
         if (((args[1] == "sff2fastq") and (len(args) != 3)) or
              ((args[1] == "demultiplex") and (len(args) < 4)) or

scripts/mg-project.py

@@ -5,6 +5,11 @@
 from argparse import ArgumentParser
 from mglib import get_auth_token, post_file, obj_from_url, VERSION, AUTH_LIST, API_URL
 
+try:
+    import raw_input as input  # python2 
+except:
+    pass
+
 prehelp = """
 NAME
     mg-project
@@ -59,6 +64,7 @@ def main(args):
     parser.add_argument("-f", "--file", dest="mdfile", default=None, help="metadata .xlsx file")
     parser.add_argument("--taxa", dest="taxa", default=None, help="metagenome_taxonomy for project: http://www.ebi.ac.uk/ena/data/view/Taxon:408169")
     parser.add_argument("--debug", dest="debug", action="store_true", default=False, help="Run in debug mode")
+    parser.add_argument("-v", "--verbose", dest="verbose", action="store_true", default=False, help="Verbose STDOUT")
     parser.add_argument("args",type=str, nargs="+", help="Action (" + ",".join(valid_actions)+")" )
 
     # get inputs
@@ -78,11 +84,11 @@ def main(args):
         sys.stderr.write("ERROR: missing Project ID\n")
         return 1
     pid = args[1]
-    
+    DEBUG = opts.verbose + opts.debug 
     # get token
     token = get_auth_token(opts)
     if not token:
-        token = raw_input('Enter your MG-RAST auth token: ')
+        token = input('Enter your MG-RAST auth token: ')
 
     # actions
     if action == "get-info":
@@ -92,7 +98,7 @@ def main(args):
         data = obj_from_url(opts.url+'/metadata/export/'+pid, auth=token)
         print(json.dumps(data, sort_keys=True, indent=4))
     elif action == "update-metadata":
-        result = post_file(opts.url+'/metadata/update', 'upload', opts.mdfile, auth=token, data=json.dumps({'project': pid}, separators=(',',':')))
+        result = post_file(opts.url+'/metadata/update', 'upload', opts.mdfile, auth=token, data=json.dumps({'project': pid}, separators=(',',':')), debug=DEBUG)
         print(json.dumps(data, sort_keys=True, indent=4))
     elif action == "make-public":
         data = obj_from_url(opts.url+'/project/'+pid+'/makepublic', auth=token)

scripts/mg-search-metagenomes.py

@@ -11,6 +11,8 @@
 VERSION
     %s
 
+    %s 
+
 SYNOPSIS
     mg-search-metagenomes [ --help, --user <user>, --passwd <password>, --token <oAuth token>, --order <field name> --direction <cv: 'asc', 'desc'> --match <cv: 'all, 'any'> --public ]
 
@@ -55,8 +57,10 @@ def main(args):
     parser.add_argument("--user", dest="user", default=None, help="OAuth username")
     parser.add_argument("--passwd", dest="passwd", default=None, help="OAuth password")
     parser.add_argument("--token", dest="token", default=None, help="OAuth token")
+    parser.add_argument("--limit", dest="limit", type=int, default=15, help="Number of results to show, if > 50 will use paginated queries to get all, default 15")
     parser.add_argument("--order", dest="order", default=None, help="field metagenomes are ordered by, default is no ordering")
     parser.add_argument("--direction", dest="direction", default="asc", help="direction of order. 'asc' for ascending order, 'desc' for descending order, default is asc")
+    parser.add_argument("--public", dest="public", action="store_true", default=False, help="return both private and pubulic data if using authenticated search, default is private only.  Non-authenticated search only returns public.")
     parser.add_argument("--match", dest="match", default="all", help="search logic. 'all' for metagenomes that match all search parameters, 'any' for metagenomes that match any search parameters, default is all")
     parser.add_argument("--status", dest="status", default="public", help="types of metagenomes to return. 'both' for all data (public and private), 'public' for public data, 'private' for users private data, default is public")
     parser.add_argument("--verbosity", dest="verbosity", default='minimal', help="amount of information to display. use keyword 'minimal' for id and name, use keyword 'full' for MIxS GSC metadata, default is minimal")
@@ -74,8 +78,6 @@ def main(args):
     maxLimit = 50
     params = [ ('limit', opts.limit if opts.limit < maxLimit else maxLimit),
                ('public', 'yes' if opts.public or (not token) else 'no') ]
-    if opts.index:
-        params.append( ('index', opts.index) )
     for sfield in SEARCH_FIELDS:
         if hasattr(opts, sfield) and getattr(opts, sfield):
             params.append((sfield, getattr(opts, sfield)))

setup.py

@@ -13,8 +13,7 @@
     packages=['mglib'],
     scripts=glob.glob('scripts/[a-z]*') + glob.glob('examples/python/*.py'),
     install_requires=  ['prettytable >= 0.7', 
-                          'requests_toolbelt >= 0.8', 'setuptools > 28.0',
-                         'poster >= 0.8.1' ]
+                          'requests_toolbelt >= 0.8', 'setuptools > 28.0' ]
      )