Merge pull request #12 from BackofenLab/add_setup

Add setup for workflows and datasets

Author: bgruening

Dockerfile

@@ -10,11 +10,27 @@ ENV ENABLE_TTS_INSTALL True
 # Enable Conda dependency resolution
 ENV GALAXY_CONFIG_CONDA_AUTO_INSTALL=True \
     GALAXY_CONFIG_CONDA_AUTO_INIT=True \
-    GALAXY_CONFIG_USE_CACHED_DEPENDENCY_MANAGER=True    
+    GALAXY_CONFIG_USE_CACHED_DEPENDENCY_MANAGER=True
 
 # Install tools
 ADD graphclust.yml $GALAXY_ROOT/tools.yaml
 RUN install-tools $GALAXY_ROOT/tools.yaml && \
     /tool_deps/_conda/bin/conda clean --tarballs
 
 ADD tour_graphclust_wf.yaml $GALAXY_ROOT/config/plugins/tours/graphclust.wf.yaml
+
+# Data libraries
+ADD setup_data_libraries.py $GALAXY_ROOT/setup_data_libraries.py
+ADD library_data.yaml $GALAXY_ROOT/library_data.yaml
+
+# Hacky script to import workflows into Galaxy after installation. I would argue this step is redundant.
+ADD import_workflows.py $GALAXY_ROOT/import_workflows.py
+ADD GraphClust_one.ga $GALAXY_ROOT/GraphClust_one.ga
+ADD GraphClust_two.ga $GALAXY_ROOT/GraphClust_two.ga
+
+# Download training data and populate the data library
+RUN startup_lite && \
+    sleep 30 && \
+    . $GALAXY_VIRTUAL_ENV/bin/activate && \
+    python $GALAXY_ROOT/setup_data_libraries.py -i $GALAXY_ROOT/library_data.yaml && \
+    python $GALAXY_ROOT/import_workflows.py

import_workflows.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python
+
+import os
+from bioblend import galaxy
+admin_email = os.environ.get('GALAXY_DEFAULT_ADMIN_USER', 'admin@galaxy.org')
+admin_pass = os.environ.get('GALAXY_DEFAULT_ADMIN_PASSWORD', 'admin')
+url = "http://localhost:8080"
+gi = galaxy.GalaxyInstance(url=url, email=admin_email, password=admin_pass)
+
+wf = galaxy.workflows.WorkflowClient(gi)
+wf.import_workflow_from_local_path('/galaxy-central/GraphClust_two.ga')
+wf.import_workflow_from_local_path('/galaxy-central/GraphClust_one.ga')

library_data.yaml

@@ -0,0 +1,5 @@
+libraries:
+    - name: "Test data"
+      files:
+        - https://raw.githubusercontent.com/eteriSokhoyan/test-data/master/cliques-high-representatives.fa
+        - https://raw.githubusercontent.com/eteriSokhoyan/test-data/master/cliques-low-representatives.fa

setup_data_libraries.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+
+import os
+import time
+import yaml
+import argparse
+import subprocess
+import logging as log
+from bioblend import galaxy
+from subprocess import CalledProcessError
+
+
+def main( data ):
+    """
+    Load files into a Galaxy data library.
+    By default all test-data tools from all installed tools
+    will be linked into a data library.
+    """
+
+    log.info("Importing data libraries.")
+
+    url = "http://localhost:8080"
+    # The environment variables are set by the parent container
+    admin_email = os.environ.get('GALAXY_DEFAULT_ADMIN_USER', 'admin@galaxy.org')
+    admin_pass = os.environ.get('GALAXY_DEFAULT_ADMIN_PASSWORD', 'admin')
+
+    # Establish connection to galaxy instance
+    gi = galaxy.GalaxyInstance(url=url, email=admin_email, password=admin_pass)
+
+    jc = galaxy.jobs.JobsClient(gi)
+
+    folders = dict()
+
+    libraries = yaml.load(data)
+    for lib in libraries['libraries']:
+        folders[lib['name']] = lib['files']
+
+    if folders:
+        log.info("Create 'Test Data' library.")
+        lib = gi.libraries.create_library('Training Data', 'Data pulled from online archives.')
+        lib_id = lib['id']
+
+        for fname, urls in folders.items():
+            log.info("Creating folder: %s" % fname)
+            folder = gi.libraries.create_folder( lib_id, fname )
+            for url in urls:
+                gi.libraries.upload_file_from_url(
+                    lib_id,
+                    url,
+                    folder_id = folder[0]['id'],
+                )
+        
+        no_break = True
+        while True:
+            no_break = False
+            for job in jc.get_jobs():
+                if job['state'] != 'ok':
+                    no_break = True
+            if not no_break:
+                break
+            time.sleep(3)
+
+
+        time.sleep(20)
+        log.info("Finished importing test data.")
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(
+        description='Populate the Galaxy data library with test data.'
+    )
+    parser.add_argument("-v", "--verbose", help="Increase output verbosity.",
+                    action="store_true")
+    parser.add_argument('-i', '--infile', type=argparse.FileType('r'))
+
+    #TODO:  Add options to override the admin_user and admin_password + specify 
+    #       files to upload via command line interface.
+
+    args = parser.parse_args()
+    if args.verbose:
+        log.basicConfig(level=log.DEBUG)
+
+    main( args.infile )
+