update scripts

Author: rcannood

scripts/get_task_execution.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+viash run src/core/fetch_task_execution/config.vsh.yaml -- \
+  --input s3://openproblems-nextflow/work/f6/8565066aee4771cc2790b92b4ac660 \
+  --output temp_debug_execution \
+  --aws_profile op \
+  --aws_credentials ~/.aws/credentials

src/core/fetch_task_execution/config.vsh.yaml

@@ -12,6 +12,11 @@ arguments:
     required: false
     direction: input
     description: The AWS profile to use for fetching the task execution directory.
+  - type: file
+    name: --aws_credentials
+    required: false
+    direction: input
+    description: The AWS credentials file to use for fetching the task execution directory.
   - type: file
     name: --output
     required: true
@@ -21,6 +26,7 @@ resources:
   - type: bash_script
     path: script.sh
 engines:
-  - type: native
+  - type: docker
+    image: amazon/aws-cli:latest
 runners:
   - type: executable

src/core/fetch_task_execution/script.sh

@@ -1,15 +1,24 @@
 ## VIASH START
 par_input=s3://openproblems-nextflow/work/f6/8565066aee4771cc2790b92b4ac660
 par_aws_profile=op
+par_aws_credentials=~/.aws/credentials
 par_output=debug
 ## VIASH END
 
-aws s3 sync \
-  ${par_aws_profile:+--profile $par_aws_profile} \
-  "$par_input" "$par_output"
+if [ -n "$par_aws_credentials" ]; then
+  export AWS_SHARED_CREDENTIALS_FILE="$par_aws_credentials"
+fi
+
+if [ -n "$par_aws_profile" ]; then
+  export AWS_PROFILE="$par_aws_profile"
+fi
+
+aws s3 sync "$par_input" "$par_output"
 
 cd "$par_output"
 
+# Replace the miniconda aws with the system aws
 sed -i 's#/home/ec2-user/miniconda/bin/aws#aws#g' .command.run
+# sed -i 's# s3 cp # s3 sync #g' .command.run
 
-AWS_DEFAULT_PROFILE=${par_aws_profile} bash .command.run nxf_stage
+bash .command.run nxf_stage