o2.conf

##################################
# Cromwell Reference Config File #
##################################

# This is the reference config file that contains all the default settings.
# Make your edits/overrides in your application.conf.

webservice {
  port = 8000
  interface = 0.0.0.0
  binding-timeout = 5s
  instance.name = "reference"
}

akka {
  actor.default-dispatcher.fork-join-executor {
    # Number of threads = min(parallelism-factor * cpus, parallelism-max)
    # Below are the default values set by Akka, uncomment to tune these

    #parallelism-factor = 3.0
    #parallelism-max = 64
  }

  dispatchers {
    # A dispatcher for actors performing blocking io operations
    # Prevents the whole system from being slowed down when waiting for responses from external resources for instance
    io-dispatcher {
      type = Dispatcher
      executor = "fork-join-executor"
      # Using the forkjoin defaults, this can be tuned if we wish
    }

    # A dispatcher for actors handling API operations
    # Keeps the API responsive regardless of the load of workflows being run
    api-dispatcher {
      type = Dispatcher
      executor = "fork-join-executor"
    }

    # A dispatcher for engine actors
    # Because backends behaviour is unpredictable (potentially blocking, slow) the engine runs
    # on its own dispatcher to prevent backends from affecting its performance.
    engine-dispatcher {
      type = Dispatcher
      executor = "fork-join-executor"
    }

    # A dispatcher used by supported backend actors
    backend-dispatcher {
      type = Dispatcher
      executor = "fork-join-executor"
    }

    # A dispatcher used for the service registry
    service-dispatcher {
      type = Dispatcher
      executor = "fork-join-executor"
    }

    # Note that without further configuration, all other actors run on the default dispatcher
  }
}

system {
  # If 'true', a SIGINT will trigger Cromwell to attempt to abort all currently running jobs before exiting
  #abort-jobs-on-terminate = false

  # If 'true' then when Cromwell starts up, it tries to restart incomplete workflows
  workflow-restart = true

  # Cromwell will cap the number of running workflows at N
  max-concurrent-workflows = 5000

  # Cromwell will launch up to N submitted workflows at a time, regardless of how many open workflow slots exist
  max-workflow-launch-count = 50

  # Number of seconds between workflow launches
  new-workflow-poll-rate = 20

  # Since the WorkflowLogCopyRouter is initialized in code, this is the number of workers
  number-of-workflow-log-copy-workers = 10

  # Default number of cache read workers
  number-of-cache-read-workers = 25
  
  io {
    # Global Throttling - This is mostly useful for GCS and can be adjusted to match
    # the quota availble on the GCS API
    number-of-requests = 100000
    per = 100 seconds
    
    # Number of times an I/O operation should be attempted before giving up and failing it.
    number-of-attempts = 5
  }

  input-read-limits {
    lines = 1280000
    bool = 7
    int = 19
    float = 50
    string = 1280000
    json = 1280000
    tsv = 1280000
    map = 1280000
    object = 1280000
  }
  
  job-rate-control {
    jobs = 2
    per = 1 second
  }
}

workflow-options {
  # These workflow options will be encrypted when stored in the database
  encrypted-fields: []

  # AES-256 key to use to encrypt the values in `encrypted-fields`
  base64-encryption-key: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="

  # Directory where to write per workflow logs
  workflow-log-dir: "cromwell-workflow-logs"

  # When true, per workflow logs will be deleted after copying
  workflow-log-temporary: true

  # Workflow-failure-mode determines what happens to other calls when a call fails. Can be either ContinueWhilePossible or NoNewCalls.
  # Can also be overridden in workflow options. Defaults to NoNewCalls. Uncomment to change:
  #workflow-failure-mode: "ContinueWhilePossible"
}

# Optional call-caching configuration.
call-caching {
  # Allows re-use of existing results for jobs you've already run
  # (default: false)
  enabled = true

  # Whether to invalidate a cache result forever if we cannot reuse them. Disable this if you expect some cache copies
  # to fail for external reasons which should not invalidate the cache (e.g. auth differences between users):
  # (default: true)
  invalidate-bad-cache-results = true
}

google {

  application-name = "cromwell"

  auths = [
    {
      name = "application-default"
      scheme = "application_default"
    },
    #{
    #  name = "user-via-refresh"
    #  scheme = "refresh_token"
    #  client-id = "secret_id"
    #  client-secret = "secret_secret"
    #},
    #{
    #  name = "service-account"
    #  scheme = "service_account"
    #  service-account-id = "my-service-account"
    #  pem-file = "/path/to/file.pem"
    #}
  ]
}

docker {
  // Set this to match your available quota against the Google Container Engine API
  gcr-api-queries-per-100-seconds = 1000
  // Time in minutes before an entry expires from the docker hashes cache and needs to be fetched again
  cache-entry-ttl = "20 minutes"
  // Maximum number of elements to be kept in the cache. If the limit is reached, old elements will be removed from the cache
  cache-size = 200
}

engine {
  # This instructs the engine which filesystems are at its disposal to perform any IO operation that it might need.
  # For instance, WDL variables declared at the Workflow level will be evaluated using the filesystems declared here.
  # If you intend to be able to run workflows with this kind of declarations:
  # workflow {
  #    String str = read_string("gs://bucket/my-file.txt")
  # }
  # You will need to provide the engine with a gcs filesystem
  # Note that the default filesystem (local) is always available.
  #filesystems {
  #  gcs {
  #    auth = "application-default"
  #  }
  #}
}

backend {
  default = "SLURM"
  providers {
    Local {
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
      config {

        # Limits the number of concurrent jobs
        #concurrent-job-limit = 5

        run-in-background = true
        # `script-epilogue` configures a shell command to run after the execution of every command block.
        #
        # If this value is not set explicitly, the default value is `sync`, equivalent to:
        # script-epilogue = "sync"
        #
        # To turn off the default `sync` behavior set this value to an empty string:
        script-epilogue = ""

        runtime-attributes = """
        String? docker
        String? docker_user
        """
        submit = "/bin/bash ${script}"
        submit-docker = """
        docker run \
          --rm -i \
          ${"--user " + docker_user} \
          -v ${cwd}:${docker_cwd} \
          ${docker} \
          /bin/bash ${script}
        """

        # Root directory where Cromwell writes job results.  This directory must be
        # visible and writeable by the Cromwell process as well as the jobs that Cromwell
        # launches.
        root = "cromwell-executions"

        filesystems {
          local {
            localization: [
              "hard-link", "soft-link", "copy"
            ]

            caching {
              # When copying a cached result, what type of file duplication should occur. Attempted in the order listed below:
              duplication-strategy: [
                "hard-link", "soft-link", "copy"
              ]

              # Possible values: file, path
              # "file" will compute an md5 hash of the file content.
              # "path" will compute an md5 hash of the file path. This strategy will only be effective if the duplication-strategy (above) is set to "soft-link",
              # in order to allow for the original file path to be hashed.
              hashing-strategy: "file"

              # When true, will check if a sibling file with the same name and the .md5 extension exists, and if it does, use the content of this file as a hash.
              # If false or the md5 does not exist, will proceed with the above-defined hashing strategy.
              check-sibling-md5: false
            }
          }
        }
      }
    }

    #TES {
    #  actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
    #  config {
    #    root = "cromwell-executions"
    #    dockerRoot = "/cromwell-executions"
    #    endpoint = "http://127.0.0.1:9000/v1/jobs"
    #  }
    #}

    #SGE {
    #  actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
    #  config {
    #
    #    # Limits the number of concurrent jobs
    #    concurrent-job-limit = 5
    #
    #    runtime-attributes = """
    #    Int cpu = 1
    #    Float? memory_gb
    #    String? sge_queue
    #    String? sge_project
    #    """
    #
    #    submit = """
    #    qsub \
    #    -terse \
    #    -V \
    #    -b n \
    #    -N ${job_name} \
    #    -wd ${cwd} \
    #    -o ${out} \
    #    -e ${err} \
    #    -pe smp ${cpu} \
    #    ${"-l mem_free=" + memory_gb + "g"} \
    #    ${"-q " + sge_queue} \
    #    ${"-P " + sge_project} \
    #    ${script}
    #    """
    #
    #    kill = "qdel ${job_id}"
    #    check-alive = "qstat -j ${job_id}"
    #    job-id-regex = "(\\d+)"
    #  }
    #}

    #LSF {
    #  actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
    #  config {
    #    submit = "bsub -J ${job_name} -cwd ${cwd} -o ${out} -e ${err} /bin/bash ${script}"
    #    kill = "bkill ${job_id}"
    #    check-alive = "bjobs ${job_id}"
    #    job-id-regex = "Job <(\\d+)>.*"
    #  }
    #}
    
    SLURM {
      actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
      config {
        concurrent-job-limit = 500
        script-epilogue = ""
        runtime-attributes = """
        Int runtime_minutes = 720
        Int cpus = 1
        Int requested_memory_mb_per_core = 8000
        String queue = "short"
        """
      
        submit = """
            sbatch -J ${job_name} -D ${cwd} -o ${out} -e ${err} -t ${runtime_minutes} -p ${queue} \
            ${"-n " + cpus} \
            --mem-per-cpu=${requested_memory_mb_per_core} \
            --constraint="groups" \
            --qos=ded_reich \
            --wrap "/bin/bash ${script}"
        """
        kill = "scancel ${job_id}"
        check-alive = "squeue -j ${job_id}"
        job-id-regex = "Submitted batch job (\\d+).*"
        
        filesystems {
          local {
            localization: [
              "hard-link", "soft-link", "copy"
            ]

            caching {
              # When copying a cached result, what type of file duplication should occur. Attempted in the order listed below:
              duplication-strategy: [
                "soft-link"
              ]

              # Possible values: file, path
              # "file" will compute an md5 hash of the file content.
              # "path" will compute an md5 hash of the file path. This strategy will only be effective if the duplication-strategy (above) is set to "soft-link",
              # in order to allow for the original file path to be hashed.
              hashing-strategy: "path"

              # When true, will check if a sibling file with the same name and the .md5 extension exists, and if it does, use the content of this file as a hash.
              # If false or the md5 does not exist, will proceed with the above-defined hashing strategy.
              check-sibling-md5: false
            }
          }
        }
      }
    }

    # Example backend that _only_ runs workflows that specify docker for every command.
    #Docker {
    #  actor-factory = "cromwell.backend.impl.sfs.config.ConfigBackendLifecycleActorFactory"
    #  config {
    #    run-in-background = true
    #    runtime-attributes = "String docker"
    #    submit-docker = "docker run --rm -v ${cwd}:${docker_cwd} -i ${docker} /bin/bash < ${script}"
    #  }
    #}

    #HtCondor {
    #  actor-factory = "cromwell.backend.impl.htcondor.HtCondorBackendFactory"
    #  config {
    #    # Root directory where Cromwell writes job results.  This directory must be
    #    # visible and writeable by the Cromwell process as well as the jobs that Cromwell
    #    # launches.
    #    root: "cromwell-executions"
    #
    #    #Placeholders:
    #    #1. Working directory.
    #    #2. Working directory volume.
    #    #3. Inputs volumes.
    #    #4. Output volume.
    #    #5. Docker image.
    #    #6. Job command.
    #    docker {
    #      #Allow soft links in dockerized jobs
    #      cmd = "docker run -w %s %s %s %s --rm %s /bin/bash -c \"%s\""
    #      defaultWorkingDir = "/workingDir/"
    #      defaultOutputDir = "/output/"
    #    }
    #
    #    cache {
    #      provider = "cromwell.backend.impl.htcondor.caching.provider.mongodb.MongoCacheActorFactory"
    #      enabled = false
    #      forceRewrite = false
    #      db {
    #        host = "127.0.0.1"
    #        port = 27017
    #        name = "htcondor"
    #        collection = "cache"
    #      }
    #    }
    #
    #    filesystems {
    #      local {
    #        localization: [
    #          "hard-link", "soft-link", "copy"
    #        ]
    #      }
    #    }
    #    # Time (in seconds) to wait before re-checking the status of the job again
    #    poll-interval = 3
    #  }
    #}

    #Spark {
    # actor-factory = "cromwell.backend.impl.spark.SparkBackendFactory"
    # config {
    #   # Root directory where Cromwell writes job results.  This directory must be
    #    # visible and writeable by the Cromwell process as well as the jobs that Cromwell
    #   # launches.
    #   root: "cromwell-executions"
    #
    #   filesystems {
    #     local {
    #       localization: [
    #         "hard-link", "soft-link", "copy"
    #       ]
    #     }
    #    }
    #      # change (master, deployMode) to (yarn, client), (yarn, cluster)
    #      #  or (spark://hostname:port, cluster) for spark standalone cluster mode
    #     master: "local"
    #     deployMode: "client"
    #  }
    # }

    #JES {
    #  actor-factory = "cromwell.backend.impl.jes.JesBackendLifecycleActorFactory"
    #  config {
    #    # Google project
    #    project = "my-cromwell-workflows"
    #
    #    # Base bucket for workflow executions
    #    root = "gs://my-cromwell-workflows-bucket"
    #
    #    # Set this to the lower of the two values "Queries per 100 seconds" and "Queries per 100 seconds per user" for
    #    # your project.
    #    #
    #    # Used to help determine maximum throughput to the Google Genomics API. Setting this value too low will
    #    # cause a drop in performance. Setting this value too high will cause QPS based locks from Google.
    #    # 1000 is the default "Queries per 100 seconds per user", 50000 is the default "Queries per 100 seconds"
    #    # See https://cloud.google.com/genomics/quotas for more information
    #    genomics-api-queries-per-100-seconds = 1000
    #
    #    # Polling for completion backs-off gradually for slower-running jobs.
    #    # This is the maximum polling interval (in seconds):
    #    maximum-polling-interval = 600
    #
    #    # Optional Dockerhub Credentials. Can be used to access private docker images.
    #    dockerhub {
    #      # account = ""
    #      # token = ""
    #    }
    #
    #    genomics {
    #      # A reference to an auth defined in the `google` stanza at the top.  This auth is used to create
    #      # Pipelines and manipulate auth JSONs.
    #      auth = "application-default"
    #
    #      # Specifies the zone(s) to use for JES jobs unless overridden by a task's runtime attributes
    #      default-zones = ["us-central1-b"]
    #
    #      // alternative service account to use on the launched compute instance
    #      // NOTE: If combined with service account authorization, both that serivce account and this service account
    #      // must be able to read and write to the 'root' GCS path
    #      compute-service-account = "default"
    #
    #      # Endpoint for APIs, no reason to change this unless directed by Google.
    #      endpoint-url = "https://genomics.googleapis.com/"
    #    }
    #
    #    filesystems {
    #      gcs {
    #        # A reference to a potentially different auth for manipulating files via engine functions.
    #        auth = "application-default"
    #      }
    #    }
    #  }
    #}

    #AWS {
    #  actor-factory = "cromwell.backend.impl.aws.AwsBackendActorFactory"
    #  config {
    #    ## These two settings are required to authenticate with the ECS service:
    #    accessKeyId = "..."
    #    secretKey = "..."
    #  }
    #}

  }
}

services {
  KeyValue {
    class = "cromwell.services.keyvalue.impl.SqlKeyValueServiceActor"
  }
  MetadataService {
    class = "cromwell.services.metadata.impl.MetadataServiceActor"
    config {
      # Set this value to "Inf" to turn off metadata summary refresh.  The default value is currently "2 seconds".
      # metadata-summary-refresh-interval = "Inf"
      # For higher scale environments, e.g. many workflows and/or jobs, DB write performance for metadata events
      # can improved by writing to the database in batches. Increasing this value can dramatically improve overall
      # performance but will both lead to a higher memory usage as well as increase the risk that metadata events
      # might not have been persisted in the event of a Cromwell crash.
      #
      # For normal usage the default value of 1 (effectively no batching) should be fine but for larger/production
      # environments we recommend a value of at least 500. There'll be no one size fits all number here so we recommend
      # benchmarking performance and tuning the value to match your environment
      # db-batch-size = 1
      #
      # Periodically the stored metadata events will be forcibly written to the DB regardless of if the batch size
      # has been reached. This is to prevent situations where events wind up never being written to an incomplete batch
      # with no new events being generated. The default value is currently 5 seconds
      # db-flush-rate = 5 seconds
    }
  }
}

database {
  # hsql default
  #driver = "slick.driver.HsqldbDriver$"
  #db {
  #  driver = "org.hsqldb.jdbcDriver"
  #  url = "jdbc:hsqldb:mem:${uniqueSchema};shutdown=false;hsqldb.tx=mvcc"
  #  connectionTimeout = 3000
  #}

  # mysql example
  profile = "slick.jdbc.MySQLProfile$"
  db {
    driver = "com.mysql.jdbc.Driver"
    url = "jdbc:mysql://dev.mysql.orchestra/devcromwellcc?rewriteBatchedStatements=true"
    user = ${USER}
    password = ${MYSQL_DB_PW}
    connectionTimeout = 5000
  }

  # For batch inserts the number of inserts to send to the DB at a time
  # insert-batch-size = 2000

  migration {
    # For databases with a very large number of symbols, selecting all the rows at once can generate a variety of
    # problems. In order to avoid any issue, the selection is paginated. This value sets how many rows should be
    # retrieved and processed at a time, before asking for the next chunk.
    read-batch-size = 100000

    # Because a symbol row can contain any arbitrary wdl value, the amount of metadata rows to insert from a single
    # symbol row can vary from 1 to several thousands (or more). To keep the size of the insert batch from growing out
    # of control we monitor its size and execute/commit when it reaches or exceeds writeBatchSize.
    write-batch-size = 100000
  }
}