tools.yaml

account: ""
path_to_input: ""
path_to_output_folder: ""

scripts:
  # Wrapper scripts to submit jobs to the cluster
  tools_submitter: ""
  # tools scripts
  tools_filter_script: ""
  tools_scheduling_script: ""
  tools_worker_script: ""
  tools_verification_script: ""

# Rules for the schedule creation
# They determine how many simultaneous downloader instances can be run on the same server
# Rules are based on the number of batches required to be downloaded from the server
# Rule is: key - number of batches, value - number of instances; if server has more than key batches, value instances can be run
# Server with 0 batches is considered to be downloaded and are ignored
# Default value is 1
# Order of the rules does not matter
schedule_rules:
  1: 1

# Structure of the output folder that will be created automatically
output_structure:
  urls_folder: ""
  logs_folder: ""
  images_folder: ""
  schedules_folder: ""
  profiles_table: ""
  ignored_table: ""
  inner_checkpoint_file: ""
  tools_folder: ""

tools_parameters:
  num_workers: 0
  max_nodes: 0
  workers_per_node: 0
  cpu_per_worker: 0
  threshold_size: 0
  new_resize_size: 0