import json
import requests
-import json
import asyncio
import aiohttp
import aiofiles
import time
+import multiprocessing as mp
+import threading
from tqdm import tqdm
-from types import SimpleNamespace as Namespace
import os
-import requests
from pathlib import Path
+from typing import List, Dict, Any, Optional
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse, quote
+from queue import Empty
from cdislogging import get_logger
+from werkzeug.security import safe_join
+
from gen3.index import Gen3Index
-from gen3.utils import DEFAULT_BACKOFF_SETTINGS, raise_for_status_and_print_error
-from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+from gen3.utils import raise_for_status_and_print_error
logging = get_logger("__name__")
MAX_RETRIES = 3
+DEFAULT_NUM_PARALLEL = 3
+DEFAULT_MAX_CONCURRENT_REQUESTS = 300
+DEFAULT_QUEUE_SIZE = 1000
@@ -80,7 +86,6 @@ Source code for gen3.file
# auth_provider legacy interface required endpoint as 1st arg
self._auth_provider = auth_provider or endpoint
self._endpoint = self._auth_provider.endpoint
- self.unsuccessful_downloads = []
[docs]
@@ -104,10 +109,29 @@ Source code for gen3.file
resp = requests.get(api_url, auth=self._auth_provider)
raise_for_status_and_print_error(resp)
- try:
- return resp.json()
- except:
- return resp.text
+ return resp.json()
+
+
+
+[docs]
+ def get_presigned_urls_batch(self, guids, protocol=None):
+ """Get presigned URLs for multiple files efficiently.
+
+ Args:
+ guids (List[str]): List of GUIDs to get presigned URLs for
+ protocol (str, optional): Protocol preference for URLs
+
+ Returns:
+ Dict[str, Dict]: Mapping of GUID to presigned URL response
+ """
+ results = {}
+ for guid in guids:
+ try:
+ results[guid] = self.get_presigned_url(guid, protocol)
+ except Exception as e:
+ logging.error(f"Failed to get presigned URL for {guid}: {e}")
+ results[guid] = None
+ return results
@@ -186,12 +210,7 @@ Source code for gen3.file
api_url, auth=self._auth_provider, json=body, headers=headers
)
raise_for_status_and_print_error(resp)
- try:
- data = json.loads(resp.text)
- except:
- return resp.text
-
- return data
+ return resp.json()
def _ensure_dirpath_exists(path: Path) -> Path:
@@ -212,16 +231,19 @@ Source code for gen3.file
[docs]
- def download_single(self, object_id, path):
+ def download_single(self, object_id, path, protocol=None):
"""
Download a single file using its GUID.
Args:
object_id (str): The file's unique ID
path (str): Path to store the downloaded file at
+
+ Returns:
+ bool: True if download successful, False otherwise
"""
try:
- url = self.get_presigned_url(object_id)
+ url = self.get_presigned_url(object_id, protocol=protocol)
except Exception as e:
logging.critical(f"Unable to get a presigned URL for download: {e}")
return False
@@ -235,9 +257,9 @@ Source code for gen3.file
# NOTE could be updated with exponential backoff
time.sleep(1)
response = requests.get(url["url"], stream=True)
- if response.status == 200:
+ if response.status_code == 200:
break
- if response.status != 200:
+ if response.status_code != 200:
logging.critical("Response status not 200, try again later")
return False
else:
@@ -251,18 +273,18 @@ Source code for gen3.file
index = Gen3Index(self._auth_provider)
record = index.get_record(object_id)
- filename = record["file_name"]
+ filename = record["file_name"] or url["url"].split("?")[0].split("/")[-1]
- out_path = Gen3File._ensure_dirpath_exists(Path(path))
+ out_path = safe_join(path, filename)
+ Gen3File._ensure_dirpath_exists(Path(os.path.dirname(out_path)))
- with open(os.path.join(out_path, filename), "wb") as f:
+ with open(out_path, "wb") as f:
for data in response.iter_content(4096):
total_downloaded += len(data)
f.write(data)
if total_size_in_bytes == total_downloaded:
logging.info(f"File {filename} downloaded successfully")
-
else:
logging.error(f"File {filename} not downloaded successfully")
return False
@@ -311,7 +333,445 @@ Source code for gen3.file
resp = requests.get(url, auth=self._auth_provider)
raise_for_status_and_print_error(resp)
return resp.json()
-
+
+
+
+[docs]
+ async def async_download_multiple(
+ self,
+ manifest_data,
+ download_path=".",
+ filename_format="original",
+ protocol=None,
+ max_concurrent_requests=DEFAULT_MAX_CONCURRENT_REQUESTS,
+ num_processes=DEFAULT_NUM_PARALLEL,
+ queue_size=DEFAULT_QUEUE_SIZE,
+ skip_completed=False,
+ rename=False,
+ no_progress=False,
+ ):
+ """Asynchronously download multiple files using multiprocessing and queues."""
+ if not manifest_data:
+ return {"succeeded": [], "failed": [], "skipped": []}
+
+ guids = []
+ for item in manifest_data:
+ guid = item.get("guid") or item.get("object_id")
+ if guid:
+ if "/" in guid:
+ guid = guid.split("/")[-1]
+ guids.append(guid)
+
+ if not guids:
+ logging.error("No valid GUIDs found in manifest data")
+ return {"succeeded": [], "failed": [], "skipped": []}
+
+ output_dir = Gen3File._ensure_dirpath_exists(Path(download_path))
+
+ input_queue = mp.Queue(maxsize=queue_size)
+ output_queue = mp.Queue()
+
+ worker_config = {
+ "endpoint": self._endpoint,
+ "auth_provider": self._auth_provider,
+ "download_path": str(output_dir),
+ "filename_format": filename_format,
+ "protocol": protocol,
+ "max_concurrent": max_concurrent_requests,
+ "skip_completed": skip_completed,
+ "rename": rename,
+ }
+
+ processes = []
+ producer_thread = None
+
+ try:
+ for i in range(num_processes):
+ p = mp.Process(
+ target=self._async_worker_process,
+ args=(input_queue, output_queue, worker_config, i),
+ )
+ p.start()
+ processes.append(p)
+
+ producer_thread = threading.Thread(
+ target=self._guid_producer,
+ args=(guids, input_queue, num_processes),
+ )
+ producer_thread.start()
+
+ results = {"succeeded": [], "failed": [], "skipped": []}
+ completed_count = 0
+
+ if not no_progress:
+ pbar = tqdm(total=len(guids), desc="Downloading")
+
+ while completed_count < len(guids):
+ try:
+ batch_results = output_queue.get()
+
+ if not batch_results:
+ continue
+
+ for result in batch_results:
+ if result["status"] == "downloaded":
+ results["succeeded"].append(result["guid"])
+ elif result["status"] == "skipped":
+ results["skipped"].append(result["guid"])
+ else:
+ results["failed"].append(result["guid"])
+
+ completed_count += 1
+ if not no_progress:
+ pbar.update(1)
+
+ except Empty:
+ logging.warning(
+ f"No more results available ({completed_count}/{len(guids)}): Queue is empty"
+ )
+ break
+ except Exception as e:
+ logging.warning(
+ f"Error waiting for results ({completed_count}/{len(guids)}): {e}"
+ )
+
+ alive_processes = [p for p in processes if p.is_alive()]
+ if not alive_processes:
+ logging.error("All worker processes have died")
+ break
+
+ if not no_progress:
+ pbar.close()
+
+ if producer_thread:
+ producer_thread.join()
+
+ except Exception as e:
+ logging.error(f"Error in download: {e}")
+ results = {"succeeded": [], "failed": [], "skipped": [], "error": str(e)}
+
+ finally:
+ for p in processes:
+ if p.is_alive():
+ p.terminate()
+
+ p.join()
+ if p.is_alive():
+ p.kill()
+
+ logging.info(
+ f"Download complete: {len(results['succeeded'])} succeeded, "
+ f"{len(results['failed'])} failed, {len(results['skipped'])} skipped"
+ )
+ return results
+
+
+ def _guid_producer(self, guids, input_queue, num_processes):
+ try:
+ for guid in guids:
+ input_queue.put(guid)
+
+ except Exception as e:
+ logging.error(f"Error in producer: {e}")
+
+ @staticmethod
+ def _async_worker_process(input_queue, output_queue, config, process_id):
+ try:
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ loop.run_until_complete(
+ Gen3File._worker_main(input_queue, output_queue, config, process_id)
+ )
+ except Exception as e:
+ logging.error(f"Error in worker process {process_id}: {e}")
+ finally:
+ try:
+ loop.close()
+ except Exception as e:
+ logging.warning(f"Error closing event loop in worker {process_id}: {e}")
+
+ @staticmethod
+ async def _worker_main(input_queue, output_queue, config, process_id):
+ endpoint = config["endpoint"]
+ auth_provider = config["auth_provider"]
+ download_path = Path(config["download_path"])
+ filename_format = config["filename_format"]
+ protocol = config["protocol"]
+ max_concurrent = config["max_concurrent"]
+ skip_completed = config["skip_completed"]
+ rename = config["rename"]
+
+ # Configure connector with optimized settings for large files
+ timeout = aiohttp.ClientTimeout(total=None, connect=3600, sock_read=3600)
+ connector = aiohttp.TCPConnector(
+ limit=max_concurrent * 2,
+ limit_per_host=max_concurrent,
+ ttl_dns_cache=300,
+ use_dns_cache=True,
+ keepalive_timeout=3600,
+ enable_cleanup_closed=True,
+ )
+ semaphore = asyncio.Semaphore(max_concurrent)
+
+ async with aiohttp.ClientSession(
+ connector=connector, timeout=timeout
+ ) as session:
+ while True:
+ try:
+ # Check if queue is empty with timeout
+ guid = input_queue.get()
+ except Empty:
+ # If queue is empty (timeout), break the loop
+ break
+
+ # Process single GUID as a batch of one
+ try:
+ batch_results = await Gen3File._process_batch(
+ session,
+ [guid],
+ endpoint,
+ auth_provider,
+ download_path,
+ filename_format,
+ protocol,
+ semaphore,
+ skip_completed,
+ rename,
+ )
+ output_queue.put(batch_results)
+ except Exception as e:
+ logging.error(
+ f"Worker {process_id}: Failed to process {guid} - {type(e).__name__}: {e}"
+ )
+ error_result = [{"guid": guid, "status": "failed", "error": str(e)}]
+ try:
+ output_queue.put(error_result)
+ except Exception as queue_error:
+ logging.error(
+ f"Worker {process_id}: Failed to send error result for {guid} - {type(queue_error).__name__}: {queue_error}"
+ )
+
+ @staticmethod
+ async def _process_batch(
+ session,
+ guids,
+ endpoint,
+ auth_provider,
+ download_path,
+ filename_format,
+ protocol,
+ semaphore,
+ skip_completed,
+ rename,
+ ):
+ """Process a batch of GUIDs for downloading."""
+ batch_results = []
+ for guid in guids:
+ async with semaphore:
+ result = await Gen3File._download_single_async(
+ session,
+ guid,
+ endpoint,
+ auth_provider,
+ download_path,
+ filename_format,
+ protocol,
+ semaphore,
+ skip_completed,
+ rename,
+ )
+ batch_results.append(result)
+ return batch_results
+
+ @staticmethod
+ async def _download_single_async(
+ session,
+ guid,
+ endpoint,
+ auth_provider,
+ download_path,
+ filename_format,
+ protocol,
+ semaphore,
+ skip_completed,
+ rename,
+ ):
+ async with semaphore:
+ try:
+ metadata = await Gen3File._get_metadata(
+ session, guid, endpoint, auth_provider.get_access_token()
+ )
+
+ original_filename = metadata.get("file_name")
+ filename = Gen3File._format_filename_static(
+ guid, original_filename, filename_format
+ )
+ filepath = download_path / filename
+
+ if skip_completed and filepath.exists():
+ return {
+ "guid": guid,
+ "status": "skipped",
+ "filepath": str(filepath),
+ "reason": "File already exists",
+ }
+
+ filepath = Gen3File._handle_conflict_static(filepath, rename)
+
+ presigned_data = await Gen3File._get_presigned_url_async(
+ session, guid, endpoint, auth_provider.get_access_token(), protocol
+ )
+
+ url = presigned_data.get("url")
+ if not url:
+ return {
+ "guid": guid,
+ "status": "failed",
+ "error": "No URL in presigned data",
+ }
+
+ filepath.parent.mkdir(parents=True, exist_ok=True)
+
+ success = await Gen3File._download_content(session, url, guid, filepath)
+ if success:
+ return {
+ "guid": guid,
+ "status": "downloaded",
+ "filepath": str(filepath),
+ "size": filepath.stat().st_size if filepath.exists() else 0,
+ }
+ else:
+ return {
+ "guid": guid,
+ "status": "failed",
+ "error": "Download failed",
+ }
+
+ except Exception as e:
+ logging.error(f"Error downloading {guid}: {e}")
+ return {
+ "guid": guid,
+ "status": "failed",
+ "error": str(e),
+ }
+
+ @staticmethod
+ async def _get_metadata(session, guid, endpoint, auth_token):
+ encoded_guid = quote(guid, safe="")
+ api_url = f"{endpoint}/index/{encoded_guid}"
+ headers = {"Authorization": f"Bearer {auth_token}"}
+
+ try:
+ async with session.get(
+ api_url, headers=headers, timeout=aiohttp.ClientTimeout(total=3600)
+ ) as resp:
+ if resp.status == 200:
+ return await resp.json()
+ raise Exception(
+ f"Failed to get metadata for {guid}: HTTP {resp.status}"
+ )
+ except aiohttp.ClientError as e:
+ raise Exception(f"Network error getting metadata for {guid}: {e}")
+ except asyncio.TimeoutError:
+ raise Exception(f"Timeout getting metadata for {guid}")
+ except Exception as e:
+ if "Failed to get metadata" not in str(e):
+ raise Exception(f"Unexpected error getting metadata for {guid}: {e}")
+ raise
+
+ @staticmethod
+ async def _get_presigned_url_async(
+ session, guid, endpoint, auth_token, protocol=None
+ ):
+ encoded_guid = quote(guid, safe="")
+ api_url = f"{endpoint}/user/data/download/{encoded_guid}"
+ headers = {"Authorization": f"Bearer {auth_token}"}
+
+ if protocol:
+ api_url += f"?protocol={protocol}"
+
+ try:
+ async with session.get(
+ api_url, headers=headers, timeout=aiohttp.ClientTimeout(total=3600)
+ ) as resp:
+ if resp.status == 200:
+ return await resp.json()
+ raise Exception(
+ f"Failed to get presigned URL for {guid}: HTTP {resp.status}"
+ )
+ except aiohttp.ClientError as e:
+ raise Exception(f"Network error getting presigned URL for {guid}: {e}")
+ except asyncio.TimeoutError:
+ raise Exception(f"Timeout getting presigned URL for {guid}")
+ except Exception as e:
+ if "Failed to get presigned URL" not in str(e):
+ raise Exception(
+ f"Unexpected error getting presigned URL for {guid}: {e}"
+ )
+ raise
+
+ @staticmethod
+ async def _download_content(session, url, guid, filepath):
+ """Download content directly to file with optimized streaming."""
+ try:
+ async with session.get(
+ url, timeout=aiohttp.ClientTimeout(total=None)
+ ) as resp:
+ if resp.status == 200:
+ async with aiofiles.open(filepath, "wb") as f:
+ chunk_size = 1024 * 1024
+ async for chunk in resp.content.iter_chunked(chunk_size):
+ await f.write(chunk)
+ return True
+ logging.error(f"Download failed for {guid}: HTTP {resp.status}")
+ return False
+ except aiohttp.ClientError as e:
+ logging.error(f"Network error downloading {guid}: {e}")
+ return False
+ except asyncio.TimeoutError:
+ logging.error(f"Timeout downloading {guid}")
+ return False
+ except OSError as e:
+ logging.error(f"File system error downloading {guid} to {filepath}: {e}")
+ return False
+ except Exception as e:
+ logging.error(
+ f"Unexpected error downloading {guid}: {type(e).__name__}: {e}"
+ )
+ return False
+
+ @staticmethod
+ def _format_filename_static(guid, original_filename, filename_format):
+ if filename_format == "guid":
+ return guid
+ elif filename_format == "combined":
+ if original_filename:
+ name, ext = os.path.splitext(original_filename)
+ return f"{name}_{guid}{ext}"
+ return guid
+ else:
+ return original_filename or guid
+
+ @staticmethod
+ def _handle_conflict_static(filepath, rename):
+ if not rename:
+ if filepath.exists():
+ logging.warning(f"File will be overwritten: {filepath}")
+ return filepath
+
+ if not filepath.exists():
+ return filepath
+
+ counter = 1
+ name = filepath.stem
+ ext = filepath.suffix
+ parent = filepath.parent
+
+ while True:
+ new_path = parent / f"{name}_{counter}{ext}"
+ if not new_path.exists():
+ return new_path
+ counter += 1
diff --git a/docs/_build/html/file.html b/docs/_build/html/file.html
index cb970d07..aae09d2d 100644
--- a/docs/_build/html/file.html
+++ b/docs/_build/html/file.html
@@ -54,6 +54,12 @@ Gen3 File Class... file = Gen3File(auth)
+
+-
+async async_download_multiple(manifest_data, download_path='.', filename_format='original', protocol=None, max_concurrent_requests=300, num_processes=3, queue_size=1000, skip_completed=False, rename=False, no_progress=False)[source]¶
+Asynchronously download multiple files using multiprocessing and queues.
+
+
-
delete_file(guid)[source]¶
@@ -91,7 +97,7 @@ Gen3 File Class
-
-download_single(object_id, path)[source]¶
+download_single(object_id, path, protocol=None)[source]¶
Download a single file using its GUID.
- Parameters:
@@ -100,6 +106,12 @@ Gen3 File ClassReturns:
+True if download successful, False otherwise
+- Return type:
+bool
+
@@ -122,6 +134,26 @@ Gen3 File Class
+
+get_presigned_urls_batch(guids, protocol=None)[source]¶
+Get presigned URLs for multiple files efficiently.
+
+- Parameters:
+
+guids (List[str]) – List of GUIDs to get presigned URLs for
protocol (str, optional) – Protocol preference for URLs
+- Returns:
+Mapping of GUID to presigned URL response
+- Return type:
+Dict[str, Dict]
+
+
-
upload_file(file_name, authz=None, protocol=None, expires_in=None, bucket=None)[source]¶
@@ -218,10 +250,12 @@ Gen3 SDK
Gen3 Auth Helper
Gen3 File Class
Gen3File
+Gen3File.async_download_multiple()Gen3File.delete_file()Gen3File.delete_file_locations()Gen3File.download_single()Gen3File.get_presigned_url()Gen3File.get_presigned_urls_batch()Gen3File.upload_file()Gen3File.upload_file_to_guid()
diff --git a/docs/_build/html/genindex.html b/docs/_build/html/genindex.html
index fc7b36ca..8c9eebe9 100644
--- a/docs/_build/html/genindex.html
+++ b/docs/_build/html/genindex.html
@@ -79,6 +79,8 @@ A
- async_delete_alias() (gen3.metadata.Gen3Metadata method)
- async_delete_aliases() (gen3.metadata.Gen3Metadata method)
+
+ - async_download_multiple() (gen3.file.Gen3File method)
- async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)
@@ -87,11 +89,11 @@ A
- async_get_aliases() (gen3.metadata.Gen3Metadata method)
- async_get_record() (gen3.index.Gen3Index method)
-
- - async_get_records_from_checksum() (gen3.index.Gen3Index method)
- get_fresh_token() (gen3.tools.download.drs_download.DownloadManager method)
- get_graphql_schema() (gen3.submission.Gen3Submission method)
@@ -369,6 +371,8 @@
G
- get_output() (gen3.jobs.Gen3Jobs method)
- get_presigned_url() (gen3.file.Gen3File method)
+
+ - get_presigned_urls_batch() (gen3.file.Gen3File method)
- get_programs() (gen3.submission.Gen3Submission method)
diff --git a/docs/_build/html/index.html b/docs/_build/html/index.html
index d329e8bc..cd7e2af8 100644
--- a/docs/_build/html/index.html
+++ b/docs/_build/html/index.html
@@ -50,10 +50,12 @@ Welcome to Gen3 SDK’s documentation!Gen3 File Class
Gen3File
diff --git a/docs/_build/html/objects.inv b/docs/_build/html/objects.inv
index 8f2fb369..cef04df4 100644
Binary files a/docs/_build/html/objects.inv and b/docs/_build/html/objects.inv differ
diff --git a/docs/_build/html/searchindex.js b/docs/_build/html/searchindex.js
index 9928858b..748fb2a7 100644
--- a/docs/_build/html/searchindex.js
+++ b/docs/_build/html/searchindex.js
@@ -1 +1 @@
-Search.setIndex({"alltitles":{"DRS Download Tools":[[10,null]],"Download":[[11,"module-gen3.tools.indexing.download_manifest"]],"Gen3 Auth Helper":[[0,null]],"Gen3 File Class":[[1,null]],"Gen3 Index Class":[[3,null]],"Gen3 Jobs Class":[[4,null]],"Gen3 Metadata Class":[[5,null]],"Gen3 Object Class":[[6,null]],"Gen3 Query Class":[[7,null]],"Gen3 Submission Class":[[8,null]],"Gen3 Tools":[[9,null]],"Gen3 Workspace Storage":[[13,null]],"Index":[[11,"module-gen3.tools.indexing.index_manifest"]],"Indexing Tools":[[11,null]],"Indices and tables":[[2,"indices-and-tables"]],"Ingest":[[12,"module-gen3.tools.metadata.ingest_manifest"]],"Metadata Tools":[[12,null]],"Verify":[[11,"module-gen3.tools.indexing.verify_manifest"]],"Welcome to Gen3 SDK\u2019s documentation!":[[2,null]]},"docnames":["auth","file","index","indexing","jobs","metadata","object","query","submission","tools","tools/drs_pull","tools/indexing","tools/metadata","wss"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1},"filenames":["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","object.rst","query.rst","submission.rst","tools.rst","tools/drs_pull.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],"indexentries":{"_manager (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable._manager",false]],"access_methods (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.access_methods",false]],"acls (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.ACLS",false]],"async_create() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_create",false]],"async_create_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_create_aliases",false]],"async_create_record() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.async_create_record",false]],"async_delete_alias() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_delete_alias",false]],"async_delete_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_delete_aliases",false]],"async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)":[[11,"gen3.tools.indexing.download_manifest.async_download_object_manifest",false]],"async_get() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_get",false]],"async_get_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_get_aliases",false]],"async_get_record() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.async_get_record",false]],"async_get_records_from_checksum() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.async_get_records_from_checksum",false]],"async_get_records_on_page() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.async_get_records_on_page",false]],"async_get_with_params() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.async_get_with_params",false]],"async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)":[[12,"gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest",false]],"async_query_urls() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.async_query_urls",false]],"async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)":[[12,"gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd",false]],"async_run_job_and_wait() (gen3.jobs.gen3jobs method)":[[4,"gen3.jobs.Gen3Jobs.async_run_job_and_wait",false]],"async_update() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_update",false]],"async_update_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_update_aliases",false]],"async_update_record() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.async_update_record",false]],"async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)":[[11,"gen3.tools.indexing.verify_manifest.async_verify_object_manifest",false]],"auth_provider (gen3.metadata.gen3metadata attribute)":[[5,"gen3.metadata.Gen3Metadata.auth_provider",false]],"authz (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.AUTHZ",false]],"batch_create() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.batch_create",false]],"cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)":[[10,"gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens",false]],"children (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.children",false]],"column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)":[[12,"gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID",false]],"commons_url (gen3.tools.download.drs_download.manifest attribute)":[[10,"gen3.tools.download.drs_download.Manifest.commons_url",false]],"copy() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.copy",false]],"create() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.create",false]],"create_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.create_aliases",false]],"create_blank() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.create_blank",false]],"create_index_key_path() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.create_index_key_path",false]],"create_job() (gen3.jobs.gen3jobs method)":[[4,"gen3.jobs.Gen3Jobs.create_job",false]],"create_new_version() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.create_new_version",false]],"create_object_list() (gen3.tools.download.drs_download.manifest static method)":[[10,"gen3.tools.download.drs_download.Manifest.create_object_list",false]],"create_program() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.create_program",false]],"create_project() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.create_project",false]],"create_record() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.create_record",false]],"created_time (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.created_time",false]],"curl() (gen3.auth.gen3auth method)":[[0,"gen3.auth.Gen3Auth.curl",false]],"current_dir (in module gen3.tools.indexing.download_manifest)":[[11,"gen3.tools.indexing.download_manifest.CURRENT_DIR",false]],"current_dir (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.CURRENT_DIR",false]],"current_dir (in module gen3.tools.indexing.verify_manifest)":[[11,"gen3.tools.indexing.verify_manifest.CURRENT_DIR",false]],"delete() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.delete",false]],"delete_alias() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.delete_alias",false]],"delete_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.delete_aliases",false]],"delete_all_guids() (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.delete_all_guids",false]],"delete_file() (gen3.file.gen3file method)":[[1,"gen3.file.Gen3File.delete_file",false]],"delete_file_locations() (gen3.file.gen3file method)":[[1,"gen3.file.Gen3File.delete_file_locations",false]],"delete_index_key_path() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.delete_index_key_path",false]],"delete_node() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.delete_node",false]],"delete_nodes() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.delete_nodes",false]],"delete_object() (gen3.object.gen3object method)":[[6,"gen3.object.Gen3Object.delete_object",false]],"delete_program() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.delete_program",false]],"delete_project() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.delete_project",false]],"delete_record() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.delete_record",false]],"delete_record() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.delete_record",false]],"delete_records() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.delete_records",false]],"download() (gen3.tools.download.drs_download.downloadable method)":[[10,"gen3.tools.download.drs_download.Downloadable.download",false]],"download() (gen3.tools.download.drs_download.downloadmanager method)":[[10,"gen3.tools.download.drs_download.DownloadManager.download",false]],"download() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.download",false]],"download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.download_files_in_drs_manifest",false]],"download_single() (gen3.file.gen3file method)":[[1,"gen3.file.Gen3File.download_single",false]],"download_url() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.download_url",false]],"downloadable (class in gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.Downloadable",false]],"downloadmanager (class in gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.DownloadManager",false]],"downloadstatus (class in gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.DownloadStatus",false]],"end_time (gen3.tools.download.drs_download.downloadstatus attribute)":[[10,"gen3.tools.download.drs_download.DownloadStatus.end_time",false]],"endpoint (gen3.metadata.gen3metadata attribute)":[[5,"gen3.metadata.Gen3Metadata.endpoint",false]],"export_node() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.export_node",false]],"export_record() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.export_record",false]],"file_name (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.file_name",false]],"file_name (gen3.tools.download.drs_download.manifest attribute)":[[10,"gen3.tools.download.drs_download.Manifest.file_name",false]],"file_size (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.file_size",false]],"file_size (gen3.tools.download.drs_download.manifest attribute)":[[10,"gen3.tools.download.drs_download.Manifest.file_size",false]],"gen3.tools":[[9,"module-gen3.tools",false]],"gen3.tools.download.drs_download":[[10,"module-gen3.tools.download.drs_download",false]],"gen3.tools.indexing.download_manifest":[[11,"module-gen3.tools.indexing.download_manifest",false]],"gen3.tools.indexing.index_manifest":[[11,"module-gen3.tools.indexing.index_manifest",false]],"gen3.tools.indexing.verify_manifest":[[11,"module-gen3.tools.indexing.verify_manifest",false]],"gen3.tools.metadata.ingest_manifest":[[12,"module-gen3.tools.metadata.ingest_manifest",false]],"gen3auth (class in gen3.auth)":[[0,"gen3.auth.Gen3Auth",false]],"gen3file (class in gen3.file)":[[1,"gen3.file.Gen3File",false]],"gen3index (class in gen3.index)":[[3,"gen3.index.Gen3Index",false]],"gen3jobs (class in gen3.jobs)":[[4,"gen3.jobs.Gen3Jobs",false]],"gen3metadata (class in gen3.metadata)":[[5,"gen3.metadata.Gen3Metadata",false]],"gen3object (class in gen3.object)":[[6,"gen3.object.Gen3Object",false]],"gen3query (class in gen3.query)":[[7,"gen3.query.Gen3Query",false]],"gen3submission (class in gen3.submission)":[[8,"gen3.submission.Gen3Submission",false]],"gen3wsstorage (class in gen3.wss)":[[13,"gen3.wss.Gen3WsStorage",false]],"get() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get",false]],"get() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.get",false]],"get_access_token() (gen3.auth.gen3auth method)":[[0,"gen3.auth.Gen3Auth.get_access_token",false]],"get_access_token_from_wts() (gen3.auth.gen3auth method)":[[0,"gen3.auth.Gen3Auth.get_access_token_from_wts",false]],"get_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.get_aliases",false]],"get_all_records() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_all_records",false]],"get_dictionary_all() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.get_dictionary_all",false]],"get_dictionary_node() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.get_dictionary_node",false]],"get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)":[[10,"gen3.tools.download.drs_download.DownloadManager.get_fresh_token",false]],"get_graphql_schema() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.get_graphql_schema",false]],"get_guids_prefix() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_guids_prefix",false]],"get_index_key_paths() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.get_index_key_paths",false]],"get_latest_version() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_latest_version",false]],"get_output() (gen3.jobs.gen3jobs method)":[[4,"gen3.jobs.Gen3Jobs.get_output",false]],"get_presigned_url() (gen3.file.gen3file method)":[[1,"gen3.file.Gen3File.get_presigned_url",false]],"get_programs() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.get_programs",false]],"get_project_dictionary() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.get_project_dictionary",false]],"get_project_manifest() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.get_project_manifest",false]],"get_projects() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.get_projects",false]],"get_record() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_record",false]],"get_record_doc() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_record_doc",false]],"get_records() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_records",false]],"get_records_on_page() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_records_on_page",false]],"get_stats() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_stats",false]],"get_status() (gen3.jobs.gen3jobs method)":[[4,"gen3.jobs.Gen3Jobs.get_status",false]],"get_urls() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_urls",false]],"get_valid_guids() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_valid_guids",false]],"get_version() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_version",false]],"get_version() (gen3.jobs.gen3jobs method)":[[4,"gen3.jobs.Gen3Jobs.get_version",false]],"get_version() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.get_version",false]],"get_versions() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_versions",false]],"get_with_params() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_with_params",false]],"graphql_query() (gen3.query.gen3query method)":[[7,"gen3.query.Gen3Query.graphql_query",false]],"guid (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.GUID",false]],"guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)":[[12,"gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT",false]],"guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)":[[12,"gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT",false]],"hostname (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.hostname",false]],"index_object_manifest() (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.index_object_manifest",false]],"indexd_record_page_size (in module gen3.tools.indexing.download_manifest)":[[11,"gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE",false]],"is_healthy() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.is_healthy",false]],"is_healthy() (gen3.jobs.gen3jobs method)":[[4,"gen3.jobs.Gen3Jobs.is_healthy",false]],"is_healthy() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.is_healthy",false]],"list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.list_access_in_drs_manifest",false]],"list_drs_object() (in module gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.list_drs_object",false]],"list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.list_files_in_drs_manifest",false]],"list_jobs() (gen3.jobs.gen3jobs method)":[[4,"gen3.jobs.Gen3Jobs.list_jobs",false]],"load() (gen3.tools.download.drs_download.manifest static method)":[[10,"gen3.tools.download.drs_download.Manifest.load",false]],"load_manifest() (gen3.tools.download.drs_download.manifest static method)":[[10,"gen3.tools.download.drs_download.Manifest.load_manifest",false]],"ls() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.ls",false]],"ls_path() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.ls_path",false]],"manifest (class in gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.Manifest",false]],"max_concurrent_requests (in module gen3.tools.indexing.download_manifest)":[[11,"gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS",false]],"max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)":[[11,"gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS",false]],"max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)":[[12,"gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS",false]],"md5 (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.MD5",false]],"md5sum (gen3.tools.download.drs_download.manifest attribute)":[[10,"gen3.tools.download.drs_download.Manifest.md5sum",false]],"module":[[9,"module-gen3.tools",false],[10,"module-gen3.tools.download.drs_download",false],[11,"module-gen3.tools.indexing.download_manifest",false],[11,"module-gen3.tools.indexing.index_manifest",false],[11,"module-gen3.tools.indexing.verify_manifest",false],[12,"module-gen3.tools.metadata.ingest_manifest",false]],"object_id (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.object_id",false]],"object_id (gen3.tools.download.drs_download.manifest attribute)":[[10,"gen3.tools.download.drs_download.Manifest.object_id",false]],"object_type (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.object_type",false]],"open_project() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.open_project",false]],"pprint() (gen3.tools.download.drs_download.downloadable method)":[[10,"gen3.tools.download.drs_download.Downloadable.pprint",false]],"prev_guid (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.PREV_GUID",false]],"query() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.query",false]],"query() (gen3.query.gen3query method)":[[7,"gen3.query.Gen3Query.query",false]],"query() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.query",false]],"query_urls() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.query_urls",false]],"raw_data_download() (gen3.query.gen3query method)":[[7,"gen3.query.Gen3Query.raw_data_download",false]],"refresh_access_token() (gen3.auth.gen3auth method)":[[0,"gen3.auth.Gen3Auth.refresh_access_token",false]],"resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)":[[10,"gen3.tools.download.drs_download.DownloadManager.resolve_objects",false]],"rm() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.rm",false]],"rm_path() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.rm_path",false]],"size (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.SIZE",false]],"start_time (gen3.tools.download.drs_download.downloadstatus attribute)":[[10,"gen3.tools.download.drs_download.DownloadStatus.start_time",false]],"status (gen3.tools.download.drs_download.downloadstatus attribute)":[[10,"gen3.tools.download.drs_download.DownloadStatus.status",false]],"submit_file() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.submit_file",false]],"submit_record() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.submit_record",false]],"threadcontrol (class in gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.ThreadControl",false]],"tmp_folder (in module gen3.tools.indexing.download_manifest)":[[11,"gen3.tools.indexing.download_manifest.TMP_FOLDER",false]],"update() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.update",false]],"update_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.update_aliases",false]],"update_blank() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.update_blank",false]],"update_record() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.update_record",false]],"updated_time (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.updated_time",false]],"upload() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.upload",false]],"upload_file() (gen3.file.gen3file method)":[[1,"gen3.file.Gen3File.upload_file",false]],"upload_file_to_guid() (gen3.file.gen3file method)":[[1,"gen3.file.Gen3File.upload_file_to_guid",false]],"upload_url() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.upload_url",false]],"urls (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.URLS",false]],"user_access() (gen3.tools.download.drs_download.downloadmanager method)":[[10,"gen3.tools.download.drs_download.DownloadManager.user_access",false]]},"objects":{"gen3":[[9,3,0,"-","tools"]],"gen3.auth":[[0,0,1,"","Gen3Auth"]],"gen3.auth.Gen3Auth":[[0,1,1,"","curl"],[0,1,1,"","get_access_token"],[0,1,1,"","get_access_token_from_wts"],[0,1,1,"","refresh_access_token"]],"gen3.file":[[1,0,1,"","Gen3File"]],"gen3.file.Gen3File":[[1,1,1,"","delete_file"],[1,1,1,"","delete_file_locations"],[1,1,1,"","download_single"],[1,1,1,"","get_presigned_url"],[1,1,1,"","upload_file"],[1,1,1,"","upload_file_to_guid"]],"gen3.index":[[3,0,1,"","Gen3Index"]],"gen3.index.Gen3Index":[[3,1,1,"","async_create_record"],[3,1,1,"","async_get_record"],[3,1,1,"","async_get_records_from_checksum"],[3,1,1,"","async_get_records_on_page"],[3,1,1,"","async_get_with_params"],[3,1,1,"","async_query_urls"],[3,1,1,"","async_update_record"],[3,1,1,"","create_blank"],[3,1,1,"","create_new_version"],[3,1,1,"","create_record"],[3,1,1,"","delete_record"],[3,1,1,"","get"],[3,1,1,"","get_all_records"],[3,1,1,"","get_guids_prefix"],[3,1,1,"","get_latest_version"],[3,1,1,"","get_record"],[3,1,1,"","get_record_doc"],[3,1,1,"","get_records"],[3,1,1,"","get_records_on_page"],[3,1,1,"","get_stats"],[3,1,1,"","get_urls"],[3,1,1,"","get_valid_guids"],[3,1,1,"","get_version"],[3,1,1,"","get_versions"],[3,1,1,"","get_with_params"],[3,1,1,"","is_healthy"],[3,1,1,"","query_urls"],[3,1,1,"","update_blank"],[3,1,1,"","update_record"]],"gen3.jobs":[[4,0,1,"","Gen3Jobs"]],"gen3.jobs.Gen3Jobs":[[4,1,1,"","async_run_job_and_wait"],[4,1,1,"","create_job"],[4,1,1,"","get_output"],[4,1,1,"","get_status"],[4,1,1,"","get_version"],[4,1,1,"","is_healthy"],[4,1,1,"","list_jobs"]],"gen3.metadata":[[5,0,1,"","Gen3Metadata"]],"gen3.metadata.Gen3Metadata":[[5,1,1,"","async_create"],[5,1,1,"","async_create_aliases"],[5,1,1,"","async_delete_alias"],[5,1,1,"","async_delete_aliases"],[5,1,1,"","async_get"],[5,1,1,"","async_get_aliases"],[5,1,1,"","async_update"],[5,1,1,"","async_update_aliases"],[5,2,1,"","auth_provider"],[5,1,1,"","batch_create"],[5,1,1,"","create"],[5,1,1,"","create_aliases"],[5,1,1,"","create_index_key_path"],[5,1,1,"","delete"],[5,1,1,"","delete_alias"],[5,1,1,"","delete_aliases"],[5,1,1,"","delete_index_key_path"],[5,2,1,"","endpoint"],[5,1,1,"","get"],[5,1,1,"","get_aliases"],[5,1,1,"","get_index_key_paths"],[5,1,1,"","get_version"],[5,1,1,"","is_healthy"],[5,1,1,"","query"],[5,1,1,"","update"],[5,1,1,"","update_aliases"]],"gen3.object":[[6,0,1,"","Gen3Object"]],"gen3.object.Gen3Object":[[6,1,1,"","delete_object"]],"gen3.query":[[7,0,1,"","Gen3Query"]],"gen3.query.Gen3Query":[[7,1,1,"","graphql_query"],[7,1,1,"","query"],[7,1,1,"","raw_data_download"]],"gen3.submission":[[8,0,1,"","Gen3Submission"]],"gen3.submission.Gen3Submission":[[8,1,1,"","create_program"],[8,1,1,"","create_project"],[8,1,1,"","delete_node"],[8,1,1,"","delete_nodes"],[8,1,1,"","delete_program"],[8,1,1,"","delete_project"],[8,1,1,"","delete_record"],[8,1,1,"","delete_records"],[8,1,1,"","export_node"],[8,1,1,"","export_record"],[8,1,1,"","get_dictionary_all"],[8,1,1,"","get_dictionary_node"],[8,1,1,"","get_graphql_schema"],[8,1,1,"","get_programs"],[8,1,1,"","get_project_dictionary"],[8,1,1,"","get_project_manifest"],[8,1,1,"","get_projects"],[8,1,1,"","open_project"],[8,1,1,"","query"],[8,1,1,"","submit_file"],[8,1,1,"","submit_record"]],"gen3.tools.download":[[10,3,0,"-","drs_download"]],"gen3.tools.download.drs_download":[[10,0,1,"","DownloadManager"],[10,0,1,"","DownloadStatus"],[10,0,1,"","Downloadable"],[10,0,1,"","Manifest"],[10,4,1,"","download_files_in_drs_manifest"],[10,4,1,"","list_access_in_drs_manifest"],[10,4,1,"","list_drs_object"],[10,4,1,"","list_files_in_drs_manifest"]],"gen3.tools.download.drs_download.DownloadManager":[[10,1,1,"","cache_hosts_wts_tokens"],[10,1,1,"","download"],[10,1,1,"","get_fresh_token"],[10,1,1,"","resolve_objects"],[10,1,1,"","user_access"]],"gen3.tools.download.drs_download.DownloadStatus":[[10,2,1,"","end_time"],[10,2,1,"","start_time"],[10,2,1,"","status"]],"gen3.tools.download.drs_download.Downloadable":[[10,2,1,"","_manager"],[10,2,1,"","access_methods"],[10,2,1,"","children"],[10,2,1,"","created_time"],[10,1,1,"","download"],[10,2,1,"","file_name"],[10,2,1,"","file_size"],[10,2,1,"","hostname"],[10,2,1,"","object_id"],[10,2,1,"","object_type"],[10,1,1,"","pprint"],[10,2,1,"","updated_time"]],"gen3.tools.download.drs_download.Manifest":[[10,2,1,"","commons_url"],[10,1,1,"","create_object_list"],[10,2,1,"","file_name"],[10,2,1,"","file_size"],[10,1,1,"","load"],[10,1,1,"","load_manifest"],[10,2,1,"","md5sum"],[10,2,1,"","object_id"]],"gen3.tools.indexing":[[11,3,0,"-","download_manifest"],[11,3,0,"-","index_manifest"],[11,3,0,"-","verify_manifest"]],"gen3.tools.indexing.download_manifest":[[11,2,1,"","CURRENT_DIR"],[11,2,1,"","INDEXD_RECORD_PAGE_SIZE"],[11,2,1,"","MAX_CONCURRENT_REQUESTS"],[11,2,1,"","TMP_FOLDER"],[11,4,1,"","async_download_object_manifest"]],"gen3.tools.indexing.index_manifest":[[11,2,1,"","ACLS"],[11,2,1,"","AUTHZ"],[11,2,1,"","CURRENT_DIR"],[11,2,1,"","GUID"],[11,2,1,"","MD5"],[11,2,1,"","PREV_GUID"],[11,2,1,"","SIZE"],[11,0,1,"","ThreadControl"],[11,2,1,"","URLS"],[11,4,1,"","delete_all_guids"],[11,4,1,"","index_object_manifest"]],"gen3.tools.indexing.verify_manifest":[[11,2,1,"","CURRENT_DIR"],[11,2,1,"","MAX_CONCURRENT_REQUESTS"],[11,4,1,"","async_verify_object_manifest"]],"gen3.tools.metadata":[[12,3,0,"-","ingest_manifest"]],"gen3.tools.metadata.ingest_manifest":[[12,2,1,"","COLUMN_TO_USE_AS_GUID"],[12,2,1,"","GUID_TYPE_FOR_INDEXED_FILE_OBJECT"],[12,2,1,"","GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"],[12,2,1,"","MAX_CONCURRENT_REQUESTS"],[12,4,1,"","async_ingest_metadata_manifest"],[12,4,1,"","async_query_urls_from_indexd"]],"gen3.wss":[[13,0,1,"","Gen3WsStorage"]],"gen3.wss.Gen3WsStorage":[[13,1,1,"","copy"],[13,1,1,"","download"],[13,1,1,"","download_url"],[13,1,1,"","ls"],[13,1,1,"","ls_path"],[13,1,1,"","rm"],[13,1,1,"","rm_path"],[13,1,1,"","upload"],[13,1,1,"","upload_url"]]},"objnames":{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},"objtypes":{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},"terms":{"0a80fada010c":11,"0a80fada096c":11,"0a80fada097c":11,"0a80fada098c":11,"0a80fada099c":11,"11e9":11,"255e396f":11,"450c":11,"473d83400bc1bc9dc635e334fadd433c":11,"473d83400bc1bc9dc635e334faddd33c":11,"473d83400bc1bc9dc635e334fadde33c":11,"473d83400bc1bc9dc635e334faddf33c":11,"6f90":8,"7d3d8d2083b4":11,"93d9af72":11,"9a07":11,"A":[1,3,4,5,6,7,8,10,11,13],"ALL":7,"AND":5,"All":11,"Be":1,"But":5,"By":11,"For":[1,5,6,7,8,9,11],"IF":11,"If":[0,1,7,11,12],"In":10,"It":10,"Most":9,"NOT":12,"OR":5,"Same":13,"Such":9,"THE":11,"THIS":11,"That":3,"The":[0,1,2,3,5,8,10,11],"There":11,"These":9,"This":[0,1,2,3,4,5,6,7,8,10,11,13],"To":11,"We":11,"When":12,"YOU":11,"_get_acl_from_row":11,"_get_authz_from_row":11,"_get_file_name_from_row":11,"_get_file_size_from_row":11,"_get_guid_for_row":12,"_get_guid_from_row":11,"_get_md5_from_row":11,"_get_urls_from_row":11,"_guid_typ":12,"_manag":[2,9,10],"_query_for_associated_indexd_record_guid":12,"_ssl":[3,4,5],"a5c6":11,"ab167e49d25b488939b1ede42752458b":3,"abov":11,"access":[0,1,3,7,10],"access_method":[2,9,10],"access_token":0,"accesstoken":0,"acl":[2,3,9,11],"across":11,"act":0,"action":[9,11],"actual":11,"add":[3,5],"addit":[3,5,10,11],"admin":[5,11],"admin_endpoint_suffix":5,"algorithm":3,"alia":[3,5],"alias":5,"aliv":7,"allow":[0,6,8,10,11,12],"allowed_data_upload_bucket":1,"along":2,"alreadi":9,"also":1,"altern":[5,11],"alway":5,"ammount":12,"amount":[1,9],"ani":[0,5,10,11],"anoth":5,"api":[0,5,8,11],"api_key":11,"appli":7,"appropri":13,"arbitrari":0,"argument":[0,13],"array":8,"asc":7,"assign":9,"assist":10,"associ":[3,5],"assum":11,"async":[3,4,5,9,11,12],"async_cr":[2,5],"async_create_alias":[2,5],"async_create_record":[2,3],"async_delete_alia":[2,5],"async_delete_alias":[2,5],"async_download_object_manifest":[2,9,11],"async_get":[2,5],"async_get_alias":[2,5],"async_get_record":[2,3],"async_get_records_from_checksum":[2,3],"async_get_records_on_pag":[2,3],"async_get_with_param":[2,3],"async_ingest_metadata_manifest":[2,9,12],"async_query_url":[2,3],"async_query_urls_from_indexd":[2,9,12],"async_run_job_and_wait":[2,4],"async_upd":[2,5],"async_update_alias":[2,5],"async_update_record":[2,3],"async_verify_object_manifest":[2,9,11],"asynchron":[3,4,5],"asyncio":[11,12],"asyncron":5,"attach":[3,5],"attempt":11,"attribut":[10,11],"auth":[1,2,3,4,5,6,7,8,10,11,12,13],"auth_provid":[1,2,3,4,5,6,7,8,13],"authbas":0,"authent":0,"author":1,"authz":[0,1,2,3,9,10,11],"auto":[0,2],"automat":0,"avail":[1,2,10,11],"az":1,"b":[5,11],"b0f1":11,"bar":10,"base":[0,1,3,4,5,6,7,8,9,11,13],"baseid":3,"basic":[3,11,12],"batch_creat":[2,5],"batch_siz":8,"behalf":0,"behavior":11,"belong":8,"blank":3,"blob":[5,7],"bodi":3,"bool":[4,5,8,10,11,12],"boolean":3,"bownload":10,"broad":9,"broken":9,"bucket":[1,6],"bundl":10,"byte":10,"c":[5,11],"cach":10,"cache_hosts_wts_token":[2,9,10],"call":[10,13],"can":[0,3,4,8,11,12],"capabl":9,"case":[0,10],"categori":9,"ccle":8,"ccle_one_record":8,"ccle_sample_nod":8,"cdis":7,"chang":[3,11],"checksum":[3,10],"checksum_typ":3,"child":10,"children":[2,9,10],"chunk_siz":8,"class":[0,2,10,11,13],"cli":10,"client":[0,3],"client_credenti":0,"client_id":0,"client_scop":0,"client_secret":0,"code":[2,8],"column":[11,12],"column_to_use_as_guid":[2,9,12],"columna":11,"columnb":11,"columnc":11,"com":7,"comma":11,"command":[10,11],"common":[0,1,3,4,5,6,7,8,9,10,11,12,13],"commons_url":[2,9,10,11,12],"complet":[4,11],"complex":7,"concat":11,"concurr":[11,12],"configur":1,"connect":12,"consist":3,"constructor":0,"contain":[0,2,5,8,9,10,11,12],"content":[3,13],"content_created_d":3,"content_updated_d":3,"continu":10,"control":3,"conveni":10,"copi":[2,13],"coroutin":11,"correspond":3,"count":3,"crdc":0,"creat":[2,3,4,5,6,8,10,11],"create_alias":[2,5],"create_blank":[2,3],"create_index_key_path":[2,5],"create_job":[2,4],"create_new_vers":[2,3],"create_object_list":[2,9,10],"create_program":[2,8],"create_project":[2,8],"create_record":[2,3],"created_tim":[2,9,10],"creation":[3,11],"cred":3,"credenti":[0,1,3,4,5,6,7,8,10,11,13],"csv":[8,11,12],"curl":[0,2],"current":[6,8,10],"current_dir":[2,9,11],"custom":11,"d":5,"d70b41b9":8,"data":[0,1,3,5,7,8,10,11],"data_spreadsheet":8,"data_typ":7,"data_upload_bucket":1,"dataa":11,"datab":11,"databas":5,"datacommon":0,"datafil":10,"datamanag":10,"date":3,"datetim":[1,3,10],"dbgap":12,"dcf":8,"def":11,"default":[0,1,3,7,8,11,12],"defin":[5,8,10],"delay":4,"delet":[0,1,2,3,5,6,8,10,11],"delete_alia":[2,5],"delete_alias":[2,5],"delete_all_guid":[2,9,11],"delete_fil":[1,2],"delete_file_loc":[1,2,6],"delete_index_key_path":[2,5],"delete_nod":[2,8],"delete_object":[2,6],"delete_program":[2,8],"delete_project":[2,8],"delete_record":[2,3,8],"delete_unpacked_packag":10,"delimet":[11,12],"delimit":11,"demograph":8,"deprec":1,"descript":[3,5],"desir":11,"dest_path":13,"dest_urlstr":13,"dest_w":13,"dest_wskey":13,"detail":[2,7,10],"determin":[10,11,12],"dev":11,"dict":[3,4,5,10,11,12],"dictionari":[3,4,5,7,8],"dids":3,"differ":5,"direct":0,"directori":[10,11],"disabl":10,"discoveri":10,"disk":13,"dispatch":4,"dist_resolut":3,"distribut":3,"doc":[7,10],"docstr":2,"document":[1,3],"doe":[0,12],"domain":[11,12],"done":4,"download":[0,1,2,3,4,5,6,7,8,9,13],"download_files_in_drs_manifest":[2,9,10],"download_list":10,"download_manifest":11,"download_singl":[1,2],"download_url":[2,13],"downloadmanag":[2,9,10],"downloadstatus":[2,9,10],"drs":[2,9],"drs_download":10,"drs_hostnam":10,"drsdownload":10,"drsobjecttyp":10,"e":[5,10],"e043ab8b77b9":8,"effici":9,"eg":3,"either":8,"elasticsearch":7,"els":[0,12],"elsewher":12,"empti":[8,11],"enabl":11,"end":[5,10],"end_tim":[2,9,10],"endpoint":[0,1,2,3,4,5,7,8,13],"entir":8,"entri":[3,11],"env":0,"environ":0,"equal":7,"error":[10,11],"error_nam":11,"etc":8,"even":11,"everi":[9,11],"everyth":11,"ex":[0,11,12],"exampl":[0,1,3,4,5,6,7,8,10,11,13],"exclud":3,"execut":[7,8,11],"exist":[1,3,5,6,9,12],"expect":[5,9,11],"experi":8,"expir":[0,1],"expires_in":1,"export":[8,10],"export_nod":[2,8],"export_record":[2,8],"f1f8":11,"factori":10,"fail":[8,10],"fals":[3,5,6,10,11],"featur":[1,6],"fenc":[0,1],"fetch":0,"field":[3,5,7,11,12],"fieldnam":11,"file":[0,2,3,4,8,9,10,11,12,13],"file_nam":[1,2,3,9,10,11],"file_s":[2,9,10,11],"file_st":3,"fileformat":8,"filenam":[0,8,10,11,12],"files":10,"fill":12,"filter":[5,7],"filter_object":7,"first":[7,8],"flag":11,"folder":11,"follow":[0,11],"forc":11,"force_metadata_columns_even_if_empti":11,"form":13,"format":[3,5,8,11],"func_to_parse_row":[11,12],"function":[2,3,4,5,9,10,11,12],"g":10,"gen3":[10,11,12],"gen3_api_key":0,"gen3_oidc_client_creds_secret":0,"gen3auth":[0,1,2,3,4,5,6,7,8,10,11,12,13],"gen3fil":[1,2],"gen3index":[2,3],"gen3job":[2,4,10],"gen3metadata":[2,5],"gen3object":[2,6],"gen3queri":[2,7],"gen3submiss":[2,8],"gen3wsstorag":[2,13],"generat":[0,1,2,3,4,5,6,7,8,10,13],"get":[0,1,2,3,4,5,8,10,11,12,13],"get_access_token":[0,2],"get_access_token_from_wt":[0,2],"get_alias":[2,5],"get_all_record":[2,3],"get_dictionary_al":[2,8],"get_dictionary_nod":[2,8],"get_fresh_token":[2,9,10],"get_graphql_schema":[2,8],"get_guid_from_fil":12,"get_guids_prefix":[2,3],"get_index_key_path":[2,5],"get_latest_vers":[2,3],"get_output":[2,4],"get_presigned_url":[1,2],"get_program":[2,8],"get_project":[2,8],"get_project_dictionari":[2,8],"get_project_manifest":[2,8],"get_record":[2,3],"get_record_doc":[2,3],"get_records_on_pag":[2,3],"get_stat":[2,3],"get_status":[2,4],"get_url":[2,3],"get_valid_guid":[2,3],"get_vers":[2,3,4,5],"get_with_param":[2,3],"giangb":11,"github":[2,7],"give":1,"given":[0,3,4,5,8,10,12,13],"global":[4,5],"good":3,"grant":0,"graph":8,"graphql":[7,8],"graphql_queri":[2,7],"group":3,"guid":[1,2,3,5,6,9,11,12],"guid_exampl":11,"guid_for_row":12,"guid_from_fil":12,"guid_type_for_indexed_file_object":[2,9,12],"guid_type_for_non_indexed_file_object":[2,9,12],"guppi":7,"handl":[3,10],"hardcod":0,"has_vers":3,"hash":[3,11],"hash_typ":3,"header":11,"healthi":[3,4,5],"help":11,"helper":2,"hit":11,"host":10,"hostnam":[2,9,10],"howto":10,"http":12,"https":[0,7,11],"id":[0,1,3,5,10,11],"idea":3,"identifi":[3,5,9,11],"idp":0,"illustr":11,"immut":3,"implement":0,"implic":11,"import":11,"includ":[0,3],"indent":10,"index":[0,2,5,9],"index_manifest":11,"index_object_manifest":[2,9,11],"indexd":[1,3,6,10,11,12],"indexd_field":[11,12],"indexd_record_page_s":[2,9,11],"indexed_file_object_guid":12,"indic":[0,11],"infil":10,"info":[3,11],"inform":[2,3,10],"ingest":[2,9],"ingest_manifest":12,"initi":[0,10],"input":[4,10,11],"input_manifest":11,"instal":[0,2,11],"instanc":[1,3,6,7,8,9,10],"instead":[1,7,11],"int":[1,3,5,7,8,10,11,12],"integ":[1,3,8],"intend":0,"interact":[1,3,4,5,6,8,13],"interest":10,"interpret":0,"introspect":8,"involv":9,"is_healthi":[2,3,4,5],"is_indexed_file_object":12,"isn":1,"issu":0,"job":2,"job_id":4,"job_input":4,"job_nam":4,"json":[0,1,3,4,5,6,7,8,10,11,13],"just":[5,11,12],"jwt":0,"key":[0,3,5,13],"know":11,"known":10,"kwarg":[3,4,5],"larg":9,"last":10,"latest":3,"least":3,"level":6,"librari":11,"like":[3,5,9,11,12],"limit":[1,3,5,12],"linear":4,"linux":10,"list":[0,1,3,4,5,7,8,10,11,13],"list_access_in_drs_manifest":[2,9,10],"list_drs_object":[2,9,10],"list_files_in_drs_manifest":[2,9,10],"list_job":[2,4],"live":[11,12],"load":[2,9,10],"load_manifest":[2,9,10],"local":[0,13],"locat":[1,6],"lock":12,"log":[8,10,11,12],"logic":[5,12],"loop":11,"ls":[2,13],"ls_path":[2,13],"maco":11,"made":3,"main":10,"make":[9,11],"manag":[1,5,10],"mani":[8,11],"manifest":[2,8,9,10,11,12],"manifest_1":10,"manifest_fil":[11,12],"manifest_file_delimit":[11,12],"manifest_row_pars":[11,12],"map":[0,11],"mark":8,"master":7,"match":[3,5,12],"max":5,"max_concurrent_request":[2,9,11,12],"max_presigned_url_ttl":1,"max_tri":8,"maximum":[11,12],"may":[0,9,11],"md":[7,10],"md5":[2,3,9,11],"md5_hash":11,"md5sum":[2,9,10],"mds":[5,12],"mean":8,"mechan":3,"merg":5,"metadata":[2,3,6,9,11],"metadata_list":5,"metadata_sourc":12,"metadata_typ":12,"metdata":12,"method":[1,7,10],"minimum":10,"minut":0,"mode":7,"modul":[2,10,11],"mostly":2,"multipl":[8,11],"must":[1,5],"my_common":10,"my_credenti":10,"my_field":7,"my_index":7,"my_program":7,"my_project":7,"name":[3,4,8,10,11,12,13],"namespac":[0,12],"necessari":[3,5],"need":[3,7,10,11],"nest":5,"net":11,"never":0,"new":[0,3],"node":8,"node_nam":8,"node_typ":8,"none":[0,1,3,4,5,6,7,8,10,11,12,13],"note":[0,3,11,12],"noth":[3,6],"now":[1,8],"num":5,"num_process":11,"num_total_fil":11,"number":[3,7,8,11,12],"object":[1,2,3,4,5,7,8,9,10,11,13],"object_id":[1,2,9,10],"object_list":10,"object_typ":[2,9,10],"objectid":10,"obtain":[0,10],"occur":10,"offset":[5,7],"oidc":0,"old":3,"one":[3,5,7,10,11],"onli":[3,5,7,8,10,11],"open":[8,10,11],"open_project":[2,8],"openid":0,"opt":0,"option":[0,1,3,4,5,6,7,8,10,11],"order":[0,8],"ordered_node_list":8,"org":10,"os":0,"otherwis":10,"output":[4,5,11,12],"output_dir":10,"output_filenam":[11,12],"overrid":[0,11,12],"overwrit":5,"packag":10,"page":[0,1,2,3,4,5,6,7,8,10,11,13],"pagin":3,"parallel":11,"param":[3,5,8,10],"paramet":[0,1,3,4,5,6,7,8,10,11,12,13],"pars":[10,11,12,13],"parser":[11,12],"particular":0,"pass":[0,7,8,10],"password":[11,12],"path":[0,1,5,10,11,13],"path_to_manifest":11,"pattern":[3,12],"pdcdatastor":11,"pend":10,"per":[11,12],"peregrin":8,"permiss":10,"persist":9,"phs0001":11,"phs0002":11,"pick":1,"pla":11,"place":11,"planx":11,"point":[0,1,3,4,5,6,7,8,10,13],"popul":[10,12],"posit":[1,7],"possibl":10,"post":[0,11],"pprint":[2,9,10],"prefix":3,"presign":1,"pretti":10,"prev_guid":[2,9,11],"previous":[3,4,11],"print":[8,10],"process":11,"processed_fil":11,"profil":[0,1,3,4,5,6,7,8,10,13],"program":[8,11],"progress":[8,10],"project":[8,11],"project_id":[7,8],"protocol":1,"provid":[0,1,3,5,7,8,12],"public":[3,5],"put":0,"py":11,"python":[2,9,11],"python3":11,"python_subprocess_command":11,"queri":[1,2,3,5,8,11,12],"query_str":7,"query_txt":[7,8],"query_url":[2,3],"quickstart":2,"rather":0,"raw":[7,11],"raw_data_download":[2,7],"rbac":3,"read":[3,5,11],"readm":2,"reason":10,"record":[1,3,5,7,8,11,12],"refresh":[0,10],"refresh_access_token":[0,2],"refresh_fil":[0,1,3,4,5,6,7,8,10,13],"refresh_token":0,"regist":8,"regular":7,"relat":9,"remov":[1,6,11,13],"replac":11,"replace_url":11,"repo":2,"repres":[3,5,10],"represent":[1,3],"request":[0,1,3,5,8,11,12],"requir":10,"resolv":10,"resolve_object":[2,9,10],"respect":7,"respons":[0,1,3,4,5],"result":[1,8,10,11],"retri":8,"retriev":[1,8,10,12],"return":[0,1,3,4,5,6,7,8,10,11],"return_full_metadata":5,"rev":3,"revers":8,"revis":3,"right":1,"rm":[2,13],"rm_path":[2,13],"root":[11,12],"row":[7,8,11,12],"row_offset":8,"rtype":3,"run":[8,11],"s":[1,4,8,10,11],"s3":[1,10,11],"safe":11,"sampl":[8,10],"sandbox":[0,1,3,4,5,6,7,8,10,13],"save":10,"save_directori":10,"schema":8,"scope":[0,1],"screen":8,"script":2,"search":[0,2,3],"second":[1,4],"secret":0,"see":[7,10,11],"self":10,"semaphon":12,"semaphor":12,"separ":[0,11],"server":10,"servic":[1,3,4,5,6,8,11,12,13],"service_loc":[3,4,5],"session":11,"set":[0,1,5,10],"setup":2,"sheepdog":8,"show":10,"show_progress":10,"shown":11,"sign":1,"signpost":3,"similar":10,"simpl":3,"simpli":11,"sinc":3,"singl":[1,5,8],"size":[2,3,9,10,11],"skip":8,"sleep":4,"someth":11,"sort":7,"sort_field":7,"sort_object":7,"sourc":[0,1,2,3,4,5,6,7,8,10,11,12,13],"space":[0,11],"specif":[5,8,11,12],"specifi":[0,1,3,11,13],"spreadsheet":8,"src_path":13,"src_urlstr":13,"src_ws":13,"src_wskey":13,"ssl":[3,4,5],"start":[4,7,8,10],"start_tim":[2,9,10],"static":10,"status":[2,4,9,10],"status_cod":10,"storag":[1,2,6],"store":[1,3,10],"str":[0,1,3,4,5,7,8,10,11,12],"string":[0,3,5,11,13],"strip":11,"sub":8,"subject":[7,8],"submiss":2,"submit":[8,11],"submit_additional_metadata_column":11,"submit_fil":[2,8],"submit_record":[2,8],"submitter_id":7,"success":10,"suffici":3,"suppli":3,"support":[0,1,5,8,11],"sure":1,"synchron":11,"syntax":7,"system":[6,7,8,9],"t":[1,5,11],"tab":11,"task":9,"temporari":11,"test":11,"test1":11,"test2":11,"test3":11,"test4":11,"test5":11,"text":[1,7,8],"thread":11,"thread_num":11,"threadcontrol":[2,9,11],"tier":7,"time":[1,3,8,10,11],"timestamp":10,"tmp_folder":[2,9,11],"token":[0,10],"tool":2,"total":11,"treat":[1,5],"tree":10,"tri":0,"true":[3,4,5,6,7,8,10,11,12],"tsv":[8,11,12],"tupl":[0,3,11,12],"type":[1,3,4,5,7,8,10,11,12],"typic":10,"uc":7,"unaccess":7,"uniqu":[1,5],"unknown":10,"unpack":10,"unpack_packag":10,"updat":[2,3,5,10,11],"update_alias":[2,5],"update_blank":[2,3],"update_record":[2,3],"updated_tim":[2,9,10],"upload":[1,2,3,8,13],"upload_fil":[1,2],"upload_file_to_guid":[1,2],"upload_url":[2,13],"url":[1,2,3,9,10,11,12,13],"urls_metadata":3,"usag":11,"use":[0,1,3,4,5,6,7,8,10,11,12,13],"use_agg_md":5,"user":[0,10,12],"user_access":[2,9,10],"usual":12,"utcnow":1,"util":9,"uuid":[1,3,8],"uuid1":8,"uuid2":8,"valid":[3,7],"valu":[0,1,3,5,7,10,11],"value_from_indexd":11,"value_from_manifest":11,"variabl":[0,7,8],"various":2,"verbos":[7,8],"verif":11,"verifi":[2,9],"verify_manifest":11,"verify_object_manifest":11,"version":[3,4,5],"vital_status":7,"wait":4,"want":[0,3,8],"warn":11,"way":10,"web":0,"whether":[3,4,5,8,11,12],"whose":5,"will":[1,3,4,5,7,10,11,12],"within":[0,2,9],"without":[3,5],"won":5,"work":[0,10],"workaround":11,"worksheet":8,"workspac":[0,2],"wrapper":10,"write":11,"ws":13,"ws_urlstr":13,"wskey":13,"wss":13,"wts":[0,10],"x":11,"xlsx":8},"titles":["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Object Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","DRS Download Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],"titleterms":{"auth":0,"class":[1,3,4,5,6,7,8],"document":2,"download":[10,11],"drs":10,"file":1,"gen3":[0,1,2,3,4,5,6,7,8,9,13],"helper":0,"index":[3,11],"indic":2,"ingest":12,"job":4,"metadata":[5,12],"object":6,"queri":7,"s":2,"sdk":2,"storag":13,"submiss":8,"tabl":2,"tool":[9,10,11,12],"verifi":11,"welcom":2,"workspac":13}})
\ No newline at end of file
+Search.setIndex({"alltitles":{"DRS Download Tools":[[10,null]],"Download":[[11,"module-gen3.tools.indexing.download_manifest"]],"Gen3 Auth Helper":[[0,null]],"Gen3 File Class":[[1,null]],"Gen3 Index Class":[[3,null]],"Gen3 Jobs Class":[[4,null]],"Gen3 Metadata Class":[[5,null]],"Gen3 Object Class":[[6,null]],"Gen3 Query Class":[[7,null]],"Gen3 Submission Class":[[8,null]],"Gen3 Tools":[[9,null]],"Gen3 Workspace Storage":[[13,null]],"Index":[[11,"module-gen3.tools.indexing.index_manifest"]],"Indexing Tools":[[11,null]],"Indices and tables":[[2,"indices-and-tables"]],"Ingest":[[12,"module-gen3.tools.metadata.ingest_manifest"]],"Metadata Tools":[[12,null]],"Verify":[[11,"module-gen3.tools.indexing.verify_manifest"]],"Welcome to Gen3 SDK\u2019s documentation!":[[2,null]]},"docnames":["auth","file","index","indexing","jobs","metadata","object","query","submission","tools","tools/drs_pull","tools/indexing","tools/metadata","wss"],"envversion":{"sphinx":66,"sphinx.domains.c":3,"sphinx.domains.changeset":1,"sphinx.domains.citation":1,"sphinx.domains.cpp":9,"sphinx.domains.index":1,"sphinx.domains.javascript":3,"sphinx.domains.math":2,"sphinx.domains.python":4,"sphinx.domains.rst":2,"sphinx.domains.std":2,"sphinx.ext.viewcode":1},"filenames":["auth.rst","file.rst","index.rst","indexing.rst","jobs.rst","metadata.rst","object.rst","query.rst","submission.rst","tools.rst","tools/drs_pull.rst","tools/indexing.rst","tools/metadata.rst","wss.rst"],"indexentries":{"_manager (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable._manager",false]],"access_methods (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.access_methods",false]],"acls (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.ACLS",false]],"async_create() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_create",false]],"async_create_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_create_aliases",false]],"async_create_record() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.async_create_record",false]],"async_delete_alias() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_delete_alias",false]],"async_delete_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_delete_aliases",false]],"async_download_multiple() (gen3.file.gen3file method)":[[1,"gen3.file.Gen3File.async_download_multiple",false]],"async_download_object_manifest() (in module gen3.tools.indexing.download_manifest)":[[11,"gen3.tools.indexing.download_manifest.async_download_object_manifest",false]],"async_get() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_get",false]],"async_get_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_get_aliases",false]],"async_get_record() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.async_get_record",false]],"async_get_records_from_checksum() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.async_get_records_from_checksum",false]],"async_get_records_on_page() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.async_get_records_on_page",false]],"async_get_with_params() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.async_get_with_params",false]],"async_ingest_metadata_manifest() (in module gen3.tools.metadata.ingest_manifest)":[[12,"gen3.tools.metadata.ingest_manifest.async_ingest_metadata_manifest",false]],"async_query_urls() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.async_query_urls",false]],"async_query_urls_from_indexd() (in module gen3.tools.metadata.ingest_manifest)":[[12,"gen3.tools.metadata.ingest_manifest.async_query_urls_from_indexd",false]],"async_run_job_and_wait() (gen3.jobs.gen3jobs method)":[[4,"gen3.jobs.Gen3Jobs.async_run_job_and_wait",false]],"async_update() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_update",false]],"async_update_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.async_update_aliases",false]],"async_update_record() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.async_update_record",false]],"async_verify_object_manifest() (in module gen3.tools.indexing.verify_manifest)":[[11,"gen3.tools.indexing.verify_manifest.async_verify_object_manifest",false]],"auth_provider (gen3.metadata.gen3metadata attribute)":[[5,"gen3.metadata.Gen3Metadata.auth_provider",false]],"authz (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.AUTHZ",false]],"batch_create() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.batch_create",false]],"cache_hosts_wts_tokens() (gen3.tools.download.drs_download.downloadmanager method)":[[10,"gen3.tools.download.drs_download.DownloadManager.cache_hosts_wts_tokens",false]],"children (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.children",false]],"column_to_use_as_guid (in module gen3.tools.metadata.ingest_manifest)":[[12,"gen3.tools.metadata.ingest_manifest.COLUMN_TO_USE_AS_GUID",false]],"commons_url (gen3.tools.download.drs_download.manifest attribute)":[[10,"gen3.tools.download.drs_download.Manifest.commons_url",false]],"copy() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.copy",false]],"create() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.create",false]],"create_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.create_aliases",false]],"create_blank() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.create_blank",false]],"create_index_key_path() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.create_index_key_path",false]],"create_job() (gen3.jobs.gen3jobs method)":[[4,"gen3.jobs.Gen3Jobs.create_job",false]],"create_new_version() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.create_new_version",false]],"create_object_list() (gen3.tools.download.drs_download.manifest static method)":[[10,"gen3.tools.download.drs_download.Manifest.create_object_list",false]],"create_program() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.create_program",false]],"create_project() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.create_project",false]],"create_record() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.create_record",false]],"created_time (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.created_time",false]],"curl() (gen3.auth.gen3auth method)":[[0,"gen3.auth.Gen3Auth.curl",false]],"current_dir (in module gen3.tools.indexing.download_manifest)":[[11,"gen3.tools.indexing.download_manifest.CURRENT_DIR",false]],"current_dir (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.CURRENT_DIR",false]],"current_dir (in module gen3.tools.indexing.verify_manifest)":[[11,"gen3.tools.indexing.verify_manifest.CURRENT_DIR",false]],"delete() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.delete",false]],"delete_alias() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.delete_alias",false]],"delete_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.delete_aliases",false]],"delete_all_guids() (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.delete_all_guids",false]],"delete_file() (gen3.file.gen3file method)":[[1,"gen3.file.Gen3File.delete_file",false]],"delete_file_locations() (gen3.file.gen3file method)":[[1,"gen3.file.Gen3File.delete_file_locations",false]],"delete_index_key_path() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.delete_index_key_path",false]],"delete_node() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.delete_node",false]],"delete_nodes() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.delete_nodes",false]],"delete_object() (gen3.object.gen3object method)":[[6,"gen3.object.Gen3Object.delete_object",false]],"delete_program() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.delete_program",false]],"delete_project() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.delete_project",false]],"delete_record() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.delete_record",false]],"delete_record() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.delete_record",false]],"delete_records() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.delete_records",false]],"download() (gen3.tools.download.drs_download.downloadable method)":[[10,"gen3.tools.download.drs_download.Downloadable.download",false]],"download() (gen3.tools.download.drs_download.downloadmanager method)":[[10,"gen3.tools.download.drs_download.DownloadManager.download",false]],"download() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.download",false]],"download_files_in_drs_manifest() (in module gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.download_files_in_drs_manifest",false]],"download_single() (gen3.file.gen3file method)":[[1,"gen3.file.Gen3File.download_single",false]],"download_url() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.download_url",false]],"downloadable (class in gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.Downloadable",false]],"downloadmanager (class in gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.DownloadManager",false]],"downloadstatus (class in gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.DownloadStatus",false]],"end_time (gen3.tools.download.drs_download.downloadstatus attribute)":[[10,"gen3.tools.download.drs_download.DownloadStatus.end_time",false]],"endpoint (gen3.metadata.gen3metadata attribute)":[[5,"gen3.metadata.Gen3Metadata.endpoint",false]],"export_node() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.export_node",false]],"export_record() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.export_record",false]],"file_name (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.file_name",false]],"file_name (gen3.tools.download.drs_download.manifest attribute)":[[10,"gen3.tools.download.drs_download.Manifest.file_name",false]],"file_size (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.file_size",false]],"file_size (gen3.tools.download.drs_download.manifest attribute)":[[10,"gen3.tools.download.drs_download.Manifest.file_size",false]],"gen3.tools":[[9,"module-gen3.tools",false]],"gen3.tools.download.drs_download":[[10,"module-gen3.tools.download.drs_download",false]],"gen3.tools.indexing.download_manifest":[[11,"module-gen3.tools.indexing.download_manifest",false]],"gen3.tools.indexing.index_manifest":[[11,"module-gen3.tools.indexing.index_manifest",false]],"gen3.tools.indexing.verify_manifest":[[11,"module-gen3.tools.indexing.verify_manifest",false]],"gen3.tools.metadata.ingest_manifest":[[12,"module-gen3.tools.metadata.ingest_manifest",false]],"gen3auth (class in gen3.auth)":[[0,"gen3.auth.Gen3Auth",false]],"gen3file (class in gen3.file)":[[1,"gen3.file.Gen3File",false]],"gen3index (class in gen3.index)":[[3,"gen3.index.Gen3Index",false]],"gen3jobs (class in gen3.jobs)":[[4,"gen3.jobs.Gen3Jobs",false]],"gen3metadata (class in gen3.metadata)":[[5,"gen3.metadata.Gen3Metadata",false]],"gen3object (class in gen3.object)":[[6,"gen3.object.Gen3Object",false]],"gen3query (class in gen3.query)":[[7,"gen3.query.Gen3Query",false]],"gen3submission (class in gen3.submission)":[[8,"gen3.submission.Gen3Submission",false]],"gen3wsstorage (class in gen3.wss)":[[13,"gen3.wss.Gen3WsStorage",false]],"get() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get",false]],"get() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.get",false]],"get_access_token() (gen3.auth.gen3auth method)":[[0,"gen3.auth.Gen3Auth.get_access_token",false]],"get_access_token_from_wts() (gen3.auth.gen3auth method)":[[0,"gen3.auth.Gen3Auth.get_access_token_from_wts",false]],"get_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.get_aliases",false]],"get_all_records() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_all_records",false]],"get_dictionary_all() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.get_dictionary_all",false]],"get_dictionary_node() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.get_dictionary_node",false]],"get_fresh_token() (gen3.tools.download.drs_download.downloadmanager method)":[[10,"gen3.tools.download.drs_download.DownloadManager.get_fresh_token",false]],"get_graphql_schema() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.get_graphql_schema",false]],"get_guids_prefix() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_guids_prefix",false]],"get_index_key_paths() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.get_index_key_paths",false]],"get_latest_version() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_latest_version",false]],"get_output() (gen3.jobs.gen3jobs method)":[[4,"gen3.jobs.Gen3Jobs.get_output",false]],"get_presigned_url() (gen3.file.gen3file method)":[[1,"gen3.file.Gen3File.get_presigned_url",false]],"get_presigned_urls_batch() (gen3.file.gen3file method)":[[1,"gen3.file.Gen3File.get_presigned_urls_batch",false]],"get_programs() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.get_programs",false]],"get_project_dictionary() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.get_project_dictionary",false]],"get_project_manifest() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.get_project_manifest",false]],"get_projects() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.get_projects",false]],"get_record() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_record",false]],"get_record_doc() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_record_doc",false]],"get_records() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_records",false]],"get_records_on_page() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_records_on_page",false]],"get_stats() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_stats",false]],"get_status() (gen3.jobs.gen3jobs method)":[[4,"gen3.jobs.Gen3Jobs.get_status",false]],"get_urls() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_urls",false]],"get_valid_guids() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_valid_guids",false]],"get_version() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_version",false]],"get_version() (gen3.jobs.gen3jobs method)":[[4,"gen3.jobs.Gen3Jobs.get_version",false]],"get_version() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.get_version",false]],"get_versions() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_versions",false]],"get_with_params() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.get_with_params",false]],"graphql_query() (gen3.query.gen3query method)":[[7,"gen3.query.Gen3Query.graphql_query",false]],"guid (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.GUID",false]],"guid_type_for_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)":[[12,"gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_INDEXED_FILE_OBJECT",false]],"guid_type_for_non_indexed_file_object (in module gen3.tools.metadata.ingest_manifest)":[[12,"gen3.tools.metadata.ingest_manifest.GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT",false]],"hostname (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.hostname",false]],"index_object_manifest() (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.index_object_manifest",false]],"indexd_record_page_size (in module gen3.tools.indexing.download_manifest)":[[11,"gen3.tools.indexing.download_manifest.INDEXD_RECORD_PAGE_SIZE",false]],"is_healthy() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.is_healthy",false]],"is_healthy() (gen3.jobs.gen3jobs method)":[[4,"gen3.jobs.Gen3Jobs.is_healthy",false]],"is_healthy() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.is_healthy",false]],"list_access_in_drs_manifest() (in module gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.list_access_in_drs_manifest",false]],"list_drs_object() (in module gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.list_drs_object",false]],"list_files_in_drs_manifest() (in module gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.list_files_in_drs_manifest",false]],"list_jobs() (gen3.jobs.gen3jobs method)":[[4,"gen3.jobs.Gen3Jobs.list_jobs",false]],"load() (gen3.tools.download.drs_download.manifest static method)":[[10,"gen3.tools.download.drs_download.Manifest.load",false]],"load_manifest() (gen3.tools.download.drs_download.manifest static method)":[[10,"gen3.tools.download.drs_download.Manifest.load_manifest",false]],"ls() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.ls",false]],"ls_path() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.ls_path",false]],"manifest (class in gen3.tools.download.drs_download)":[[10,"gen3.tools.download.drs_download.Manifest",false]],"max_concurrent_requests (in module gen3.tools.indexing.download_manifest)":[[11,"gen3.tools.indexing.download_manifest.MAX_CONCURRENT_REQUESTS",false]],"max_concurrent_requests (in module gen3.tools.indexing.verify_manifest)":[[11,"gen3.tools.indexing.verify_manifest.MAX_CONCURRENT_REQUESTS",false]],"max_concurrent_requests (in module gen3.tools.metadata.ingest_manifest)":[[12,"gen3.tools.metadata.ingest_manifest.MAX_CONCURRENT_REQUESTS",false]],"md5 (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.MD5",false]],"md5sum (gen3.tools.download.drs_download.manifest attribute)":[[10,"gen3.tools.download.drs_download.Manifest.md5sum",false]],"module":[[9,"module-gen3.tools",false],[10,"module-gen3.tools.download.drs_download",false],[11,"module-gen3.tools.indexing.download_manifest",false],[11,"module-gen3.tools.indexing.index_manifest",false],[11,"module-gen3.tools.indexing.verify_manifest",false],[12,"module-gen3.tools.metadata.ingest_manifest",false]],"object_id (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.object_id",false]],"object_id (gen3.tools.download.drs_download.manifest attribute)":[[10,"gen3.tools.download.drs_download.Manifest.object_id",false]],"object_type (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.object_type",false]],"open_project() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.open_project",false]],"pprint() (gen3.tools.download.drs_download.downloadable method)":[[10,"gen3.tools.download.drs_download.Downloadable.pprint",false]],"prev_guid (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.PREV_GUID",false]],"query() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.query",false]],"query() (gen3.query.gen3query method)":[[7,"gen3.query.Gen3Query.query",false]],"query() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.query",false]],"query_urls() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.query_urls",false]],"raw_data_download() (gen3.query.gen3query method)":[[7,"gen3.query.Gen3Query.raw_data_download",false]],"refresh_access_token() (gen3.auth.gen3auth method)":[[0,"gen3.auth.Gen3Auth.refresh_access_token",false]],"resolve_objects() (gen3.tools.download.drs_download.downloadmanager method)":[[10,"gen3.tools.download.drs_download.DownloadManager.resolve_objects",false]],"rm() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.rm",false]],"rm_path() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.rm_path",false]],"size (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.SIZE",false]],"start_time (gen3.tools.download.drs_download.downloadstatus attribute)":[[10,"gen3.tools.download.drs_download.DownloadStatus.start_time",false]],"status (gen3.tools.download.drs_download.downloadstatus attribute)":[[10,"gen3.tools.download.drs_download.DownloadStatus.status",false]],"submit_file() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.submit_file",false]],"submit_record() (gen3.submission.gen3submission method)":[[8,"gen3.submission.Gen3Submission.submit_record",false]],"threadcontrol (class in gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.ThreadControl",false]],"tmp_folder (in module gen3.tools.indexing.download_manifest)":[[11,"gen3.tools.indexing.download_manifest.TMP_FOLDER",false]],"update() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.update",false]],"update_aliases() (gen3.metadata.gen3metadata method)":[[5,"gen3.metadata.Gen3Metadata.update_aliases",false]],"update_blank() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.update_blank",false]],"update_record() (gen3.index.gen3index method)":[[3,"gen3.index.Gen3Index.update_record",false]],"updated_time (gen3.tools.download.drs_download.downloadable attribute)":[[10,"gen3.tools.download.drs_download.Downloadable.updated_time",false]],"upload() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.upload",false]],"upload_file() (gen3.file.gen3file method)":[[1,"gen3.file.Gen3File.upload_file",false]],"upload_file_to_guid() (gen3.file.gen3file method)":[[1,"gen3.file.Gen3File.upload_file_to_guid",false]],"upload_url() (gen3.wss.gen3wsstorage method)":[[13,"gen3.wss.Gen3WsStorage.upload_url",false]],"urls (in module gen3.tools.indexing.index_manifest)":[[11,"gen3.tools.indexing.index_manifest.URLS",false]],"user_access() (gen3.tools.download.drs_download.downloadmanager method)":[[10,"gen3.tools.download.drs_download.DownloadManager.user_access",false]]},"objects":{"gen3":[[9,3,0,"-","tools"]],"gen3.auth":[[0,0,1,"","Gen3Auth"]],"gen3.auth.Gen3Auth":[[0,1,1,"","curl"],[0,1,1,"","get_access_token"],[0,1,1,"","get_access_token_from_wts"],[0,1,1,"","refresh_access_token"]],"gen3.file":[[1,0,1,"","Gen3File"]],"gen3.file.Gen3File":[[1,1,1,"","async_download_multiple"],[1,1,1,"","delete_file"],[1,1,1,"","delete_file_locations"],[1,1,1,"","download_single"],[1,1,1,"","get_presigned_url"],[1,1,1,"","get_presigned_urls_batch"],[1,1,1,"","upload_file"],[1,1,1,"","upload_file_to_guid"]],"gen3.index":[[3,0,1,"","Gen3Index"]],"gen3.index.Gen3Index":[[3,1,1,"","async_create_record"],[3,1,1,"","async_get_record"],[3,1,1,"","async_get_records_from_checksum"],[3,1,1,"","async_get_records_on_page"],[3,1,1,"","async_get_with_params"],[3,1,1,"","async_query_urls"],[3,1,1,"","async_update_record"],[3,1,1,"","create_blank"],[3,1,1,"","create_new_version"],[3,1,1,"","create_record"],[3,1,1,"","delete_record"],[3,1,1,"","get"],[3,1,1,"","get_all_records"],[3,1,1,"","get_guids_prefix"],[3,1,1,"","get_latest_version"],[3,1,1,"","get_record"],[3,1,1,"","get_record_doc"],[3,1,1,"","get_records"],[3,1,1,"","get_records_on_page"],[3,1,1,"","get_stats"],[3,1,1,"","get_urls"],[3,1,1,"","get_valid_guids"],[3,1,1,"","get_version"],[3,1,1,"","get_versions"],[3,1,1,"","get_with_params"],[3,1,1,"","is_healthy"],[3,1,1,"","query_urls"],[3,1,1,"","update_blank"],[3,1,1,"","update_record"]],"gen3.jobs":[[4,0,1,"","Gen3Jobs"]],"gen3.jobs.Gen3Jobs":[[4,1,1,"","async_run_job_and_wait"],[4,1,1,"","create_job"],[4,1,1,"","get_output"],[4,1,1,"","get_status"],[4,1,1,"","get_version"],[4,1,1,"","is_healthy"],[4,1,1,"","list_jobs"]],"gen3.metadata":[[5,0,1,"","Gen3Metadata"]],"gen3.metadata.Gen3Metadata":[[5,1,1,"","async_create"],[5,1,1,"","async_create_aliases"],[5,1,1,"","async_delete_alias"],[5,1,1,"","async_delete_aliases"],[5,1,1,"","async_get"],[5,1,1,"","async_get_aliases"],[5,1,1,"","async_update"],[5,1,1,"","async_update_aliases"],[5,2,1,"","auth_provider"],[5,1,1,"","batch_create"],[5,1,1,"","create"],[5,1,1,"","create_aliases"],[5,1,1,"","create_index_key_path"],[5,1,1,"","delete"],[5,1,1,"","delete_alias"],[5,1,1,"","delete_aliases"],[5,1,1,"","delete_index_key_path"],[5,2,1,"","endpoint"],[5,1,1,"","get"],[5,1,1,"","get_aliases"],[5,1,1,"","get_index_key_paths"],[5,1,1,"","get_version"],[5,1,1,"","is_healthy"],[5,1,1,"","query"],[5,1,1,"","update"],[5,1,1,"","update_aliases"]],"gen3.object":[[6,0,1,"","Gen3Object"]],"gen3.object.Gen3Object":[[6,1,1,"","delete_object"]],"gen3.query":[[7,0,1,"","Gen3Query"]],"gen3.query.Gen3Query":[[7,1,1,"","graphql_query"],[7,1,1,"","query"],[7,1,1,"","raw_data_download"]],"gen3.submission":[[8,0,1,"","Gen3Submission"]],"gen3.submission.Gen3Submission":[[8,1,1,"","create_program"],[8,1,1,"","create_project"],[8,1,1,"","delete_node"],[8,1,1,"","delete_nodes"],[8,1,1,"","delete_program"],[8,1,1,"","delete_project"],[8,1,1,"","delete_record"],[8,1,1,"","delete_records"],[8,1,1,"","export_node"],[8,1,1,"","export_record"],[8,1,1,"","get_dictionary_all"],[8,1,1,"","get_dictionary_node"],[8,1,1,"","get_graphql_schema"],[8,1,1,"","get_programs"],[8,1,1,"","get_project_dictionary"],[8,1,1,"","get_project_manifest"],[8,1,1,"","get_projects"],[8,1,1,"","open_project"],[8,1,1,"","query"],[8,1,1,"","submit_file"],[8,1,1,"","submit_record"]],"gen3.tools.download":[[10,3,0,"-","drs_download"]],"gen3.tools.download.drs_download":[[10,0,1,"","DownloadManager"],[10,0,1,"","DownloadStatus"],[10,0,1,"","Downloadable"],[10,0,1,"","Manifest"],[10,4,1,"","download_files_in_drs_manifest"],[10,4,1,"","list_access_in_drs_manifest"],[10,4,1,"","list_drs_object"],[10,4,1,"","list_files_in_drs_manifest"]],"gen3.tools.download.drs_download.DownloadManager":[[10,1,1,"","cache_hosts_wts_tokens"],[10,1,1,"","download"],[10,1,1,"","get_fresh_token"],[10,1,1,"","resolve_objects"],[10,1,1,"","user_access"]],"gen3.tools.download.drs_download.DownloadStatus":[[10,2,1,"","end_time"],[10,2,1,"","start_time"],[10,2,1,"","status"]],"gen3.tools.download.drs_download.Downloadable":[[10,2,1,"","_manager"],[10,2,1,"","access_methods"],[10,2,1,"","children"],[10,2,1,"","created_time"],[10,1,1,"","download"],[10,2,1,"","file_name"],[10,2,1,"","file_size"],[10,2,1,"","hostname"],[10,2,1,"","object_id"],[10,2,1,"","object_type"],[10,1,1,"","pprint"],[10,2,1,"","updated_time"]],"gen3.tools.download.drs_download.Manifest":[[10,2,1,"","commons_url"],[10,1,1,"","create_object_list"],[10,2,1,"","file_name"],[10,2,1,"","file_size"],[10,1,1,"","load"],[10,1,1,"","load_manifest"],[10,2,1,"","md5sum"],[10,2,1,"","object_id"]],"gen3.tools.indexing":[[11,3,0,"-","download_manifest"],[11,3,0,"-","index_manifest"],[11,3,0,"-","verify_manifest"]],"gen3.tools.indexing.download_manifest":[[11,2,1,"","CURRENT_DIR"],[11,2,1,"","INDEXD_RECORD_PAGE_SIZE"],[11,2,1,"","MAX_CONCURRENT_REQUESTS"],[11,2,1,"","TMP_FOLDER"],[11,4,1,"","async_download_object_manifest"]],"gen3.tools.indexing.index_manifest":[[11,2,1,"","ACLS"],[11,2,1,"","AUTHZ"],[11,2,1,"","CURRENT_DIR"],[11,2,1,"","GUID"],[11,2,1,"","MD5"],[11,2,1,"","PREV_GUID"],[11,2,1,"","SIZE"],[11,0,1,"","ThreadControl"],[11,2,1,"","URLS"],[11,4,1,"","delete_all_guids"],[11,4,1,"","index_object_manifest"]],"gen3.tools.indexing.verify_manifest":[[11,2,1,"","CURRENT_DIR"],[11,2,1,"","MAX_CONCURRENT_REQUESTS"],[11,4,1,"","async_verify_object_manifest"]],"gen3.tools.metadata":[[12,3,0,"-","ingest_manifest"]],"gen3.tools.metadata.ingest_manifest":[[12,2,1,"","COLUMN_TO_USE_AS_GUID"],[12,2,1,"","GUID_TYPE_FOR_INDEXED_FILE_OBJECT"],[12,2,1,"","GUID_TYPE_FOR_NON_INDEXED_FILE_OBJECT"],[12,2,1,"","MAX_CONCURRENT_REQUESTS"],[12,4,1,"","async_ingest_metadata_manifest"],[12,4,1,"","async_query_urls_from_indexd"]],"gen3.wss":[[13,0,1,"","Gen3WsStorage"]],"gen3.wss.Gen3WsStorage":[[13,1,1,"","copy"],[13,1,1,"","download"],[13,1,1,"","download_url"],[13,1,1,"","ls"],[13,1,1,"","ls_path"],[13,1,1,"","rm"],[13,1,1,"","rm_path"],[13,1,1,"","upload"],[13,1,1,"","upload_url"]]},"objnames":{"0":["py","class","Python class"],"1":["py","method","Python method"],"2":["py","attribute","Python attribute"],"3":["py","module","Python module"],"4":["py","function","Python function"]},"objtypes":{"0":"py:class","1":"py:method","2":"py:attribute","3":"py:module","4":"py:function"},"terms":{"0a80fada010c":11,"0a80fada096c":11,"0a80fada097c":11,"0a80fada098c":11,"0a80fada099c":11,"11e9":11,"255e396f":11,"450c":11,"473d83400bc1bc9dc635e334fadd433c":11,"473d83400bc1bc9dc635e334faddd33c":11,"473d83400bc1bc9dc635e334fadde33c":11,"473d83400bc1bc9dc635e334faddf33c":11,"6f90":8,"7d3d8d2083b4":11,"93d9af72":11,"9a07":11,"A":[1,3,4,5,6,7,8,10,11,13],"ALL":7,"AND":5,"All":11,"Be":1,"But":5,"By":11,"For":[1,5,6,7,8,9,11],"IF":11,"If":[0,1,7,11,12],"In":10,"It":10,"Most":9,"NOT":12,"OR":5,"Same":13,"Such":9,"THE":11,"THIS":11,"That":3,"The":[0,1,2,3,5,8,10,11],"There":11,"These":9,"This":[0,1,2,3,4,5,6,7,8,10,11,13],"To":11,"We":11,"When":12,"YOU":11,"_get_acl_from_row":11,"_get_authz_from_row":11,"_get_file_name_from_row":11,"_get_file_size_from_row":11,"_get_guid_for_row":12,"_get_guid_from_row":11,"_get_md5_from_row":11,"_get_urls_from_row":11,"_guid_typ":12,"_manag":[2,9,10],"_query_for_associated_indexd_record_guid":12,"_ssl":[3,4,5],"a5c6":11,"ab167e49d25b488939b1ede42752458b":3,"abov":11,"access":[0,1,3,7,10],"access_method":[2,9,10],"access_token":0,"accesstoken":0,"acl":[2,3,9,11],"across":11,"act":0,"action":[9,11],"actual":11,"add":[3,5],"addit":[3,5,10,11],"admin":[5,11],"admin_endpoint_suffix":5,"algorithm":3,"alia":[3,5],"alias":5,"aliv":7,"allow":[0,6,8,10,11,12],"allowed_data_upload_bucket":1,"along":2,"alreadi":9,"also":1,"altern":[5,11],"alway":5,"ammount":12,"amount":[1,9],"ani":[0,5,10,11],"anoth":5,"api":[0,5,8,11],"api_key":11,"appli":7,"appropri":13,"arbitrari":0,"argument":[0,13],"array":8,"asc":7,"assign":9,"assist":10,"associ":[3,5],"assum":11,"async":[1,3,4,5,9,11,12],"async_cr":[2,5],"async_create_alias":[2,5],"async_create_record":[2,3],"async_delete_alia":[2,5],"async_delete_alias":[2,5],"async_download_multipl":[1,2],"async_download_object_manifest":[2,9,11],"async_get":[2,5],"async_get_alias":[2,5],"async_get_record":[2,3],"async_get_records_from_checksum":[2,3],"async_get_records_on_pag":[2,3],"async_get_with_param":[2,3],"async_ingest_metadata_manifest":[2,9,12],"async_query_url":[2,3],"async_query_urls_from_indexd":[2,9,12],"async_run_job_and_wait":[2,4],"async_upd":[2,5],"async_update_alias":[2,5],"async_update_record":[2,3],"async_verify_object_manifest":[2,9,11],"asynchron":[1,3,4,5],"asyncio":[11,12],"asyncron":5,"attach":[3,5],"attempt":11,"attribut":[10,11],"auth":[1,2,3,4,5,6,7,8,10,11,12,13],"auth_provid":[1,2,3,4,5,6,7,8,13],"authbas":0,"authent":0,"author":1,"authz":[0,1,2,3,9,10,11],"auto":[0,2],"automat":0,"avail":[1,2,10,11],"az":1,"b":[5,11],"b0f1":11,"bar":10,"base":[0,1,3,4,5,6,7,8,9,11,13],"baseid":3,"basic":[3,11,12],"batch_creat":[2,5],"batch_siz":8,"behalf":0,"behavior":11,"belong":8,"blank":3,"blob":[5,7],"bodi":3,"bool":[1,4,5,8,10,11,12],"boolean":3,"bownload":10,"broad":9,"broken":9,"bucket":[1,6],"bundl":10,"byte":10,"c":[5,11],"cach":10,"cache_hosts_wts_token":[2,9,10],"call":[10,13],"can":[0,3,4,8,11,12],"capabl":9,"case":[0,10],"categori":9,"ccle":8,"ccle_one_record":8,"ccle_sample_nod":8,"cdis":7,"chang":[3,11],"checksum":[3,10],"checksum_typ":3,"child":10,"children":[2,9,10],"chunk_siz":8,"class":[0,2,10,11,13],"cli":10,"client":[0,3],"client_credenti":0,"client_id":0,"client_scop":0,"client_secret":0,"code":[2,8],"column":[11,12],"column_to_use_as_guid":[2,9,12],"columna":11,"columnb":11,"columnc":11,"com":7,"comma":11,"command":[10,11],"common":[0,1,3,4,5,6,7,8,9,10,11,12,13],"commons_url":[2,9,10,11,12],"complet":[4,11],"complex":7,"concat":11,"concurr":[11,12],"configur":1,"connect":12,"consist":3,"constructor":0,"contain":[0,2,5,8,9,10,11,12],"content":[3,13],"content_created_d":3,"content_updated_d":3,"continu":10,"control":3,"conveni":10,"copi":[2,13],"coroutin":11,"correspond":3,"count":3,"crdc":0,"creat":[2,3,4,5,6,8,10,11],"create_alias":[2,5],"create_blank":[2,3],"create_index_key_path":[2,5],"create_job":[2,4],"create_new_vers":[2,3],"create_object_list":[2,9,10],"create_program":[2,8],"create_project":[2,8],"create_record":[2,3],"created_tim":[2,9,10],"creation":[3,11],"cred":3,"credenti":[0,1,3,4,5,6,7,8,10,11,13],"csv":[8,11,12],"curl":[0,2],"current":[6,8,10],"current_dir":[2,9,11],"custom":11,"d":5,"d70b41b9":8,"data":[0,1,3,5,7,8,10,11],"data_spreadsheet":8,"data_typ":7,"data_upload_bucket":1,"dataa":11,"datab":11,"databas":5,"datacommon":0,"datafil":10,"datamanag":10,"date":3,"datetim":[1,3,10],"dbgap":12,"dcf":8,"def":11,"default":[0,1,3,7,8,11,12],"defin":[5,8,10],"delay":4,"delet":[0,1,2,3,5,6,8,10,11],"delete_alia":[2,5],"delete_alias":[2,5],"delete_all_guid":[2,9,11],"delete_fil":[1,2],"delete_file_loc":[1,2,6],"delete_index_key_path":[2,5],"delete_nod":[2,8],"delete_object":[2,6],"delete_program":[2,8],"delete_project":[2,8],"delete_record":[2,3,8],"delete_unpacked_packag":10,"delimet":[11,12],"delimit":11,"demograph":8,"deprec":1,"descript":[3,5],"desir":11,"dest_path":13,"dest_urlstr":13,"dest_w":13,"dest_wskey":13,"detail":[2,7,10],"determin":[10,11,12],"dev":11,"dict":[1,3,4,5,10,11,12],"dictionari":[3,4,5,7,8],"dids":3,"differ":5,"direct":0,"directori":[10,11],"disabl":10,"discoveri":10,"disk":13,"dispatch":4,"dist_resolut":3,"distribut":3,"doc":[7,10],"docstr":2,"document":[1,3],"doe":[0,12],"domain":[11,12],"done":4,"download":[0,1,2,3,4,5,6,7,8,9,13],"download_files_in_drs_manifest":[2,9,10],"download_list":10,"download_manifest":11,"download_path":1,"download_singl":[1,2],"download_url":[2,13],"downloadmanag":[2,9,10],"downloadstatus":[2,9,10],"drs":[2,9],"drs_download":10,"drs_hostnam":10,"drsdownload":10,"drsobjecttyp":10,"e":[5,10],"e043ab8b77b9":8,"effici":[1,9],"eg":3,"either":8,"elasticsearch":7,"els":[0,12],"elsewher":12,"empti":[8,11],"enabl":11,"end":[5,10],"end_tim":[2,9,10],"endpoint":[0,1,2,3,4,5,7,8,13],"entir":8,"entri":[3,11],"env":0,"environ":0,"equal":7,"error":[10,11],"error_nam":11,"etc":8,"even":11,"everi":[9,11],"everyth":11,"ex":[0,11,12],"exampl":[0,1,3,4,5,6,7,8,10,11,13],"exclud":3,"execut":[7,8,11],"exist":[1,3,5,6,9,12],"expect":[5,9,11],"experi":8,"expir":[0,1],"expires_in":1,"export":[8,10],"export_nod":[2,8],"export_record":[2,8],"f1f8":11,"factori":10,"fail":[8,10],"fals":[1,3,5,6,10,11],"featur":[1,6],"fenc":[0,1],"fetch":0,"field":[3,5,7,11,12],"fieldnam":11,"file":[0,2,3,4,8,9,10,11,12,13],"file_nam":[1,2,3,9,10,11],"file_s":[2,9,10,11],"file_st":3,"fileformat":8,"filenam":[0,8,10,11,12],"filename_format":1,"files":10,"fill":12,"filter":[5,7],"filter_object":7,"first":[7,8],"flag":11,"folder":11,"follow":[0,11],"forc":11,"force_metadata_columns_even_if_empti":11,"form":13,"format":[3,5,8,11],"func_to_parse_row":[11,12],"function":[2,3,4,5,9,10,11,12],"g":10,"gen3":[10,11,12],"gen3_api_key":0,"gen3_oidc_client_creds_secret":0,"gen3auth":[0,1,2,3,4,5,6,7,8,10,11,12,13],"gen3fil":[1,2],"gen3index":[2,3],"gen3job":[2,4,10],"gen3metadata":[2,5],"gen3object":[2,6],"gen3queri":[2,7],"gen3submiss":[2,8],"gen3wsstorag":[2,13],"generat":[0,1,2,3,4,5,6,7,8,10,13],"get":[0,1,2,3,4,5,8,10,11,12,13],"get_access_token":[0,2],"get_access_token_from_wt":[0,2],"get_alias":[2,5],"get_all_record":[2,3],"get_dictionary_al":[2,8],"get_dictionary_nod":[2,8],"get_fresh_token":[2,9,10],"get_graphql_schema":[2,8],"get_guid_from_fil":12,"get_guids_prefix":[2,3],"get_index_key_path":[2,5],"get_latest_vers":[2,3],"get_output":[2,4],"get_presigned_url":[1,2],"get_presigned_urls_batch":[1,2],"get_program":[2,8],"get_project":[2,8],"get_project_dictionari":[2,8],"get_project_manifest":[2,8],"get_record":[2,3],"get_record_doc":[2,3],"get_records_on_pag":[2,3],"get_stat":[2,3],"get_status":[2,4],"get_url":[2,3],"get_valid_guid":[2,3],"get_vers":[2,3,4,5],"get_with_param":[2,3],"giangb":11,"github":[2,7],"give":1,"given":[0,3,4,5,8,10,12,13],"global":[4,5],"good":3,"grant":0,"graph":8,"graphql":[7,8],"graphql_queri":[2,7],"group":3,"guid":[1,2,3,5,6,9,11,12],"guid_exampl":11,"guid_for_row":12,"guid_from_fil":12,"guid_type_for_indexed_file_object":[2,9,12],"guid_type_for_non_indexed_file_object":[2,9,12],"guppi":7,"handl":[3,10],"hardcod":0,"has_vers":3,"hash":[3,11],"hash_typ":3,"header":11,"healthi":[3,4,5],"help":11,"helper":2,"hit":11,"host":10,"hostnam":[2,9,10],"howto":10,"http":12,"https":[0,7,11],"id":[0,1,3,5,10,11],"idea":3,"identifi":[3,5,9,11],"idp":0,"illustr":11,"immut":3,"implement":0,"implic":11,"import":11,"includ":[0,3],"indent":10,"index":[0,2,5,9],"index_manifest":11,"index_object_manifest":[2,9,11],"indexd":[1,3,6,10,11,12],"indexd_field":[11,12],"indexd_record_page_s":[2,9,11],"indexed_file_object_guid":12,"indic":[0,11],"infil":10,"info":[3,11],"inform":[2,3,10],"ingest":[2,9],"ingest_manifest":12,"initi":[0,10],"input":[4,10,11],"input_manifest":11,"instal":[0,2,11],"instanc":[1,3,6,7,8,9,10],"instead":[1,7,11],"int":[1,3,5,7,8,10,11,12],"integ":[1,3,8],"intend":0,"interact":[1,3,4,5,6,8,13],"interest":10,"interpret":0,"introspect":8,"involv":9,"is_healthi":[2,3,4,5],"is_indexed_file_object":12,"isn":1,"issu":0,"job":2,"job_id":4,"job_input":4,"job_nam":4,"json":[0,1,3,4,5,6,7,8,10,11,13],"just":[5,11,12],"jwt":0,"key":[0,3,5,13],"know":11,"known":10,"kwarg":[3,4,5],"larg":9,"last":10,"latest":3,"least":3,"level":6,"librari":11,"like":[3,5,9,11,12],"limit":[1,3,5,12],"linear":4,"linux":10,"list":[0,1,3,4,5,7,8,10,11,13],"list_access_in_drs_manifest":[2,9,10],"list_drs_object":[2,9,10],"list_files_in_drs_manifest":[2,9,10],"list_job":[2,4],"live":[11,12],"load":[2,9,10],"load_manifest":[2,9,10],"local":[0,13],"locat":[1,6],"lock":12,"log":[8,10,11,12],"logic":[5,12],"loop":11,"ls":[2,13],"ls_path":[2,13],"maco":11,"made":3,"main":10,"make":[9,11],"manag":[1,5,10],"mani":[8,11],"manifest":[2,8,9,10,11,12],"manifest_1":10,"manifest_data":1,"manifest_fil":[11,12],"manifest_file_delimit":[11,12],"manifest_row_pars":[11,12],"map":[0,1,11],"mark":8,"master":7,"match":[3,5,12],"max":5,"max_concurrent_request":[1,2,9,11,12],"max_presigned_url_ttl":1,"max_tri":8,"maximum":[11,12],"may":[0,9,11],"md":[7,10],"md5":[2,3,9,11],"md5_hash":11,"md5sum":[2,9,10],"mds":[5,12],"mean":8,"mechan":3,"merg":5,"metadata":[2,3,6,9,11],"metadata_list":5,"metadata_sourc":12,"metadata_typ":12,"metdata":12,"method":[1,7,10],"minimum":10,"minut":0,"mode":7,"modul":[2,10,11],"mostly":2,"multipl":[1,8,11],"multiprocess":1,"must":[1,5],"my_common":10,"my_credenti":10,"my_field":7,"my_index":7,"my_program":7,"my_project":7,"name":[3,4,8,10,11,12,13],"namespac":[0,12],"necessari":[3,5],"need":[3,7,10,11],"nest":5,"net":11,"never":0,"new":[0,3],"no_progress":1,"node":8,"node_nam":8,"node_typ":8,"none":[0,1,3,4,5,6,7,8,10,11,12,13],"note":[0,3,11,12],"noth":[3,6],"now":[1,8],"num":5,"num_process":[1,11],"num_total_fil":11,"number":[3,7,8,11,12],"object":[1,2,3,4,5,7,8,9,10,11,13],"object_id":[1,2,9,10],"object_list":10,"object_typ":[2,9,10],"objectid":10,"obtain":[0,10],"occur":10,"offset":[5,7],"oidc":0,"old":3,"one":[3,5,7,10,11],"onli":[3,5,7,8,10,11],"open":[8,10,11],"open_project":[2,8],"openid":0,"opt":0,"option":[0,1,3,4,5,6,7,8,10,11],"order":[0,8],"ordered_node_list":8,"org":10,"origin":1,"os":0,"otherwis":[1,10],"output":[4,5,11,12],"output_dir":10,"output_filenam":[11,12],"overrid":[0,11,12],"overwrit":5,"packag":10,"page":[0,1,2,3,4,5,6,7,8,10,11,13],"pagin":3,"parallel":11,"param":[3,5,8,10],"paramet":[0,1,3,4,5,6,7,8,10,11,12,13],"pars":[10,11,12,13],"parser":[11,12],"particular":0,"pass":[0,7,8,10],"password":[11,12],"path":[0,1,5,10,11,13],"path_to_manifest":11,"pattern":[3,12],"pdcdatastor":11,"pend":10,"per":[11,12],"peregrin":8,"permiss":10,"persist":9,"phs0001":11,"phs0002":11,"pick":1,"pla":11,"place":11,"planx":11,"point":[0,1,3,4,5,6,7,8,10,13],"popul":[10,12],"posit":[1,7],"possibl":10,"post":[0,11],"pprint":[2,9,10],"prefer":1,"prefix":3,"presign":1,"pretti":10,"prev_guid":[2,9,11],"previous":[3,4,11],"print":[8,10],"process":11,"processed_fil":11,"profil":[0,1,3,4,5,6,7,8,10,13],"program":[8,11],"progress":[8,10],"project":[8,11],"project_id":[7,8],"protocol":1,"provid":[0,1,3,5,7,8,12],"public":[3,5],"put":0,"py":11,"python":[2,9,11],"python3":11,"python_subprocess_command":11,"queri":[1,2,3,5,8,11,12],"query_str":7,"query_txt":[7,8],"query_url":[2,3],"queue":1,"queue_siz":1,"quickstart":2,"rather":0,"raw":[7,11],"raw_data_download":[2,7],"rbac":3,"read":[3,5,11],"readm":2,"reason":10,"record":[1,3,5,7,8,11,12],"refresh":[0,10],"refresh_access_token":[0,2],"refresh_fil":[0,1,3,4,5,6,7,8,10,13],"refresh_token":0,"regist":8,"regular":7,"relat":9,"remov":[1,6,11,13],"renam":1,"replac":11,"replace_url":11,"repo":2,"repres":[3,5,10],"represent":[1,3],"request":[0,1,3,5,8,11,12],"requir":10,"resolv":10,"resolve_object":[2,9,10],"respect":7,"respons":[0,1,3,4,5],"result":[1,8,10,11],"retri":8,"retriev":[1,8,10,12],"return":[0,1,3,4,5,6,7,8,10,11],"return_full_metadata":5,"rev":3,"revers":8,"revis":3,"right":1,"rm":[2,13],"rm_path":[2,13],"root":[11,12],"row":[7,8,11,12],"row_offset":8,"rtype":3,"run":[8,11],"s":[1,4,8,10,11],"s3":[1,10,11],"safe":11,"sampl":[8,10],"sandbox":[0,1,3,4,5,6,7,8,10,13],"save":10,"save_directori":10,"schema":8,"scope":[0,1],"screen":8,"script":2,"search":[0,2,3],"second":[1,4],"secret":0,"see":[7,10,11],"self":10,"semaphon":12,"semaphor":12,"separ":[0,11],"server":10,"servic":[1,3,4,5,6,8,11,12,13],"service_loc":[3,4,5],"session":11,"set":[0,1,5,10],"setup":2,"sheepdog":8,"show":10,"show_progress":10,"shown":11,"sign":1,"signpost":3,"similar":10,"simpl":3,"simpli":11,"sinc":3,"singl":[1,5,8],"size":[2,3,9,10,11],"skip":8,"skip_complet":1,"sleep":4,"someth":11,"sort":7,"sort_field":7,"sort_object":7,"sourc":[0,1,2,3,4,5,6,7,8,10,11,12,13],"space":[0,11],"specif":[5,8,11,12],"specifi":[0,1,3,11,13],"spreadsheet":8,"src_path":13,"src_urlstr":13,"src_ws":13,"src_wskey":13,"ssl":[3,4,5],"start":[4,7,8,10],"start_tim":[2,9,10],"static":10,"status":[2,4,9,10],"status_cod":10,"storag":[1,2,6],"store":[1,3,10],"str":[0,1,3,4,5,7,8,10,11,12],"string":[0,3,5,11,13],"strip":11,"sub":8,"subject":[7,8],"submiss":2,"submit":[8,11],"submit_additional_metadata_column":11,"submit_fil":[2,8],"submit_record":[2,8],"submitter_id":7,"success":[1,10],"suffici":3,"suppli":3,"support":[0,1,5,8,11],"sure":1,"synchron":11,"syntax":7,"system":[6,7,8,9],"t":[1,5,11],"tab":11,"task":9,"temporari":11,"test":11,"test1":11,"test2":11,"test3":11,"test4":11,"test5":11,"text":[1,7,8],"thread":11,"thread_num":11,"threadcontrol":[2,9,11],"tier":7,"time":[1,3,8,10,11],"timestamp":10,"tmp_folder":[2,9,11],"token":[0,10],"tool":2,"total":11,"treat":[1,5],"tree":10,"tri":0,"true":[1,3,4,5,6,7,8,10,11,12],"tsv":[8,11,12],"tupl":[0,3,11,12],"type":[1,3,4,5,7,8,10,11,12],"typic":10,"uc":7,"unaccess":7,"uniqu":[1,5],"unknown":10,"unpack":10,"unpack_packag":10,"updat":[2,3,5,10,11],"update_alias":[2,5],"update_blank":[2,3],"update_record":[2,3],"updated_tim":[2,9,10],"upload":[1,2,3,8,13],"upload_fil":[1,2],"upload_file_to_guid":[1,2],"upload_url":[2,13],"url":[1,2,3,9,10,11,12,13],"urls_metadata":3,"usag":11,"use":[0,1,3,4,5,6,7,8,10,11,12,13],"use_agg_md":5,"user":[0,10,12],"user_access":[2,9,10],"usual":12,"utcnow":1,"util":9,"uuid":[1,3,8],"uuid1":8,"uuid2":8,"valid":[3,7],"valu":[0,1,3,5,7,10,11],"value_from_indexd":11,"value_from_manifest":11,"variabl":[0,7,8],"various":2,"verbos":[7,8],"verif":11,"verifi":[2,9],"verify_manifest":11,"verify_object_manifest":11,"version":[3,4,5],"vital_status":7,"wait":4,"want":[0,3,8],"warn":11,"way":10,"web":0,"whether":[3,4,5,8,11,12],"whose":5,"will":[1,3,4,5,7,10,11,12],"within":[0,2,9],"without":[3,5],"won":5,"work":[0,10],"workaround":11,"worksheet":8,"workspac":[0,2],"wrapper":10,"write":11,"ws":13,"ws_urlstr":13,"wskey":13,"wss":13,"wts":[0,10],"x":11,"xlsx":8},"titles":["Gen3 Auth Helper","Gen3 File Class","Welcome to Gen3 SDK\u2019s documentation!","Gen3 Index Class","Gen3 Jobs Class","Gen3 Metadata Class","Gen3 Object Class","Gen3 Query Class","Gen3 Submission Class","Gen3 Tools","DRS Download Tools","Indexing Tools","Metadata Tools","Gen3 Workspace Storage"],"titleterms":{"auth":0,"class":[1,3,4,5,6,7,8],"document":2,"download":[10,11],"drs":10,"file":1,"gen3":[0,1,2,3,4,5,6,7,8,9,13],"helper":0,"index":[3,11],"indic":2,"ingest":12,"job":4,"metadata":[5,12],"object":6,"queri":7,"s":2,"sdk":2,"storag":13,"submiss":8,"tabl":2,"tool":[9,10,11,12],"verifi":11,"welcom":2,"workspac":13}})
\ No newline at end of file
diff --git a/docs/howto/asyncDownloadMultiple.md b/docs/howto/asyncDownloadMultiple.md
new file mode 100644
index 00000000..4dee1153
--- /dev/null
+++ b/docs/howto/asyncDownloadMultiple.md
@@ -0,0 +1,186 @@
+## Asynchronous Multiple File Downloads
+
+The Gen3 SDK provides an optimized asynchronous download method `async_download_multiple` for efficiently downloading large numbers of files with high throughput and memory efficiency.
+
+## Overview
+
+The `async_download_multiple` method implements a hybrid architecture combining:
+
+- **Multiprocessing**: Multiple Python subprocesses for CPU utilization
+- **Asyncio**: High I/O concurrency within each process
+- **Queue-based memory management**: Efficient handling of large file sets
+- **Just-in-time presigned URL generation**: Optimized authentication flow
+
+## Architecture
+
+### Concurrency Model
+
+The implementation uses a three-tier architecture:
+
+1. **Producer Thread**: Feeds GUIDs to worker processes via bounded queues
+2. **Worker Processes**: Multiple Python subprocesses with asyncio event loops
+3. **Queue System**: Memory-efficient streaming of work items
+
+```python
+# Architecture overview
+Producer Thread → Input Queue → Worker Processes → Output Queue → Results
+ (1) (configurable) (configurable) (configurable) (Final)
+```
+
+### Key Features
+
+- **Memory Efficiency**: Bounded queues prevent memory explosion with large file sets
+- **True Parallelism**: Multiprocessing bypasses Python GIL limitations
+- **High Concurrency**: Configurable concurrent downloads per process
+- **Resume Support**: Skip completed files with `--skip-completed` flag
+- **Progress Tracking**: Real-time progress bars and detailed reporting
+
+## Usage
+
+### Command Line Interface
+
+Download multiple files using a manifest:
+
+```bash
+gen3 --endpoint my-commons.org --auth credentials.json download-multiple \
+ --manifest files.json \
+ --download-path ./downloads \
+ --max-concurrent-requests 10 \
+ --filename-format original \
+ --skip-completed \
+ --no-prompt
+```
+
+### Python API
+
+The `async_download_multiple` method is available in the `Gen3File` class for programmatic use. Refer to the Python SDK documentation for the complete API reference.
+
+## Parameters
+
+For detailed parameter information and current default values, run:
+
+```bash
+gen3 download-multiple --help
+```
+
+The command supports various options for customizing download behavior, including concurrency settings, file naming strategies, and progress controls.
+
+## Performance Characteristics
+
+### Throughput Optimization
+
+The method is optimized for high-throughput scenarios:
+
+- **Concurrent Downloads**: Configurable number of simultaneous downloads
+- **Memory Usage**: Bounded by queue sizes (typically < 100MB)
+- **CPU Utilization**: Leverages multiple CPU cores
+- **Network Efficiency**: Just-in-time presigned URL generation
+
+### Scalability
+
+Performance scales with:
+
+- **File Count**: Linear time complexity with constant memory usage
+- **File Size**: Independent of individual file sizes
+- **Network Bandwidth**: Limited by available bandwidth and concurrent connections
+- **System Resources**: Scales with available CPU cores and memory
+
+## Error Handling
+
+### Robust Error Recovery
+
+The implementation includes comprehensive error handling:
+
+- **Network Failures**: Automatic retry with exponential backoff
+- **Authentication Errors**: Token refresh and retry
+- **File System Errors**: Graceful handling of permission and space issues
+- **Process Failures**: Automatic worker process restart
+
+### Result Reporting
+
+The method returns a structured result object containing lists of succeeded, failed, and skipped downloads with detailed information about each operation.
+
+## Best Practices
+
+### Configuration Recommendations
+
+For optimal performance, adjust the concurrency and process settings based on your specific use case:
+
+- **Small files**: Use higher concurrent request limits
+- **Large files**: Use lower concurrent request limits to avoid overwhelming the system
+- **High-bandwidth networks**: Increase the number of worker processes
+- **Limited memory**: Reduce queue sizes to manage memory usage
+
+
+## Comparison with Synchronous Downloads
+
+### Performance Advantages
+
+| Metric | Synchronous | Asynchronous |
+| ------------------ | ---------------------------- | ---------------------------- |
+| Memory Usage | O(n) - grows with file count | O(1) - bounded by queue size |
+| CPU Utilization | Single core | Multiple cores |
+| Network Efficiency | Sequential | Parallel |
+| Scalability | Limited by GIL | Scales with CPU cores |
+
+## Troubleshooting
+
+### Common Issues
+
+**Slow Downloads:**
+
+- Check network bandwidth and server limits
+- Reduce concurrent request limits if server is overwhelmed
+
+**Memory Issues:**
+
+- Reduce queue sizes and batch sizes
+- Lower the number of worker processes if system memory is limited
+- Monitor system memory usage during downloads
+
+**Authentication Errors:**
+
+- Verify credentials file is valid and not expired
+- Check endpoint URL is correct
+- Ensure proper permissions for target files
+
+**Process Failures:**
+
+- Check system resources (CPU, memory, file descriptors)
+- Verify network connectivity to Gen3 commons
+- Review logs for specific error messages
+
+### Debugging
+
+Enable verbose logging for detailed debugging:
+
+```bash
+gen3 -vv --endpoint my-commons.org --auth credentials.json download-multiple \
+ --manifest files.json \
+ --download-path ./downloads
+```
+
+## Examples
+
+### Basic Usage
+
+```bash
+# Download files with default settings
+gen3 --endpoint data.commons.io --auth creds.json download-multiple \
+ --manifest my_files.json \
+ --download-path ./data
+```
+
+### High-Performance Configuration
+
+```bash
+# Optimized for high-throughput downloads
+gen3 --endpoint data.commons.io --auth creds.json download-multiple \
+ --manifest large_dataset.json \
+ --download-path ./large_downloads \
+ --max-concurrent-requests 8 \
+ --no-progress \
+ --skip-completed
+```
+
+**Note**: The specific values shown in examples (like `--max-concurrent-requests 8`) are for demonstration only. For current parameter options and default values, always refer to the command line help: `gen3 download-multiple --help`
diff --git a/gen3/cli/__main__.py b/gen3/cli/__main__.py
index 5a51be11..a701efb5 100644
--- a/gen3/cli/__main__.py
+++ b/gen3/cli/__main__.py
@@ -15,6 +15,7 @@
import gen3.cli.drs_pull as drs_pull
import gen3.cli.users as users
import gen3.cli.wrap as wrap
+import gen3.cli.download as download
import gen3
from gen3 import logging as sdklogging
from gen3.cli import nih
@@ -142,6 +143,8 @@ def main(
main.add_command(objects.objects)
main.add_command(drs_pull.drs_pull)
main.add_command(file.file)
+main.add_command(download.download_single, name="download-single")
+main.add_command(download.download_multiple, name="download-multiple")
main.add_command(nih.nih)
main.add_command(users.users)
main.add_command(wrap.run)
diff --git a/gen3/cli/download.py b/gen3/cli/download.py
new file mode 100644
index 00000000..ab8bb91b
--- /dev/null
+++ b/gen3/cli/download.py
@@ -0,0 +1,305 @@
+"""
+Gen3 download commands for CLI.
+"""
+
+import asyncio
+import json
+from datetime import datetime
+from typing import List, Dict, Any
+import os
+
+import click
+
+from cdislogging import get_logger
+from gen3.file import Gen3File
+
+logging = get_logger("__name__")
+
+MAX_CONCURRENT_REQUESTS = 20
+
+
+def get_or_create_event_loop_for_thread():
+ """Get or create event loop for current thread."""
+ try:
+ loop = asyncio.get_event_loop()
+ except RuntimeError:
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ return loop
+
+
+def load_manifest(manifest_path: str) -> List[Dict[str, Any]]:
+ """Load manifest from JSON file.
+
+ Args:
+ manifest_path (str): Path to the manifest JSON file.
+
+ Returns:
+ List[Dict[str, Any]]: List of dictionaries containing file information.
+
+ Raises:
+ FileNotFoundError: If the manifest file does not exist.
+ json.JSONDecodeError: If the manifest file contains invalid JSON.
+ """
+ try:
+ with open(manifest_path, "r") as f:
+ return json.load(f)
+ except (FileNotFoundError, json.JSONDecodeError) as e:
+ raise click.ClickException(f"Error loading manifest: {e}")
+
+
+def validate_manifest(manifest_data: List[Dict[str, Any]]) -> bool:
+ """Validate manifest structure.
+
+ Args:
+ manifest_data (List[Dict[str, Any]]): List of dictionaries to validate.
+
+ Returns:
+ bool: True if manifest is valid, False otherwise.
+ """
+ if not isinstance(manifest_data, list):
+ return False
+
+ for item in manifest_data:
+ if not isinstance(item, dict):
+ return False
+ if not (item.get("guid") or item.get("object_id")):
+ return False
+
+ return True
+
+
+@click.command()
+@click.argument("guid")
+@click.option(
+ "--download-path",
+ default=f"./download_{datetime.now().strftime('%d_%b_%Y')}",
+ help="Directory to download file to (default: timestamped folder)",
+)
+@click.option(
+ "--protocol",
+ default=None,
+ help="Protocol for presigned URL (e.g., s3) (default: auto-detect)",
+)
+@click.option(
+ "--filename-format",
+ default="original",
+ type=click.Choice(["original", "guid", "combined"]),
+ help="Filename format (default: original)",
+)
+@click.option(
+ "--skip-completed",
+ is_flag=True,
+ default=False,
+ help="Skip file if it already exists (default: false)",
+)
+@click.option(
+ "--rename",
+ is_flag=True,
+ help="Rename file if it already exists (default: false)",
+)
+@click.option(
+ "--no-progress",
+ is_flag=True,
+ help="Disable progress bar (default: false)",
+)
+@click.pass_context
+def download_single(
+ ctx,
+ guid,
+ download_path,
+ protocol,
+ filename_format,
+ skip_completed,
+ rename,
+ no_progress,
+):
+ """Download a single file by GUID using the async download pipeline."""
+ auth = ctx.obj["auth_factory"].get()
+
+ download_path = os.path.abspath(download_path)
+ os.makedirs(download_path, exist_ok=True)
+
+ # Build a single-item manifest compatible with async_download_multiple
+ manifest_data = [{"guid": guid}]
+
+ try:
+ file_client = Gen3File(auth_provider=auth)
+
+ loop = get_or_create_event_loop_for_thread()
+ result = loop.run_until_complete(
+ file_client.async_download_multiple(
+ manifest_data=manifest_data,
+ download_path=download_path,
+ filename_format=filename_format,
+ protocol=protocol,
+ max_concurrent_requests=1,
+ num_processes=1,
+ queue_size=1,
+ skip_completed=skip_completed,
+ rename=rename,
+ no_progress=no_progress,
+ )
+ )
+
+ if result["succeeded"]:
+ click.echo(f"✓ Downloaded GUID {guid} to {download_path}")
+ elif result["skipped"]:
+ click.echo(f"- Skipped GUID {guid} (already exists)")
+ else:
+ click.echo(f"✗ Failed to download GUID {guid}")
+ if result.get("failed"):
+ logging.error(f"Failure details: {result['failed']}")
+
+ except Exception as e:
+ logging.error(f"Download failed: {e}")
+ raise click.ClickException(f"Download failed: {e}")
+
+
+@click.command()
+@click.option("--manifest", required=True, help="Path to manifest JSON file")
+@click.option(
+ "--download-path",
+ default=f"download_{datetime.now().strftime('%d_%b_%Y')}",
+ help="Directory to download files to (default: timestamped folder)",
+)
+@click.option(
+ "--filename-format",
+ default="original",
+ type=click.Choice(["original", "guid", "combined"]),
+ help="Filename format: 'original' uses the original filename from metadata, 'guid' uses only the file GUID, 'combined' uses original filename with GUID appended (default: original)",
+)
+@click.option(
+ "--protocol",
+ default=None,
+ help="Protocol for presigned URLs (e.g., s3) (default: auto-detect)",
+)
+@click.option(
+ "--max-concurrent-requests",
+ default=MAX_CONCURRENT_REQUESTS,
+ help="Maximum concurrent async downloads per process (default: 20)",
+ type=int,
+)
+@click.option(
+ "--numparallel",
+ default=None,
+ help="Same as max-concurrent-requests, for compatibility with previous gen3-client arguments",
+ type=int,
+)
+@click.option(
+ "--num-processes",
+ default=3,
+ help="Number of worker processes for parallel downloads (default: 3)",
+ type=int,
+)
+@click.option(
+ "--queue-size",
+ default=1000,
+ help="Maximum items in input queue (default: 1000)",
+ type=int,
+)
+@click.option(
+ "--skip-completed",
+ type=bool,
+ default=True,
+ help="Skip files that already exist (default: true)",
+)
+@click.option(
+ "--rename", is_flag=True, help="Rename files if they already exist (default: false)"
+)
+@click.option(
+ "--no-prompt", is_flag=True, help="Do not prompt for confirmations (default: false)"
+)
+@click.option(
+ "--no-progress", is_flag=True, help="Disable progress bar (default: false)"
+)
+@click.pass_context
+def download_multiple(
+ ctx,
+ manifest,
+ download_path,
+ filename_format,
+ protocol,
+ max_concurrent_requests,
+ numparallel,
+ num_processes,
+ queue_size,
+ skip_completed=True,
+ rename=False,
+ no_prompt=False,
+ no_progress=False,
+):
+ """
+ Asynchronously download multiple files from a manifest with just-in-time presigned URL generation.
+ """
+ auth = ctx.obj["auth_factory"].get()
+
+ # Use numparallel as max_concurrent_requests if provided (for gen3-client compatibility)
+ if (
+ numparallel is not None and max_concurrent_requests == MAX_CONCURRENT_REQUESTS
+ ): # 20 is the default
+ max_concurrent_requests = numparallel
+
+ try:
+ manifest_data = load_manifest(manifest)
+
+ if not validate_manifest(manifest_data):
+ raise click.ClickException("Invalid manifest format")
+
+ if not manifest_data:
+ click.echo("No files to download")
+ return
+
+ if not no_prompt:
+ click.echo(f"Found {len(manifest_data)} files to download")
+ if not click.confirm("Continue with async download?"):
+ click.echo("Download cancelled")
+ return
+
+ file_client = Gen3File(auth_provider=auth)
+
+ # Debug logging for input parameters
+ logging.debug(
+ f"Async download parameters: manifest_data={len(manifest_data)} items, download_path={download_path}, filename_format={filename_format}, protocol={protocol}, max_concurrent_requests={max_concurrent_requests}, skip_completed={skip_completed}, rename={rename}, no_progress={no_progress}"
+ )
+
+ loop = get_or_create_event_loop_for_thread()
+ result = loop.run_until_complete(
+ file_client.async_download_multiple(
+ manifest_data=manifest_data,
+ download_path=download_path,
+ filename_format=filename_format,
+ protocol=protocol,
+ max_concurrent_requests=max_concurrent_requests,
+ num_processes=num_processes,
+ queue_size=queue_size,
+ skip_completed=skip_completed,
+ rename=rename,
+ no_progress=no_progress,
+ )
+ )
+
+ click.echo("\nAsync Download Results:")
+ click.echo(f"✓ Succeeded: {len(result['succeeded'])}")
+
+ if result["skipped"] and len(result["skipped"]) > 0:
+ click.echo(f"- Skipped: {len(result['skipped'])}")
+
+ if result["failed"] and len(result["failed"]) > 0:
+ click.echo(f"✗ Failed: {len(result['failed'])}")
+
+ if result["failed"]:
+ click.echo(
+ "\nTo retry failed downloads, run the same command with --skip-completed true:"
+ )
+
+ success_rate = (
+ (len(result["succeeded"]) + len(result["skipped"]))
+ / len(manifest_data)
+ * 100
+ )
+ click.echo(f"\nSuccess rate: {success_rate:.1f}%")
+
+ except Exception as e:
+ logging.error(f"Async batch download failed: {e}")
+ raise click.ClickException(f"Async batch download failed: {e}")
diff --git a/gen3/file.py b/gen3/file.py
index 75161574..357d1134 100644
--- a/gen3/file.py
+++ b/gen3/file.py
@@ -1,26 +1,32 @@
import json
import requests
-import json
import asyncio
import aiohttp
import aiofiles
import time
+import multiprocessing as mp
+import threading
from tqdm import tqdm
-from types import SimpleNamespace as Namespace
import os
-import requests
from pathlib import Path
+from typing import List, Dict, Any, Optional
+from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse, quote
+from queue import Empty
from cdislogging import get_logger
+from werkzeug.security import safe_join
+
from gen3.index import Gen3Index
-from gen3.utils import DEFAULT_BACKOFF_SETTINGS, raise_for_status_and_print_error
-from urllib.parse import parse_qsl, urlencode, urlparse, urlunparse
+from gen3.utils import raise_for_status_and_print_error
logging = get_logger("__name__")
MAX_RETRIES = 3
+DEFAULT_NUM_PARALLEL = 3
+DEFAULT_MAX_CONCURRENT_REQUESTS = 300
+DEFAULT_QUEUE_SIZE = 1000
class Gen3File:
@@ -45,7 +51,6 @@ def __init__(self, endpoint=None, auth_provider=None):
# auth_provider legacy interface required endpoint as 1st arg
self._auth_provider = auth_provider or endpoint
self._endpoint = self._auth_provider.endpoint
- self.unsuccessful_downloads = []
def get_presigned_url(self, guid, protocol=None):
"""Generates a presigned URL for a file.
@@ -67,10 +72,26 @@ def get_presigned_url(self, guid, protocol=None):
resp = requests.get(api_url, auth=self._auth_provider)
raise_for_status_and_print_error(resp)
- try:
- return resp.json()
- except:
- return resp.text
+ return resp.json()
+
+ def get_presigned_urls_batch(self, guids, protocol=None):
+ """Get presigned URLs for multiple files efficiently.
+
+ Args:
+ guids (List[str]): List of GUIDs to get presigned URLs for
+ protocol (str, optional): Protocol preference for URLs
+
+ Returns:
+ Dict[str, Dict]: Mapping of GUID to presigned URL response
+ """
+ results = {}
+ for guid in guids:
+ try:
+ results[guid] = self.get_presigned_url(guid, protocol)
+ except Exception as e:
+ logging.error(f"Failed to get presigned URL for {guid}: {e}")
+ results[guid] = None
+ return results
def delete_file(self, guid):
"""
@@ -140,12 +161,7 @@ def upload_file(
api_url, auth=self._auth_provider, json=body, headers=headers
)
raise_for_status_and_print_error(resp)
- try:
- data = json.loads(resp.text)
- except:
- return resp.text
-
- return data
+ return resp.json()
def _ensure_dirpath_exists(path: Path) -> Path:
"""Utility to create a directory if missing.
@@ -163,16 +179,19 @@ def _ensure_dirpath_exists(path: Path) -> Path:
return out_path
- def download_single(self, object_id, path):
+ def download_single(self, object_id, path, protocol=None):
"""
Download a single file using its GUID.
Args:
object_id (str): The file's unique ID
path (str): Path to store the downloaded file at
+
+ Returns:
+ bool: True if download successful, False otherwise
"""
try:
- url = self.get_presigned_url(object_id)
+ url = self.get_presigned_url(object_id, protocol=protocol)
except Exception as e:
logging.critical(f"Unable to get a presigned URL for download: {e}")
return False
@@ -186,9 +205,9 @@ def download_single(self, object_id, path):
# NOTE could be updated with exponential backoff
time.sleep(1)
response = requests.get(url["url"], stream=True)
- if response.status == 200:
+ if response.status_code == 200:
break
- if response.status != 200:
+ if response.status_code != 200:
logging.critical("Response status not 200, try again later")
return False
else:
@@ -202,18 +221,18 @@ def download_single(self, object_id, path):
index = Gen3Index(self._auth_provider)
record = index.get_record(object_id)
- filename = record["file_name"]
+ filename = record["file_name"] or url["url"].split("?")[0].split("/")[-1]
- out_path = Gen3File._ensure_dirpath_exists(Path(path))
+ out_path = safe_join(path, filename)
+ Gen3File._ensure_dirpath_exists(Path(os.path.dirname(out_path)))
- with open(os.path.join(out_path, filename), "wb") as f:
+ with open(out_path, "wb") as f:
for data in response.iter_content(4096):
total_downloaded += len(data)
f.write(data)
if total_size_in_bytes == total_downloaded:
logging.info(f"File {filename} downloaded successfully")
-
else:
logging.error(f"File {filename} not downloaded successfully")
return False
@@ -259,3 +278,438 @@ def upload_file_to_guid(
resp = requests.get(url, auth=self._auth_provider)
raise_for_status_and_print_error(resp)
return resp.json()
+
+ async def async_download_multiple(
+ self,
+ manifest_data,
+ download_path=".",
+ filename_format="original",
+ protocol=None,
+ max_concurrent_requests=DEFAULT_MAX_CONCURRENT_REQUESTS,
+ num_processes=DEFAULT_NUM_PARALLEL,
+ queue_size=DEFAULT_QUEUE_SIZE,
+ skip_completed=False,
+ rename=False,
+ no_progress=False,
+ ):
+ """Asynchronously download multiple files using multiprocessing and queues."""
+ if not manifest_data:
+ return {"succeeded": [], "failed": [], "skipped": []}
+
+ guids = []
+ for item in manifest_data:
+ guid = item.get("guid") or item.get("object_id")
+ if guid:
+ if "/" in guid:
+ guid = guid.split("/")[-1]
+ guids.append(guid)
+
+ if not guids:
+ logging.error("No valid GUIDs found in manifest data")
+ return {"succeeded": [], "failed": [], "skipped": []}
+
+ output_dir = Gen3File._ensure_dirpath_exists(Path(download_path))
+
+ input_queue = mp.Queue(maxsize=queue_size)
+ output_queue = mp.Queue()
+
+ worker_config = {
+ "endpoint": self._endpoint,
+ "auth_provider": self._auth_provider,
+ "download_path": str(output_dir),
+ "filename_format": filename_format,
+ "protocol": protocol,
+ "max_concurrent": max_concurrent_requests,
+ "skip_completed": skip_completed,
+ "rename": rename,
+ }
+
+ processes = []
+ producer_thread = None
+
+ try:
+ for i in range(num_processes):
+ p = mp.Process(
+ target=self._async_worker_process,
+ args=(input_queue, output_queue, worker_config, i),
+ )
+ p.start()
+ processes.append(p)
+
+ producer_thread = threading.Thread(
+ target=self._guid_producer,
+ args=(guids, input_queue, num_processes),
+ )
+ producer_thread.start()
+
+ results = {"succeeded": [], "failed": [], "skipped": []}
+ completed_count = 0
+
+ if not no_progress:
+ pbar = tqdm(total=len(guids), desc="Downloading")
+
+ while completed_count < len(guids):
+ try:
+ batch_results = output_queue.get()
+
+ if not batch_results:
+ continue
+
+ for result in batch_results:
+ if result["status"] == "downloaded":
+ results["succeeded"].append(result["guid"])
+ elif result["status"] == "skipped":
+ results["skipped"].append(result["guid"])
+ else:
+ results["failed"].append(result["guid"])
+
+ completed_count += 1
+ if not no_progress:
+ pbar.update(1)
+
+ except Empty:
+ logging.warning(
+ f"No more results available ({completed_count}/{len(guids)}): Queue is empty"
+ )
+ break
+ except Exception as e:
+ logging.warning(
+ f"Error waiting for results ({completed_count}/{len(guids)}): {e}"
+ )
+
+ alive_processes = [p for p in processes if p.is_alive()]
+ if not alive_processes:
+ logging.error("All worker processes have died")
+ break
+
+ if not no_progress:
+ pbar.close()
+
+ if producer_thread:
+ producer_thread.join()
+
+ except Exception as e:
+ logging.error(f"Error in download: {e}")
+ results = {"succeeded": [], "failed": [], "skipped": [], "error": str(e)}
+
+ finally:
+ for p in processes:
+ if p.is_alive():
+ p.terminate()
+
+ p.join()
+ if p.is_alive():
+ p.kill()
+
+ logging.info(
+ f"Download complete: {len(results['succeeded'])} succeeded, "
+ f"{len(results['failed'])} failed, {len(results['skipped'])} skipped"
+ )
+ return results
+
+ def _guid_producer(self, guids, input_queue, num_processes):
+ try:
+ for guid in guids:
+ input_queue.put(guid)
+
+ except Exception as e:
+ logging.error(f"Error in producer: {e}")
+
+ @staticmethod
+ def _async_worker_process(input_queue, output_queue, config, process_id):
+ try:
+ loop = asyncio.new_event_loop()
+ asyncio.set_event_loop(loop)
+ loop.run_until_complete(
+ Gen3File._worker_main(input_queue, output_queue, config, process_id)
+ )
+ except Exception as e:
+ logging.error(f"Error in worker process {process_id}: {e}")
+ finally:
+ try:
+ loop.close()
+ except Exception as e:
+ logging.warning(f"Error closing event loop in worker {process_id}: {e}")
+
+ @staticmethod
+ async def _worker_main(input_queue, output_queue, config, process_id):
+ endpoint = config["endpoint"]
+ auth_provider = config["auth_provider"]
+ download_path = Path(config["download_path"])
+ filename_format = config["filename_format"]
+ protocol = config["protocol"]
+ max_concurrent = config["max_concurrent"]
+ skip_completed = config["skip_completed"]
+ rename = config["rename"]
+
+ # Configure connector with optimized settings for large files
+ timeout = aiohttp.ClientTimeout(total=None, connect=3600, sock_read=3600)
+ connector = aiohttp.TCPConnector(
+ limit=max_concurrent * 2,
+ limit_per_host=max_concurrent,
+ ttl_dns_cache=300,
+ use_dns_cache=True,
+ keepalive_timeout=3600,
+ enable_cleanup_closed=True,
+ )
+ semaphore = asyncio.Semaphore(max_concurrent)
+
+ async with aiohttp.ClientSession(
+ connector=connector, timeout=timeout
+ ) as session:
+ while True:
+ try:
+ # Check if queue is empty with timeout
+ guid = input_queue.get()
+ except Empty:
+ # If queue is empty (timeout), break the loop
+ break
+
+ # Process single GUID as a batch of one
+ try:
+ batch_results = await Gen3File._process_batch(
+ session,
+ [guid],
+ endpoint,
+ auth_provider,
+ download_path,
+ filename_format,
+ protocol,
+ semaphore,
+ skip_completed,
+ rename,
+ )
+ output_queue.put(batch_results)
+ except Exception as e:
+ logging.error(
+ f"Worker {process_id}: Failed to process {guid} - {type(e).__name__}: {e}"
+ )
+ error_result = [{"guid": guid, "status": "failed", "error": str(e)}]
+ try:
+ output_queue.put(error_result)
+ except Exception as queue_error:
+ logging.error(
+ f"Worker {process_id}: Failed to send error result for {guid} - {type(queue_error).__name__}: {queue_error}"
+ )
+
+ @staticmethod
+ async def _process_batch(
+ session,
+ guids,
+ endpoint,
+ auth_provider,
+ download_path,
+ filename_format,
+ protocol,
+ semaphore,
+ skip_completed,
+ rename,
+ ):
+ """Process a batch of GUIDs for downloading."""
+ batch_results = []
+ for guid in guids:
+ async with semaphore:
+ result = await Gen3File._download_single_async(
+ session,
+ guid,
+ endpoint,
+ auth_provider,
+ download_path,
+ filename_format,
+ protocol,
+ semaphore,
+ skip_completed,
+ rename,
+ )
+ batch_results.append(result)
+ return batch_results
+
+ @staticmethod
+ async def _download_single_async(
+ session,
+ guid,
+ endpoint,
+ auth_provider,
+ download_path,
+ filename_format,
+ protocol,
+ semaphore,
+ skip_completed,
+ rename,
+ ):
+ async with semaphore:
+ try:
+ metadata = await Gen3File._get_metadata(
+ session, guid, endpoint, auth_provider.get_access_token()
+ )
+
+ original_filename = metadata.get("file_name")
+ filename = Gen3File._format_filename_static(
+ guid, original_filename, filename_format
+ )
+ filepath = download_path / filename
+
+ if skip_completed and filepath.exists():
+ return {
+ "guid": guid,
+ "status": "skipped",
+ "filepath": str(filepath),
+ "reason": "File already exists",
+ }
+
+ filepath = Gen3File._handle_conflict_static(filepath, rename)
+
+ presigned_data = await Gen3File._get_presigned_url_async(
+ session, guid, endpoint, auth_provider.get_access_token(), protocol
+ )
+
+ url = presigned_data.get("url")
+ if not url:
+ return {
+ "guid": guid,
+ "status": "failed",
+ "error": "No URL in presigned data",
+ }
+
+ filepath.parent.mkdir(parents=True, exist_ok=True)
+
+ success = await Gen3File._download_content(session, url, guid, filepath)
+ if success:
+ return {
+ "guid": guid,
+ "status": "downloaded",
+ "filepath": str(filepath),
+ "size": filepath.stat().st_size if filepath.exists() else 0,
+ }
+ else:
+ return {
+ "guid": guid,
+ "status": "failed",
+ "error": "Download failed",
+ }
+
+ except Exception as e:
+ logging.error(f"Error downloading {guid}: {e}")
+ return {
+ "guid": guid,
+ "status": "failed",
+ "error": str(e),
+ }
+
+ @staticmethod
+ async def _get_metadata(session, guid, endpoint, auth_token):
+ encoded_guid = quote(guid, safe="")
+ api_url = f"{endpoint}/index/{encoded_guid}"
+ headers = {"Authorization": f"Bearer {auth_token}"}
+
+ try:
+ async with session.get(
+ api_url, headers=headers, timeout=aiohttp.ClientTimeout(total=3600)
+ ) as resp:
+ if resp.status == 200:
+ return await resp.json()
+ raise Exception(
+ f"Failed to get metadata for {guid}: HTTP {resp.status}"
+ )
+ except aiohttp.ClientError as e:
+ raise Exception(f"Network error getting metadata for {guid}: {e}")
+ except asyncio.TimeoutError:
+ raise Exception(f"Timeout getting metadata for {guid}")
+ except Exception as e:
+ if "Failed to get metadata" not in str(e):
+ raise Exception(f"Unexpected error getting metadata for {guid}: {e}")
+ raise
+
+ @staticmethod
+ async def _get_presigned_url_async(
+ session, guid, endpoint, auth_token, protocol=None
+ ):
+ encoded_guid = quote(guid, safe="")
+ api_url = f"{endpoint}/user/data/download/{encoded_guid}"
+ headers = {"Authorization": f"Bearer {auth_token}"}
+
+ if protocol:
+ api_url += f"?protocol={protocol}"
+
+ try:
+ async with session.get(
+ api_url, headers=headers, timeout=aiohttp.ClientTimeout(total=3600)
+ ) as resp:
+ if resp.status == 200:
+ return await resp.json()
+ raise Exception(
+ f"Failed to get presigned URL for {guid}: HTTP {resp.status}"
+ )
+ except aiohttp.ClientError as e:
+ raise Exception(f"Network error getting presigned URL for {guid}: {e}")
+ except asyncio.TimeoutError:
+ raise Exception(f"Timeout getting presigned URL for {guid}")
+ except Exception as e:
+ if "Failed to get presigned URL" not in str(e):
+ raise Exception(
+ f"Unexpected error getting presigned URL for {guid}: {e}"
+ )
+ raise
+
+ @staticmethod
+ async def _download_content(session, url, guid, filepath):
+ """Download content directly to file with optimized streaming."""
+ try:
+ async with session.get(
+ url, timeout=aiohttp.ClientTimeout(total=None)
+ ) as resp:
+ if resp.status == 200:
+ async with aiofiles.open(filepath, "wb") as f:
+ chunk_size = 1024 * 1024
+ async for chunk in resp.content.iter_chunked(chunk_size):
+ await f.write(chunk)
+ return True
+ logging.error(f"Download failed for {guid}: HTTP {resp.status}")
+ return False
+ except aiohttp.ClientError as e:
+ logging.error(f"Network error downloading {guid}: {e}")
+ return False
+ except asyncio.TimeoutError:
+ logging.error(f"Timeout downloading {guid}")
+ return False
+ except OSError as e:
+ logging.error(f"File system error downloading {guid} to {filepath}: {e}")
+ return False
+ except Exception as e:
+ logging.error(
+ f"Unexpected error downloading {guid}: {type(e).__name__}: {e}"
+ )
+ return False
+
+ @staticmethod
+ def _format_filename_static(guid, original_filename, filename_format):
+ if filename_format == "guid":
+ return guid
+ elif filename_format == "combined":
+ if original_filename:
+ name, ext = os.path.splitext(original_filename)
+ return f"{name}_{guid}{ext}"
+ return guid
+ else:
+ return original_filename or guid
+
+ @staticmethod
+ def _handle_conflict_static(filepath, rename):
+ if not rename:
+ if filepath.exists():
+ logging.warning(f"File will be overwritten: {filepath}")
+ return filepath
+
+ if not filepath.exists():
+ return filepath
+
+ counter = 1
+ name = filepath.stem
+ ext = filepath.suffix
+ parent = filepath.parent
+
+ while True:
+ new_path = parent / f"{name}_{counter}{ext}"
+ if not new_path.exists():
+ return new_path
+ counter += 1
diff --git a/poetry.lock b/poetry.lock
index 6fd94941..c7572f16 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 2.2.1 and should not be changed by hand.
[[package]]
name = "aiofiles"
@@ -1768,7 +1768,7 @@ version = "3.0.3"
description = "Safely add untrusted strings to HTML/XML markup."
optional = false
python-versions = ">=3.9"
-groups = ["dev"]
+groups = ["main", "dev"]
files = [
{file = "markupsafe-3.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:2f981d352f04553a7171b8e44369f2af4055f888dfb147d55e42d29e29e74559"},
{file = "markupsafe-3.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:e1c1493fb6e50ab01d20a22826e57520f1284df32f2d8601fdd90b6304601419"},
@@ -3118,18 +3118,18 @@ zstd = ["zstandard (>=0.18.0)"]
[[package]]
name = "werkzeug"
-version = "3.1.3"
+version = "3.1.8"
description = "The comprehensive WSGI web application library."
optional = false
python-versions = ">=3.9"
-groups = ["dev"]
+groups = ["main", "dev"]
files = [
- {file = "werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e"},
- {file = "werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746"},
+ {file = "werkzeug-3.1.8-py3-none-any.whl", hash = "sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50"},
+ {file = "werkzeug-3.1.8.tar.gz", hash = "sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44"},
]
[package.dependencies]
-MarkupSafe = ">=2.1.1"
+markupsafe = ">=2.1.1"
[package.extras]
watchdog = ["watchdog (>=2.3)"]
@@ -3318,4 +3318,4 @@ fhir = ["fhirclient"]
[metadata]
lock-version = "2.1"
python-versions = ">=3.9, <4"
-content-hash = "0ef223739ee0f4e9faf3bf60aa2c6ca6ea2757f7b84023e93c838109379a6915"
+content-hash = "78dbea877df3184bb2e29fb27df7fa74e615204a72dd0bf98452c097783d5e63"
diff --git a/pyproject.toml b/pyproject.toml
index 9a2caddd..54408cab 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -42,6 +42,7 @@ gen3users = "*"
# A list of all of the optional dependencies, some of which are included in the
# below `extras`. They can be opted into by apps.
fhirclient = { version = "*", optional = true }
+werkzeug = "^3.1.6"
[tool.poetry.extras]
fhir = ["fhirclient"]
diff --git a/tests/download_tests/test_async_download.py b/tests/download_tests/test_async_download.py
index f42442b3..5f4b8c91 100644
--- a/tests/download_tests/test_async_download.py
+++ b/tests/download_tests/test_async_download.py
@@ -1,4 +1,4 @@
-from unittest.mock import patch
+from unittest.mock import patch, MagicMock, AsyncMock
import json
import pytest
from pathlib import Path
@@ -264,3 +264,30 @@ def test_load_manifest_bad_format(self):
manifest_list = _load_manifest(Path(DIR, "resources/bad_format.json"))
assert manifest_list == None
+
+ @pytest.mark.asyncio
+ async def test_async_download_multiple_empty_manifest(self, mock_gen3_auth):
+ """
+ Test async_download_multiple with an empty manifest.
+ Verifies it returns empty results without errors.
+ """
+ file_tool = Gen3File(mock_gen3_auth)
+ result = await file_tool.async_download_multiple(manifest_data=[])
+
+ assert result == {"succeeded": [], "failed": [], "skipped": []}
+
+ @pytest.mark.asyncio
+ async def test_async_download_multiple_invalid_guids(self, mock_gen3_auth):
+ """
+ Test async_download_multiple with invalid GUIDs.
+ Verifies it returns empty results for missing GUIDs.
+ """
+ file_tool = Gen3File(mock_gen3_auth)
+
+ # Manifest with missing guid/object_id fields
+ manifest_data = [{"file_name": "test.txt"}, {}]
+
+ result = await file_tool.async_download_multiple(manifest_data=manifest_data)
+
+ assert result == {"succeeded": [], "failed": [], "skipped": []}
+
diff --git a/tests/test_file.py b/tests/test_file.py
index a02a80eb..8ea04bf2 100644
--- a/tests/test_file.py
+++ b/tests/test_file.py
@@ -1,9 +1,11 @@
"""
Tests gen3.file.Gen3File for calls
"""
-from unittest.mock import patch
-import json
+
+from unittest.mock import patch, MagicMock
import pytest
+import tempfile
+from pathlib import Path
from requests import HTTPError
@@ -248,20 +250,37 @@ def test_upload_file(
expected response to compare with mock
"""
with patch("gen3.file.requests") as mock_request:
- mock_request.status_code = status_code
- mock_request.post().text = response_text
- res = gen3_file.upload_file(
- file_name="file.txt",
- authz=authz,
- protocol=supported_protocol,
- expires_in=expires_in,
+ mock_response = MagicMock()
+ mock_response.status_code = status_code
+ mock_response.text = response_text
+ mock_response.json.return_value = (
+ expected_response if status_code == 201 else {}
)
+
+ # Make raise_for_status() raise HTTPError for non-2xx status codes
+ if status_code >= 400:
+ mock_response.raise_for_status.side_effect = HTTPError()
+
+ mock_request.post.return_value = mock_response
+
if status_code == 201:
+ res = gen3_file.upload_file(
+ file_name="file.txt",
+ authz=authz,
+ protocol=supported_protocol,
+ expires_in=expires_in,
+ )
# check that the SDK is getting fence
assert res.get("url") == expected_response["url"]
else:
- # check the error message
- assert expected_response in res
+ # For non-201 status codes, the method should raise an exception
+ with pytest.raises(HTTPError):
+ gen3_file.upload_file(
+ file_name="file.txt",
+ authz=authz,
+ protocol=supported_protocol,
+ expires_in=expires_in,
+ )
@pytest.mark.parametrize(
@@ -326,7 +345,7 @@ def test_upload_file_no_refresh_token(gen3_file, supported_protocol, authz, expi
def test_upload_file_no_api_key(gen3_file, supported_protocol, authz, expires_in):
"""
Upload files for a Gen3File given a protocol, authz, and expires_in
- without an api_key in the refresh token, which should return a 401
+ without an api_key in the refresh token, which should raise an HTTPError
:param gen3.file.Gen3File gen3_file:
Gen3File object
@@ -341,15 +360,19 @@ def test_upload_file_no_api_key(gen3_file, supported_protocol, authz, expires_in
gen3_file._auth_provider._refresh_token = {"not_api_key": "123"}
with patch("gen3.file.requests") as mock_request:
- mock_request.status_code = 401
- mock_request.post().text = "Failed to upload data file."
- res = gen3_file.upload_file(
- file_name="file.txt",
- authz=authz,
- protocol=supported_protocol,
- expires_in=expires_in,
- )
- assert res == "Failed to upload data file."
+ mock_response = MagicMock()
+ mock_response.status_code = 401
+ mock_response.text = "Failed to upload data file."
+ mock_response.raise_for_status.side_effect = HTTPError()
+ mock_request.post.return_value = mock_response
+
+ with pytest.raises(HTTPError):
+ gen3_file.upload_file(
+ file_name="file.txt",
+ authz=authz,
+ protocol=supported_protocol,
+ expires_in=expires_in,
+ )
@pytest.mark.parametrize(
@@ -371,7 +394,7 @@ def test_upload_file_no_api_key(gen3_file, supported_protocol, authz, expires_in
def test_upload_file_wrong_api_key(gen3_file, supported_protocol, authz, expires_in):
"""
Upload files for a Gen3File given a protocol, authz, and expires_in
- with the wrong value for the api_key in the refresh token, which should return a 401
+ with the wrong value for the api_key in the refresh token, which should raise an HTTPError
:param gen3.file.Gen3File gen3_file:
Gen3File object
@@ -386,12 +409,187 @@ def test_upload_file_wrong_api_key(gen3_file, supported_protocol, authz, expires
gen3_file._auth_provider._refresh_token = {"api_key": "wrong_value"}
with patch("gen3.file.requests") as mock_request:
- mock_request.status_code = 401
- mock_request.post().text = "Failed to upload data file."
- res = gen3_file.upload_file(
- file_name="file.txt",
- authz=authz,
- protocol=supported_protocol,
- expires_in=expires_in,
+ mock_response = MagicMock()
+ mock_response.status_code = 401
+ mock_response.text = "Failed to upload data file."
+ mock_response.raise_for_status.side_effect = HTTPError()
+ mock_request.post.return_value = mock_response
+
+ with pytest.raises(HTTPError):
+ gen3_file.upload_file(
+ file_name="file.txt",
+ authz=authz,
+ protocol=supported_protocol,
+ expires_in=expires_in,
+ )
+
+
+@pytest.fixture
+def mock_manifest_data():
+ return [
+ {"guid": "test-guid-1", "file_name": "file1.txt"},
+ {"guid": "test-guid-2", "file_name": "file2.txt"},
+ {"object_id": "test-guid-3", "file_name": "file3.txt"},
+ ]
+
+
+def test_download_single_success(gen3_file):
+ """
+ Test download_single basic functionality with synchronous download.
+
+ Verifies that download_single downloads a file successfully using
+ synchronous requests and returns True.
+ """
+ gen3_file._auth_provider._refresh_token = {"api_key": "123"}
+
+ with patch.object(gen3_file, "get_presigned_url") as mock_presigned, patch(
+ "gen3.file.requests.get"
+ ) as mock_get, patch("gen3.index.Gen3Index.get_record") as mock_index:
+ mock_presigned.return_value = {"url": "https://fake-url.com/file"}
+ mock_index.return_value = {"file_name": "test-file.txt"}
+ mock_response = MagicMock()
+ mock_response.status_code = 200
+ mock_response.headers = {"content-length": "12"}
+ mock_response.iter_content = lambda size: [b"test content"]
+ mock_get.return_value = mock_response
+
+ result = gen3_file.download_single(
+ object_id="test-guid", path="/tmp", protocol="s3"
+ )
+
+ assert result == True
+ mock_presigned.assert_called_once_with("test-guid", protocol="s3")
+ mock_index.assert_called_once_with("test-guid")
+
+
+def test_download_single_failed(gen3_file):
+ """
+ Test download_single basic functionality with synchronous download.
+
+ Verifies that download_single downloads a file successfully using
+ synchronous requests and returns True.
+ """
+ gen3_file._auth_provider._refresh_token = {"api_key": "123"}
+
+ with patch.object(gen3_file, "get_presigned_url") as mock_presigned, patch(
+ "gen3.file.requests.get"
+ ) as mock_get, patch("gen3.index.Gen3Index.get_record") as mock_index:
+ mock_presigned.return_value = {"url": "https://fake-url.com/file"}
+ mock_index.return_value = {"file_name": "test-file.txt"}
+ mock_response = MagicMock()
+ mock_response.status_code = 404
+ mock_response.headers = {"content-length": "12"}
+ mock_response.iter_content = lambda size: [b"test content"]
+ mock_get.return_value = mock_response
+
+ result = gen3_file.download_single(
+ object_id="test-guid", path="/tmp", protocol="s3"
)
- assert res == "Failed to upload data file."
+
+ assert result == False
+ mock_presigned.assert_called_once_with("test-guid", protocol="s3")
+ mock_index.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_async_download_multiple_empty_manifest(gen3_file):
+ """
+ Test async_download_multiple with an empty manifest.
+
+ Verifies that calling async_download_multiple with an empty manifest
+ returns empty succeeded, failed, and skipped lists.
+ """
+ result = await gen3_file.async_download_multiple(manifest_data=[])
+ assert result == {"succeeded": [], "failed": [], "skipped": []}
+
+
+@pytest.mark.asyncio
+async def test_async_download_multiple_success(gen3_file, mock_manifest_data):
+ """
+ Test successful async download of multiple files.
+
+ Verifies that async_download_multiple correctly processes a manifest with
+ multiple files and returns all downloads as successful.
+ """
+ gen3_file._auth_provider._refresh_token = {"api_key": "123"}
+ gen3_file._auth_provider.get_access_token = MagicMock(return_value="fake_token")
+
+ with (
+ patch("gen3.file.mp.Process"),
+ patch("gen3.file.mp.Queue") as mock_queue,
+ patch("threading.Thread"),
+ ):
+ mock_input_queue = MagicMock()
+ mock_output_queue = MagicMock()
+ mock_queue.side_effect = [mock_input_queue, mock_output_queue]
+
+ mock_output_queue.get.side_effect = [
+ [{"guid": "test-guid-1", "status": "downloaded"}],
+ [{"guid": "test-guid-2", "status": "downloaded"}],
+ [{"guid": "test-guid-3", "status": "downloaded"}],
+ ]
+
+ result = await gen3_file.async_download_multiple(
+ manifest_data=mock_manifest_data, download_path="/tmp"
+ )
+
+ assert len(result["succeeded"]) == 3
+
+
+def test_get_presigned_urls_batch(gen3_file):
+ """
+ Test batch retrieval of presigned URLs for multiple GUIDs.
+
+ Verifies that get_presigned_urls_batch correctly calls get_presigned_url
+ for each GUID and returns a mapping of results.
+ """
+ gen3_file._auth_provider._refresh_token = {"api_key": "123"}
+
+ with patch.object(gen3_file, "get_presigned_url") as mock_get_url:
+ mock_get_url.return_value = {"url": "https://example.com/presigned"}
+
+ results = gen3_file.get_presigned_urls_batch(["guid1", "guid2"])
+
+ assert len(results) == 2
+ assert mock_get_url.call_count == 2
+
+
+def test_format_filename_static():
+ """
+ Test the static _format_filename_static method with different filename formats.
+
+ Verifies that files can be formatted as original, guid-only, or combined
+ (filename_guidXXX.ext) based on the format parameter.
+ """
+ from gen3.file import Gen3File
+
+ assert (
+ Gen3File._format_filename_static("guid123", "test.txt", "original")
+ == "test.txt"
+ )
+ assert Gen3File._format_filename_static("guid123", "test.txt", "guid") == "guid123"
+ assert (
+ Gen3File._format_filename_static("guid123", "test.txt", "combined")
+ == "test_guid123.txt"
+ )
+
+
+def test_handle_conflict_static():
+ """
+ Test the static _handle_conflict_static method for file conflict resolution.
+
+ Verifies that existing files can be either kept or renamed with a numeric
+ suffix based on the rename parameter.
+ """
+ from gen3.file import Gen3File
+
+ with tempfile.TemporaryDirectory() as temp_dir:
+ temp_path = Path(temp_dir)
+ existing_file = temp_path / "existing.txt"
+ existing_file.write_text("test")
+
+ result = Gen3File._handle_conflict_static(existing_file, rename=False)
+ assert result == existing_file
+
+ result = Gen3File._handle_conflict_static(existing_file, rename=True)
+ assert result.name == "existing_1.txt"