Skip to content

Commit 9ae6192

Browse files
frano-mclaude
frano-m
and
claude
authored
feat: add download guide pages for anvil-cmg (#4758)
* feat: add download guide pages for anvil-cmg (#4757) Add four new guide sub-pages under /guides for anvil-cmg only: - Data Download Options (overview of download methods) - Dataset Manifest Download (TSV manifest guide with screenshots) - curl Command Data Download (curl-based download guide with screenshots) - Individual File Download (single file download guide with screenshots) Add guide sub-menu to anvil-cmg header navigation with responsive visibility (desktop shows "Guides" link, tablet/mobile shows sub-items). Convert all guide images to WebP format. Map MDX hr to MUI Divider and add content styles for hr and pre+p spacing. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * feat: add sidebar navigation to guide pages (#4757) Add Nav component to all guide sub-pages with active state highlighting for the current page. Extract navigation items to shared constants. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: add exact selectedMatch to guides overview nav item (#4757) Prevents the Overview item from being highlighted when on sub-pages. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: rename guides overview label to "About AnVIL Explorer" (#4757) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: use portalURL variable for requesting data access link (#4757) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * feat: replace curl command page with file manifest download page (#4757) Remove curl command data download guide and add file manifest download guide. Update navigation across all guide pages. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * feat: data download options option to file manifest (#4757) * fix: change file manifest to data download (#4757) * feat: tsv file download paths (#4757) * fix: return notFound from getContentStaticProps for proper 404 status (#4757) Return { notFound: true } instead of null mdxSource so Next.js serves a real 404 status when content is missing. Remove unused NotFoundPage fallback from guide pages. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> * fix: rename GUIDES nav item to ABOUT_ANVIL_EXPLORER (#4757) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Fran McDade <18710366+frano-m@users.noreply.github.com> Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 755af90 commit 9ae6192

32 files changed

Lines changed: 570 additions & 17 deletions

‎app/components/Layout/components/Content/content.styles.ts‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
import { css } from "@emotion/react";
2-
import styled from "@emotion/styled";
31
import { FONT } from "@databiosphere/findable-ui/lib/styles/common/constants/font";
4-
import { ThemeProps } from "@databiosphere/findable-ui/lib/theme/types";
52
import { typographyToCSS } from "@databiosphere/findable-ui/lib/styles/common/mixins/typography";
3+
import { ThemeProps } from "@databiosphere/findable-ui/lib/theme/types";
4+
import { css } from "@emotion/react";
5+
import styled from "@emotion/styled";
66

77
const muiAlert = ({ theme }: ThemeProps) => css`
88
.MuiAlert-root {
@@ -89,6 +89,10 @@ export const Content = styled.div`
8989
}
9090
}
9191
92+
hr {
93+
margin: 20px 0;
94+
}
95+
9296
p {
9397
font: ${FONT.BODY_LARGE_400_2_LINES};
9498
margin: 0 0 16px;
@@ -98,6 +102,10 @@ export const Content = styled.div`
98102
}
99103
}
100104
105+
pre + p {
106+
margin-top: 16px;
107+
}
108+
101109
ul {
102110
font: ${FONT.BODY_LARGE_400_2_LINES};
103111
margin: 16px 0;

‎app/components/common/Figure/figure.styles.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
import styled from "@emotion/styled";
2-
import { PALETTE } from "@databiosphere/findable-ui/lib/styles/common/constants/palette";
31
import { FONT } from "@databiosphere/findable-ui/lib/styles/common/constants/font";
2+
import { PALETTE } from "@databiosphere/findable-ui/lib/styles/common/constants/palette";
3+
import styled from "@emotion/styled";
44

55
export const Figure = styled.figure`
66
margin: 32px 0;

‎app/content/anvil-cmg/guides/data-download-options.mdx‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
<Breadcrumbs
2+
breadcrumbs={[
3+
{ path: "/datasets", text: "AnVIL Data Explorer" },
4+
{ path: "/guides", text: "Guides" },
5+
{ path: "", text: "Data Download Options" },
6+
]}
7+
/>
8+
9+
# Data Download Options
10+
11+
There are several ways to download files for use on local, institutional, or other computational services.
12+
13+
With support from Amazon's Open Data Sponsorship Program, the AnVIL open-access datasets are available with no-cost egress and for use within the AWS environment.
14+
15+
Managed-access datasets can be downloaded from Google Cloud Platform on a requester-pays basis. For more information, refer to the ["Requesting Data Access"]({portalURL}/learn/find-data/requesting-data-access#requester-pays) document.
16+
17+
The following options are available through the AnVIL Data Explorer:
18+
19+
- **TSV File Manifest Downloads**
20+
- Available for all datasets, including open and managed access datasets.
21+
- Manifest can include one or more datasets based on the search criteria used in the AnVIL Data Explorer.
22+
23+
- **Data Download via curl**
24+
- Available only for open-access datasets.
25+
- `curl` Command downloads can be full datasets or include select file types from one or more open-access datasets.
26+
27+
- **Individual File Download**
28+
- Available only for files in open-access datasets.

‎app/content/anvil-cmg/guides/data-download-via-curl.mdx‎

Lines changed: 103 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,103 @@
1+
<Breadcrumbs
2+
breadcrumbs={[
3+
{ path: "/datasets", text: "AnVIL Data Explorer" },
4+
{ path: "/guides", text: "Guides" },
5+
{ path: "", text: "Data Download via curl" },
6+
]}
7+
/>
8+
9+
# Data Download via curl
10+
11+
The **Download Open-Access Data (curl Command)** enables the user to select the organism type and file formats they wish to transfer to a local or institutional system. Complete datasets can be downloaded by selecting all available file types.
12+
13+
**NOTE:** At this time, this option is available only for open-access datasets.
14+
15+
## Prerequisites
16+
17+
curl must be installed on the destination system where the command will be run. Most Mac, Linux, Windows 10 & 11 systems include curl by default. Older Windows users can download it from the curl website or use Windows Subsystem for Linux (WSL).
18+
19+
## Example
20+
21+
### Downloading The Full Dataset
22+
23+
1. Visit the dataset of interest by clicking on the dataset name in the Data Explorer.
24+
25+
<Figure
26+
alt="Visit the dataset of interest"
27+
src="/guides/curl-command-download/single-dataset-download-01.webp"
28+
width="100%"
29+
/>
30+
31+
2. On the dataset description page, click on the "Export" button in the upper right-hand corner of that page.
32+
33+
<Figure
34+
alt="Click the Export button"
35+
src="/guides/curl-command-download/single-dataset-download-02.webp"
36+
width="100%"
37+
/>
38+
39+
3. Then click on "Download Open-Access Data Files (No Data Transfer Fees)" in the "Download" section near the bottom of the page.
40+
41+
<Figure
42+
alt="Click Download Open-Access Data Files"
43+
src="/guides/curl-command-download/single-dataset-download-03.webp"
44+
width="100%"
45+
/>
46+
47+
4. This will display a screen that allows some refinement of the data to download.
48+
49+
<Figure
50+
alt="Refine the data to download"
51+
src="/guides/curl-command-download/single-dataset-download-04.webp"
52+
width="100%"
53+
/>
54+
55+
5. Select all of the organism type(s) at the top of the page.
56+
57+
6. Check the box next to the Name heading. This will select all of the file types.
58+
- If the user wants to download only specific file types, select only those file types and leave the others unchecked.
59+
60+
7. Select Bash<sup>[1](#footnote-1)</sup> if you are on Mac, Linux, or Windows Subsystem for Linux; select cmd.exe if you are on Windows Command Prompt.
61+
62+
8. Click on the Request curl Command button.
63+
64+
<Figure
65+
alt="Click the Request curl Command button"
66+
src="/guides/curl-command-download/single-dataset-download-08.webp"
67+
width="100%"
68+
/>
69+
70+
This will generate a curl manifest and the command needed to transfer the files. The resulting command will be similar to this:
71+
72+
```
73+
curl --location --fail https://service.explore.anvilproject.org/manifest/files/ksQylKdhbnZpbDEzpGN1cmzEEKxolyZNG12_p9nHuKrRpbDEEH2f6ZDL2lSzofvXZ80pfgXEIJHlLajfJ07ut9ZEMwSwDDAdmSZQam5pZbCxG3WZeFBl | curl --retry 15 --retry-delay 10 --config -
74+
```
75+
76+
On the destination system, issue the specified curl command. Clicking the text box containing the curl command copies it to your clipboard so you can paste it into a terminal window.
77+
78+
<Figure
79+
alt="Copy the curl command to clipboard"
80+
src="/guides/curl-command-download/single-dataset-download-08b.webp"
81+
width="100%"
82+
/>
83+
84+
For single-dataset downloads, a series of subdirectories will be created containing the selected files from that dataset.
85+
86+
### Downloading Files From Multiple Datasets
87+
88+
Downloading files from multiple datasets works the same way as downloading from a single dataset, except for how you select the datasets.
89+
90+
In this case, on the Data Explorer's main page, use the faceted search feature in the right-hand column to select the datasets of interest and then click on the "Export" button on the top right of the page.
91+
92+
<Figure
93+
alt="Select datasets and click Export"
94+
src="/guides/curl-command-download/multiple-datasets-download-01.webp"
95+
width="100%"
96+
/>
97+
98+
From this point on, the interface is the same as the single dataset download above. Continue with Step 3 above.
99+
100+
---
101+
102+
<sup id="footnote-1">1</sup> The Bash shell will work for most of the common
103+
Unix/Linux command-line shells.

‎app/content/anvil-cmg/guides/individual-file-download.mdx‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
<Breadcrumbs
2+
breadcrumbs={[
3+
{ path: "/datasets", text: "AnVIL Data Explorer" },
4+
{ path: "/guides", text: "Guides" },
5+
{ path: "", text: "Individual File Download" },
6+
]}
7+
/>
8+
9+
# Individual File Download
10+
11+
Individual file downloads from the AnVIL Data Explorer are available for open-access files. Files can also be downloaded with information in the dataset manifests.
12+
13+
## Example
14+
15+
Downloading individual files.
16+
17+
1. Use the faceted search in the left-hand column to limit the scope of the files listed.
18+
19+
<Figure
20+
alt="Use the faceted search to limit the scope of files"
21+
src="/guides/individual-download/individual-files-01.webp"
22+
width="100%"
23+
/>
24+
25+
2. Select the "Files" tab at the top of the list of datasets. This will change the display to list the available files.
26+
27+
<Figure
28+
alt="Select the Files tab"
29+
src="/guides/individual-download/individual-files-02.webp"
30+
width="100%"
31+
/>
32+
33+
3. To download the file, click the download button next to the file of interest. This will start the Download folder as specified in the browser configuration.
34+
35+
<Figure
36+
alt="Click the download button next to the file"
37+
src="/guides/individual-download/individual-files-03.webp"
38+
width="100%"
39+
/>
40+
41+
Note that if there are files in the list that are not available for download (e.g., files from a managed access dataset), the download icon will be grayed out.
42+
43+
<Figure
44+
alt="Disabled download icon for files that are not available for download"
45+
src="/guides/individual-download/individual-files-04.webp"
46+
width="100%"
47+
/>

‎app/content/anvil-cmg/guides/tsv-file-manifest-download.mdx‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
<Breadcrumbs
2+
breadcrumbs={[
3+
{ path: "/datasets", text: "AnVIL Data Explorer" },
4+
{ path: "/guides", text: "Guides" },
5+
{ path: "", text: "TSV File Manifest Download" },
6+
]}
7+
/>
8+
9+
# TSV File Manifest Download
10+
11+
Manifest downloads are available for all of the datasets listed in the AnVIL Data Explorer, including both open-access and managed-access datasets. A tab-separated-value file (.tsv) is generated based on the data selected.
12+
13+
The downloaded manifest contains a number of columns. Depending on how the data will be accessed and used, some of the key columns are:
14+
15+
- **dataset.title**, which contains the name of the dataset that the file belongs to.
16+
- A manifest can contain files from multiple datasets, depending on how the file is generated.
17+
- **datasets.consent_group** and **datasets.data_use_permission**, which contain the dataset's consent and use codes.
18+
- **files.file_size**, which contains the file size in bytes.
19+
- **files.name**, which contains the file name.
20+
- **files.drs_url**, which contains the DRS URL for use within the Terra environment.
21+
- **files.azul_url**, which is a URL that allows HTTP access to the individual file.
22+
- Files in open-access datasets are available via this link.
23+
- At this time, AnVIL requires requester-pays for managed-access datasets, so the files are not accessible through this URL.
24+
- **files.azul_mirror_url**, which contains the URI to the Amazon Web Services S3 bucket for that file.
25+
- Please note that the file name in the bucket is a hash to reduce storage requirements in case there is file duplication.
26+
- This field will be blank if the file is not present through the AWS Open Data Sponsorship Program.
27+
28+
## Example
29+
30+
### Downloading The Manifest For A Single Dataset
31+
32+
1. Visit the dataset of interest by clicking on the dataset name in the Data Explorer.
33+
34+
<Figure
35+
alt="Visit the dataset of interest"
36+
src="/guides/dataset-manifest-download/single-dataset-download-01.webp"
37+
width="100%"
38+
/>
39+
40+
2. On the dataset description page, click on the "Export" button in the upper right-hand corner of that page.
41+
42+
<Figure
43+
alt="Click the Export button"
44+
src="/guides/dataset-manifest-download/single-dataset-download-02.webp"
45+
width="100%"
46+
/>
47+
48+
3. Then click on "Download TSV Manifest" in the "Download" section near the bottom of the page.
49+
50+
<Figure
51+
alt="Click Download TSV Manifest"
52+
src="/guides/dataset-manifest-download/single-dataset-download-03.webp"
53+
width="100%"
54+
/>
55+
56+
4. This will display a screen to request the generation of the manifest. Click on the "Request Link" button.
57+
58+
<Figure
59+
alt="Click the Request Link button"
60+
src="/guides/dataset-manifest-download/single-dataset-download-04.webp"
61+
width="100%"
62+
/>
63+
64+
5. Once the manifest is generated, you can either download it directly by clicking the download icon or copy its URL by clicking the copy icon.
65+
66+
<Figure
67+
alt="Download or copy the manifest link"
68+
src="/guides/dataset-manifest-download/single-dataset-download-05.webp"
69+
width="100%"
70+
/>
71+
72+
The manifest can be viewed with any utilities that can import tab-separated-value files. It can additionally be processed with scripts depending on the need.
73+
74+
### Downloading A Manifest For Multiple Datasets
75+
76+
Downloading files from multiple datasets works the same way as downloading from a single dataset, except for how you select the datasets.
77+
78+
In this case, on the Data Explorer's main page, use the faceted search feature in the right-hand column to select the datasets of interest and then click on the "Export" button on the top right of the page.
79+
80+
<Figure
81+
alt="Select datasets and click Export"
82+
src="/guides/dataset-manifest-download/multiple-datasets-download-01.webp"
83+
width="100%"
84+
/>
85+
86+
From this point on, the interface is the same as the single dataset download above. Continue with Step 3 above.

‎app/content/common/constants.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { ANCHOR_TARGET } from "@databiosphere/findable-ui/lib/components/Links/common/entities";
2+
import { Divider } from "@mui/material";
23
import * as C from "../../components";
34
import { Figure } from "../../components/common/Figure/figure";
45
import { Link } from "../../components/Layout/components/Content/components/Link/link";
@@ -11,6 +12,7 @@ export const MDX_COMPONENTS = {
1112
Breadcrumbs: C.Breadcrumbs,
1213
Figure,
1314
a: Link,
15+
hr: Divider,
1416
};
1517

1618
export const MDX_SCOPE = { ANCHOR_TARGET };

‎app/content/common/contentPages.ts‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,13 +23,7 @@ export async function getContentStaticProps(
2323
!contentPathname ||
2424
!isContentPathnameExists(contentPathname, slug)
2525
) {
26-
return {
27-
props: {
28-
mdxSource: null,
29-
pageTitle,
30-
slug: null,
31-
},
32-
};
26+
return { notFound: true };
3327
}
3428
const markdownPathname = getMarkdownPathname(contentPathname, slug);
3529
const markdownWithMeta = fs.readFileSync(markdownPathname, "utf-8");

‎pages/guides/data-download-options.tsx‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
import { Main } from "@databiosphere/findable-ui/lib/components/Layout/components/ContentLayout/components/Main/main";
2+
import { Nav } from "@databiosphere/findable-ui/lib/components/Layout/components/Nav/nav";
3+
import { ContentView } from "@databiosphere/findable-ui/lib/views/ContentView/contentView";
4+
import { GetStaticProps, InferGetStaticPropsType } from "next";
5+
import { MDXRemote } from "next-mdx-remote";
6+
import { JSX } from "react";
7+
import { Content } from "../../app/components/Layout/components/Content/content";
8+
import { MDX_COMPONENTS } from "../../app/content/common/constants";
9+
import { getContentStaticProps } from "../../app/content/common/contentPages";
10+
import {
11+
ABOUT_ANVIL_EXPLORER,
12+
DATA_DOWNLOAD_OPTIONS,
13+
DATA_DOWNLOAD_VIA_CURL,
14+
INDIVIDUAL_FILE_DOWNLOAD,
15+
TSV_FILE_MANIFEST_DOWNLOAD,
16+
} from "../../site-config/anvil-cmg/dev/layout/navigationItem";
17+
const slug = ["guides", "data-download-options"];
18+
19+
export const getStaticProps: GetStaticProps = async () => {
20+
return getContentStaticProps({ params: { slug } }, "Data Download Options");
21+
};
22+
23+
const Page = ({
24+
layoutStyle,
25+
mdxSource,
26+
}: InferGetStaticPropsType<typeof getStaticProps>): JSX.Element => {
27+
return (
28+
<ContentView
29+
content={
30+
<Content>
31+
<MDXRemote {...mdxSource} components={MDX_COMPONENTS} />
32+
</Content>
33+
}
34+
navigation={
35+
<Nav
36+
navigation={[
37+
ABOUT_ANVIL_EXPLORER,
38+
{ active: true, ...DATA_DOWNLOAD_OPTIONS },
39+
TSV_FILE_MANIFEST_DOWNLOAD,
40+
DATA_DOWNLOAD_VIA_CURL,
41+
INDIVIDUAL_FILE_DOWNLOAD,
42+
]}
43+
/>
44+
}
45+
layoutStyle={layoutStyle ?? undefined}
46+
/>
47+
);
48+
};
49+
50+
Page.Main = Main;
51+
52+
export default Page;

0 commit comments

Comments
 (0)