Merge pull request #93 from elixir-europe/multi-auth-provider-credentials

Multi auth provider credentials

Author: kdp-cloud

.github/workflows/test-mars.yml

@@ -15,7 +15,7 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest, windows-latest]
-        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.13"]
     runs-on: ${{ matrix.os }}
     env:
       working-directory: ./mars-cli
@@ -48,6 +48,9 @@ jobs:
         run: ruff check mars_lib/
         working-directory: ${{ env.working-directory }}
 
+      - name: Create mypy cache directory
+        run: mkdir -p /tmp/mypy_cache
+
       - name: Type checking
-        run: mypy --install-types --non-interactive mars_lib/
+        run: mypy --install-types --non-interactive --cache-dir /tmp/mypy_cache mars_lib/
         working-directory: ${{ env.working-directory }}

mars-cli/mars_cli.py

@@ -197,10 +197,22 @@ def cli(ctx, development):
 
 @cli.command()
 @click.option(
-    "--credential-service-name", type=click.STRING, help="service name from the keyring"
+    "--webin-username",
+    type=click.STRING,
+    help="Username for webin authentication",
+    envvar="WEBIN_USERNAME",
 )
 @click.option(
-    "--username-credentials", type=click.STRING, help="Username from the keyring"
+    "--metabolights-username",
+    type=click.STRING,
+    help="Username for MetaboLights metadata submission",
+    envvar="METABOLIGHTS_USERNAME",
+)
+@click.option(
+    "--metabolights-ftp-username",
+    type=click.STRING,
+    help="Username for MetaboLights data submission",
+    envvar="METABOLIGHTS_FTP_USERNAME",
 )
 @click.option(
     "--credentials-file",
@@ -218,8 +230,10 @@ def cli(ctx, development):
 @click.option("--submit-to-ena", type=click.BOOL, default=True, help="Submit to ENA.")
 @click.option(
     "--file-transfer",
-    type=click.STRING,
-    help="provide the name of a file transfer solution, like ftp or aspera",
+    type=click.Choice(["ftp", "aspera"], case_sensitive=False),
+    required=True,
+    default="ftp",
+    help="provide the name of a file transfer solution, like ftp or aspera. The default is ftp.",
 )
 @click.option(
     "--data-files",
@@ -247,8 +261,9 @@ def cli(ctx, development):
 @click.pass_context
 def submit(
     ctx,
-    credential_service_name,
-    username_credentials,
+    webin_username,
+    metabolights_username,
+    metabolights_ftp_username,
     credentials_file,
     isa_json_file,
     submit_to_biosamples,
@@ -280,8 +295,9 @@ def submit(
     data_file_paths = [f.name for f in data_files] if file_transfer else []
 
     submission(
-        credential_service_name,
-        username_credentials,
+        webin_username,
+        metabolights_username,
+        metabolights_ftp_username,
         credentials_file,
         isa_json_file.name,
         target_repositories,
@@ -361,12 +377,14 @@ def validate_isa_json(isa_json_file, investigation_is_root, validation_schema):
 
 @cli.command()
 @click.option(
-    "--service-name",
-    type=click.STRING,
+    "--auth-provider",
+    type=click.Choice(
+        ["webin", "metabolights_metadata", "metabolights_data"], case_sensitive=False
+    ),
     is_flag=False,
     flag_value="value",
-    default=f"mars-cli_{datetime.now().strftime('%Y-%m-%dT%H:%M:%S')}",
-    help='You are advised to include service name to match the credentials to. If empty, it defaults to "mars-cli_{DATESTAMP}"',
+    required=True,
+    help="",
 )
 @click.argument(
     "username",
@@ -380,9 +398,9 @@ def validate_isa_json(isa_json_file, investigation_is_root, validation_schema):
     confirmation_prompt=True,
     help="The password to store. Note: You are required to confirm the password.",
 )
-def set_password(service_name, username, password):
+def set_password(auth_provider, username, password):
     """Store a password in the keyring."""
-    CredentialManager(service_name).set_password_keyring(username, password)
+    CredentialManager(auth_provider).set_password_keyring(username, password)
 
 
 if __name__ == "__main__":

mars-cli/mars_lib/authentication.py

@@ -1,6 +1,68 @@
-from typing import Optional
+import io
+from typing import Optional, Union
 import requests
 import json
+from enum import Enum
+
+
+class AuthProvider(Enum):
+    """
+    Holds constants, tied to the repository authentication providers.
+    """
+
+    WEBIN = "webin"
+    METABOLIGHTS_METADATA = "metabolights_metadata"
+    METABOLIGHTS_DATA = "metabolights_data"
+
+    @classmethod
+    def available_providers(cls):
+        return {item.value for item in cls}
+
+    @classmethod
+    def is_valid_provider(cls, provider: str):
+        return provider in cls.available_providers()
+
+
+def load_credentials(
+    credentials_file: Union[io.TextIOWrapper, str]
+) -> dict[str, dict[str, str]]:
+    """
+    Validate the credentials.
+
+    Args:
+    credentials_file (_): The credentials in file formate.
+
+    Raises:
+    ValueError: If the credentials are not valid.
+
+    Returns:
+    dict: The credentials.
+    """
+    if isinstance(credentials_file, str):
+        with open(credentials_file, "r") as file:
+            credentials = json.load(file)
+    elif isinstance(credentials_file, io.TextIOWrapper):
+        with open(credentials_file.name, "r") as file:
+            credentials = json.load(file)
+    else:
+        raise TypeError("Credentials file must be of type str or io.TextIOWrapper.")
+
+    if not all(
+        repo in AuthProvider.available_providers() for repo in credentials.keys()
+    ):
+        raise ValueError(
+            f"Credentials dictionary must have valid keys. Valid keys are:\n{AuthProvider.available_providers()}"
+        )
+
+    if not all(
+        key in ["username", "password"]
+        for repo, creds in credentials.items()
+        for key in creds.keys()
+    ):
+        raise ValueError(
+            "Credentials dictionary must contain 'username' and 'password' keys."
+        )
+    return credentials
 
 
 def get_webin_auth_token(
@@ -45,7 +107,10 @@ def get_webin_auth_token(
 
 def get_metabolights_auth_token(
     credentials_dict: dict[str, str],
-    headers: dict[str, str] = {"Content-Type": "application/x-www-form-urlencoded"},
+    headers: dict[str, str] = {
+        "Content-Type": "application/x-www-form-urlencoded",
+        "Accept": "application/json",
+    },
     auth_url: str = "https://www-test.ebi.ac.uk/metabolights/mars/ws3/auth/token",
 ) -> Optional[str]:
     """
@@ -59,10 +124,6 @@ def get_metabolights_auth_token(
     Returns:
     str: The obtained token.
     """
-    headers = {
-        "Content-Type": "application/x-www-form-urlencoded",
-        "Accept": "application/json",
-    }
     form_data = f'grant_type=password&username={credentials_dict["username"]}&password={credentials_dict["password"]}'
     try:
         response = requests.post(

mars-cli/mars_lib/credential.py

@@ -3,6 +3,8 @@
 import getpass
 import keyring.util.platform_ as keyring_platform
 
+from mars_lib.authentication import AuthProvider
+
 """
 Credential Manager Module
 =========================
@@ -52,8 +54,11 @@
 
 
 class CredentialManager:
-    def __init__(self, service_name: str) -> None:
-        self.service_name = service_name
+    def __init__(self, auth_provider: str) -> None:
+        if not AuthProvider.is_valid_provider(auth_provider):
+            raise ValueError(f"Invalid authentication provider: {auth_provider}")
+
+        self.service_name = auth_provider
 
     def get_credential_env(self, username: str) -> str:
         """

mars-cli/mars_lib/models/isa_json.py

@@ -255,7 +255,7 @@ class MaterialAttribute(IsaBase):
 
 
 class Study(CommentedIsaBase):
-    id: str = Field(alias="@id", default=None)
+    id: Optional[str] = Field(alias="@id", default=None)
     assays: List[Assay] = []
     characteristicCategories: List[MaterialAttribute] = []
     description: Optional[str] = None
@@ -284,7 +284,7 @@ def validate_filename(cls, v: str) -> Union[str, None]:
 
 
 class Investigation(CommentedIsaBase):
-    id: str = Field(alias="@id", default=None)
+    id: Optional[str] = Field(alias="@id", default=None)
     description: Optional[str] = None
     filename: Optional[str] = None
     identifier: Optional[str] = None

mars-cli/mars_lib/submit.py

@@ -6,7 +6,12 @@
 import requests
 import json
 from typing import Any
-from mars_lib.authentication import get_metabolights_auth_token, get_webin_auth_token
+from mars_lib.authentication import (
+    get_metabolights_auth_token,
+    get_webin_auth_token,
+    load_credentials,
+    AuthProvider,
+)
 from mars_lib.biosamples_external_references import (
     get_header,
     biosamples_endpoints,
@@ -44,8 +49,9 @@ def save_step_to_file(time_stamp: float, filename: str, isa_json: IsaJson):
 
 
 def submission(
-    credential_service_name: str,
-    username_credentials: str,
+    webin_username: str,
+    metabolights_username: str,
+    metabolights_ftp_username: str,
     credentials_file: TextIOWrapper,
     isa_json_file: str,
     target_repositories: list[str],
@@ -59,17 +65,24 @@ def submission(
     # Get password from the credential manager
     # Else:
     # read credentials from file
-    if not (credential_service_name is None or username_credentials is None):
-        cm = CredentialManager(credential_service_name)
+    if all([webin_username, metabolights_username, metabolights_ftp_username]):
         user_credentials = {
-            "username": username_credentials,
-            "password": cm.get_password_keyring(username_credentials),
+            cred_pair[0]: {
+                "username": cred_pair[1],
+                "password": CredentialManager(cred_pair[0]).get_password_keyring(
+                    cred_pair[1]
+                ),
+            }
+            for cred_pair in zip(
+                AuthProvider.available_providers(),
+                [webin_username, metabolights_username, metabolights_ftp_username],
+            )
         }
     else:
         if credentials_file == "":
             raise ValueError("No credentials found")
 
-        user_credentials = json.load(credentials_file)
+        user_credentials = load_credentials(credentials_file)
 
     isa_json = load_isa_json(isa_json_file, investigation_is_root)
 
@@ -101,7 +114,7 @@ def submission(
         # Submit to Biosamples
         biosamples_result = submit_to_biosamples(
             isa_json=isa_json,
-            biosamples_credentials=user_credentials,
+            biosamples_credentials=user_credentials[AuthProvider.WEBIN.value],
             biosamples_url=urls["BIOSAMPLES"]["SUBMISSION"],
             webin_token_url=urls["WEBIN"]["TOKEN"],
         )
@@ -124,7 +137,7 @@ def submission(
                 file_paths=[
                     Path(df) for df in data_file_map[TargetRepository.ENA.value]
                 ],
-                user_credentials=user_credentials,
+                user_credentials=user_credentials[AuthProvider.WEBIN.value],
                 submission_url=urls["ENA"]["DATA-SUBMISSION"],
                 file_transfer=file_transfer,
             )
@@ -135,7 +148,7 @@ def submission(
         # Step 2 : submit isa-json to ena
         ena_result = submit_to_ena(
             isa_json=isa_json,
-            user_credentials=user_credentials,
+            user_credentials=user_credentials[AuthProvider.WEBIN.value],
             submission_url=urls["ENA"]["SUBMISSION"],
         )
         print_and_log(
@@ -159,7 +172,9 @@ def submission(
             file_paths=data_file_map[TargetRepository.METABOLIGHTS.value],
             file_transfer=file_transfer,
             isa_json=isa_json,
-            metabolights_credentials=user_credentials,
+            metabolights_credentials=user_credentials[
+                AuthProvider.METABOLIGHTS_METADATA.value
+            ],
             metabolights_url=urls["METABOLIGHTS"]["SUBMISSION"],
             metabolights_token_url=urls["METABOLIGHTS"]["TOKEN"],
         )
@@ -252,9 +267,9 @@ def upload_to_metabolights(
         "accept": "application/json",
         "Authorization": f"Bearer {token}",
     }
-    isa_json_str = isa_json.investigation.model_dump_json(
-        by_alias=True, exclude_none=True
-    )
+    isa_json_str = reduce_isa_json_for_target_repo(
+        isa_json, TargetRepository.METABOLIGHTS
+    ).investigation.model_dump_json(by_alias=True, exclude_none=True)
     json_file = io.StringIO(isa_json_str)
 
     files = {"isa_json_file": ("isa_json.json", json_file)}
@@ -360,7 +375,7 @@ def submit_to_ena(
             else result.request.body or ""
         )
         raise requests.HTTPError(
-            f"Request towards ENA failed!\nRequest:\nMethod:{result.request.method}\nStatus:{result.status_code}\nURL:{result.request.url}\nHeaders:{result.request.headers}\nBody:{body}"
+            f"Request towards ENA failed!\nRequest:\nMethod:{result.request.method}\nStatus:{result.status_code}\nURL:{submission_url}\nParams: ['webinUserName': {params.get('webinUserName')}, 'webinPassword': ****]\nHeaders:{result.request.headers}\nBody:{body}"
         )
 
     return result
@@ -372,11 +387,8 @@ def upload_to_ena(
     submission_url: str,
     file_transfer: str,
 ):
-    ALLOWED_FILE_TRANSFER_SOLUTIONS = {"ftp", "aspera"}
     file_transfer = file_transfer.lower()
 
-    if file_transfer not in ALLOWED_FILE_TRANSFER_SOLUTIONS:
-        raise ValueError(f"Unsupported transfer protocol: {file_transfer}")
     if file_transfer == "ftp":
         uploader = FTPUploader(
             submission_url,

mars-cli/tests/fixtures/bad_credentials_file.json

@@ -0,0 +1,18 @@
+{
+    "webin": {
+        "username": "put-your-username-here",
+        "password": "put-your-password-here"
+    },
+    "metabolights_metadata": {
+        "username": "put-your-username-here",
+        "password": "put-your-password-here"
+    },
+    "metabolights_data": {
+        "username": "put-your-username-here",
+        "password": "put-your-password-here"
+    },
+    "blahblahblah_repo": {
+      "username": "put-your-username",
+      "password": "put-your-password"
+    }
+}