diff --git a/README.md b/README.md index 8e7f625c..9644fd86 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Given a readme file (or a GitHub/Gitlab/Codeberg/Bitbucket repository) SOMEF wil - **Citation**: Preferred citation(s) as the authors have stated in their readme file. SOMEF recognizes Bibtex, Citation File Format files and other means by which authors cite their papers (e.g., by in-text citation). For CITATION.cff files, SOMEF now generates two separate entries: one for the software tool and another for the preferred citation (if available). This ensures metadata like DOI or version is correctly assigned to each entity. SOMEF now performs citation reconciliation: scholarly publications (articles) are assigned in codemeta to `referencePublication`, while the software itself is credited in `creditText`. (See https://somef.readthedocs.io/en/latest/output/#codemeta-format). -We recognize the following properties: +When using `-e`, publication metadata is enriched via OpenAlex. We recognize the following properties: - Title: Title of the publication - Author: list of author names in the publication - URL: URL of the publication @@ -37,6 +37,7 @@ We recognize the following properties: - Journal: Journal name where the paper was published - Year: Year of publication - Pages: Page range in the journal + - `openalex_id`: OpenAlex ID of the publication - **Code of conduct**: Link to the code of conduct of the project - **Code repository**: Link to the GitHub/GitLab/Codeberg and Bitbucket repository used for the extraction - **Contact**: Contact person responsible for maintaining a software component @@ -55,8 +56,14 @@ We recognize the following properties: - **Forks url**: Links to forks made of the project - **Full name**: Name + owner (owner/name) - **Full title**: If the repository is a short name, we will attempt to extract the longer version of the repository name -- **Funding**: Funding information associated with the project. **Note**: Currently, this information is only extracted from existing `codemeta.json` files within the repository. -- **Identifier**: Identifier associated with the software (if any), such as Digital Object Identifiers and Software Heritage identifiers (SWH). DOIs associated with publications will also be detected. +- **Funding**: Funding information associated with the project. **Note**: This information is extracted from existing `codemeta.json` files within the repository. When using `-e`, the project data is enriched with OpenAIRE, adding: + - `project_code`: Project code + - `project_title`: Project title + - `project_acronym`: Project acronym + - `grant_id`: Call/grant identifier +- **Identifier**: Identifier associated with the software (if any), such as Digital Object Identifiers and Software Heritage identifiers (SWH). DOIs associated with publications will also be detected. When using `-e`, the following enrichment identifiers are also added: + - `openaire_id`: URL to the OpenAIRE explore page for the software + - `swhid`: Software Heritage identifier (for Zenodo DOIs) - **Images**: Images used to illustrate the software component - **Installation instructions**: A set of instructions that indicate how to install a target repository - **Invocation**: Execution command(s) needed to run a scientific software component @@ -372,6 +379,8 @@ Options: -h, --help Show this message and exit. + -e, --enrichment Enrich metadata with external APIs (OpenAlex, OpenAIRE, Zenodo) + --github-token TEXT GitHub personal access token (if invalid, stored config is used instead) @@ -391,6 +400,7 @@ Options: -b, --branch name branch Branch of the repository to analyze. Overrides the default branch. --tag text Tag of the repository to analyze. Cannot be used together with --branch. + ``` Alternatively, you can set tokens via environment variables or by running `somef configure`, which stores them permanently. @@ -436,6 +446,14 @@ This includes identifying dependencies, runtime requirements, and development to SOMEF is designed to work primarily with repositories written in English. Repositories in other languages may not be processed as effectively, and results could be incomplete or less accurate. +### Enrichment with `-e` + +The `-e` (or `--enrichment`) flag queries external APIs to complete the extracted metadata: +- **OpenAlex**: adds `openalex_id` to DOIs of publications. +- **OpenAIRE**: adds `openaire_id` and enriches funding information (project code, title, acronym, grant id). +- **Zenodo**: adds `swhid` (Software Heritage ID) for Zenodo DOIs. + +**Note:** Enrichment makes additional network requests to external services, which may slow down the overall execution time. Use this flag only when you need the extra metadata. ## Repository versions: default behavior, branch and tag diff --git a/src/somef/__main__.py b/src/somef/__main__.py index 3cbe8681..1a2fca07 100644 --- a/src/somef/__main__.py +++ b/src/somef/__main__.py @@ -210,6 +210,13 @@ def configure(auto, base_uri): default=None, help="Tag of the repository to analyze. Incompatible with --branch" ) +@click.option( + "--enrich", + "-e", + is_flag=True, + default=False, + help="Enrich metadata with external APIs (OpenAlex, OpenAIRE, Zenodo)" + ) @click.option( "--github-token", type=str, diff --git a/src/somef/somef_cli.py b/src/somef/somef_cli.py index 9ba9b2b2..59b6fcc2 100644 --- a/src/somef/somef_cli.py +++ b/src/somef/somef_cli.py @@ -13,7 +13,7 @@ from . import header_analysis, regular_expressions, process_repository, configuration, process_files, \ supervised_classification from .process_results import Result -from .utils import constants, markdown_utils +from .utils import constants, markdown_utils, enrichment from .parser import mardown_parser, create_excerpts from .export.turtle_export import DataGraph from .export import json_export @@ -278,6 +278,7 @@ def run_cli(*, reconcile_authors=False, branch=None, tag=None, + enrich=False, github_token=None, gitlab_token=None, codeberg_token=None, @@ -328,6 +329,9 @@ def run_cli(*, repo_data = json_export.unify_results(repo_data.results) + if enrich: + repo_data = enrichment.run_enrichment(repo_data) + if output is not None: output = output.replace(".json","") output = output + "_" + encoded_url + ".json" @@ -371,6 +375,8 @@ def run_cli(*, repo_data = repo_data.get_json() repo_data = json_export.unify_results(repo_data.results) + if enrich: + repo_data = enrichment.run_enrichment(repo_data) if output is not None: json_export.save_json_output(repo_data, output, missing, pretty=pretty) diff --git a/src/somef/test/test_enrichment.py b/src/somef/test/test_enrichment.py new file mode 100644 index 00000000..2e4c17bd --- /dev/null +++ b/src/somef/test/test_enrichment.py @@ -0,0 +1,169 @@ +import json +import os +import unittest +from unittest.mock import patch +from pathlib import Path +from .. import somef_cli +from ..utils import constants +from ..utils.enrichment import run_enrichment + +test_data_path = str(Path(__file__).parent / "test_data") + os.path.sep + + +class TestEnrichment(unittest.TestCase): + + @unittest.skipIf(os.getenv("CI") == "true", "Skipped in CI because it requires external APIs") + def test_enrichment_integration(self): + """Tests that --enrich adds openalex_id, openaire_id, swhid, + orcid identifier and funding project properties to the output.""" + + somef_cli.run_cli(threshold=0.8, + repo_url="https://github.com/oeg-upm/rsfc", + output=test_data_path + "test-enrich.json", + enrich=True, + pretty=True) + + with open(test_data_path + "test-enrich.json") as f: + data = json.load(f) + + citations = data.get("citation", []) + # self.assertTrue(any("openalex_id" in c["result"] for c in citations)) + # self.assertTrue(any("openaire_id" in c["result"] for c in citations)) + + identifiers = data.get(constants.CAT_IDENTIFIER, []) + constants.PROP_OPENAIRE_ID + self.assertTrue(any( + constants.PROP_OPENAIRE_ID in c["result"] or constants.PROP_OPENALEX_ID in c["result"] + for c in citations + )) + self.assertTrue(any( + constants.PROP_OPENAIRE_ID in i["result"] or constants.PROP_OPENALEX_ID in i["result"] + for i in identifiers + )) + self.assertTrue(any(constants.PROP_SWHID in i["result"] for i in identifiers)) + + authors = data.get("author", []) + self.assertTrue(any( + constants.PROP_IDENTIFIER in a["result"] and "orcid" in a["result"].get(constants.PROP_IDENTIFIER, "").lower() + for a in authors + )) + + fundings = data.get("funding", []) + if fundings: + self.assertTrue(any(constants.PROP_PROJECT_CODE in f["result"] for f in fundings)) + self.assertTrue(any(constants.PROP_GRANT_ID in f["result"] for f in fundings)) + + os.remove(test_data_path + "test-enrich.json") + + + @unittest.skipIf(os.getenv("CI") == "true", "Skipped in CI") + def test_enrichment_funding(self): + """Tests funding enrichment with a repo that has codemeta.json with funding.""" + + somef_cli.run_cli(threshold=0.8, + repo_url="https://github.com/codemeta/codemeta", + output=test_data_path + "test-enrich-funding.json", + enrich=True, + pretty=True) + + with open(test_data_path + "test-enrich-funding.json") as f: + data = json.load(f) + + fundings = data.get("funding", []) + self.assertTrue(any("project_code" in f["result"] for f in fundings)) + self.assertTrue(any("project_title" in f["result"] for f in fundings)) + + os.remove(test_data_path + "test-enrich-funding.json") + + + @patch("somef.utils.enrichment.search_openalex_author") + def test_enrichment_author_orcid(self, mock_search): + """Tests that an author without ORCID gets one via search_openalex_author""" + + mock_search.return_value = "https://orcid.org/0000-0003-0454-7145" + + results = { + constants.CAT_AUTHORS: [ + { + "result": { + constants.PROP_NAME: "Daniel Garijo", + } + } + ], + constants.CAT_CONTRIBUTORS: [], + constants.CAT_CITATION: [], + constants.PROP_FUNDING: [], + constants.PROP_IDENTIFIER: [], + } + + enriched = run_enrichment(results) + + author_result = enriched[constants.CAT_AUTHORS][0]["result"] + self.assertIn(constants.PROP_IDENTIFIER, author_result) + self.assertIn("orcid", author_result[constants.PROP_IDENTIFIER].lower()) + + + @patch("somef.utils.enrichment.get_openaire_project") + def test_enrichment_funding_project(self, mock_project): + """Tests that a funding entry gets enriched with project metadata from OpenAIRE""" + + mock_project.return_value = { + constants.PROP_PROJECT_CODE: "12345", + constants.PROP_PROJECT_TITLE: "Test Project", + constants.PROP_PROJECT_ACRONYM: "TP", + constants.PROP_GRANT_ID: "EU.H2020.123", + constants.PROP_FUNDER: "European Commission", + constants.PROP_START_DATE: "2020-01-01", + constants.PROP_END_DATE: "2023-12-31", + } + + results = { + constants.PROP_FUNDING: [ + { + "result": { + constants.PROP_FUNDING: "EU.H2020.123", + } + } + ], + constants.CAT_CITATION: [], + constants.PROP_IDENTIFIER: [], + constants.CAT_AUTHORS: [], + constants.CAT_CONTRIBUTORS: [], + } + + enriched = run_enrichment(results) + + fund_result = enriched[constants.PROP_FUNDING][0]["result"] + self.assertEqual(fund_result[constants.PROP_PROJECT_CODE], "12345") + self.assertEqual(fund_result[constants.PROP_PROJECT_TITLE], "Test Project") + self.assertEqual(fund_result[constants.PROP_GRANT_ID], "EU.H2020.123") + + + @patch("somef.utils.enrichment.get_openalex_id") + @patch("somef.utils.enrichment.get_openaire_id") + def test_enrichment_openaire_fallback(self, mock_openaire, mock_openalex): + """Tests that OpenAlex is used as fallback when OpenAIRE returns nothing for a DOI""" + + mock_openaire.return_value = None + mock_openalex.return_value = "https://openalex.org/W123456" + + results = { + constants.CAT_CITATION: [ + { + "result": { + "doi": "10.1007/978-3-319-68204-4_9", + } + } + ], + constants.PROP_IDENTIFIER: [], + constants.CAT_AUTHORS: [], + constants.CAT_CONTRIBUTORS: [], + constants.PROP_FUNDING: [], + } + + enriched = run_enrichment(results) + + cit_result = enriched[constants.CAT_CITATION][0]["result"] + self.assertNotIn(constants.PROP_OPENAIRE_ID, cit_result) + self.assertIn(constants.PROP_OPENALEX_ID, cit_result) + self.assertEqual(cit_result[constants.PROP_OPENALEX_ID], "https://openalex.org/W123456") \ No newline at end of file diff --git a/src/somef/utils/constants.py b/src/somef/utils/constants.py index 4d47eeb1..6094c6f6 100644 --- a/src/somef/utils/constants.py +++ b/src/somef/utils/constants.py @@ -98,6 +98,7 @@ REGEXP_ALL_DOIS = r'10\.\d{4,9}/[-._;()/:A-Z0-9]+' # Detect zenodo latest doi in readme. +ZENODO_API_BASE = "https://zenodo.org/api" REGEXP_ZENODO_LATEST_DOI = r':target:\s*(https://zenodo\.org/badge/latestdoi/\d+)' REGEXP_ZENODO_DOI = r'https://zenodo\.org/badge/DOI/\d+' REGEXP_ZENODO_JSON_LD = r"]*type=['\"]application/ld\+json['\"][^>]*>(.*?)" @@ -708,3 +709,21 @@ class RepositoryType(Enum): CAT_REQUIREMENTS, CAT_INSTALLATION, } + +# Enrichment +OPENALEX_BASE = "https://api.openalex.org" +OPENAIRE_BASE = "https://api.openaire.eu" +OPENAIRE_EXPLORE = "https://explore.openaire.eu" +OPENAIRE_NAMESPACE = "http://namespace.openaire.eu/oaf" +REGEXP_DOI_IN_URL = r'(10\.\d{4,9}/[-._;()/:A-Za-z0-9]+)' +REGEXP_FIND_ZENODO = r'zenodo\.(\d+)' +PROP_OPENALEX_ID = "openalex_id" +PROP_OPENAIRE_ID = "openaire_id" +PROP_SWHID = "swhid" +PROP_PROJECT_CODE = "project_code" +PROP_PROJECT_TITLE = "project_title" +PROP_PROJECT_ACRONYM = "project_acronym" +PROP_GRANT_ID = "grant_id" +PROP_FUNDER = "funder" +PROP_START_DATE = "start_date" +PROP_END_DATE = "end_date" diff --git a/src/somef/utils/enrichment.py b/src/somef/utils/enrichment.py new file mode 100644 index 00000000..802a2211 --- /dev/null +++ b/src/somef/utils/enrichment.py @@ -0,0 +1,221 @@ + +import requests +import re +import logging +from ..utils import constants +import xml.etree.ElementTree as ET + +def get_openalex_id(doi): + url = f"{constants.OPENALEX_BASE}/works/doi:{doi}" + resp = requests.get(url) + if resp.status_code != 200: + return None + return resp.json().get("id") + +def get_openaire_id(doi) -> dict | None: + url = f"{constants.OPENAIRE_BASE}/search/researchProducts?doi={doi}&format=json" + resp = requests.get(url) + if resp.status_code != 200: + return None + data = resp.json() + results = data.get("response", {}).get("results", {}).get("result", []) + if results: + raw_id = results[0].get("header", {}).get("dri:objIdentifier", {}).get("$") + if raw_id: + return f"{constants.OPENAIRE_EXPLORE}/search/software?orpId={raw_id}" + return None + +def get_zenodo_swhid(doi): + """Get SWHID from a Zenodo DOI""" + + match = re.search(constants.REGEXP_FIND_ZENODO, doi) + if not match: + return None + record_id = match.group(1) + + url = f"{constants.ZENODO_API_BASE}/records/{record_id}" + resp = requests.get(url) + if resp.status_code != 200: + return None + + data = resp.json() + + swh = data.get("swh", {}) + if swh: + return swh.get("id") or swh.get(constants.PROP_SWHID) + + for rel_id in data.get("metadata", {}).get("related_identifiers", []): + if rel_id.get("identifier", "").startswith("swh:"): + return rel_id["identifier"] + + return None + + +def extract_doi(result): + """Extract a DOI from a result dict which may contain a DOI in different fields and formats.""" + + doi = result.get("doi") + if doi: + return doi + + for entry in result.get("identifier", []): + if entry.get("type") == "doi": + return entry.get("value") + + for key in ("url", "value"): + doi_url = re.search(constants.REGEXP_DOI_IN_URL, result.get(key, "")) + if doi_url: + return doi_url.group(1) + + return None + + +def search_openalex_author(name): + url = f"{constants.OPENALEX_BASE}/authors?search={requests.utils.quote(name)}" + resp = requests.get(url) + if resp.status_code != 200: + return None + results = resp.json().get("results", []) + if results: + return results[0].get("orcid") + return None + + +def collect_existing_orcids(results): + """ + Collect ORCIDs already present in the data. + Looks for ORCIDs in citation authors (url field) and in author/contributor entries (identifier and url fields). + """ + orcid_map = {} + + for citation in results.get(constants.CAT_CITATION, []): + for author in citation["result"].get(constants.PROP_AUTHOR, []): + add_orcid_to_map(orcid_map, author.get(constants.PROP_NAME, ""), author.get(constants.PROP_URL, "")) + add_orcid_to_map(orcid_map, author.get(constants.PROP_NAME, ""), author.get(constants.PROP_IDENTIFIER, "")) + + for category in (constants.CAT_AUTHORS, constants.CAT_CONTRIBUTORS): + for entry in results.get(category, []): + result = entry["result"] + add_orcid_to_map(orcid_map, result.get(constants.PROP_NAME, ""), result.get(constants.PROP_IDENTIFIER, "")) + add_orcid_to_map(orcid_map, result.get(constants.PROP_NAME, ""), result.get(constants.PROP_URL, "")) + return orcid_map + + +def add_orcid_to_map(orcid_map, name, value): + """Add an ORCID to the map if value contains an ORCID and name is not empty.""" + if value and "orcid" in value.lower() and name: + orcid_map[name.lower().strip()] = value + + +def clean_name(name): + """Remove newlines and surrounding whitespace from a name string.""" + return name.replace("\n", "").strip() + + +def has_orcid(result): + """Check if a result already has an ORCID.""" + for key in ("identifier", "url"): + val = result.get(key, "") + if "orcid" in val.lower(): + return True + return False + + +def get_openaire_project(identifier): + """Search for a project in OpenAIRE by grant ID or call identifierr""" + url = f"{constants.OPENAIRE_BASE}/search/projects?keywords={requests.utils.quote(identifier)}" + resp = requests.get(url) + if resp.status_code != 200: + return None + + root = ET.fromstring(resp.text) + ns = constants.OPENAIRE_NAMESPACE + + project = root.find(f".//{{{ns}}}project") + if project is None: + return None + + return { + constants.PROP_PROJECT_CODE: project.findtext("code"), + constants.PROP_PROJECT_TITLE: project.findtext("title"), + constants.PROP_PROJECT_ACRONYM: project.findtext("acronym"), + constants.PROP_GRANT_ID: project.findtext("callidentifier"), + constants.PROP_FUNDER: project.findtext(".//funder/shortname"), + constants.PROP_START_DATE: project.findtext("startdate"), + constants.PROP_END_DATE: project.findtext("enddate"), + } + +def run_enrichment(results) -> dict: + + logging.info("Enrichment process started.") + + for citation in results.get(constants.CAT_CITATION, []): + doi = extract_doi(citation["result"]) + if doi: + openaire_id = get_openaire_id(doi) + if openaire_id: + citation["result"][constants.PROP_OPENAIRE_ID] = openaire_id + else: + citation["result"][constants.PROP_OPENALEX_ID] = get_openalex_id(doi) + + for identifier in results.get(constants.PROP_IDENTIFIER, []): + value = identifier["result"].get("value", "") + m = re.search(constants.REGEXP_DOI_IN_URL, value) + if m: + doi = m.group(0) + if doi: + openaire_id = get_openaire_id(doi) + if openaire_id: + identifier["result"][constants.PROP_OPENAIRE_ID] = openaire_id + else: + identifier["result"][constants.PROP_OPENALEX_ID] = get_openalex_id(doi) + + if "zenodo" in doi.lower(): + if constants.PROP_SWHID not in identifier["result"]: + identifier["result"][constants.PROP_SWHID] = get_zenodo_swhid(doi) + + orcid_map = collect_existing_orcids(results) + + for category in (constants.CAT_AUTHORS, constants.CAT_CONTRIBUTORS): + for entry in results.get(category, []): + result = entry["result"] + if has_orcid(result): + continue + name = clean_name(result.get(constants.PROP_NAME) or result.get("value", "")) + if not name: + continue + + orcid = orcid_map.get(name.lower()) + + if not orcid: + orcid = search_openalex_author(name) + + if orcid: + result[constants.PROP_IDENTIFIER] = orcid + + for funding in results.get(constants.PROP_FUNDING, []): + result = funding["result"] + identifier = result.get(constants.PROP_FUNDING) + if identifier: + identifier = identifier.split(";")[0].strip() + else: + funder = result.get(constants.PROP_FUNDER) + if isinstance(funder, dict): + identifier = funder.get(constants.PROP_NAME) + else: + identifier = funder + + if identifier: + project = get_openaire_project(identifier) + # logging.info(f"Enrichment found project for funding identifier '{identifier}': {project}") + if project: + if project[constants.PROP_PROJECT_CODE]: + result[constants.PROP_PROJECT_CODE] = project[constants.PROP_PROJECT_CODE] + if project[constants.PROP_PROJECT_TITLE]: + result[constants.PROP_PROJECT_TITLE] = project[constants.PROP_PROJECT_TITLE] + if project[constants.PROP_PROJECT_ACRONYM]: + result[constants.PROP_PROJECT_ACRONYM] = project[constants.PROP_PROJECT_ACRONYM] + if project[constants.PROP_GRANT_ID]: + result[constants.PROP_GRANT_ID] = project[constants.PROP_GRANT_ID] + + return results \ No newline at end of file