diff --git a/.gitignore b/.gitignore index 7858c187..c02e1e33 100644 --- a/.gitignore +++ b/.gitignore @@ -24,5 +24,7 @@ repos.txt !package_neors.json !package_npm.json !test_data/api_responses/*.json +!**/test_data/api_responses/codeberg/*.json +!**/test_data/api_responses/bitbucket/*.json uv.lock .python-version diff --git a/README.md b/README.md index 5577d24b..657dd92d 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ A command line interface for automatically extracting relevant metadata from cod ## Features -Given a readme file (or a GitHub/Gitlab repository) SOMEF will extract the following categories (if present), listed in alphabetical order: +Given a readme file (or a GitHub/Gitlab/Codeberg/Bitbucket repository) SOMEF will extract the following categories (if present), listed in alphabetical order: - **Acknowledgement**: Text acknowledging funding sources or contributors - **Application domain**: The application domain of the repository. Current supported domains include: Astrophysics, Audio, Computer vision, Graphs, Natural language processing, Reinforcement learning, Semantc web, Sequential. Domains are not mutually exclusive. These domains have been extracted from [awesome lists](https://github.com/topics/awesome-list) and [Papers with code](https://paperswithcode.com/). Find more information in our [documentation](https://somef.readthedocs.io/en/latest/) @@ -38,7 +38,7 @@ We recognize the following properties: - Year: Year of publication - Pages: Page range in the journal - **Code of conduct**: Link to the code of conduct of the project -- **Code repository**: Link to the GitHub/GitLab repository used for the extraction +- **Code repository**: Link to the GitHub/GitLab/Codeberg and Bitbucket repository used for the extraction - **Contact**: Contact person responsible for maintaining a software component - **Continuous integration**: Link to continuous integration service(s) - **Contribution guidelines**: Text indicating how to contribute to this code repository @@ -72,7 +72,7 @@ We recognize the following properties: - **Package files**: Links to package files used to wrap the project in a package. - **Programming languages**: Languages used in the repository - **Related papers**: URL to possible related papers within the repository stated within the readme file (from Arxiv) -- **Releases** (GitHub only): Pointer to the available versions of a software component. For each release, somef will track the following properties: +- **Releases**: Pointer to the available versions of a software component. For each release, somef will track the following properties: - Description: Release notes - Author: Agent responsible of creating the release - Name: Name of the release @@ -93,7 +93,7 @@ We recognize the following properties: - **Usage examples**: Assumptions and considerations recorded by the authors when executing a software component, or examples on how to use it - **Workflows**: URL and path to the computational workflow files present in the repository -We use different supervised classifiers, header analysis, regular expressions, the GitHub/Gitlab API to retrieve all these fields (more than one technique may be used for each field) and language specific metadata parsers (e.g., for package files). Each extraction records its provenance, with the confidence and technique used on each step. For more information check the [output format description](https://somef.readthedocs.io/en/latest/output/) +We use different supervised classifiers, header analysis, regular expressions, the GitHub/Gitlab/Codeberg and Bitbucket API to retrieve all these fields (more than one technique may be used for each field) and language specific metadata parsers (e.g., for package files). Each extraction records its provenance, with the confidence and technique used on each step. For more information check the [output format description](https://somef.readthedocs.io/en/latest/output/) ### Confidence values in header analysis @@ -265,10 +265,14 @@ somef configure And you will be asked to provide the following: -- A GitHub authentication token [**optional, leave blank if not used**], which SOMEF uses to retrieve metadata from GitHub. If you don't include an authentication token, you can still use SOMEF. However, you may be limited to a series of requests per hour. For more information, see [https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) -- The path to the trained classifiers (pickle files). If you have your own classifiers, you can provide them here. Otherwise, you can leave it blank +- A **GitHub** authentication token [**optional, leave blank if not used**], which SOMEF uses to retrieve metadata from GitHub. If you don't include an authentication token, you can still use SOMEF. However, you may be limited to a series of requests per hour. For more information, see [https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) +- A **GitLab** authentication token [**optional**], used for GitLab.com and self-hosted GitLab instances (e.g., `gitlab.in2p3.fr`). Tokens are per-instance. Note: **a token from GitLab.com does not work for self-hosted servers**. Create one at `https://gitlab.com/-/user_settings/personal_access_tokens` (scope: `read_api`). Without a token, some self-hosted GitLab instances may not return rate limit information. +- A **Codeberg** authentication token [**optional**], used to retrieve metadata from Codeberg. Create one at `https://codeberg.org/user/settings/applications` (permissions: `read:repository`, `read:user`). Codeberg (Forgejo) does not expose rate limit headers even with a token. +- A **Bitbucket** authentication token [**optional**], used for Bitbucket Cloud. Create an API token with scopes at `https://bitbucket.org/account/settings/api-tokens/` (permissions: `read:repository:bitbucket`, `read:account`). You will also need to provide your Atlassian account email, as Bitbucket API tokens use Basic authentication (`email:token` encoded in base64). Without a token you are limited to 60 requests/hour. +- The path to the trained classifiers (pickle files). If you have your own classifiers, you can provide them here. Otherwise, you can leave it blank. -If you want somef to be automatically configured (without GitHUb authentication key and using the default classifiers) just type: + +If you want SOMEF to be automatically configured (without any tokens and using the default classifiers) just type: ```bash somef configure -a @@ -310,10 +314,10 @@ Usage: somef describe [OPTIONS] Options: -t, --threshold FLOAT Threshold to classify the text [required] Input: [mutually_exclusive, required] - -r, --repo_url URL Github/Gitlab Repository URL + -r, --repo_url URL Github/Gitlab/Codeberg/Bitbucket Repository URL -d, --doc_src PATH Path to the README file source -i, --in_file PATH A file of newline separated links to GitHub/ - Gitlab repositories + Gitlab/Codeberg/Bitbucket repositories -l, --local_repo PATH Path to the local repository source. No APIs will be used Output: [required_any] @@ -360,6 +364,21 @@ Options: requests and increase execution time -h, --help Show this message and exit. + + --github-token TEXT GitHub personal access token (if invalid, + stored config is used instead) + + --gitlab-token TEXT GitLab personal access token (if invalid, + stored config is used instead) + + --codeberg-token TEXT Codeberg personal access token (if invalid, + stored config is used instead) + + --bitbucket-token TEXT Bitbucket app password (if invalid, stored + config is used instead) + + --bitbucket-email TEXT Bitbucket Atlassian account email (required + with --bitbucket-token) Repoository versions [mutually_exclusive] (see section *Repository versions*t): -b, --branch name branch Branch of the repository to analyze. Overrides the default branch. @@ -367,6 +386,10 @@ Options: --tag text Tag of the repository to analyze. Cannot be used together with --branch. ``` +Alternatively, you can set tokens via environment variables or by running `somef configure`, which stores them permanently. +The CLI flags take precedence over stored config when valid. + + ## Usage example: The following command extracts all metadata available from [https://github.com/dgarijo/Widoco/](https://github.com/dgarijo/Widoco/). diff --git a/docs/bitbucket.md b/docs/bitbucket.md new file mode 100644 index 00000000..7c8eeba2 --- /dev/null +++ b/docs/bitbucket.md @@ -0,0 +1,21 @@ +When analyzing a Bitbucket repository, SOMEF uses the [Bitbucket Cloud API](https://developer.atlassian.com/cloud/bitbucket/rest/api-group-repositories/) +(`GET /2.0/repositories/{workspace}/{repo_slug}`) to retrieve metadata. The table below shows how Bitbucket API +fields map to SOMEF categories: + +| SOMEF category | Bitbucket API field | Notes | +|---|---|---| +| `name` | `slug` | | +| `description` | `description` | | +| `full_name` | `full_name` | Format: `{workspace}/{slug}` | +| `code_repository` | `links.html.href` | | +| `owner` | `owner.nickname` | Falls back to `owner.username` for team workspaces | +| `date_created` | `created_on` | | +| `date_updated` | `updated_on` | | +| `homepage` | `website` | | +| `forks_url` | `links.forks.href` | | +| `download_url` | *(constructed)* | Built as `{html_url}/downloads` | +| `issue_tracker` | *(constructed)* | Built as `{html_url}/issues` when `has_issues` is true | +| `programming_languages` | `language` | Single string, not a dictionary with sizes | +| `releases` | `/refs/tags` | Bitbucket has no dedicated releases endpoint; uses the tags endpoint | +| `stars` | *(not available)* | Bitbucket does not have a stargazers feature | +| `forks_count` | *(not available)* | Bitbucket does not expose fork counts in its API | \ No newline at end of file diff --git a/docs/codeberg.md b/docs/codeberg.md new file mode 100644 index 00000000..302826f3 --- /dev/null +++ b/docs/codeberg.md @@ -0,0 +1,29 @@ +When analyzing a Codeberg repository, SOMEF uses the [Codeberg API](https://codeberg.org/api/v1/swagger) +(`GET /api/v1/repos/{owner}/{repo}`) to retrieve metadata. The table below shows how Codeberg API +fields map to SOMEF categories: + +| SOMEF category | Codeberg API field | Notes | +|---|---|---| +| `name` | `name` | | +| `description` | `description` | | +| `code_repository` | `html_url` | | +| `owner` | `owner.login` | | +| `date_created` | `created_at` | | +| `date_updated` | `updated_at` | | +| `stars` | `stars_count` | In GitHub this field is `stargazers_count` | +| `forks_count` | `forks_count` | | +| `homepage` | `website` | In GitHub this field is `homepage` | +| `keywords` | `topics` | | +| `issue_tracker` | *(constructed)* | Built as `{html_url}/issues` | +| `license` | *(content API)* | *1* | +| `programming_languages` | `languages_url` | Additional GET request to the languages endpoint | +| `releases` | `/repos/{owner}/{repo}/releases` | Additional GET request | + +For releases, the field mapping is identical to GitHub. The only differences are that Codeberg +uses `attachments` instead of `assets` for release files, and it does not provide +`author.type` (`AGENT_TYPE`) for release authors. + +--------------- + +*1* +Extracted by fetching the LICENSE file via `GET /api/v1/repos/{owner}/{repo}/contents/{filename}` (tries `LICENSE`, `LICENSE.md`, `LICENCE`, `COPYING`). The content is base64-decoded and analyzed with `detect_license_spdx()` to obtain the SPDX identifier, name and URL. Technique: `Codeberg_API`. \ No newline at end of file diff --git a/docs/index.md b/docs/index.md index e3ac76e7..00fe5ed8 100644 --- a/docs/index.md +++ b/docs/index.md @@ -46,7 +46,7 @@ We recognize the following properties: - Year: Year of publication - Pages: Page range in the journal - **Code of conduct**: Link to the code of conduct of the project -- **Code repository**: Link to the GitHub/GitLab repository used for the extraction +- **Code repository**: Link to the GitHub/GitLab/Codeberg/Bitbucket repository used for the extraction - **Contact**: Contact person responsible for maintaining a software component - **Continuous integration**: Link to continuous integration service(s) - **Contribution guidelines**: Text indicating how to contribute to this code repository @@ -80,7 +80,7 @@ We recognize the following properties: - **Package files**: Links to package files used to wrap the project in a package. - **Programming languages**: Languages used in the repository - **Related papers**: URL to possible related papers within the repository stated within the readme file (from Arxiv) -- **Releases** (GitHub and Gitlab): Pointer to the available versions of a software component. For each release, somef will track the following properties: +- **Releases** (GitHub, Gitlab, Codeberg and Bitbucket): Pointer to the available versions of a software component. For each release, somef will track the following properties: - Assets: files attached to the release - Description: Release notes - Author: Agent responsible of creating the release @@ -102,7 +102,7 @@ We recognize the following properties: - **Usage examples**: Assumptions and considerations recorded by the authors when executing a software component, or examples on how to use it - **Workflows**: URL and path to the computational workflow files present in the repository -We use different supervised classifiers, header analysis, regular expressions, the GitHub/Gitlab API to retrieve all these fields (more than one technique may be used for each field) and language specific metadata parsers (e.g., for package files). Each extraction records its provenance, with the confidence and technique used on each step. For more information check the [output format description](https://somef.readthedocs.io/en/latest/output/) +We use different supervised classifiers, header analysis, regular expressions, the GitHub/Gitlab/Codeberg/Bitbucket API to retrieve all these fields (more than one technique may be used for each field) and language specific metadata parsers (e.g., for package files). Each extraction records its provenance, with the confidence and technique used on each step. For more information check the [output format description](https://somef.readthedocs.io/en/latest/output/) 1 The available application domains currently are: diff --git a/docs/install.md b/docs/install.md index 014e1eb5..fa73552c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -83,25 +83,25 @@ docker run -it --rm -v $PWD/:/out kcapd/somef /bin/bash ``` If you move any files produced by somef into `/out`, then you will be able to see them in your current directory. + ## Configure -Before running SOMEF, you must configure it appropriately. Run: -```bash -python -m nltk.downloader wordnet -python -m nltk.downloader omw-1.4 -``` -To download two wordnet modules needed. Then run: +Before running SOMEF for the first time, you must **configure** it appropriately (you only need to do this once). Run: ```bash somef configure ``` -And you will be asked to provide the following: +And you will be asked to provide the following: -- A GitHub authentication token [**optional, leave blank if not used**], which SOMEF uses to retrieve metadata from GitHub. If you don't include an authentication token, you can still use SOMEF. However, you may be limited to a series of requests per hour. For more information, see [https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) -- The path to the trained classifiers (pickle files). If you have your own classifiers, you can provide them here. Otherwise, you can leave it blank +- A **GitHub** authentication token [**optional, leave blank if not used**], which SOMEF uses to retrieve metadata from GitHub. If you don't include an authentication token, you can still use SOMEF. However, you may be limited to a series of requests per hour. For more information, see [https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line](https://help.github.com/en/github/authenticating-to-github/creating-a-personal-access-token-for-the-command-line) +- A **GitLab** authentication token [**optional**], used for GitLab.com and self-hosted GitLab instances (e.g., `gitlab.in2p3.fr`). Tokens are per-instance. Note: **a token from GitLab.com does not work for self-hosted servers**. Create one at `https://gitlab.com/-/user_settings/personal_access_tokens` (scope: `read_api`). Without a token, some self-hosted GitLab instances may not return rate limit information. +- A **Codeberg** authentication token [**optional**], used to retrieve metadata from Codeberg. Create one at `https://codeberg.org/user/settings/applications` (permissions: `read:repository`, `read:user`). Codeberg (Forgejo) does not expose rate limit headers even with a token. +- A **Bitbucket** authentication token [**optional**], used for Bitbucket Cloud. Create an API token with scopes at `https://bitbucket.org/account/settings/api-tokens/` (permissions: `read:repository:bitbucket`, `read:account`). You will also need to provide your Atlassian account email, as Bitbucket API tokens use Basic authentication (`email:token` encoded in base64). Without a token you are limited to 60 requests/hour. +- The path to the trained classifiers (pickle files). If you have your own classifiers, you can provide them here. Otherwise, you can leave it blank. -If you want to configure SOMEF with the default parameters, just type: + +If you want SOMEF to be automatically configured (without any tokens and using the default classifiers) just type: ```bash somef configure -a @@ -112,6 +112,7 @@ For showing help about the available options, run: ```bash somef configure --help ``` + Which displays: ```bash diff --git a/docs/output.md b/docs/output.md index f34f1281..9d374c50 100644 --- a/docs/output.md +++ b/docs/output.md @@ -73,7 +73,7 @@ SOMEF aims to recognize the following categories (in alphabetical order): - `code_of_conduct`: Link to the code of conduct file of the project - `code_repository`: Link to the source code (typically the repository where the readme can be found) - `contact`: Contact person responsible for maintaining a software component. -- `continuous_integration`: Link to continuous integration service, supported on GitHub as well as in GitLab. +- `continuous_integration`: Link to continuous integration service, supported on GitHub as well as in GitLab, Codeberg and Bitbucket. - `contributing guidelines`: Guidelines indicating how to contribute to a software component. - `contributor`: Contributors to this software. Note: Contributor metadata is exported from metadata files (e.g., CodeMeta, CONTRIBUTORS, etc.) not from git logs. - `copyright_holder`: Entity or individual owning the rights to the software. The year is also extracted, if available. @@ -180,7 +180,7 @@ Depending on the `type` of the result, additional properties may be found. The following object `types` are currently supported: -- `Release`: software releases of the current code repository, as available from GitHub. +- `Release`: software releases of the current code repository, as available from GitHub, GitLab and Codeberg - `Programming_language`: Programming language used in the repository. - `License`: object representing all the metadata SOMEF extracts from a license. - `Agent`: user (typically, a person) or organization responsible for authoring a software release or a paper. @@ -330,6 +330,8 @@ The techniques can be of several types: - `file_exploration`: the result comes from an exploration of the files in the repository - `GitHub_API`: the result was obtained from the GitHub API. - `GitLab_API`: the result was obtained from the GitLab API. +- `Codeberg_API`: the result was obtained from the Codeberg API. +- `Bitbucket_API`: the result was obtained from the Bitbucket API. - `regular_expression`: the result was obtained after performing regular expressions on the files in the repository. - `software_type_heuristics`: the result was obtained from analysis of the repository based on various heuristics from the README, code and extension analysis. - `supervised_classification`: the results were obtained after running text classifiers trained for detecting that type of header. diff --git a/docs/usage.md b/docs/usage.md index b212e972..c5b06245 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -19,7 +19,7 @@ Commands: The options to run somef are through the `describe` command: ```bash - somef describe --help +$ somef describe --help SOMEF Command Line Interface Usage: somef describe [OPTIONS] @@ -28,21 +28,25 @@ Usage: somef describe [OPTIONS] Options: -t, --threshold FLOAT Threshold to classify the text [required] Input: [mutually_exclusive, required] - -r, --repo_url URL Github Repository URL + -r, --repo_url URL Github/Gitlab/Codeberg/Bitbucket Repository URL -d, --doc_src PATH Path to the README file source - -i, --in_file PATH A file of newline separated links to GitHub - repositories + -i, --in_file PATH A file of newline separated links to GitHub/ + Gitlab/Codeberg/Bitbucket repositories + -l, --local_repo PATH Path to the local repository source. No APIs will be used Output: [required_any] -o, --output PATH Path to the output file. If supplied, the output will be in JSON - - -g, --graph_out PATH Path to the output Knowledge Graph file. If - supplied, the output will be a Knowledge - Graph, in the format given in the --format - option - -c, --codemeta_out PATH Path to an output codemeta file (in JSON-LD) - + -c, --codemeta_out PATH Path to an output codemeta file + -g, --graph_out PATH Path to the output Knowledge Graph export + file. If supplied, the output will be a + Knowledge Graph, in the format given in the + --format option chosen (turtle, json-ld) + -gc, --google_codemeta_out PATH Path to a Google-compliant Codemeta JSON-LD + file. This output transforms the standard + Codemeta to follow Google’s expected JSON-LD + structure. + -f, --graph_format [turtle|json-ld] If the --graph_out option is given, this is the format that the graph will be stored in @@ -51,18 +55,58 @@ Options: is easy to compare to another JSON output file. - -m, --missing JSON report with the missing metadata fields - SOMEF was not able to find. The report will - be placed in $PATH_missing.json, where - $PATH is -o, -c or -g. + -m, --missing The JSON will include a field + somef_missing_categories to report with the + missing metadata fields that SOMEF was not + able to find. + -kt, --keep_tmp PATH SOMEF will NOT delete the temporary folder where files are stored for analysis. Files will be stored at the desired path + -all, --requirements_all Export all detected requirements, including + text and libraries (default). + + -v, --requirements_v Export only requirements from structured + sources (pom.xml, requirements.txt, etc.) + + + -ra, --reconcile_authors SOMEF will extract additional information + from certain files like CODEOWNERS. + This may require extra API + requests and increase execution time + -h, --help Show this message and exit. + + --github-token TEXT GitHub personal access token (if invalid, + stored config is used instead) + + --gitlab-token TEXT GitLab personal access token (if invalid, + stored config is used instead) + + --codeberg-token TEXT Codeberg personal access token (if invalid, + stored config is used instead) + + --bitbucket-token TEXT Bitbucket app password (if invalid, stored + config is used instead) + + --bitbucket-email TEXT Bitbucket Atlassian account email (required + with --bitbucket-token) + + Repoository versions [mutually_exclusive] (see section *Repository versions*t): + -b, --branch name branch Branch of the repository to analyze. Overrides the default branch. + + --tag text Tag of the repository to analyze. Cannot be used together with --branch. ``` +Note about tokens: GitHub, Codeberg and Bitbucket APIs can be used without authentication for basic metadata extraction. GitLab may require a token for self-hosted instances. User enrichment via CODEOWNERS (--reconcile_authors) works for +GitHub, GitLab and Codeberg; Bitbucket does not expose a public user API, so enrichment is not supported for that platform. + +Alternatively, you can set tokens via environment variables or by running `somef configure`, which stores them permanently. +The CLI flags take precedence over stored config when valid. + + ## Usage example: The following command extracts all metadata available from [https://github.com/dgarijo/Widoco/](https://github.com/dgarijo/Widoco/). diff --git a/src/somef/__main__.py b/src/somef/__main__.py index 0b666830..b62c2c51 100644 --- a/src/somef/__main__.py +++ b/src/somef/__main__.py @@ -19,7 +19,7 @@ def cli(): click.echo("SOftware Metadata Extraction Framework (SOMEF) Command Line Interface") -@cli.command(help="Configure GitHub credentials and classifiers file path") +@cli.command(help="Configure repository credentials and classifiers file path") @click.option('-a', '--auto', help="Automatically configure SOMEF", is_flag=True, default=False) @click.option('-b', '--base_uri', type=URLParamType(), help="Base URI for somef transformations", default=constants.CONF_DEFAULT_BASE_URI) @@ -32,14 +32,24 @@ def configure(auto, base_uri): elif base_uri is not constants.CONF_DEFAULT_BASE_URI: configuration.update_base_uri(base_uri) else: - authorization = click.prompt("Authorization", default="") + # authorization = click.prompt("Authorization", default="") + github_authorization = click.prompt("GitHub Authorization token (leave blank to skip)", default="") + gitlab_authorization = click.prompt( + "GitLab Authorization token (leave blank to skip; works for gitlab.com and self-hosted instances)", + default="", + ) + codeberg_authorization = click.prompt("Codeberg Authorization token (leave blank to skip)", default="") + + bitbucket_authorization = click.prompt("Bitbucket Authorization token (leave blank to skip)", default="") + bitbucket_email = click.prompt("Bitbucket Atlassian account email (for API token auth)", default="") + description = click.prompt("Documentation classifier model file", default=configuration.default_description) invocation = click.prompt("Invocation classifier model file", default=configuration.default_invocation) installation = click.prompt("Installation classifier model file", default=configuration.default_installation) citation = click.prompt("Citation classifier model file", default=configuration.default_citation) base_uri = click.prompt("Base URI for RDF generation", default=base_uri) # configuration.configure() - configuration.configure(authorization, description, invocation, installation, citation, base_uri) + configuration.configure(github_authorization, gitlab_authorization, codeberg_authorization, bitbucket_authorization, bitbucket_email, description, invocation, installation, citation, base_uri) click.secho(f"Success", fg="green") @@ -197,6 +207,38 @@ def configure(auto, base_uri): default=None, help="Tag of the repository to analyze. Incompatible with --branch" ) +@click.option( + "--github-token", + type=str, + default=None, + help="GitHub personal access token (if invalid, stored config is used instead)" +) +@click.option( + "--gitlab-token", + "-gt", + type=str, + default=None, + help="GitLab personal access token (if invalid, stored config is used instead)" +) +@click.option( + "--codeberg-token", + type=str, + default=None, + help="Codeberg personal access token (if invalid, stored config is used instead)" +) +@click.option( + "--bitbucket-token", + type=str, + default=None, + help="Bitbucket API token (if invalid, stored config is used instead)" +) +@click.option( + "--bitbucket-email", + type=str, + default=None, + help="Bitbucket Atlassian account email (required with --bitbucket-token)" +) + def describe(requirements_v, requirements_all, **kwargs): # import so missing packages get installed when appropriate if requirements_v: diff --git a/src/somef/configuration.py b/src/somef/configuration.py index 599228fe..ab71f99c 100644 --- a/src/somef/configuration.py +++ b/src/somef/configuration.py @@ -3,6 +3,7 @@ import json import sys import logging +import base64 from .utils import constants @@ -50,13 +51,20 @@ def update_base_uri(base_uri): json.dump(data, fh) -def configure(authorization="", - description=default_description, - invocation=default_invocation, - installation=default_installation, - citation=default_citation, - base_uri=constants.CONF_DEFAULT_BASE_URI, - similarity_threshold=constants.CONF_DEFAULT_SIMILARITY_THRESHOLD): +def configure( + # authorization="", + github_authorization="", + gitlab_authorization="", + codeberg_authorization="", + bitbucket_authorization="", + bitbucket_email="", + description=default_description, + invocation=default_invocation, + installation=default_installation, + citation=default_citation, + base_uri=constants.CONF_DEFAULT_BASE_URI, + similarity_threshold=constants.CONF_DEFAULT_SIMILARITY_THRESHOLD): + """ Function to configure the main program""" import nltk nltk.download('wordnet') @@ -75,7 +83,7 @@ def configure(authorization="", # data = json.load(fh) # else: data = { - constants.CONF_AUTHORIZATION: "token " + authorization, + # constants.CONF_AUTHORIZATION: "token " + authorization, constants.CONF_DESCRIPTION: description, constants.CONF_INVOCATION: invocation, constants.CONF_INSTALLATION: installation, @@ -84,8 +92,32 @@ def configure(authorization="", constants.CONF_SIMILARITY_THRESHOLD: similarity_threshold } - if data[constants.CONF_AUTHORIZATION] == "token ": - del data[constants.CONF_AUTHORIZATION] + # if data[constants.CONF_AUTHORIZATION] == "token ": + # del data[constants.CONF_AUTHORIZATION] + + if github_authorization: + data[constants.CONF_GITHUB_AUTHORIZATION] = "token " + github_authorization + + if gitlab_authorization: + token = gitlab_authorization + if not token.lower().startswith("bearer "): + token = "Bearer " + token + data[constants.CONF_GITLAB_AUTHORIZATION] = token + + if codeberg_authorization: + token = codeberg_authorization + if not token.lower().startswith("token "): + token = "token " + token + data[constants.CONF_CODEBERG_AUTHORIZATION] = token + + + if bitbucket_authorization: + token = bitbucket_authorization + email = bitbucket_email + if not token.lower().startswith("basic "): + raw = f"{email}:{token}" + token = "Basic " + base64.b64encode(raw.encode()).decode() + data[constants.CONF_BITBUCKET_AUTHORIZATION] = token with credentials_file.open("w") as fh: credentials_file.parent.chmod(0o700) diff --git a/src/somef/parser/codeowners_parser.py b/src/somef/parser/codeowners_parser.py index 1c7c92a7..2eb6b6a9 100644 --- a/src/somef/parser/codeowners_parser.py +++ b/src/somef/parser/codeowners_parser.py @@ -105,16 +105,26 @@ def parse_codeowners_file(file_path, metadata_result: Result, source, reconcile_ # return None -def enrich_user(username, repo_type, server_url=None): +def enrich_user(username, repo_type, server_url=None, gitlab_authorization=None): """ Enrich user metadata using the appropriate platform API. - + Parameters ---------- - username : str Username to enrich. - repo_type : str "GITHUB" or "GITLAB" + username : str + Username to enrich. + repo_type : RepositoryType + "GITHUB", "GITLAB" or "CODEBERG". server_url : str, optional - Base URL of GitLab instance if repo_type is "GITLAB" + Base URL of GitLab instance if repo_type is "GITLAB". + gitlab_authorization : str, optional + GitLab personal access token. Only needed for GitLab + (self-hosted instances or higher rate limits). + GitHub and Codeberg user APIs are public and do not + require authentication. + Bitbucket does not expose a public user endpoint, + so user enrichment is not supported for that platform. + Returns ------- @@ -146,7 +156,15 @@ def enrich_user(username, repo_type, server_url=None): if not server_url.startswith("http"): server_url = "https://" + server_url api_url = f"{server_url.rstrip('/')}/api/v4/users?username={username}" - response = requests.get(api_url, timeout=5) + # response = requests.get(api_url, timeout=5) + # Build auth header — same token works for gitlab.com and self-hosted instances + gl_headers = {} + if gitlab_authorization: + token = gitlab_authorization + if not token.startswith("Bearer "): + token = "Bearer " + token + gl_headers["Authorization"] = token + response = requests.get(api_url, headers=gl_headers, timeout=5) if response.status_code != 200: logging.warning(f"GitLab API request failed for {username}: {response.status_code}") return None @@ -163,6 +181,17 @@ def enrich_user(username, repo_type, server_url=None): logging.error(f"Error enriching GitLab user {username}: {e}") return None + elif repo_type == constants.RepositoryType.CODEBERG: + url = f"https://codeberg.org/api/v1/users/{username}" + response = requests.get(url, timeout=5) + if response.status_code != 200: + return None + data = response.json() + return { + constants.PROP_CODEOWNERS_NAME: data.get("full_name"), + constants.PROP_CODEOWNERS_EMAIL: data.get("email"), + } + else: logging.warning(f"Unsupported repo_type {repo_type}") return None diff --git a/src/somef/process_files.py b/src/somef/process_files.py index 2b80ad0f..ae3daa6e 100644 --- a/src/somef/process_files.py +++ b/src/somef/process_files.py @@ -328,6 +328,7 @@ def process_repository_files(repo_dir, metadata_result: Result, repo_type, owner # if repo_type == constants.RepositoryType.GITLAB: if filename.endswith(".yml"): + category = None if repo_type == constants.RepositoryType.GITLAB: analysis = extract_workflows.is_file_continuous_integration_gitlab(os.path.join(repo_dir, file_path)) if analysis: @@ -345,26 +346,29 @@ def process_repository_files(repo_dir, metadata_result: Result, repo_type, owner { constants.PROP_VALUE: workflow_url, constants.PROP_TYPE: constants.URL - }, 1, constants.TECHNIQUE_FILE_EXPLORATION) + }, 1, constants.TECHNIQUE_FILE_EXPLORATION) + elif repo_type == constants.RepositoryType.CODEBERG: + if (file_path.startswith(".forgejo/workflows/") or file_path.startswith(".gitea/workflows/")): + category = constants.CAT_CONTINUOUS_INTEGRATION + else: + category = None + elif repo_type == constants.RepositoryType.BITBUCKET: + if os.path.basename(file_path) == "bitbucket-pipelines.yml": + category = constants.CAT_CONTINUOUS_INTEGRATION + else: + category = None elif repo_type == constants.RepositoryType.GITHUB: - # if file_path.startswith(".github/workflows/"): - # category = constants.CAT_WORKFLOWS - # elif filename in [".travis.yml", "azure-pipelines.yml", "jenkinsfile"] or file_path.startswith(".circleci/"): - # category = constants.CAT_CONTINUOUS_INTEGRATION - # else: - # category = None if file_path.startswith(".github/workflows/"): category = constants.CAT_CONTINUOUS_INTEGRATION else: category = None - if category: - workflow_url = get_file_link(repo_type, file_path, owner, repo_name, repo_default_branch, - repo_dir, repo_relative_path, filename) - metadata_result.add_result(category, - {constants.PROP_VALUE: workflow_url, constants.PROP_TYPE: constants.URL}, - 1, constants.TECHNIQUE_FILE_EXPLORATION) - + if category: + workflow_url = get_file_link(repo_type, file_path, owner, repo_name, repo_default_branch, + repo_dir, repo_relative_path, filename) + metadata_result.add_result(category, + {constants.PROP_VALUE: workflow_url, constants.PROP_TYPE: constants.URL}, + 1, constants.TECHNIQUE_FILE_EXPLORATION) if filename.endswith(".ga") or filename.endswith(".cwl") or filename.endswith(".nf") or ( filename.endswith(".snake") or filename.endswith( ".smk") or "Snakefile" == filename_no_ext) or filename.endswith(".knwf") or filename.endswith( @@ -413,6 +417,10 @@ def process_repository_files(repo_dir, metadata_result: Result, repo_type, owner docs_url = f"https://github.com/{owner}/{repo_name}/tree/{urllib.parse.quote(repo_default_branch)}/{docs_path}" elif repo_type == constants.RepositoryType.GITLAB: docs_url = f"https://{domain_gitlab}/{owner}/{repo_name}/-/tree/{urllib.parse.quote(repo_default_branch)}/{docs_path}" + elif repo_type == constants.RepositoryType.CODEBERG: + docs_url = f"https://codeberg.org/{owner}/{repo_name}/src/branch/{urllib.parse.quote(repo_default_branch)}/{docs_path}" + elif repo_type == constants.RepositoryType.BITBUCKET: + docs_url = f"https://bitbucket.org/{owner}/{repo_name}/src/{urllib.parse.quote(repo_default_branch)}/{docs_path}" else: docs_url = os.path.join(repo_dir, docs_path) # docs.append(docs_url) @@ -452,6 +460,10 @@ def get_file_link(repo_type, file_path, owner, repo_name, repo_default_branch, r return convert_to_raw_user_content_github(file_path, owner, repo_name, repo_default_branch) elif repo_type == constants.RepositoryType.GITLAB: return convert_to_raw_user_content_gitlab(file_path, owner, repo_name, repo_default_branch) + elif repo_type == constants.RepositoryType.CODEBERG: + return convert_to_raw_user_content_codeberg(file_path, owner, repo_name, repo_default_branch) + elif repo_type == constants.RepositoryType.BITBUCKET: + return convert_to_raw_user_content_bitbucket(file_path, owner, repo_name, repo_default_branch) else: return os.path.join(repo_dir, repo_relative_path, filename) @@ -695,6 +707,24 @@ def convert_to_raw_user_content_github(partial, owner, repo_name, repo_ref): return f"https://raw.githubusercontent.com/{owner}/{repo_name}/{repo_ref}/{urllib.parse.quote(partial)}" +def convert_to_raw_user_content_codeberg(partial, owner, repo_name, repo_ref): + """Converts Codeberg paths into raw content URLs""" + if partial.startswith("./"): + partial = partial.replace("./", "") + if partial.startswith(".\\"): + partial = partial.replace(".\\", "") + return f"https://codeberg.org/{owner}/{repo_name}/raw/branch/{repo_ref}/{urllib.parse.quote(partial)}" + + +def convert_to_raw_user_content_bitbucket(partial, owner, repo_name, repo_ref): + """Converts Bitbucket paths into raw content URLs""" + if partial.startswith("./"): + partial = partial.replace("./", "") + if partial.startswith(".\\"): + partial = partial.replace(".\\", "") + return f"https://bitbucket.org/{owner}/{repo_name}/raw/{repo_ref}/{urllib.parse.quote(partial)}" + + def convert_to_raw_user_content_gitlab(partial, owner, repo_name, repo_ref): """Converts GitLab paths into raw.githubuser content URLs, accessible by users""" if partial.startswith("./"): diff --git a/src/somef/process_repository.py b/src/somef/process_repository.py index 6cbd6e06..7adcd4e1 100644 --- a/src/somef/process_repository.py +++ b/src/somef/process_repository.py @@ -5,6 +5,7 @@ import requests import sys import re +import base64 from datetime import datetime from urllib.parse import urlparse, quote from .utils import constants @@ -21,9 +22,9 @@ def header_template(authorization=None): header = {} file_paths = configuration.get_configuration_file() if authorization is not None: - header[constants.CONF_AUTHORIZATION] = authorization - elif constants.CONF_AUTHORIZATION in file_paths.keys(): - header[constants.CONF_AUTHORIZATION] = file_paths[constants.CONF_AUTHORIZATION] + header[constants.PROP_AUTHORIZATION] = authorization + elif constants.CONF_GITHUB_AUTHORIZATION in file_paths: + header[constants.PROP_AUTHORIZATION] = file_paths[constants.CONF_GITHUB_AUTHORIZATION] return header @@ -40,7 +41,7 @@ def is_gitlab(gitlab_server): # the same as requests.get(args).json(), but protects against rate limiting def rate_limit_get(*args, backoff_rate=2, initial_backoff=1, size_limit_mb=constants.SIZE_DOWNLOAD_LIMIT_MB, **kwargs): # def rate_limit_get(*args, backoff_rate=2, initial_backoff=1, **kwargs): - """Function to obtain how many requests we have pending with the GitHub API""" + """Function to obtain how many requests we have pending with the repository API""" """GET request that handles rate limiting and prevents downloading excessively large files""" size_limit_bytes = size_limit_mb * 1024 * 1024 @@ -49,7 +50,13 @@ def rate_limit_get(*args, backoff_rate=2, initial_backoff=1, size_limit_mb=const raise ValueError("Missing URL in rate_limit_get") parsed = urlparse(url) - is_api_request = "api.github.com" in parsed.netloc + # is_api_request = "api.github.com" in parsed.netloc + is_api_request = any([ + "api.github.com" in parsed.netloc, + "/api/v1" in parsed.path, + "api.bitbucket.org" in parsed.netloc, + "/api/v4" in parsed.path, + ]) content_length = None # Check file size before downloading the full body (skip for GitHub API requests, # which are always small JSON payloads). @@ -85,27 +92,38 @@ def rate_limit_get(*args, backoff_rate=2, initial_backoff=1, size_limit_mb=const stream=use_stream, **kwargs ) - # Detect invalid or insufficient GitHub token + # Detect invalid or insufficient token if response.status_code == 401: - raise Exception("Invalid GitHub token. Run `somef configure` to set a valid token.") + raise Exception("Invalid token. Run `somef configure` to set a valid token.") if response.status_code == 403: - raise Exception("GitHub token lacks required permissions or scopes.") + raise Exception("Token lacks required permissions or scopes.") date = response.headers.get("Date", "") # Show rate limit information if available - if "X-RateLimit-Remaining" in response.headers: - rate_limit_remaining = response.headers["X-RateLimit-Remaining"] - epochtime = int(response.headers["X-RateLimit-Reset"]) + rate_limit_remaining = response.headers.get("X-RateLimit-Remaining") or response.headers.get("RateLimit-Remaining") + rate_limit_reset = response.headers.get("X-RateLimit-Reset") or response.headers.get("RateLimit-Reset") + + # if "X-RateLimit-Remaining" in response.headers: + # rate_limit_remaining = response.headers["X-RateLimit-Remaining"] + if rate_limit_remaining is not None and rate_limit_reset is not None: + # epochtime = int(response.headers["X-RateLimit-Reset"]) + epochtime = int(rate_limit_reset) + + if epochtime < 1000000000: + epochtime = int(time.time()) + epochtime + date_reset = datetime.fromtimestamp(epochtime) + logging.info( - "Remaining GitHub API requests: " + rate_limit_remaining + " ### Next rate limit reset at: " + str( + "Remaining repository API requests: " + rate_limit_remaining + " ### Next rate limit reset at: " + str( date_reset)) if not use_stream: try: json_data = response.json() msg = json_data.get("message", "") - if "API rate limit exceeded" in msg: + # if "API rate limit exceeded" in msg: + if "rate limit" in msg.lower(): rate_limited = True logging.warning(f"Rate limited. Backing off for {initial_backoff} seconds") time.sleep(initial_backoff) @@ -132,12 +150,10 @@ def rate_limit_get(*args, backoff_rate=2, initial_backoff=1, size_limit_mb=const content.extend(chunk) total_read += len(chunk) - # Comprobar límite de tamaño if total_read > size_limit_bytes: logging.warning(f"Downloaded content exceeded {size_limit_bytes} bytes. Aborting.") return None, None - # Comprobar límite de tiempo elapsed = time.monotonic() - start_time if elapsed > constants.DOWNLOAD_TIMEOUT_SECONDS: logging.warning(f"Download exceeded time limit ({elapsed:.2f}s). Aborting.") @@ -152,7 +168,7 @@ def rate_limit_get(*args, backoff_rate=2, initial_backoff=1, size_limit_mb=const return response, date -def load_gitlab_repository_metadata(repo_metadata: Result, repository_url): +def load_gitlab_repository_metadata(repo_metadata: Result, repository_url, authorization=None): """ Function uses the repository_url provided to load required information from gitlab. Information kept from the repository is written in keep_keys. @@ -160,6 +176,7 @@ def load_gitlab_repository_metadata(repo_metadata: Result, repository_url): ---------- @param repo_metadata: Result object with the metadata found in the repository so far @param repository_url: URL of the Gitlab repository to analyze + @param authorization: Authorization token for GitLab API access Returns ------- @@ -215,25 +232,25 @@ def load_gitlab_repository_metadata(repo_metadata: Result, repository_url): # could be gitlab.com or some gitlab self-hosted GitLab servers like gitlab.in2p3.fr if repository_url.rfind("gitlab.com") > 0: - project_id = get_project_id(repository_url, False) + project_id = get_project_id(repository_url, False, authorization) project_api_url = f"https://gitlab.com/api/v4/projects/{project_id}" else: project_path = url.path.lstrip("/") # "gammalearn/gammalearn" encoded_project_path = quote(project_path, safe="") # Codifica "/" como "%2F" # Build url of api to get id api_url = f"https://{url.netloc}/api/v4/projects/{encoded_project_path}" - project_id = get_project_id(api_url, True) + project_id = get_project_id(api_url, True, authorization) logging.info(f'Project_id: {project_id}') project_api_url = f"https://{url.netloc}/api/v4/projects/{project_id}" logging.info(f"Downloading {project_api_url}") - details = requests.get(project_api_url) + details, date = rate_limit_get(project_api_url, headers=gitlab_header_template(authorization)) project_details = details.json() - date = details.headers["date"] + # date = details.headers["date"] repo_api_base_url = f"{repository_url}" # releases = get_gitlab_releases(project_id, f"https://{url.netloc}") - all_releases = get_all_gitlab_releases(project_api_url) + all_releases = get_all_gitlab_releases(project_api_url, authorization) release_list_filtered = [do_crosswalk(release, constants.release_gitlab_crosswalk_table) for release in all_releases] for release in release_list_filtered: @@ -411,9 +428,9 @@ def load_gitlab_repository_metadata(repo_metadata: Result, repository_url): -def download_gitlab_files(directory, owner, repo_name, repo_branch, repo_ref): +def download_gitlab_files(directory, owner, repo_name, repo_branch, repo_ref, authorization=None): """ - Download all repository files from a GitHub repository + Download all repository files from a GitLab repository Parameters ---------- @param repo_branch: Branch of the repo we are analysing @@ -421,6 +438,7 @@ def download_gitlab_files(directory, owner, repo_name, repo_branch, repo_ref): @param repo_name: name of the repo @param owner: owner of the GitLab repo @param directory: directory where to extract all downloaded files + @param authorization: Authorization token for GitLab API access Returns ------- @rtype: string @@ -440,7 +458,7 @@ def download_gitlab_files(directory, owner, repo_name, repo_branch, repo_ref): ) logging.info(f"Downloading {repo_archive_url}") - repo_download = requests.get(repo_archive_url) + repo_download = requests.get(repo_archive_url, headers=gitlab_header_template(authorization)) repo_zip = repo_download.content repo_zip_file = os.path.join(directory, "repo.zip") @@ -483,12 +501,28 @@ def download_readme(owner, repo_name, default_branch, repo_type, authorization, elif repo_type is constants.RepositoryType.GITHUB: primary_url = f"https://raw.githubusercontent.com/{owner}/{repo_name}/{default_branch}/README.md" secondary_url = f"https://raw.githubusercontent.com/{owner}/{repo_name}/master/README.md" + elif repo_type is constants.RepositoryType.CODEBERG: + primary_url = f"https://codeberg.org/{owner}/{repo_name}/raw/branch/{default_branch}/README.md" + secondary_url = f"https://codeberg.org/{owner}/{repo_name}/raw/branch/master/README.md" + elif repo_type is constants.RepositoryType.BITBUCKET: + primary_url = f"https://bitbucket.org/{owner}/{repo_name}/raw/{default_branch}/README.md" + secondary_url = f"https://bitbucket.org/{owner}/{repo_name}/raw/master/README.md" else: logging.error("Repository type not supported") return None logging.info(f"Downloading {primary_url}") - repo_download, date = rate_limit_get(primary_url, headers=header_template(authorization)) + + if repo_type is constants.RepositoryType.GITLAB: + headers = gitlab_header_template(authorization) + elif repo_type is constants.RepositoryType.CODEBERG: + headers = codeberg_header_template(authorization) + elif repo_type is constants.RepositoryType.BITBUCKET: + headers = bitbucket_header_template(authorization) + else: + headers = header_template(authorization) + + repo_download, date = rate_limit_get(primary_url, headers=headers) if repo_download is None: logging.warning(f"Repository archive skipped due to size limit: {constants.SIZE_DOWNLOAD_LIMIT_MB} MB or content-lenght none") @@ -496,7 +530,8 @@ def download_readme(owner, repo_name, default_branch, repo_type, authorization, if repo_download.status_code == 404: logging.error(f"Error: Archive request failed with HTTP {repo_download.status_code}") logging.info(f"Trying to download {secondary_url}") - repo_download, date = rate_limit_get(secondary_url, headers=header_template(authorization)) + # repo_download, date = rate_limit_get(secondary_url, headers=header_template(authorization)) + repo_download, date = rate_limit_get(secondary_url, headers=headers) if repo_download is None: logging.warning(f"Repository archive skipped due to size limit: {constants.SIZE_DOWNLOAD_LIMIT_MB} MB or content-lenght none") return None @@ -530,7 +565,11 @@ def load_online_repository_metadata(repository_metadata: Result, repository_url, @return: Result object with the available metadata from online APIs plus its owner, repo name and default branch """ if repo_type == constants.RepositoryType.GITLAB: - return load_gitlab_repository_metadata(repository_metadata, repository_url) + return load_gitlab_repository_metadata(repository_metadata, repository_url, authorization) + elif repo_type == constants.RepositoryType.CODEBERG: + return load_codeberg_repository_metadata(repository_metadata, repository_url, authorization) + elif repo_type == constants.RepositoryType.BITBUCKET: + return load_bitbucket_repository_metadata(repository_metadata, repository_url, authorization) elif repo_type == constants.RepositoryType.LOCAL: logging.warning("Trying to download metadata from a local repository") return None @@ -768,7 +807,7 @@ def do_crosswalk(data, crosswalk_table): if value is not None: output[somef_key] = value else: - logging.error(f"Error: key {path} not present in github repository") + logging.debug(f"Error: key {path} not present in repository") return output @@ -795,90 +834,16 @@ def download_repository_files(owner, repo_name, default_branch, repo_type, targe if repo_type == constants.RepositoryType.GITHUB: return download_github_files(target_dir, owner, repo_name, default_branch, authorization) elif repo_type == constants.RepositoryType.GITLAB: - return download_gitlab_files(target_dir, owner, repo_name, default_branch, repo_ref) + return download_gitlab_files(target_dir, owner, repo_name, default_branch, repo_ref, authorization) + elif repo_type == constants.RepositoryType.CODEBERG: + return download_codeberg_files(target_dir, owner, repo_name, default_branch, authorization) + elif repo_type == constants.RepositoryType.BITBUCKET: + return download_bitbucket_files(target_dir, owner, repo_name, default_branch, authorization) else: logging.error("Cannot download files from a local repository!") return None -# def download_github_files(directory, owner, repo_name, repo_ref, authorization): -# """ -# Download all repository files from a GitHub repository -# Parameters -# ---------- -# repo_ref: link to branch of the repo -# repo_name: name of the repo -# owner: GitHub owner -# directory: directory where to extract all downloaded files -# authorization: GitHub authorization token - -# Returns -# ------- -# path to the folder where all files have been downloaded -# """ -# # download the repo at the selected branch with the link -# repo_archive_url = f"https://github.com/{owner}/{repo_name}/archive/{repo_ref}.zip" -# logging.info(f"Downloading {repo_archive_url}") - -# repo_download, date = rate_limit_get(repo_archive_url, headers=header_template(authorization)) - -# if repo_download is None: -# logging.warning(f"Repository archive skipped due to size limit: {constants.SIZE_DOWNLOAD_LIMIT_MB} MB or not content lenght.") -# return None - -# if repo_download.status_code == 300: -# logging.warning(f"Ambiguous ref detected for {repo_ref}, trying tags/heads resolution") - -# for ref_type in ["tags", "heads"]: -# repo_archive_url = f"https://github.com/{owner}/{repo_name}/archive/refs/{ref_type}/{repo_ref}.zip" -# logging.info(f"Trying to download {repo_archive_url}") - -# repo_download, date = rate_limit_get(repo_archive_url, headers=header_template(authorization)) - -# if repo_download is None: -# logging.warning(f"Repository archive skipped due to size limit: {constants.SIZE_DOWNLOAD_LIMIT_MB} MB or not content length.") -# return None - -# if repo_download.status_code == 200: -# break - -# if repo_download.status_code == 404: -# logging.error(f"Error: Archive request failed with HTTP {repo_download.status_code}") -# repo_archive_url = f"https://github.com/{owner}/{repo_name}/archive/main.zip" -# logging.info(f"Trying to download {repo_archive_url}") -# repo_download, date = rate_limit_get(repo_archive_url, headers=header_template(authorization)) -# if repo_download is None: -# logging.warning(f"Repository archive skipped due to size limit: {constants.SIZE_DOWNLOAD_LIMIT_MB} MB or not content lenght.") -# return None - -# if repo_download.status_code != 200: -# logging.error(f"Error: Archive request failed with HTTP {repo_download.status_code}") -# return None - -# repo_zip = repo_download.content - -# repo_name_full = owner + "_" + repo_name -# repo_zip_file = os.path.join(directory, repo_name_full + ".zip") -# repo_extract_dir = os.path.join(directory, repo_name_full) - -# with open(repo_zip_file, "wb") as f: -# f.write(repo_zip) - -# try: -# with zipfile.ZipFile(repo_zip_file, "r") as zip_ref: -# zip_ref.extractall(repo_extract_dir) -# except zipfile.BadZipFile: -# logging.error("Downloaded archive is not a valid zip (repo may be empty)") -# return None - -# repo_folders = os.listdir(repo_extract_dir) -# if not repo_folders: -# logging.warning("Repository archive is empty") -# return None - -# repo_dir = os.path.join(repo_extract_dir, repo_folders[0]) -# return repo_dir - def download_github_files(directory, owner, repo_name, repo_ref, authorization): """ Download all repository files from a GitHub repository. @@ -974,7 +939,7 @@ def download_github_files(directory, owner, repo_name, repo_ref, authorization): return repo_dir -def get_project_id(repository_url,self_hosted): +def get_project_id(repository_url,self_hosted, authorization=None): """ Function to download a repository, given its URL Parameters: @@ -985,7 +950,9 @@ def get_project_id(repository_url,self_hosted): """ logging.info(f"Downloading {repository_url}") - response = requests.get(repository_url) + # response = requests.get(repository_url) + headers = gitlab_header_template(authorization) + response = requests.get(repository_url, headers=headers) project_id = "-1" if self_hosted: @@ -1014,14 +981,15 @@ def get_project_id(repository_url,self_hosted): project_id = response_str[start:end] return project_id -def get_all_gitlab_releases(repo_api_base_url): +def get_all_gitlab_releases(repo_api_base_url, authorization=None): all_releases = [] page = 1 while True: url = f"{repo_api_base_url}/releases?page={page}&per_page=100" logging.info(f"Getting releases from: {url}") - response = requests.get(url) + headers = gitlab_header_template(authorization) + response = requests.get(url, headers=headers) logging.info(f"Response: {response.status_code}") content_type = response.headers.get("Content-Type", "") if response.status_code != 200 or "application/json" not in content_type: @@ -1063,7 +1031,7 @@ def get_all_gitlab_releases(repo_api_base_url): # print(release_list_filtered) return all_releases -def get_gitlab_releases(project_id, base_url): +def get_gitlab_releases(project_id, base_url, authorization=None): """ Retrieves the releases of a GitLab repository without authentication. @@ -1075,8 +1043,9 @@ def get_gitlab_releases(project_id, base_url): logging.info(f"Getting releases from: {releases_url}") - response = requests.get(releases_url) - + headers = gitlab_header_template(authorization) + response = requests.get(releases_url, headers=headers) + if response.status_code != 200: logging.error(f"Error getting releases: {response.text}") return [] @@ -1143,3 +1112,346 @@ def get_all_paginated_results(base_url, headers, per_page=100): return all_results + +def load_codeberg_repository_metadata(repo_metadata: Result, repository_url, authorization=None): + logging.info(f"Loading Repository {repository_url} Information....") + + file_paths = configuration.get_configuration_file() + headers = codeberg_header_template(authorization) + + if repository_url[-1] == '/': + repository_url = repository_url[:-1] + url = urlparse(repository_url) + + path_components = [p for p in url.path.split('/') if p] + if len(path_components) < 2: + logging.error("Codeberg link is not correct. Expected https://codeberg.org//") + return repo_metadata, "", "", "", "" + + owner = path_components[0] + repo_name = path_components[1] + default_branch = None + + if len(path_components) >= 4 and path_components[2] == "tree": + default_branch = path_components[3] + + repo_api_url = f"{constants.CODEBERG_API}/{owner}/{repo_name}" + # resp = requests.get(repo_api_url) + resp, _ = rate_limit_get(repo_api_url, headers=headers) + if resp.status_code != 200: + logging.error(f"Error fetching Codeberg repository: {resp.status_code}") + return repo_metadata, "", "", "", "" + general_resp = resp.json() + + if default_branch is None: + default_branch = general_resp.get('default_branch', 'main') + + filtered_resp = do_crosswalk(general_resp, constants.codeberg_crosswalk_table) + if 'html_url' in general_resp: + filtered_resp[constants.CAT_ISSUE_TRACKER] = f"{general_resp['html_url']}/issues" + + filtered_resp[constants.CAT_DOWNLOAD_URL] = f"https://codeberg.org/{owner}/{repo_name}/releases" + + detected_license_info = None + for license_filename in ["LICENSE", "LICENSE.md", "LICENCE", "COPYING"]: + license_url = f"{constants.CODEBERG_API}/{owner}/{repo_name}/contents/{license_filename}?ref={default_branch}" + lic_resp, _ = rate_limit_get(license_url, headers=headers) + if lic_resp.status_code == 200: + lic_data = lic_resp.json() + raw_b64 = lic_data.get("content", "") + if raw_b64: + license_text = base64.b64decode(raw_b64).decode("utf-8") + license_info = detect_license_spdx(license_text, 'JSON') + if license_info: + result = { + constants.PROP_VALUE: license_info["spdx_id"], + constants.PROP_NAME: license_info["name"], + constants.PROP_SPDX_ID: license_info["spdx_id"], + constants.PROP_TYPE: "License", + constants.PROP_URL: license_info["url"], + constants.PROP_IDENTIFIER: license_info["identifier"], + + } + repo_metadata.add_result(constants.CAT_LICENSE, result, 1, constants.TECHNIQUE_CODEBERG_API) + break + + for category, value in filtered_resp.items(): + value_type = constants.STRING + if category in constants.all_categories: + if category == constants.CAT_ISSUE_TRACKER: + value = value.replace("{/number}", "") if isinstance(value, str) else value + if category == constants.CAT_OWNER: + value_type = "User" + if category == constants.CAT_KEYWORDS: + value = '%s,' % (', '.join(value)) + value = value.rstrip(',') + if category in [constants.CAT_CODE_REPOSITORY, constants.CAT_ISSUE_TRACKER, + constants.CAT_DOWNLOAD_URL, constants.CAT_HOMEPAGE]: + value_type = constants.URL + if category in [constants.CAT_DATE_CREATED, constants.CAT_DATE_UPDATED]: + value_type = constants.DATE + if category in [constants.CAT_FORK_COUNTS, constants.CAT_STARS]: + value_type = constants.NUMBER + + + result = { + constants.PROP_VALUE: value, + constants.PROP_TYPE: value_type + } + if result['value']: + repo_metadata.add_result(category, result, 1, constants.TECHNIQUE_CODEBERG_API) + + if 'languages_url' in filtered_resp: + lang_resp, _ = rate_limit_get(filtered_resp['languages_url'], headers=headers) + if lang_resp.status_code == 200: + languages = lang_resp.json() + for l, s in languages.items(): + result = { + constants.PROP_VALUE: l, + constants.PROP_NAME: l, + constants.PROP_TYPE: constants.LANGUAGE, + constants.PROP_SIZE: s, + } + repo_metadata.add_result(constants.CAT_PROGRAMMING_LANGUAGES, result, 1, + constants.TECHNIQUE_CODEBERG_API) + + releases_url = f"{constants.CODEBERG_API}/{owner}/{repo_name}/releases" + releases_resp, _ = rate_limit_get(releases_url, headers=headers) + if releases_resp.status_code == 200: + releases_list = releases_resp.json() + release_list_filtered = [do_crosswalk(r, constants.release_codeberg_crosswalk_table) + for r in releases_list] + for release in release_list_filtered: + release_obj = { + constants.PROP_TYPE: constants.RELEASE, + constants.PROP_VALUE: release.get(constants.PROP_URL, "") + } + for category, value in release.items(): + if category == constants.PROP_AUTHOR: + value = { + constants.PROP_NAME: value, + constants.PROP_TYPE: release.get(constants.AGENT_TYPE, "Person") + } + if value: + release_obj[category] = value + if category == constants.CAT_ASSETS and isinstance(value, list): + assets_filtered = [do_crosswalk(a, constants.release_assets_codeberg) for a in value] + key_mapping = { + constants.PROP_BROWSER_URL: constants.PROP_CONTENT_URL, + constants.PROP_SIZE: constants.PROP_CONTENT_SIZE, + constants.PROP_CONTENT_TYPE: constants.PROP_ENCODING_FORMAT, + constants.PROP_DATE_CREATED_AT: constants.PROP_UPLOAD_DATE + } + assets_filtered = [{key_mapping.get(k, k): v for k, v in a.items()} for a in assets_filtered] + release_obj[category] = assets_filtered + repo_metadata.add_result(constants.CAT_RELEASES, release_obj, 1, constants.TECHNIQUE_CODEBERG_API) + + logging.info("Repository information successfully loaded.\n") + return repo_metadata, owner, repo_name, default_branch, "/".join(path_components) + + +def download_codeberg_files(directory, owner, repo_name, repo_branch,authorization=None): + """ + Download all repository files from a Codeberg repository. + """ + repo_archive_url = f"https://codeberg.org/{owner}/{repo_name}/archive/{repo_branch}.zip" + logging.info(f"Downloading {repo_archive_url}") + + headers = codeberg_header_template(authorization) + + repo_download, _ = rate_limit_get(repo_archive_url, headers=headers) + if repo_download.status_code != 200: + logging.error(f"Error downloading Codeberg archive: HTTP {repo_download.status_code}") + return None + + repo_zip = repo_download.content + + repo_name_full = owner + "_" + repo_name + repo_zip_file = os.path.join(directory, repo_name_full + ".zip") + repo_extract_dir = os.path.join(directory, repo_name_full) + + with open(repo_zip_file, "wb") as f: + f.write(repo_zip) + + try: + with zipfile.ZipFile(repo_zip_file, "r") as zip_ref: + zip_ref.extractall(repo_extract_dir) + except zipfile.BadZipFile: + logging.error("Downloaded archive is not a valid zip") + return None + + repo_folders = os.listdir(repo_extract_dir) + if not repo_folders: + logging.warning("Repository archive is empty") + return None + + repo_dir = os.path.join(repo_extract_dir, repo_folders[0]) + return repo_dir + + +def gitlab_header_template(authorization=None): + header = {} + file_paths = configuration.get_configuration_file() + if authorization is not None: + header[constants.PROP_AUTHORIZATION] = authorization + logging.info("GitLab: using authorization token passed directly") + elif constants.CONF_GITLAB_AUTHORIZATION in file_paths: + header[constants.PROP_AUTHORIZATION] = file_paths[constants.CONF_GITLAB_AUTHORIZATION] + logging.info("GitLab: authorization token found in config") + return header + +def codeberg_header_template(authorization=None): + header = {} + file_paths = configuration.get_configuration_file() + if authorization is not None: + header[constants.PROP_AUTHORIZATION] = authorization + logging.info("Codeberg: using authorization token passed directly") + elif constants.CONF_CODEBERG_AUTHORIZATION in file_paths: + header[constants.PROP_AUTHORIZATION] = file_paths[constants.CONF_CODEBERG_AUTHORIZATION] + logging.info("Codeberg: authorization token found in config") + return header + + +def bitbucket_header_template(authorization=None): + header = {} + file_paths = configuration.get_configuration_file() + if authorization is not None: + header[constants.PROP_AUTHORIZATION] = authorization + logging.info("Bitbucket: using authorization token passed directly") + elif constants.CONF_BITBUCKET_AUTHORIZATION in file_paths: + header[constants.PROP_AUTHORIZATION] = file_paths[constants.CONF_BITBUCKET_AUTHORIZATION] + logging.info("Bitbucket: authorization token found in config") + return header + + +def load_bitbucket_repository_metadata(repo_metadata: Result, repository_url, authorization=None): + logging.info(f"Loading Repository {repository_url} Information....") + if repository_url[-1] == '/': + repository_url = repository_url[:-1] + url = urlparse(repository_url) + + path_components = [p for p in url.path.split('/') if p] + if len(path_components) < 2: + logging.error("Bitbucket link is not correct. Expected https://bitbucket.org//") + return repo_metadata, "", "", "", "" + + owner = path_components[0] + repo_name = path_components[1] + default_branch = None + + if len(path_components) >= 4 and path_components[2] == "tree": + default_branch = path_components[3] + + # API call + repo_api_url = f"{constants.BITBUCKET_API}/{owner}/{repo_name}" + headers = bitbucket_header_template(authorization) + resp, _ = rate_limit_get(repo_api_url, headers=headers) + if resp.status_code != 200: + logging.error(f"Error fetching Bitbucket repository: {resp.status_code}") + return repo_metadata, "", "", "", "" + general_resp = resp.json() + + if default_branch is None: + default_branch = general_resp.get('mainbranch', {}).get('name', 'main') + + filtered_resp = do_crosswalk(general_resp, constants.bitbucket_crosswalk_table) + + if constants.CAT_OWNER not in filtered_resp or not filtered_resp[constants.CAT_OWNER]: + owner_obj = general_resp.get('owner', {}) + owner_val = owner_obj.get('nickname') or owner_obj.get('username') + if owner_val: + filtered_resp[constants.CAT_OWNER] = owner_val + + # Issue tracker + if general_resp.get('has_issues', False) and 'links' in general_resp and 'html' in general_resp['links']: + html_url = general_resp['links']['html']['href'] + filtered_resp[constants.CAT_ISSUE_TRACKER] = f"{html_url}/issues" + + + if 'language' in general_resp and general_resp['language']: + lang_value = general_resp['language'] + result = { + constants.PROP_VALUE: lang_value, + constants.PROP_NAME: lang_value, + constants.PROP_TYPE: constants.LANGUAGE, + } + repo_metadata.add_result(constants.CAT_PROGRAMMING_LANGUAGES, result, 1, + constants.TECHNIQUE_BITBUCKET_API) + + + if 'links' in general_resp and 'html' in general_resp['links']: + filtered_resp[constants.CAT_DOWNLOAD_URL] = f"{general_resp['links']['html']['href']}/downloads" + + for category, value in filtered_resp.items(): + value_type = constants.STRING + if category in constants.all_categories: + if category == constants.CAT_OWNER: + value_type = "User" + if category in [constants.CAT_CODE_REPOSITORY, constants.CAT_ISSUE_TRACKER, + constants.CAT_DOWNLOAD_URL, constants.CAT_HOMEPAGE, constants.CAT_FORKS_URLS]: + value_type = constants.URL + if category in [constants.CAT_DATE_CREATED, constants.CAT_DATE_UPDATED]: + value_type = constants.DATE + if category == constants.CAT_PROGRAMMING_LANGUAGES: + value_type = constants.LANGUAGE + + result = { + constants.PROP_VALUE: value, + constants.PROP_TYPE: value_type + } + if result['value']: + repo_metadata.add_result(category, result, 1, constants.TECHNIQUE_BITBUCKET_API) + + # Releases from /refs/tags + tags_url = f"{constants.BITBUCKET_API}/{owner}/{repo_name}/refs/tags" + tags_resp, _ = rate_limit_get(tags_url, headers=headers) + if tags_resp.status_code == 200: + tags_data = tags_resp.json() + tags_list = tags_data.get('values', []) + for tag in tags_list: + release_obj = do_crosswalk(tag, constants.release_bitbucket_crosswalk_table) + release_obj[constants.PROP_TYPE] = constants.RELEASE + release_obj[constants.PROP_VALUE] = tag.get('name', '') + repo_metadata.add_result(constants.CAT_RELEASES, release_obj, 1, + constants.TECHNIQUE_BITBUCKET_API) + + logging.info("Repository information successfully loaded.\n") + return repo_metadata, owner, repo_name, default_branch, "/".join(path_components) + + +def download_bitbucket_files(directory, owner, repo_name, repo_branch, authorization=None): + repo_archive_url = f"https://bitbucket.org/{owner}/{repo_name}/get/{repo_branch}.zip" + logging.info(f"Downloading {repo_archive_url}") + + headers = bitbucket_header_template(authorization) + repo_download, _ = rate_limit_get(repo_archive_url, headers=headers) + if repo_download is None: + logging.warning(f"Repository archive skipped due to size limit: {constants.SIZE_DOWNLOAD_LIMIT_MB} MB or no content-length") + return None + if repo_download.status_code != 200: + logging.error(f"Error downloading Bitbucket archive: HTTP {repo_download.status_code}") + return None + + repo_zip = repo_download.content + + repo_name_full = owner + "_" + repo_name + repo_zip_file = os.path.join(directory, repo_name_full + ".zip") + repo_extract_dir = os.path.join(directory, repo_name_full) + + with open(repo_zip_file, "wb") as f: + f.write(repo_zip) + + try: + with zipfile.ZipFile(repo_zip_file, "r") as zip_ref: + zip_ref.extractall(repo_extract_dir) + except zipfile.BadZipFile: + logging.error("Downloaded archive is not a valid zip") + return None + + repo_folders = os.listdir(repo_extract_dir) + if not repo_folders: + logging.warning("Repository archive is empty") + return None + + repo_dir = os.path.join(repo_extract_dir, repo_folders[0]) + return repo_dir \ No newline at end of file diff --git a/src/somef/somef_cli.py b/src/somef/somef_cli.py index 98efaab4..7daadc0c 100644 --- a/src/somef/somef_cli.py +++ b/src/somef/somef_cli.py @@ -6,6 +6,8 @@ import os import tempfile import urllib.parse +import requests +import base64 from os import path from . import header_analysis, regular_expressions, process_repository, configuration, process_files, \ @@ -79,16 +81,24 @@ def cli_get_data(threshold, ignore_classifiers, repo_url=None, doc_src=None, loc url = urlparse(repo_url) servidor = url.netloc bGitLab = False + bCodeberg = False + bBitbucket = False if process_repository.is_gitlab(servidor): logging.info(f"{servidor} is GitLab.") + repo_type = constants.RepositoryType.GITLAB bGitLab = True - # if reconcile_authors: - # logging.info("Author enrichment disabled: GitLab repositories are not supported for GitHub user enrichment.") - # reconcile_authors = False + logging.info(f"DEBUG: {servidor} is_gitlab = {bGitLab}") + elif servidor == constants.CODEBERG_DOMAIN: + repo_type = constants.RepositoryType.CODEBERG + bCodeberg = True + logging.info(f"DEBUG: {servidor} is_codeberg = {bCodeberg}") + elif "bitbucket.org" in servidor: + repo_type = constants.RepositoryType.BITBUCKET + bBitbucket = True + - logging.info(f"DEBUG: {servidor} is_gitlab = {bGitLab}") - if bGitLab: - repo_type = constants.RepositoryType.GITLAB + # if bGitLab: + # repo_type = constants.RepositoryType.GITLAB logging.info("Processing repository metadata.") repository_metadata, owner, repo_name, def_branch, project_path = process_repository.load_online_repository_metadata( @@ -267,7 +277,12 @@ def run_cli(*, requirements_mode="all", reconcile_authors=False, branch=None, - tag=None + tag=None, + github_token=None, + gitlab_token=None, + codeberg_token=None, + bitbucket_token=None, + bitbucket_email=None ): """Function to run all the required components of the cli for a repository""" # check if it is a valid url @@ -297,11 +312,13 @@ def run_cli(*, # urls_to_process] for repo_url in urls_to_process: try: + authorization = verify_and_resolve_token( + repo_url, github_token, gitlab_token, codeberg_token, bitbucket_token, bitbucket_email) encoded_url = urllib.parse.quote(repo_url, safe='') encoded_url = encoded_url.replace(".","") #removing dots just in case repo_data = cli_get_data(threshold=threshold, ignore_classifiers=ignore_classifiers, repo_url=repo_url, ignore_github_metadata=ignore_github_metadata, readme_only=readme_only, - keep_tmp=keep_tmp, ignore_test_folder=ignore_test_folder, requirements_mode=requirements_mode, reconcile_authors=reconcile_authors, + keep_tmp=keep_tmp, authorization=authorization, ignore_test_folder=ignore_test_folder, requirements_mode=requirements_mode, reconcile_authors=reconcile_authors, branch=branch, tag=tag) if hasattr(repo_data, "get_json"): @@ -333,13 +350,15 @@ def run_cli(*, else: if repo_url: + authorization = verify_and_resolve_token( + repo_url, github_token, gitlab_token, codeberg_token, bitbucket_token, bitbucket_email) repo_data = cli_get_data(threshold=threshold, ignore_classifiers=ignore_classifiers, repo_url=repo_url, ignore_github_metadata=ignore_github_metadata, readme_only=readme_only, - keep_tmp=keep_tmp, ignore_test_folder=ignore_test_folder, reconcile_authors=reconcile_authors, + keep_tmp=keep_tmp, authorization=authorization, ignore_test_folder=ignore_test_folder, reconcile_authors=reconcile_authors, branch=branch, tag=tag) elif local_repo: repo_data = cli_get_data(threshold=threshold, ignore_classifiers=ignore_classifiers, - local_repo=local_repo, keep_tmp=keep_tmp, ignore_test_folder=ignore_test_folder, reconcile_authors=reconcile_authors, + local_repo=local_repo, keep_tmp=keep_tmp, ignore_test_folder=ignore_test_folder, reconcile_authors=reconcile_authors, branch=branch, tag=tag) else: repo_data = cli_get_data(threshold=threshold, ignore_classifiers=ignore_classifiers, @@ -369,3 +388,68 @@ def run_cli(*, data_graph.export_to_file(graph_out, graph_format) +def verify_and_resolve_token(repo_url, github_token, gitlab_token, codeberg_token, bitbucket_token, bitbucket_email): + if repo_url is None: + return None + + servidor = urlparse(repo_url).netloc + + if process_repository.is_gitlab(servidor): + platform = "gitlab" + elif servidor == constants.CODEBERG_DOMAIN: + platform = "codeberg" + elif "bitbucket.org" in servidor: + platform = "bitbucket" + else: + platform = "github" + + # Warnings if tokens not matching the platform of the repo + for p, flag in [("github", github_token), ("gitlab", gitlab_token), + ("codeberg", codeberg_token), ("bitbucket", bitbucket_token)]: + if platform != p and flag is not None: + logging.warning(f"--{p}-token provided but repo is not on {p.capitalize()}. Ignoring token.") + + authorization = None + verify_url = None + if platform == "github" and github_token is not None: + authorization = "token " + github_token + verify_url = "https://api.github.com/user" + elif platform == "gitlab" and gitlab_token is not None: + t = gitlab_token + if not t.lower().startswith("bearer "): + t = "Bearer " + t + authorization = t + verify_url = f"https://{servidor}/api/v4/user" + elif platform == "codeberg" and codeberg_token is not None: + authorization = "token " + codeberg_token + verify_url = "https://codeberg.org/api/v1/user" + elif platform == "bitbucket" and bitbucket_token is not None: + if not bitbucket_email: + logging.error("--bitbucket-email is required with --bitbucket-token.") + sys.exit(1) + raw = f"{bitbucket_email}:{bitbucket_token}" + authorization = "Basic " + base64.b64encode(raw.encode()).decode() + path_parts = [p for p in urlparse(repo_url).path.split('/') if p] + workspace = path_parts[0] if path_parts else "" + repo_slug = path_parts[1] if len(path_parts) > 1 else "" + verify_url = f"https://api.bitbucket.org/2.0/repositories/{workspace}/{repo_slug}" + + if authorization is None: + return None + + # verify + try: + resp = requests.get(verify_url, headers={constants.PROP_AUTHORIZATION: authorization}, timeout=10) + if resp.status_code == 401: + logging.error(f"{platform.capitalize()} token is invalid (401). " + "Run `somef configure` or provide a correct token flag.") + # sys.exit(1) + return None + elif resp.status_code == 403: + logging.warning(f"{platform.capitalize()} token lacks permissions (403). Proceeding anyway.") + except requests.RequestException as e: + logging.warning(f"Could not verify {platform} token: {e}. Proceeding anyway.") + + return authorization + + diff --git a/src/somef/test/test_bitbucket.py b/src/somef/test/test_bitbucket.py new file mode 100644 index 00000000..c729c662 --- /dev/null +++ b/src/somef/test/test_bitbucket.py @@ -0,0 +1,137 @@ +import os +import unittest +import json +from pathlib import Path +from .. import somef_cli +from ..utils import constants +from .. import process_repository +from ..process_results import Result +from unittest.mock import patch, MagicMock + +test_data_path = str(Path(__file__).parent / "test_data") + os.path.sep + + +def _make_mock_response(status_code, content=b""): + """Helper: create a minimal mock requests.Response.""" + resp = MagicMock() + resp.status_code = status_code + resp.content = content + resp.headers = {} + try: + resp.json.return_value = json.loads(content) + except json.JSONDecodeError: + pass + return resp + + +class TestBitbucketRepository(unittest.TestCase): + + @patch("somef.process_repository.requests.get") + def test_load_bitbucket_metadata(self, mock_get): + repo_json = json.load(open(test_data_path + "api_responses/bitbucket/bitbucket_response.json")) + tags_json = json.load(open(test_data_path + "api_responses/bitbucket/bitbucket_tags.json")) + + mock_get.side_effect = [ + _make_mock_response(200, json.dumps(repo_json).encode()), + _make_mock_response(200, json.dumps(tags_json).encode()) + ] + + result, owner, repo_name, branch, path = \ + process_repository.load_bitbucket_repository_metadata( + Result(), "https://bitbucket.org/bitbucketpipelines/pipelines-guide-python" + ) + + self.assertIn(constants.CAT_NAME, result.results) + self.assertIn(constants.CAT_DESCRIPTION, result.results) + self.assertIn(constants.CAT_CODE_REPOSITORY, result.results) + self.assertIn(constants.CAT_FULL_NAME, result.results) + self.assertIn(constants.CAT_FORKS_URLS, result.results) + self.assertIn(constants.CAT_PROGRAMMING_LANGUAGES, result.results) + # this repo has not issues + self.assertNotIn(constants.CAT_ISSUE_TRACKER, result.results) + # this repo has not releases + self.assertIn(constants.CAT_RELEASES, result.results) + self.assertEqual(len(result.results[constants.CAT_RELEASES]), 1) + # print(result.results[constants.CAT_RELEASES]) + self.assertEqual(result.results[constants.CAT_RELEASES][0]["result"]["value"], "0.0.0-5-10-0-SNAPSHOT-002-epic-AUI-4224-skate-responsive-header") + self.assertEqual(owner, "bitbucketpipelines") + self.assertEqual(repo_name, "pipelines-guide-python") + self.assertEqual(branch, "master") + + + @patch("somef.process_repository.requests.get") + def test_bitbucket_api_error_returns_empty(self, mock_get): + mock_get.return_value = _make_mock_response(404) + result, owner, repo_name, branch, path = \ + process_repository.load_bitbucket_repository_metadata( + Result(), "https://bitbucket.org/nonexistent/repo" + ) + self.assertEqual(owner, "") + self.assertNotIn(constants.CAT_NAME, result.results) + + + @unittest.skipIf(os.getenv("CI") == "true", "Skipped in CI because it is already verified locally") + def test_bitbucket_integration_cli(self): + output_file = test_data_path + "test-bitbucket-integration.json" + somef_cli.run_cli( + threshold=0.8, + ignore_classifiers=False, + repo_url="https://bitbucket.org/bitbucketpipelines/pipelines-guide-python", + local_repo=None, + doc_src=None, + in_file=None, + output=output_file, + graph_out=None, + graph_format="turtle", + codemeta_out=None, + pretty=True, + missing=True, + readme_only=False, + reconcile_authors=False + ) + + with open(output_file, "r") as f: + json_content = json.load(f) + + name_entries = json_content.get(constants.CAT_NAME, []) + self.assertTrue( + any(e["technique"] == constants.TECHNIQUE_BITBUCKET_API and e["result"]["value"] == "pipelines-guide-python" + for e in name_entries), + "Name from Bitbucket_API should be 'pipelines-guide-python'" + ) + + code_repo_entries = json_content.get(constants.CAT_CODE_REPOSITORY, []) + self.assertTrue( + any(e["technique"] == constants.TECHNIQUE_BITBUCKET_API and e["result"]["value"] == "https://bitbucket.org/bitbucketpipelines/pipelines-guide-python" + for e in code_repo_entries) + ) + + full_name_entries = json_content.get(constants.CAT_FULL_NAME, []) + self.assertTrue( + any(e["technique"] == constants.TECHNIQUE_BITBUCKET_API and e["result"]["value"] == "bitbucketpipelines/pipelines-guide-python" + for e in full_name_entries) + ) + + pl_entries = json_content.get(constants.CAT_PROGRAMMING_LANGUAGES, []) + self.assertTrue( + any(e["technique"] == constants.TECHNIQUE_BITBUCKET_API and e["result"]["value"] == "python" + for e in pl_entries) + ) + + + ft_entries = json_content.get(constants.CAT_FULL_TITLE, []) + self.assertTrue( + any(e.get("result", {}).get("value") == "Pipelines Python" + for e in ft_entries) + ) + + # print(json_content[constants.CAT_FULL_TITLE]) + # print(json_content[constants.CAT_FORKS_URLS]) + self.assertTrue(any(e["technique"] == constants.TECHNIQUE_BITBUCKET_API + for e in json_content.get(constants.CAT_FORKS_URLS, []))) + + self.assertTrue(any(e["technique"] == "file_exploration" + for e in json_content.get(constants.CAT_HAS_BUILD_FILE, []))) + + os.remove(output_file) + diff --git a/src/somef/test/test_codeberg.py b/src/somef/test/test_codeberg.py new file mode 100644 index 00000000..e3a6aa2a --- /dev/null +++ b/src/somef/test/test_codeberg.py @@ -0,0 +1,183 @@ +import os +import unittest +import json +import base64 +from pathlib import Path +from .. import somef_cli +from ..utils import constants +from .. import process_repository +from ..parser import codeowners_parser +from ..process_results import Result +from unittest.mock import patch, MagicMock + +test_data_path = str(Path(__file__).parent / "test_data") + os.path.sep + + +def _make_mock_response(status_code, content=b""): + """Helper: create a minimal mock requests.Response.""" + resp = MagicMock() + resp.status_code = status_code + resp.content = content + resp.headers = {} + try: + resp.json.return_value = json.loads(content) + except json.JSONDecodeError: + pass + return resp + +class TestCodebergRepository(unittest.TestCase): + + @patch("somef.process_repository.detect_license_spdx") + @patch("somef.process_repository.requests.get") + def test_load_codeberg_metadata(self, mock_get, mock_detect_license): + """Load Codeberg repository metadata via mocked API. From the API response previously saved in local""" + + mock_detect_license.return_value = { + "name": "MIT License", + "spdx_id": "MIT", + "@id": "https://spdx.org/licenses/MIT", + "url": "https://spdx.org/licenses/MIT", + "identifier": "https://spdx.org/licenses/MIT" + } + # load api response + repo_json = json.load(open(test_data_path + "api_responses/codeberg/codeberg_forgejo.json")) + lang_json = json.load(open(test_data_path + "api_responses/codeberg/codeberg_forgejo_languages.json")) + releases_json = json.load(open(test_data_path + "api_responses/codeberg/codeberg_forgejo_releases.json")) + + license_response = { + "name": "LICENSE", + "path": "LICENSE", + "content": base64.b64encode(b"MIT License").decode(), + "encoding": "base64" + } + + mock_get.side_effect = [ + _make_mock_response(200, json.dumps(repo_json).encode()), + _make_mock_response(200, json.dumps(license_response).encode()), + _make_mock_response(200, json.dumps(lang_json).encode()), + _make_mock_response(200, json.dumps(releases_json).encode()) + ] + + result, owner, repo_name, branch, path = process_repository.load_codeberg_repository_metadata( + Result(), "https://codeberg.org/forgejo/forgejo" + ) + + self.assertIn(constants.CAT_NAME, result.results) + self.assertEqual(result.results[constants.CAT_NAME][0]["result"]["value"], "forgejo") + self.assertIn(constants.CAT_STARS, result.results) + self.assertIn(constants.CAT_DESCRIPTION, result.results) + self.assertIn(constants.CAT_ISSUE_TRACKER, result.results) + self.assertEqual(owner, "forgejo") + self.assertEqual(repo_name, "forgejo") + # language + self.assertIn(constants.CAT_PROGRAMMING_LANGUAGES, result.results) + # releases + self.assertIn(constants.CAT_RELEASES, result.results) + + self.assertIn(constants.CAT_LICENSE, result.results) + license_entry = result.results[constants.CAT_LICENSE][0] + self.assertEqual(license_entry["technique"], constants.TECHNIQUE_CODEBERG_API) + self.assertEqual(license_entry["result"]["spdx_id"], "MIT") + self.assertEqual(license_entry["result"]["value"], "MIT") + self.assertEqual(license_entry["result"]["name"], "MIT License") + + @patch("somef.process_repository.requests.get") + def test_codeberg_api_error_returns_empty(self, mock_get): + """HTTP error must return empty tuples, not crash.""" + mock_get.return_value = _make_mock_response(404) + result, owner, repo_name, branch, path = \ + process_repository.load_codeberg_repository_metadata( + Result(), "https://codeberg.org/nonexistent/repo" + ) + self.assertEqual(owner, "") + self.assertNotIn(constants.CAT_NAME, result.results) + + @unittest.skipIf(os.getenv("CI") == "true", "Skipped in CI because it is already verified locally") + def test_codeberg_integration_cli(self): + """End-to-end CLI test against a real Codeberg repository.""" + output_file = test_data_path + "test-codeberg-integration.json" + + somef_cli.run_cli( + threshold=0.8, + ignore_classifiers=False, + repo_url="https://codeberg.org/forgejo/forgejo", + local_repo=None, + doc_src=None, + in_file=None, + output=output_file, + graph_out=None, + graph_format="turtle", + codemeta_out=None, + pretty=True, + missing=True, + readme_only=False, + reconcile_authors=False + ) + + with open(output_file, "r") as f: + json_content = json.load(f) + + name_entries = json_content.get(constants.CAT_NAME, []) + self.assertTrue( + any(e["technique"] == constants.TECHNIQUE_CODEBERG_API and e["result"]["value"] == "forgejo" + for e in name_entries), + "Name from Codeberg_API should be 'forgejo'" + ) + + desc_entries = json_content.get(constants.CAT_DESCRIPTION, []) + self.assertTrue( + any(e["technique"] == constants.TECHNIQUE_CODEBERG_API and e["result"]["value"] == "Beyond coding. We forge." + for e in desc_entries), + "Description from Codeberg_API should match" + ) + + code_repo_entries = json_content.get(constants.CAT_CODE_REPOSITORY, []) + self.assertTrue( + any(e["technique"] == constants.TECHNIQUE_CODEBERG_API and + e["result"]["value"] == "https://codeberg.org/forgejo/forgejo" + for e in code_repo_entries) + ) + + homepage_entries = json_content.get(constants.CAT_HOMEPAGE, []) + self.assertTrue( + any(e["technique"] == constants.TECHNIQUE_CODEBERG_API and + e["result"]["value"] == "https://forgejo.org" + for e in homepage_entries) + ) + self.assertIn(constants.CAT_STARS, json_content) + self.assertIn(constants.CAT_FORK_COUNTS, json_content) + + license_entries = json_content.get(constants.CAT_LICENSE, []) + # print("License entries:", license_entries) + self.assertTrue( + any("Codeberg_API" in e["technique"] and + e["result"].get("spdx_id") == "GPL-3.0" + for e in license_entries), + "License from Codeberg_API should have spdx_id GPL-3.0" + ) + ci_entries = json_content.get(constants.CAT_CONTINUOUS_INTEGRATION, []) + self.assertTrue(len(ci_entries) >= 10, f"Expected at least 10 CI workflows, got {len(ci_entries)}") + + os.remove(output_file) + + @patch("somef.parser.codeowners_parser.requests.get") + def test_enrich_user_codeberg(self, mock_get): + response = MagicMock() + response.status_code = 200 + response.json.return_value = { + "full_name": "Forgejo", + "email": "forgejo@noreply.codeberg.org", + } + mock_get.return_value = response + + result = codeowners_parser.enrich_user( + "forgejo", + constants.RepositoryType.CODEBERG, + ) + + mock_get.assert_called_once_with( + "https://codeberg.org/api/v1/users/forgejo", + timeout=5, + ) + self.assertEqual(result[constants.PROP_CODEOWNERS_NAME], "Forgejo") + self.assertEqual(result[constants.PROP_CODEOWNERS_EMAIL], "forgejo@noreply.codeberg.org") \ No newline at end of file diff --git a/src/somef/test/test_data/api_responses/bitbucket/bitbucket_response.json b/src/somef/test/test_data/api_responses/bitbucket/bitbucket_response.json new file mode 100644 index 00000000..6d4f1d31 --- /dev/null +++ b/src/somef/test/test_data/api_responses/bitbucket/bitbucket_response.json @@ -0,0 +1,127 @@ +{ + "type": "repository", + "full_name": "bitbucketpipelines/pipelines-guide-python", + "links": { + "self": { + "href": "https://api.bitbucket.org/2.0/repositories/bitbucketpipelines/pipelines-guide-python" + }, + "html": { + "href": "https://bitbucket.org/bitbucketpipelines/pipelines-guide-python" + }, + "avatar": { + "href": "https://bytebucket.org/ravatar/%7Bcfe0b566-69b6-44a1-a3cf-c124a7ce0f24%7D?ts=python" + }, + "pullrequests": { + "href": "https://api.bitbucket.org/2.0/repositories/bitbucketpipelines/pipelines-guide-python/pullrequests" + }, + "commits": { + "href": "https://api.bitbucket.org/2.0/repositories/bitbucketpipelines/pipelines-guide-python/commits" + }, + "forks": { + "href": "https://api.bitbucket.org/2.0/repositories/bitbucketpipelines/pipelines-guide-python/forks" + }, + "watchers": { + "href": "https://api.bitbucket.org/2.0/repositories/bitbucketpipelines/pipelines-guide-python/watchers" + }, + "branches": { + "href": "https://api.bitbucket.org/2.0/repositories/bitbucketpipelines/pipelines-guide-python/refs/branches" + }, + "tags": { + "href": "https://api.bitbucket.org/2.0/repositories/bitbucketpipelines/pipelines-guide-python/refs/tags" + }, + "downloads": { + "href": "https://api.bitbucket.org/2.0/repositories/bitbucketpipelines/pipelines-guide-python/downloads" + }, + "source": { + "href": "https://api.bitbucket.org/2.0/repositories/bitbucketpipelines/pipelines-guide-python/src" + }, + "clone": [ + { + "name": "https", + "href": "https://bitbucket.org/bitbucketpipelines/pipelines-guide-python.git" + }, + { + "name": "ssh", + "href": "git@bitbucket.org:bitbucketpipelines/pipelines-guide-python.git" + } + ], + "hooks": { + "href": "https://api.bitbucket.org/2.0/repositories/bitbucketpipelines/pipelines-guide-python/hooks" + } + }, + "name": "pipelines-guide-python", + "slug": "pipelines-guide-python", + "description": "This is an example repo showing pipelines with python", + "scm": "git", + "website": "", + "owner": { + "display_name": "Bitbucket Pipelines", + "links": { + "self": { + "href": "https://api.bitbucket.org/2.0/workspaces/%7Bb2919ebc-eb34-4395-b60f-cd937aaa001c%7D" + }, + "avatar": { + "href": "https://bitbucket.org/account/bitbucketpipelines/avatar/" + }, + "html": { + "href": "https://bitbucket.org/%7Bb2919ebc-eb34-4395-b60f-cd937aaa001c%7D/" + } + }, + "type": "team", + "uuid": "{b2919ebc-eb34-4395-b60f-cd937aaa001c}", + "username": "bitbucketpipelines" + }, + "workspace": { + "type": "workspace", + "uuid": "{b2919ebc-eb34-4395-b60f-cd937aaa001c}", + "name": "Bitbucket Pipelines", + "slug": "bitbucketpipelines", + "links": { + "avatar": { + "href": "https://bitbucket.org/workspaces/bitbucketpipelines/avatar/?ts=1569518620" + }, + "html": { + "href": "https://bitbucket.org/bitbucketpipelines/" + }, + "self": { + "href": "https://api.bitbucket.org/2.0/workspaces/bitbucketpipelines" + } + } + }, + "is_private": false, + "project": { + "type": "project", + "key": "DOC", + "uuid": "{92800567-05cc-4bbd-8485-114b0e689717}", + "name": "documentation", + "links": { + "self": { + "href": "https://api.bitbucket.org/2.0/workspaces/bitbucketpipelines/projects/DOC" + }, + "html": { + "href": "https://bitbucket.org/bitbucketpipelines/workspace/projects/DOC" + }, + "avatar": { + "href": "https://bitbucket.org/bitbucketpipelines/workspace/projects/DOC/avatar/32?ts=1506484642" + } + } + }, + "fork_policy": "allow_forks", + "created_on": "2018-10-05T04:53:15.008411+00:00", + "updated_on": "2026-01-29T16:48:15.149268+00:00", + "size": 1373327, + "language": "python", + "uuid": "{cfe0b566-69b6-44a1-a3cf-c124a7ce0f24}", + "mainbranch": { + "name": "master", + "type": "branch" + }, + "override_settings": { + "default_merge_strategy": false, + "branching_model": false + }, + "parent": null, + "enforced_signed_commits": null, + "has_issues": false, + "has_wiki": false +} \ No newline at end of file diff --git a/src/somef/test/test_data/api_responses/bitbucket/bitbucket_tags.json b/src/somef/test/test_data/api_responses/bitbucket/bitbucket_tags.json new file mode 100644 index 00000000..d3750eac --- /dev/null +++ b/src/somef/test/test_data/api_responses/bitbucket/bitbucket_tags.json @@ -0,0 +1,85 @@ +{ + "values": [ + { + "name": "0.0.0-5-10-0-SNAPSHOT-002-epic-AUI-4224-skate-responsive-header", + "type": "tag", + "message": "Release 0.0.0-5-10-0-SNAPSHOT-002-epic-AUI-4224-skate-responsive-header.\n", + "date": "2016-06-15T06:37:45+00:00", + "tagger": { + "type": "author", + "raw": "bambooagent ", + "user": { + "display_name": "Build Agent", + "links": { + "self": {"href": "https://api.bitbucket.org/2.0/users/%7Bf87c782c-77b3-4ad7-bfb2-1054b759bd5f%7D"}, + "avatar": {"href": "https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/initials/BA-4.png"}, + "html": {"href": "https://bitbucket.org/%7Bf87c782c-77b3-4ad7-bfb2-1054b759bd5f%7D/"} + }, + "type": "user", + "uuid": "{f87c782c-77b3-4ad7-bfb2-1054b759bd5f}", + "account_id": "557057:2ff479f7-af2d-45d0-aaaf-94d454f548f7", + "nickname": "bamboo_hg_user" + } + }, + "target": { + "type": "commit", + "hash": "427be59da7b7323591364a1efe3bb681fb3cd4b9", + "date": "2016-06-15T06:37:44+00:00", + "author": { + "type": "author", + "raw": "bambooagent ", + "user": { + "display_name": "Build Agent", + "links": { + "self": {"href": "https://api.bitbucket.org/2.0/users/%7Bf87c782c-77b3-4ad7-bfb2-1054b759bd5f%7D"}, + "avatar": {"href": "https://avatar-management--avatars.us-west-2.prod.public.atl-paas.net/initials/BA-4.png"}, + "html": {"href": "https://bitbucket.org/%7Bf87c782c-77b3-4ad7-bfb2-1054b759bd5f%7D/"} + }, + "type": "user", + "uuid": "{f87c782c-77b3-4ad7-bfb2-1054b759bd5f}", + "account_id": "557057:2ff479f7-af2d-45d0-aaaf-94d454f548f7", + "nickname": "bamboo_hg_user" + } + }, + "message": "0.0.0-5-10-0-SNAPSHOT-002-epic-AUI-4224-skate-responsive-header\n", + "parents": [ + { + "hash": "8535751e4890bde895e327cd52b0c336fb4946d5", + "type": "commit", + "links": { + "self": {"href": "https://api.bitbucket.org/2.0/repositories/atlassian/aui/commit/8535751e4890bde895e327cd52b0c336fb4946d5"}, + "html": {"href": "https://bitbucket.org/atlassian/aui/commits/8535751e4890bde895e327cd52b0c336fb4946d5"} + } + } + ], + "repository": { + "type": "repository", + "full_name": "atlassian/aui", + "name": "aui", + "uuid": "{585074de-7b60-4fd1-81ed-e0bc7fafbda5}", + "links": { + "self": {"href": "https://api.bitbucket.org/2.0/repositories/atlassian/aui"}, + "html": {"href": "https://bitbucket.org/atlassian/aui"}, + "avatar": {"href": "https://bytebucket.org/ravatar/%7B585074de-7b60-4fd1-81ed-e0bc7fafbda5%7D?ts=86317"} + } + }, + "links": { + "self": {"href": "https://api.bitbucket.org/2.0/repositories/atlassian/aui/commit/427be59da7b7323591364a1efe3bb681fb3cd4b9"}, + "html": {"href": "https://bitbucket.org/atlassian/aui/commits/427be59da7b7323591364a1efe3bb681fb3cd4b9"}, + "diff": {"href": "https://api.bitbucket.org/2.0/repositories/atlassian/aui/diff/427be59da7b7323591364a1efe3bb681fb3cd4b9"}, + "approve": {"href": "https://api.bitbucket.org/2.0/repositories/atlassian/aui/commit/427be59da7b7323591364a1efe3bb681fb3cd4b9/approve"}, + "comments": {"href": "https://api.bitbucket.org/2.0/repositories/atlassian/aui/commit/427be59da7b7323591364a1efe3bb681fb3cd4b9/comments"}, + "patch": {"href": "https://api.bitbucket.org/2.0/repositories/atlassian/aui/patch/427be59da7b7323591364a1efe3bb681fb3cd4b9"} + } + }, + "links": { + "self": {"href": "https://api.bitbucket.org/2.0/repositories/atlassian/aui/refs/tags/0.0.0-5-10-0-SNAPSHOT-002-epic-AUI-4224-skate-responsive-header"}, + "commits": {"href": "https://api.bitbucket.org/2.0/repositories/atlassian/aui/commits/0.0.0-5-10-0-SNAPSHOT-002-epic-AUI-4224-skate-responsive-header"}, + "html": {"href": "https://bitbucket.org/atlassian/aui/commits/tag/0.0.0-5-10-0-SNAPSHOT-002-epic-AUI-4224-skate-responsive-header"} + } + } + ], + "pagelen": 10, + "page": 1, + "size": 1 +} \ No newline at end of file diff --git a/src/somef/test/test_data/api_responses/codeberg/codeberg_forgejo.json b/src/somef/test/test_data/api_responses/codeberg/codeberg_forgejo.json new file mode 100644 index 00000000..20610ef3 --- /dev/null +++ b/src/somef/test/test_data/api_responses/codeberg/codeberg_forgejo.json @@ -0,0 +1,104 @@ +{ + "id": 73144, + "owner": { + "id": 70422, + "login": "forgejo", + "login_name": "", + "source_id": 0, + "full_name": "Forgejo", + "email": "forgejo@noreply.codeberg.org", + "avatar_url": "https://codeberg.org/avatars/dae8ab126a96f6fbd6942cf08ab92382", + "html_url": "https://codeberg.org/forgejo", + "language": "", + "is_admin": false, + "last_login": "0001-01-01T00:00:00Z", + "created": "2022-11-06T07:18:11+01:00", + "restricted": false, + "active": false, + "prohibit_login": false, + "location": "", + "pronouns": "", + "website": "https://forgejo.org", + "description": "Beyond coding. We forge.", + "visibility": "public", + "followers_count": 519, + "following_count": 0, + "starred_repos_count": 0, + "username": "forgejo" + }, + "name": "forgejo", + "full_name": "forgejo/forgejo", + "description": "Beyond coding. We forge.", + "empty": false, + "private": false, + "fork": false, + "template": false, + "parent": null, + "mirror": false, + "size": 328744, + "language": "Go", + "languages_url": "https://codeberg.org/api/v1/repos/forgejo/forgejo/languages", + "html_url": "https://codeberg.org/forgejo/forgejo", + "url": "https://codeberg.org/api/v1/repos/forgejo/forgejo", + "link": "", + "ssh_url": "ssh://git@codeberg.org/forgejo/forgejo.git", + "clone_url": "https://codeberg.org/forgejo/forgejo.git", + "original_url": "https://github.com/go-gitea/gitea", + "website": "https://forgejo.org", + "stars_count": 4804, + "forks_count": 827, + "watchers_count": 118, + "open_issues_count": 1320, + "open_pr_counter": 139, + "release_counter": 105, + "default_branch": "forgejo", + "archived": false, + "created_at": "2022-11-06T07:24:57+01:00", + "updated_at": "2026-06-02T01:13:56+02:00", + "archived_at": "1970-01-01T01:00:00+01:00", + "permissions": { + "admin": false, + "push": false, + "pull": true + }, + "has_issues": true, + "internal_tracker": { + "enable_time_tracker": false, + "allow_only_contributors_to_track_time": true, + "enable_issue_dependencies": true + }, + "has_wiki": false, + "has_wiki_contents": false, + "wiki_branch": "master", + "wiki_ssh_url": "ssh://git@codeberg.org/forgejo/forgejo.wiki.git", + "wiki_clone_url": "https://codeberg.org/forgejo/forgejo.wiki.git", + "globally_editable_wiki": false, + "has_pull_requests": true, + "has_projects": true, + "has_releases": true, + "has_packages": true, + "has_actions": true, + "ignore_whitespace_conflicts": false, + "allow_merge_commits": true, + "allow_rebase": true, + "allow_rebase_explicit": true, + "allow_squash_merge": true, + "allow_fast_forward_only_merge": false, + "allow_rebase_update": true, + "default_delete_branch_after_merge": true, + "default_merge_style": "squash", + "default_allow_maintainer_edit": true, + "default_update_style": "merge", + "avatar_url": "https://codeberg.org/repo-avatars/73144-c883a242dec5299fbc06bbe3ee71d8c6", + "internal": false, + "mirror_interval": "", + "object_format_name": "sha1", + "mirror_updated": "0001-01-01T00:00:00Z", + "repo_transfer": null, + "topics": [ + "forge", + "forgejo", + "git", + "self-hosted" + ] +} \ No newline at end of file diff --git a/src/somef/test/test_data/api_responses/codeberg/codeberg_forgejo_languages.json b/src/somef/test/test_data/api_responses/codeberg/codeberg_forgejo_languages.json new file mode 100644 index 00000000..02acc451 --- /dev/null +++ b/src/somef/test/test_data/api_responses/codeberg/codeberg_forgejo_languages.json @@ -0,0 +1,16 @@ +{ + "Go": 16886294, + "go-html-template": 2256939, + "JavaScript": 563991, + "CSS": 395828, + "TypeScript": 377005, + "Vue": 136433, + "Roff": 58206, + "Makefile": 46509, + "Shell": 44058, + "Jsonnet": 15443, + "Dockerfile": 7283, + "Less": 5467, + "Scheme": 2253, + "Nix": 705 +} \ No newline at end of file diff --git a/src/somef/test/test_data/api_responses/codeberg/codeberg_forgejo_releases.json b/src/somef/test/test_data/api_responses/codeberg/codeberg_forgejo_releases.json new file mode 100644 index 00000000..7bc17202 --- /dev/null +++ b/src/somef/test/test_data/api_responses/codeberg/codeberg_forgejo_releases.json @@ -0,0 +1,261 @@ +[ + { + "id": 9279765, + "tag_name": "v15.0.2", + "target_commitish": "", + "name": "v15.0.2", + "body": "See https://codeberg.org/forgejo/forgejo/src/branch/forgejo/release-notes-published/15.0.2.md", + "url": "https://codeberg.org/api/v1/repos/forgejo/forgejo/releases/9279765", + "html_url": "https://codeberg.org/forgejo/forgejo/releases/tag/v15.0.2", + "tarball_url": "https://codeberg.org/forgejo/forgejo/archive/v15.0.2.tar.gz", + "zipball_url": "https://codeberg.org/forgejo/forgejo/archive/v15.0.2.zip", + "hide_archive_links": false, + "upload_url": "https://codeberg.org/api/v1/repos/forgejo/forgejo/releases/9279765/assets", + "draft": false, + "prerelease": false, + "created_at": "2026-05-12T12:09:16+02:00", + "published_at": "2026-05-12T12:09:16+02:00", + "author": { + "id": 70541, + "login": "release-team", + "login_name": "", + "source_id": 0, + "full_name": "Forgejo Release Team", + "email": "release-team@noreply.codeberg.org", + "avatar_url": "https://codeberg.org/avatars/83623afe083e82955f79b2dfeea58bd9f34897fae0f6fd001bc282d3a40efc8b", + "html_url": "https://codeberg.org/release-team", + "language": "", + "is_admin": false, + "last_login": "0001-01-01T00:00:00Z", + "created": "2022-11-07T10:25:48+01:00", + "restricted": false, + "active": false, + "prohibit_login": false, + "location": "", + "pronouns": "", + "website": "https://codeberg.org/forgejo/forgejo", + "description": "Account for publishing Forgejo releases using the Forgejo CI", + "visibility": "public", + "followers_count": 16, + "following_count": 0, + "starred_repos_count": 0, + "username": "release-team" + }, + "assets": [ + { + "id": 1404560, + "name": "forgejo-15.0.2-linux-amd64", + "size": 0, + "download_count": 0, + "created_at": "2026-06-03T14:03:49+02:00", + "uuid": "4ba17d97-2885-4c38-a5cc-7c9a7f8be084", + "browser_download_url": "https://code.forgejo.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-amd64", + "type": "external" + }, + { + "id": 1291356, + "name": "forgejo-15.0.2-linux-amd64.asc", + "size": 228, + "download_count": 1261, + "created_at": "2026-05-12T12:09:19+02:00", + "uuid": "4c7724d9-d9d4-41fc-9758-e2abf194fd61", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-amd64.asc", + "type": "attachment" + }, + { + "id": 1291359, + "name": "forgejo-15.0.2-linux-amd64.sha256", + "size": 93, + "download_count": 6882, + "created_at": "2026-05-12T12:09:20+02:00", + "uuid": "814194b6-26ee-4ce8-aa58-e9b854f0ef40", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-amd64.sha256", + "type": "attachment" + }, + { + "id": 1404536, + "name": "forgejo-15.0.2-linux-amd64.xz", + "size": 33843308, + "download_count": 68, + "created_at": "2026-06-03T13:57:40+02:00", + "uuid": "de77717e-973e-44aa-847f-37b5bdc9c0dc", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-amd64.xz", + "type": "attachment" + }, + { + "id": 1291365, + "name": "forgejo-15.0.2-linux-amd64.xz.asc", + "size": 228, + "download_count": 158, + "created_at": "2026-05-12T12:09:21+02:00", + "uuid": "9d6858ef-6b0b-44c3-931f-e643448d71e2", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-amd64.xz.asc", + "type": "attachment" + }, + { + "id": 1291368, + "name": "forgejo-15.0.2-linux-amd64.xz.sha256", + "size": 96, + "download_count": 251, + "created_at": "2026-05-12T12:09:22+02:00", + "uuid": "7e553e73-d979-4bd6-9b08-d1f0c14446c4", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-amd64.xz.sha256", + "type": "attachment" + }, + { + "id": 1404557, + "name": "forgejo-15.0.2-linux-arm-6", + "size": 0, + "download_count": 0, + "created_at": "2026-06-03T14:03:49+02:00", + "uuid": "5fd81cd6-8afe-470b-9705-53a2b19908b2", + "browser_download_url": "https://code.forgejo.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-arm-6", + "type": "external" + }, + { + "id": 1291374, + "name": "forgejo-15.0.2-linux-arm-6.asc", + "size": 228, + "download_count": 18, + "created_at": "2026-05-12T12:09:25+02:00", + "uuid": "c5cf70db-8b76-4ddb-aebf-9a08164a7d88", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-arm-6.asc", + "type": "attachment" + }, + { + "id": 1291377, + "name": "forgejo-15.0.2-linux-arm-6.sha256", + "size": 93, + "download_count": 21, + "created_at": "2026-05-12T12:09:25+02:00", + "uuid": "51a849fa-543a-456b-81f0-70d2b54c08ac", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-arm-6.sha256", + "type": "attachment" + }, + { + "id": 1404542, + "name": "forgejo-15.0.2-linux-arm-6.xz", + "size": 30797300, + "download_count": 1, + "created_at": "2026-06-03T13:59:10+02:00", + "uuid": "4bfb8fad-8795-4da2-acb3-121bbec2aa8c", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-arm-6.xz", + "type": "attachment" + }, + { + "id": 1291383, + "name": "forgejo-15.0.2-linux-arm-6.xz.asc", + "size": 228, + "download_count": 16, + "created_at": "2026-05-12T12:09:26+02:00", + "uuid": "2f3e578d-6fef-4335-9ca0-e5c61432d836", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-arm-6.xz.asc", + "type": "attachment" + }, + { + "id": 1291386, + "name": "forgejo-15.0.2-linux-arm-6.xz.sha256", + "size": 96, + "download_count": 19, + "created_at": "2026-05-12T12:09:27+02:00", + "uuid": "4ef9e8fc-8a72-4f0c-8133-2ce6042c0159", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-arm-6.xz.sha256", + "type": "attachment" + }, + { + "id": 1404563, + "name": "forgejo-15.0.2-linux-arm64", + "size": 0, + "download_count": 0, + "created_at": "2026-06-03T14:03:49+02:00", + "uuid": "c3e8726e-087a-415d-bc73-4a9630f83c19", + "browser_download_url": "https://code.forgejo.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-arm64", + "type": "external" + }, + { + "id": 1291392, + "name": "forgejo-15.0.2-linux-arm64.asc", + "size": 228, + "download_count": 118, + "created_at": "2026-05-12T12:09:30+02:00", + "uuid": "ce472907-e213-41a1-bc58-c9998737cce4", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-arm64.asc", + "type": "attachment" + }, + { + "id": 1291395, + "name": "forgejo-15.0.2-linux-arm64.sha256", + "size": 93, + "download_count": 165, + "created_at": "2026-05-12T12:09:30+02:00", + "uuid": "df1a91a9-1c94-4545-bd35-710f87e695ad", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-arm64.sha256", + "type": "attachment" + }, + { + "id": 1404539, + "name": "forgejo-15.0.2-linux-arm64.xz", + "size": 30275260, + "download_count": 6, + "created_at": "2026-06-03T13:59:10+02:00", + "uuid": "e7ef71c6-93a2-44ff-8af0-c07ba4d12a01", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-arm64.xz", + "type": "attachment" + }, + { + "id": 1291401, + "name": "forgejo-15.0.2-linux-arm64.xz.asc", + "size": 228, + "download_count": 44, + "created_at": "2026-05-12T12:09:32+02:00", + "uuid": "6f0b2d62-1a06-4bc2-bb4d-2d632a6b68e4", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-arm64.xz.asc", + "type": "attachment" + }, + { + "id": 1291404, + "name": "forgejo-15.0.2-linux-arm64.xz.sha256", + "size": 96, + "download_count": 84, + "created_at": "2026-05-12T12:09:32+02:00", + "uuid": "a6b6e43d-0e40-4ff5-bf8d-f0793658b90a", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-15.0.2-linux-arm64.xz.sha256", + "type": "attachment" + }, + { + "id": 1404545, + "name": "forgejo-src-15.0.2.tar.gz", + "size": 49711959, + "download_count": 20, + "created_at": "2026-06-03T14:00:54+02:00", + "uuid": "45465223-70f5-4705-9289-c2d5fe7d269c", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-src-15.0.2.tar.gz", + "type": "attachment" + }, + { + "id": 1291410, + "name": "forgejo-src-15.0.2.tar.gz.asc", + "size": 228, + "download_count": 42, + "created_at": "2026-05-12T12:09:34+02:00", + "uuid": "6db70858-d902-4d68-8bf2-10359c325599", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-src-15.0.2.tar.gz.asc", + "type": "attachment" + }, + { + "id": 1291413, + "name": "forgejo-src-15.0.2.tar.gz.sha256", + "size": 92, + "download_count": 36, + "created_at": "2026-05-12T12:09:34+02:00", + "uuid": "5e710ccf-62f2-499e-96e9-f4833eaa9c72", + "browser_download_url": "https://codeberg.org/forgejo/forgejo/releases/download/v15.0.2/forgejo-src-15.0.2.tar.gz.sha256", + "type": "attachment" + } + ], + "archive_download_count": { + "zip": 224, + "tar_gz": 363 + } + } +] \ No newline at end of file diff --git a/src/somef/test/test_gitlab_token_handling.py b/src/somef/test/test_gitlab_token_handling.py new file mode 100644 index 00000000..af145671 --- /dev/null +++ b/src/somef/test/test_gitlab_token_handling.py @@ -0,0 +1,118 @@ +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from .. import somef_cli +from ..parser import codeowners_parser +from ..process_results import Result +from ..utils import constants + + +class TestGitlabTokenHandling(unittest.TestCase): + @patch("somef.parser.codeowners_parser.requests.get") + def test_enrich_user_gitlab_adds_bearer_prefix(self, mock_get): + # Simulate a successful GitLab user lookup response. + response = MagicMock() + response.status_code = 200 + response.json.return_value = [ + { + "name": "Alice", + "organization": "Example Org", + "public_email": "alice@example.org", + } + ] + mock_get.return_value = response + + # Pass a raw token and verify the request header is normalized to Bearer. + result = codeowners_parser.enrich_user( + "alice", + constants.RepositoryType.GITLAB, + server_url="gitlab.example.org", + gitlab_authorization="my-token", + ) + + # Ensure request URL, auth header, and timeout are built correctly. + mock_get.assert_called_once_with( + "https://gitlab.example.org/api/v4/users?username=alice", + headers={"Authorization": "Bearer my-token"}, + timeout=5, + ) + # Ensure selected fields from the GitLab payload are mapped into SOMEF output. + self.assertEqual(result[constants.PROP_CODEOWNERS_NAME], "Alice") + self.assertEqual(result[constants.PROP_CODEOWNERS_COMPANY], "Example Org") + self.assertEqual(result[constants.PROP_CODEOWNERS_EMAIL], "alice@example.org") + + @patch("somef.parser.codeowners_parser.requests.get") + def test_enrich_user_gitlab_preserves_existing_bearer_prefix(self, mock_get): + # Simulate a valid response; this test focuses on header formatting only. + response = MagicMock() + response.status_code = 200 + response.json.return_value = [{"name": "Bob"}] + mock_get.return_value = response + + # Pass an already-prefixed token and verify no duplicate prefix is introduced. + codeowners_parser.enrich_user( + "bob", + constants.RepositoryType.GITLAB, + server_url="https://gitlab.example.org", + gitlab_authorization="Bearer already-prefixed", + ) + + mock_get.assert_called_once_with( + "https://gitlab.example.org/api/v4/users?username=bob", + headers={"Authorization": "Bearer already-prefixed"}, + timeout=5, + ) + + + @patch("somef.somef_cli.verify_and_resolve_token", return_value="Bearer token-456") + @patch("somef.somef_cli.json_export.unify_results", return_value={}) + @patch("somef.somef_cli.cli_get_data") + @patch("somef.somef_cli.validators.url", return_value=True) + def test_run_cli_forwards_gitlab_token_in_file_mode(self, _mock_valid_url, mock_cli_get_data, _mock_unify, _mock_verify_token): + # Avoid network/repository processing and only assert argument forwarding. + mock_cli_get_data.return_value = Result() + + # Build a temporary input file to trigger batch mode. + with tempfile.TemporaryDirectory() as tmp_dir: + input_file = Path(tmp_dir) / "repos.txt" + input_file.write_text("https://gitlab.com/group/project\n", encoding="utf-8") + + # Run file mode with a GitLab token. + somef_cli.run_cli( + in_file=str(input_file), + output=None, + codemeta_out=None, + google_codemeta_out=None, + gitlab_token="token-456", + ) + + # Ensure CLI propagates the token in batch mode as well. + self.assertEqual(mock_cli_get_data.call_count, 1) + kwargs = mock_cli_get_data.call_args.kwargs + self.assertEqual(kwargs["authorization"], "Bearer token-456") + + + @patch("somef.somef_cli.verify_and_resolve_token", return_value="Bearer token-123") + @patch("somef.somef_cli.json_export.unify_results", return_value={}) + @patch("somef.somef_cli.cli_get_data") + @patch("somef.somef_cli.validators.url", return_value=True) + def test_run_cli_forwards_gitlab_token_single_repo(self, _mock_valid_url, mock_cli_get_data, _mock_unify, _mock_verify_token): + mock_cli_get_data.return_value = Result() + + somef_cli.run_cli( + repo_url="https://gitlab.com/group/project", + output=None, + codemeta_out=None, + google_codemeta_out=None, + gitlab_token="token-123", + ) + + self.assertEqual(mock_cli_get_data.call_count, 1) + kwargs = mock_cli_get_data.call_args.kwargs + self.assertEqual(kwargs["authorization"], "Bearer token-123") + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/src/somef/test/test_verify_token.py b/src/somef/test/test_verify_token.py new file mode 100644 index 00000000..6a1b18aa --- /dev/null +++ b/src/somef/test/test_verify_token.py @@ -0,0 +1,62 @@ +import sys +import unittest +from unittest.mock import patch, MagicMock + +from .. import somef_cli +from ..process_results import Result + + +class TestTokenVerification(unittest.TestCase): + + @patch("somef.somef_cli.process_repository.is_gitlab", return_value=True) + def test_github_token_on_gitlab_repo(self, mock_is_gitlab): + """--github-token on a GitLab repo returns None (mismatch warning)""" + result = somef_cli.verify_and_resolve_token( + "https://gitlab.com/group/project", + github_token="some-token", gitlab_token=None, + codeberg_token=None, bitbucket_token=None, bitbucket_email=None + ) + self.assertIsNone(result) + + @patch("somef.somef_cli.requests.get") + def test_github_token_invalid_401(self, mock_get): + """Invalid token (API 401) returns None (falls back to config)""" + mock_resp = MagicMock() + mock_resp.status_code = 401 + mock_get.return_value = mock_resp + + result = somef_cli.verify_and_resolve_token( + "https://github.com/owner/repo", + github_token="bad-token", gitlab_token=None, + codeberg_token=None, bitbucket_token=None, bitbucket_email=None + ) + self.assertIsNone(result) + + @patch("somef.somef_cli.requests.get") + def test_bitbucket_token_without_email(self, mock_get): + """Bitbucket token without email exits with error""" + with self.assertRaises(SystemExit): + somef_cli.verify_and_resolve_token( + "https://bitbucket.org/workspace/repo", + github_token=None, gitlab_token=None, + codeberg_token=None, bitbucket_token="some-token", + bitbucket_email=None + ) + + @patch("somef.somef_cli.cli_get_data") + @patch("somef.somef_cli.verify_and_resolve_token", return_value="token test-abc") + @patch("somef.somef_cli.validators.url", return_value=True) + def test_run_cli_forwards_authorization(self, mock_url, mock_verify, mock_get_data): + """Token resolves into authorization and is forwarded to cli_get_data""" + mock_get_data.return_value = Result() + somef_cli.run_cli( + repo_url="https://github.com/owner/repo", + github_token="test-abc", + threshold=0.8, output=None + ) + kwargs = mock_get_data.call_args.kwargs + self.assertEqual(kwargs["authorization"], "token test-abc") + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/src/somef/test/test_wiki.py b/src/somef/test/test_wiki.py index 9818d00d..7de51bbf 100644 --- a/src/somef/test/test_wiki.py +++ b/src/somef/test/test_wiki.py @@ -38,6 +38,7 @@ def test_wiki_with_content(self): os.remove(json_file_path) + @unittest.skipIf(os.getenv("CI") == "true", "Skipped in CI: prone to rate limiting failures") def test_wiki_without_content(self): """Checks if codemeta file has been exported with wiki content""" somef_cli.run_cli(threshold=0.8, diff --git a/src/somef/utils/constants.py b/src/somef/utils/constants.py index df086fc7..1b73f1dd 100644 --- a/src/somef/utils/constants.py +++ b/src/somef/utils/constants.py @@ -3,7 +3,15 @@ from pathlib import Path # constants about SOMEF configuration -CONF_AUTHORIZATION = "Authorization" +# CONF_AUTHORIZATION = "Authorization" +CONF_GITHUB_AUTHORIZATION = "GitHubAuthorization" +CONF_GITLAB_AUTHORIZATION = "GitlabAuthorization" +CONF_CODEBERG_AUTHORIZATION = "CodebergAuthorization" +CONF_BITBUCKET_AUTHORIZATION = "BitbucketAuthorization" +# Backward-compatible key used in legacy code paths for GitHub authorization. +CONF_AUTHORIZATION = CONF_GITHUB_AUTHORIZATION +PROP_AUTHORIZATION = "Authorization" + CONF_DESCRIPTION = "description" CONF_INVOCATION = "invocation" CONF_INSTALLATION = "installation" @@ -63,7 +71,8 @@ #License spdx # REGEXP_APACHE = r'(?i)apache\s+license\s*,?\s*version\s*2\.0' REGEXP_APACHE = r'(?i)apache(?:\s+license)?\s*(?:,?\s*version\s*)?2\.0' -REGEXP_GPL3 = r'(?i)gnu\s+general\s+public\s+license\s*,?\s*version\s*3\.0' +REGEXP_GPL3 = r'(?i)gnu\s+general\s+public\s+license\s*,?\s*version\s*3(?:\.0)?' + # REGEXP_MIT = r'(?i)mit\s+license' REGEXP_MIT = r'(?i)(mit\s+license|permission\s+is\s+hereby\s+granted|THE\s+SOFTWARE\s+IS\s+PROVIDED\s+"AS\s+IS")' REGEXP_BSD2 = r'(?i)bsd\s*-?\s*2-?clause(?:\s*license)?' @@ -72,7 +81,7 @@ REGEXP_CC0 = r'(?i)creative\s+commons\s+zero\s+v?1\.0\s+universal' REGEXP_EPL2 = r'(?i)eclipse\s+public\s+license\s*,?\s*version\s*2\.0' REGEXP_AGPL3 = r'(?i)gnu\s+affero\s+general\s+public\s+license\s*(?:v(?:ersion)?\.?\s*3(?:\.0)?)' -REGEXP_GPL2 = r'(?i)gnu\s+general\s+public\s+license\s*,?\s*version\s*2\.0' +REGEXP_GPL2 = r'(?i)gnu\s+general\s+public\s+license\s*,?\s*version\s*2(?:\.0)?' REGEXP_LGPL1 = r'(?i)gnu\s+lesser\s+general\s+public\s+license\s*,?\s*version\s*1\.0' REGEXP_MPL2 = r'(?i)mozilla\s+public\s+license\s*,?\s*version\s*2\.0' REGEXP_UNLICENSE = r'(?i)the\s+unlicense' @@ -311,6 +320,8 @@ TECHNIQUE_CODE_CONFIG_PARSER = "code_parser" TECHNIQUE_GITHUB_API = "GitHub_API" TECHNIQUE_GITLAB_API = "GitLab_API" +TECHNIQUE_CODEBERG_API = "Codeberg_API" +TECHNIQUE_BITBUCKET_API = "Bitbucket_API" TECHNIQUE_HEURISTICS = "software_type_heuristics" # GitHub properties @@ -318,6 +329,14 @@ GITHUB_ACCEPT_HEADER = "application/vnd.github.v3+json" GITHUB_API = "https://api.github.com/repos" +#Codeberg properties +CODEBERG_DOMAIN = "codeberg.org" +CODEBERG_API = "https://codeberg.org/api/v1/repos" + +# Bitbucket properties +BITBUCKET_DOMAIN = "bitbucket.org" +BITBUCKET_API = "https://api.bitbucket.org/2.0/repositories" + # Software Heritage SWH_ROOT = "https://archive.softwareheritage.org/" REGEXP_SWH = r'\[\!\[SWH\]([^\]]+)\]\(([^)]+)\)' @@ -349,6 +368,38 @@ CAT_HOMEPAGE: "homepage" } +# Crosswalk to retrieve easily contents of interest from the codeberg response +codeberg_crosswalk_table = { + CAT_CODE_REPOSITORY: "html_url", + "languages_url": "languages_url", + CAT_OWNER: ["owner", "login"], + # AGENT_TYPE: ["owner", "type"], + CAT_DATE_CREATED: "created_at", + CAT_DATE_UPDATED: "updated_at", + CAT_DESCRIPTION: "description", + CAT_NAME: "name", + CAT_FULL_NAME: "full_name", + # CAT_ISSUE_TRACKER: "issues_url", + CAT_STARS: "stars_count", + CAT_KEYWORDS: "topics", + CAT_FORK_COUNTS: "forks_count", + CAT_HOMEPAGE: "website" +} + + +bitbucket_crosswalk_table = { + CAT_FULL_NAME: "full_name", + CAT_NAME: "name", + CAT_DESCRIPTION: "description", + CAT_DATE_CREATED: "created_on", + CAT_DATE_UPDATED: "updated_on", + CAT_OWNER: ["owner", "nickname"], + CAT_CODE_REPOSITORY: ["links", "html", "href"], + CAT_HOMEPAGE: "website", + CAT_FORKS_URLS: ["links", "forks", "href"] + # CAT_PROGRAMMING_LANGUAGES: "language", +} + # Mapping for releases release_crosswalk_table = { PROP_TAG: 'tag_name', @@ -391,6 +442,36 @@ PROP_DOWNLOAD_COUNT: "download_count" } +release_assets_codeberg = { + PROP_URL: "url", + PROP_NAME: "name", + PROP_SIZE: "size", + PROP_BROWSER_URL: "browser_download_url", + PROP_CONTENT_TYPE: "content_type", + PROP_DATE_CREATED_AT: "created_at", + PROP_DOWNLOAD_COUNT: "download_count" +} + +release_codeberg_crosswalk_table = { + PROP_TAG: 'tag_name', + PROP_NAME: 'name', + PROP_AUTHOR: ['author', 'login'], + # AGENT_TYPE: ['author', 'type'], + PROP_DESCRIPTION: 'body', + PROP_TARBALL_URL: 'tarball_url', + PROP_ZIPBALL_URL: 'zipball_url', + PROP_HTML_URL: 'html_url', + PROP_URL: 'url', + PROP_RELEASE_ID: 'id', + PROP_DATE_CREATED: 'created_at', + PROP_DATE_PUBLISHED: "published_at", + # CAT_ASSETS: "attachments" +} + +release_bitbucket_crosswalk_table = { + PROP_TAG: "name", + PROP_NAME: "name", +} # Minimum percentage of total bytes a programming language must have to be considered relevant in CodeMeta file. MINIMUM_PERCENTAGE_LANGUAGE_PROGRAMMING = 10 @@ -423,6 +504,8 @@ class RepositoryType(Enum): GITHUB = 1 GITLAB = 2 LOCAL = 3 + CODEBERG = 4 + BITBUCKET = 5 # Media/script/non-software sets workflow_extensions=('.ga','.cwl','.nf','.knwf','.t2flow','.dag','.kar','.wdl',".smk",".snake")