Implement wave container commands#3954
Conversation
JulianFlesch
changed the title
| ] | ||
| return containers_flat | ||
|
|
||
| def _resolve_module_dir(self, module: str | Path) -> Path: |
There was a problem hiding this comment.
do we not have this funciton already somehwere else?
There was a problem hiding this comment.
I might be overlooking something, but not really ... As far as I can tell, we have ModulesInfo, which was used at some point, but it never provides the environment.yml or meta.yml paths, or confirms their existance.
Maybe this should be in ModulesInfo?
| return module_dir | ||
|
|
||
| @staticmethod | ||
| def _environment_path(module_dir: Path) -> Path: |
There was a problem hiding this comment.
because we lint for this, I think we should be able to reuse something there.
|
|
||
|
|
||
| @contextmanager | ||
| def set_wd_tempdir() -> Generator[None, None, None]: |
There was a problem hiding this comment.
this function is for sure created somewhere else already.
There was a problem hiding this comment.
Yes, In pipeline downloads! But that version throws Download-specific errors ...
The plan was move it out of the utils.py over there into the general utils.
But now we don't use it at all anymore ...
Should we remove it again?
|
|
Added the meta_yml sorting now. I also switched the default value of |
mashehu
force-pushed
the
feature/3952-create-waveyml
branch
from
1de5380 to
b92491a
Compare
| sort_by="test", | ||
| local=False, | ||
| fix_version=False, | ||
| ): |
|
|
||
| corrected_data = mod_io_data.copy() | ||
|
|
||
| if io_type == "input": |
| if skip_file.is_file(): | ||
| with open(skip_file) as fh: | ||
| data = json.load(fh) | ||
| for system in CONTAINER_SYSTEMS + ["conda"]: |
There was a problem hiding this comment.
I would first check if the module name is actually in the json file. and then start looping through it
|
|
||
| # Load containers from meta.yml into the component before linting | ||
| meta_path = Path(self.nfcore_component.component_dir, "meta.yml") | ||
| if meta_path.exists(): |
There was a problem hiding this comment.
no need for this if clause, we already check that in read_meta_yml()
mashehu
force-pushed
the
feature/3952-create-waveyml
branch
3 times, most recently
from
47ed327 to
b71b250
Compare
| log.error("No modules found to build containers for") | ||
| sys.exit(1) | ||
|
|
||
| log.info(f"Building containers for {len(manager.available_modules)} module(s)") |
There was a problem hiding this comment.
make this another progress bar (not a spinner this time)
| sys.exit(1) | ||
|
|
||
|
|
||
| def modules_containers_conda_lock(ctx, module, platform=CONTAINER_PLATFORMS[0]): |
There was a problem hiding this comment.
do we actually want this externalized? it is part of nf-core m container c anyway
| maintainers: | ||
| - "{{ author }}" | ||
|
|
||
| # TODO container-conversion: Add "containers" section |
There was a problem hiding this comment.
will be automatically generated.
| # TODO container-conversion: Add "containers" section |
| mod.get_inputs_from_main_nf() | ||
| mod.get_outputs_from_main_nf() | ||
| mod.get_topics_from_main_nf() | ||
| # TODO container-conversion: get_containers from main_nf |
There was a problem hiding this comment.
use module_containers.list.containers here
mashehu
force-pushed
the
feature/3952-create-waveyml
branch
from
5d3a49b to
3e6859d
Compare
| # Get latest build | ||
| build = _get_build(response) | ||
| def _fix_module_version(self, current_version, latest_version, response): | ||
| """Updates the module conda version in main.nf.""" |
There was a problem hiding this comment.
should look into environment.yml not main.nf
There was a problem hiding this comment.
and then need to run containers.create on it
| from nf_core.modules.lint.main_nf import _fix_module_version | ||
|
|
||
| fixed = _fix_module_version(module, bioconda_version, last_ver, response) |
There was a problem hiding this comment.
rewrite this part to not touch main.nf but enviroment.yml
| if skip_file.is_file(): | ||
| with open(skip_file) as fh: | ||
| data = json.load(fh) | ||
| for system in CONTAINER_SYSTEMS + ["conda"]: |
There was a problem hiding this comment.
rewrite CONTAINER_SYSTEMS+ ["conda"] to something like SYSTEMS
| return formatted_topics | ||
|
|
||
|
|
||
| def obtain_containers(_, containers: dict) -> dict: |
There was a problem hiding this comment.
replace with get_containers_from_meta or pydantic model
| name_hash = tag.rsplit("--", 1)[1] | ||
| else: | ||
| tag_lower = tag.lower() | ||
| if len(tag_lower) >= 8 and all(c in "0123456789abcdef" for c in tag_lower): |
There was a problem hiding this comment.
isn't there a better way to check for hexadecimal characters?
| ( | ||
| "meta_yml", | ||
| "containers_build_id_hash", | ||
| f"Build ID hash does not match {system} container tag for {platform}", |
There was a problem hiding this comment.
| f"Build ID hash does not match {system} container tag for {platform}", | |
| f"Build ID `{build_hash}` hash does not match {system} container tag `{name_hash}` for {platform}", |
| ( | ||
| "meta_yml", | ||
| "containers_build_id_hash", | ||
| f"Could not compare build ID hash with {system} container tag for {platform}", |
There was a problem hiding this comment.
| f"Could not compare build ID hash with {system} container tag for {platform}", | |
| f"Could not compare build ID hash with {system} container tag for {platform}, because either build_hash:`{build_hash}` or name_hash:`{name_hash}` is `None`.", |
| pass | ||
|
|
||
|
|
||
| def get_container_with_regex(main_nf_path: Path, component_name: str | None = None) -> str: |
There was a problem hiding this comment.
add test for this function
| ) | ||
| _, success = manager.create(await_build, progress_bar=progress_bar, task_id=module_task_id, force=force) | ||
| if success: | ||
| manager.update_containers_in_meta() |
There was a problem hiding this comment.
if in pipline repo, this should also generate pipeline container configs, I think try_generate_container_configs
| from ..test_modules import TestModules | ||
|
|
||
|
|
||
| class TestModuleContainers: |
There was a problem hiding this comment.
| class TestModuleContainers: | |
| class TestModuleContainers(TestModules): |
I think
| def _write_meta(self, module_dir: Path, meta: dict) -> None: | ||
| (module_dir / "meta.yml").write_text(yaml.safe_dump(meta), encoding="utf-8") | ||
|
|
||
| def _containers_by_system(self, prefix: str = "testC") -> dict: |
There was a problem hiding this comment.
replace with pydantic model
| assert manager.module_directory == module_dir | ||
| assert manager.environment_yml == module_dir / "environment.yml" | ||
| assert manager.meta_yml == module_dir / "meta.yml" | ||
|
|
There was a problem hiding this comment.
add one integration test to check that the wave logic actually works and doesn't change in the background. maybe with --dry-run
| with mock.patch.object(manager, "get_containers_from_meta", return_value=containers): | ||
| listed = manager.list_containers() |
There was a problem hiding this comment.
don't mock this, run this against the gitlab test repo with one specific module
Co-authored-by: Matthias Hörtenhuber <mashehu@users.noreply.github.com>
3 participants
Closes #3952