diff --git a/dvc/dependency/base.py b/dvc/dependency/base.py index 57a11a67ee..2b46e8e94d 100644 --- a/dvc/dependency/base.py +++ b/dvc/dependency/base.py @@ -1,6 +1,10 @@ +from typing import Optional + from dvc.exceptions import DvcException from dvc.fs import download as fs_download +from dvc.ignore import DvcIgnoreFilter from dvc.output import Output +from dvc_objects.fs.scheme import Schemes class DependencyDoesNotExistError(DvcException): @@ -27,6 +31,26 @@ class Dependency(Output): IsNotFileOrDirError: type[DvcException] = DependencyIsNotFileOrDirError IsStageFileError: type[DvcException] = DependencyIsStageFileError + @property + def dvcignore(self) -> Optional[DvcIgnoreFilter]: + """ + For dependencies we override the dvcignore to be part of + SCM root as well. Outputs cannot be saved outside the DVC repo. + However, you can have dependency for subdir DVC repos. + + Returns: + Optional[DvcIgnoreFilter]: DVC repo root or SCM root dvcignore. + """ + if self.fs.protocol != Schemes.LOCAL: + return None + + assert self.repo + if self.fs.isin_or_eq(self.fs_path, self.repo.root_dir): + return self.repo.dvcignore + if self.fs.isin_or_eq(self.fs_path, self.repo.scm.root_dir): + return self.repo.scm_dvcignore + return None + def workspace_status(self) -> dict[str, str]: if self.fs.version_aware: old_fs_path = self.fs_path diff --git a/dvc/output.py b/dvc/output.py index 384ba77cd8..b878faa7cc 100644 --- a/dvc/output.py +++ b/dvc/output.py @@ -34,9 +34,11 @@ from dvc_data.hashfile.transfer import transfer as otransfer from dvc_data.hashfile.tree import Tree, du from dvc_objects.errors import ObjectFormatError +from dvc_objects.fs.local import LocalFileSystem +from dvc_objects.fs.scheme import Schemes from .annotations import ANNOTATION_FIELDS, ANNOTATION_SCHEMA, Annotation -from .fs import LocalFileSystem, RemoteMissingDepsError, Schemes, get_cloud_fs +from .fs import RemoteMissingDepsError, get_cloud_fs from .fs.callbacks import DEFAULT_CALLBACK, Callback, TqdmCallback from .utils import relpath from .utils.fs import path_isin @@ -351,7 +353,7 @@ def __init__( # noqa: PLR0913 self.fs = fs_cls(**fs_config) if ( - self.fs.protocol == "local" + self.fs.protocol == Schemes.LOCAL and stage and isinstance(stage.repo.fs, LocalFileSystem) and path_isin(path, stage.repo.root_dir) @@ -363,7 +365,7 @@ def __init__( # noqa: PLR0913 if ( self.repo - and self.fs.protocol == "local" + and self.fs.protocol == Schemes.LOCAL and not self.fs.isabs(self.def_path) ): self.fs = self.repo.fs @@ -461,7 +463,7 @@ def __repr__(self): return f"{type(self).__name__}: {self.def_path!r}" def __str__(self): - if self.fs.protocol != "local": + if self.fs.protocol != Schemes.LOCAL: return self.def_path if ( @@ -643,7 +645,7 @@ def changed(self) -> bool: @property def dvcignore(self) -> Optional["DvcIgnoreFilter"]: - if self.fs.protocol == "local": + if self.fs.protocol == Schemes.LOCAL: assert self.repo return self.repo.dvcignore return None @@ -891,7 +893,7 @@ def dumpd(self, **kwargs): # noqa: C901, PLR0912 return ret def verify_metric(self): - if self.fs.protocol != "local": + if self.fs.protocol != Schemes.LOCAL: raise DvcException(f"verify metric is not supported for {self.protocol}") if not self.metric: return @@ -1003,7 +1005,7 @@ def move(self, out: "Output") -> None: else: logger.warning("%r missing", self.fspath) - if self.protocol == "local" and self.use_scm_ignore: + if self.protocol == Schemes.LOCAL and self.use_scm_ignore: assert self.repo self.repo.scm_context.ignore_remove(self.fspath) diff --git a/dvc/repo/__init__.py b/dvc/repo/__init__.py index a7c3d4d7ca..be37fbeff6 100644 --- a/dvc/repo/__init__.py +++ b/dvc/repo/__init__.py @@ -332,6 +332,12 @@ def scm_context(self) -> "SCMContext": def dvcignore(self) -> DvcIgnoreFilter: return DvcIgnoreFilter(self.fs, self.root_dir) + @cached_property + def scm_dvcignore(self) -> DvcIgnoreFilter: + if self.root_dir == self.scm.root_dir: + return self.dvcignore + return DvcIgnoreFilter(self.fs, self.scm.root_dir) + def get_rev(self): from dvc.fs import GitFileSystem, LocalFileSystem diff --git a/tests/func/test_ignore.py b/tests/func/test_ignore.py index 86b40fb488..665f608459 100644 --- a/tests/func/test_ignore.py +++ b/tests/func/test_ignore.py @@ -7,7 +7,7 @@ from dvc.ignore import DvcIgnore, DvcIgnorePatterns from dvc.output import OutputIsIgnoredError from dvc.pathspec_math import PatternInfo, merge_patterns -from dvc.repo import Repo +from dvc.repo import Repo, lock_repo from dvc.testing.tmp_dir import TmpDir from dvc_data.hashfile.build import IgnoreInCollectedDirError from dvc_data.hashfile.utils import get_mtime_and_size @@ -227,6 +227,38 @@ def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory): assert dvc.dvcignore.is_ignored_file(os.fspath(ext_dir / "y.backup")) is False +def test_subdir_repo_uses_scm_root_dvcignore_for_external_dependency(tmp_dir, scm): + tmp_dir.gen( + { + ".dvcignore": "*.pyc\n", + "main_lib": {"foo.py": "VALUE = 1\n"}, + "sudirs_monorepo": {"proj1": {}}, + } + ) + + subrepo_dir = tmp_dir / "sudirs_monorepo" / "proj1" + with subrepo_dir.chdir(): + repo = Repo.init(subdir=True) + stage = repo.stage.create( + name="check-main-lib", + cmd="echo check", + deps=[os.path.join("..", "..", "main_lib")], + outs=[], + ) + stage.save(run_cache=False) + + pycache_dir = tmp_dir / "main_lib" / "__pycache__" + pycache_dir.mkdir() + (pycache_dir / "foo.pyc").write_bytes(b"ignored bytecode") + + with lock_repo(repo): + assert not stage.changed_deps() + + (tmp_dir / "main_lib" / "new.txt").write_text("not ignored") + with lock_repo(repo): + assert stage.changed_deps() + + def test_ignore_resurface_subrepo(tmp_dir, scm, dvc): tmp_dir.dvc_gen({"foo": "foo"}, commit="add foo") subrepo_dir = tmp_dir / "subdir"