Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions dvc/dependency/base.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
from typing import Optional

from dvc.exceptions import DvcException
from dvc.fs import download as fs_download
from dvc.ignore import DvcIgnoreFilter
from dvc.output import Output
from dvc_objects.fs.scheme import Schemes


class DependencyDoesNotExistError(DvcException):
Expand All @@ -27,6 +31,26 @@ class Dependency(Output):
IsNotFileOrDirError: type[DvcException] = DependencyIsNotFileOrDirError
IsStageFileError: type[DvcException] = DependencyIsStageFileError

@property
def dvcignore(self) -> Optional[DvcIgnoreFilter]:
"""
For dependencies we override the dvcignore to be part of
SCM root as well. Outputs cannot be saved outside the DVC repo.
However, you can have dependency for subdir DVC repos.

Returns:
Optional[DvcIgnoreFilter]: DVC repo root or SCM root dvcignore.
"""
if self.fs.protocol != Schemes.LOCAL:
return None

assert self.repo
if self.fs.isin_or_eq(self.fs_path, self.repo.root_dir):
return self.repo.dvcignore
if self.fs.isin_or_eq(self.fs_path, self.repo.scm.root_dir):
return self.repo.scm_dvcignore
return None

def workspace_status(self) -> dict[str, str]:
if self.fs.version_aware:
old_fs_path = self.fs_path
Expand Down
16 changes: 9 additions & 7 deletions dvc/output.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,9 +34,11 @@
from dvc_data.hashfile.transfer import transfer as otransfer
from dvc_data.hashfile.tree import Tree, du
from dvc_objects.errors import ObjectFormatError
from dvc_objects.fs.local import LocalFileSystem
from dvc_objects.fs.scheme import Schemes

from .annotations import ANNOTATION_FIELDS, ANNOTATION_SCHEMA, Annotation
from .fs import LocalFileSystem, RemoteMissingDepsError, Schemes, get_cloud_fs
from .fs import RemoteMissingDepsError, get_cloud_fs
from .fs.callbacks import DEFAULT_CALLBACK, Callback, TqdmCallback
from .utils import relpath
from .utils.fs import path_isin
Expand Down Expand Up @@ -351,7 +353,7 @@ def __init__( # noqa: PLR0913
self.fs = fs_cls(**fs_config)

if (
self.fs.protocol == "local"
self.fs.protocol == Schemes.LOCAL
and stage
and isinstance(stage.repo.fs, LocalFileSystem)
and path_isin(path, stage.repo.root_dir)
Expand All @@ -363,7 +365,7 @@ def __init__( # noqa: PLR0913

if (
self.repo
and self.fs.protocol == "local"
and self.fs.protocol == Schemes.LOCAL
and not self.fs.isabs(self.def_path)
):
self.fs = self.repo.fs
Expand Down Expand Up @@ -461,7 +463,7 @@ def __repr__(self):
return f"{type(self).__name__}: {self.def_path!r}"

def __str__(self):
if self.fs.protocol != "local":
if self.fs.protocol != Schemes.LOCAL:
return self.def_path

if (
Expand Down Expand Up @@ -643,7 +645,7 @@ def changed(self) -> bool:

@property
def dvcignore(self) -> Optional["DvcIgnoreFilter"]:
if self.fs.protocol == "local":
if self.fs.protocol == Schemes.LOCAL:
assert self.repo
return self.repo.dvcignore
return None
Expand Down Expand Up @@ -891,7 +893,7 @@ def dumpd(self, **kwargs): # noqa: C901, PLR0912
return ret

def verify_metric(self):
if self.fs.protocol != "local":
if self.fs.protocol != Schemes.LOCAL:
raise DvcException(f"verify metric is not supported for {self.protocol}")
if not self.metric:
return
Expand Down Expand Up @@ -1003,7 +1005,7 @@ def move(self, out: "Output") -> None:
else:
logger.warning("%r missing", self.fspath)

if self.protocol == "local" and self.use_scm_ignore:
if self.protocol == Schemes.LOCAL and self.use_scm_ignore:
assert self.repo
self.repo.scm_context.ignore_remove(self.fspath)

Expand Down
6 changes: 6 additions & 0 deletions dvc/repo/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,12 @@ def scm_context(self) -> "SCMContext":
def dvcignore(self) -> DvcIgnoreFilter:
return DvcIgnoreFilter(self.fs, self.root_dir)

@cached_property
def scm_dvcignore(self) -> DvcIgnoreFilter:
if self.root_dir == self.scm.root_dir:
return self.dvcignore
return DvcIgnoreFilter(self.fs, self.scm.root_dir)

def get_rev(self):
from dvc.fs import GitFileSystem, LocalFileSystem

Expand Down
34 changes: 33 additions & 1 deletion tests/func/test_ignore.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from dvc.ignore import DvcIgnore, DvcIgnorePatterns
from dvc.output import OutputIsIgnoredError
from dvc.pathspec_math import PatternInfo, merge_patterns
from dvc.repo import Repo
from dvc.repo import Repo, lock_repo
from dvc.testing.tmp_dir import TmpDir
from dvc_data.hashfile.build import IgnoreInCollectedDirError
from dvc_data.hashfile.utils import get_mtime_and_size
Expand Down Expand Up @@ -227,6 +227,38 @@ def test_ignore_external(tmp_dir, scm, dvc, tmp_path_factory):
assert dvc.dvcignore.is_ignored_file(os.fspath(ext_dir / "y.backup")) is False


def test_subdir_repo_uses_scm_root_dvcignore_for_external_dependency(tmp_dir, scm):
tmp_dir.gen(
{
".dvcignore": "*.pyc\n",
"main_lib": {"foo.py": "VALUE = 1\n"},
"sudirs_monorepo": {"proj1": {}},
}
)

subrepo_dir = tmp_dir / "sudirs_monorepo" / "proj1"
with subrepo_dir.chdir():
repo = Repo.init(subdir=True)
stage = repo.stage.create(
name="check-main-lib",
cmd="echo check",
deps=[os.path.join("..", "..", "main_lib")],
outs=[],
)
stage.save(run_cache=False)

pycache_dir = tmp_dir / "main_lib" / "__pycache__"
pycache_dir.mkdir()
(pycache_dir / "foo.pyc").write_bytes(b"ignored bytecode")

with lock_repo(repo):
assert not stage.changed_deps()

(tmp_dir / "main_lib" / "new.txt").write_text("not ignored")
with lock_repo(repo):
assert stage.changed_deps()


def test_ignore_resurface_subrepo(tmp_dir, scm, dvc):
tmp_dir.dvc_gen({"foo": "foo"}, commit="add foo")
subrepo_dir = tmp_dir / "subdir"
Expand Down
Loading