Skip to content

Commit e468f02

Browse files
committed
pytest_plugin(fix): Add atomic_init to *_repo fixtures for race-free master copy
why: Multiple pytest-xdist workers could initialize the master copy simultaneously, causing duplicate work and potential race conditions. what: - Use atomic_init for master copy creation in git_repo, hg_repo, svn_repo - Apply same fix to async_git_repo, async_hg_repo, async_svn_repo - Exclude .libvcs_master_initialized marker from shutil.copytree - All workers now get unique copies from master, never share master directly
1 parent 7ab501d commit e468f02

1 file changed

Lines changed: 139 additions & 109 deletions

File tree

src/libvcs/pytest_plugin.py

Lines changed: 139 additions & 109 deletions
Original file line numberDiff line numberDiff line change
@@ -979,37 +979,42 @@ def git_repo(
979979
# Unified master copy shared with async_git_repo
980980
master_copy = remote_repos_path / "git_repo_master"
981981

982-
if master_copy.exists():
983-
shutil.copytree(master_copy, new_checkout_path)
984-
repo = GitSync(url=remote_url, path=str(new_checkout_path))
985-
return RepoFixtureResult(
986-
repo=repo,
987-
path=new_checkout_path,
988-
remote_url=remote_url,
989-
master_copy_path=master_copy,
990-
created_at=created_at,
991-
from_cache=True,
982+
def create_master() -> None:
983+
"""Create master copy atomically - only one worker does this."""
984+
repo = GitSync(
985+
url=remote_url,
986+
path=master_copy,
987+
remotes={
988+
"origin": GitRemote(
989+
name="origin",
990+
push_url=remote_url,
991+
fetch_url=remote_url,
992+
),
993+
},
992994
)
995+
repo.obtain()
993996

994-
repo = GitSync(
995-
url=remote_url,
996-
path=master_copy,
997-
remotes={
998-
"origin": GitRemote(
999-
name="origin",
1000-
push_url=remote_url,
1001-
fetch_url=remote_url,
1002-
),
1003-
},
997+
# atomic_init returns True if this process did the init, False if waited
998+
from_cache = not atomic_init(
999+
master_copy,
1000+
create_master,
1001+
marker_name=".libvcs_master_initialized",
10041002
)
1005-
repo.obtain()
1003+
1004+
# All workers get a unique copy from master (exclude marker file)
1005+
shutil.copytree(
1006+
master_copy,
1007+
new_checkout_path,
1008+
ignore=shutil.ignore_patterns(".libvcs_master_initialized"),
1009+
)
1010+
repo = GitSync(url=remote_url, path=str(new_checkout_path))
10061011
return RepoFixtureResult(
10071012
repo=repo,
1008-
path=master_copy,
1013+
path=new_checkout_path,
10091014
remote_url=remote_url,
10101015
master_copy_path=master_copy,
10111016
created_at=created_at,
1012-
from_cache=False,
1017+
from_cache=from_cache,
10131018
)
10141019

10151020

@@ -1031,27 +1036,32 @@ def hg_repo(
10311036
# Unified master copy shared with async_hg_repo
10321037
master_copy = remote_repos_path / "hg_repo_master"
10331038

1034-
if master_copy.exists():
1035-
shutil.copytree(master_copy, new_checkout_path)
1036-
repo = HgSync(url=remote_url, path=str(new_checkout_path))
1037-
return RepoFixtureResult(
1038-
repo=repo,
1039-
path=new_checkout_path,
1040-
remote_url=remote_url,
1041-
master_copy_path=master_copy,
1042-
created_at=created_at,
1043-
from_cache=True,
1044-
)
1039+
def create_master() -> None:
1040+
"""Create master copy atomically - only one worker does this."""
1041+
repo = HgSync(url=remote_url, path=master_copy)
1042+
repo.obtain()
1043+
1044+
# atomic_init returns True if this process did the init, False if waited
1045+
from_cache = not atomic_init(
1046+
master_copy,
1047+
create_master,
1048+
marker_name=".libvcs_master_initialized",
1049+
)
10451050

1046-
repo = HgSync(url=remote_url, path=master_copy)
1047-
repo.obtain()
1051+
# All workers get a unique copy from master (exclude marker file)
1052+
shutil.copytree(
1053+
master_copy,
1054+
new_checkout_path,
1055+
ignore=shutil.ignore_patterns(".libvcs_master_initialized"),
1056+
)
1057+
repo = HgSync(url=remote_url, path=str(new_checkout_path))
10481058
return RepoFixtureResult(
10491059
repo=repo,
1050-
path=master_copy,
1060+
path=new_checkout_path,
10511061
remote_url=remote_url,
10521062
master_copy_path=master_copy,
10531063
created_at=created_at,
1054-
from_cache=False,
1064+
from_cache=from_cache,
10551065
)
10561066

10571067

@@ -1072,27 +1082,32 @@ def svn_repo(
10721082
# Unified master copy shared with async_svn_repo
10731083
master_copy = remote_repos_path / "svn_repo_master"
10741084

1075-
if master_copy.exists():
1076-
shutil.copytree(master_copy, new_checkout_path)
1077-
repo = SvnSync(url=remote_url, path=str(new_checkout_path))
1078-
return RepoFixtureResult(
1079-
repo=repo,
1080-
path=new_checkout_path,
1081-
remote_url=remote_url,
1082-
master_copy_path=master_copy,
1083-
created_at=created_at,
1084-
from_cache=True,
1085-
)
1085+
def create_master() -> None:
1086+
"""Create master copy atomically - only one worker does this."""
1087+
repo = SvnSync(url=remote_url, path=str(master_copy))
1088+
repo.obtain()
1089+
1090+
# atomic_init returns True if this process did the init, False if waited
1091+
from_cache = not atomic_init(
1092+
master_copy,
1093+
create_master,
1094+
marker_name=".libvcs_master_initialized",
1095+
)
10861096

1087-
repo = SvnSync(url=remote_url, path=str(master_copy))
1088-
repo.obtain()
1097+
# All workers get a unique copy from master (exclude marker file)
1098+
shutil.copytree(
1099+
master_copy,
1100+
new_checkout_path,
1101+
ignore=shutil.ignore_patterns(".libvcs_master_initialized"),
1102+
)
1103+
repo = SvnSync(url=remote_url, path=str(new_checkout_path))
10891104
return RepoFixtureResult(
10901105
repo=repo,
1091-
path=master_copy,
1106+
path=new_checkout_path,
10921107
remote_url=remote_url,
10931108
master_copy_path=master_copy,
10941109
created_at=created_at,
1095-
from_cache=False,
1110+
from_cache=from_cache,
10961111
)
10971112

10981113

@@ -1129,38 +1144,43 @@ async def async_git_repo(
11291144
# Unified master copy shared with git_repo
11301145
master_copy = remote_repos_path / "git_repo_master"
11311146

1132-
if master_copy.exists():
1133-
shutil.copytree(master_copy, new_checkout_path)
1134-
repo = AsyncGitSync(url=remote_url, path=new_checkout_path)
1135-
yield RepoFixtureResult(
1136-
repo=repo,
1137-
path=new_checkout_path,
1138-
remote_url=remote_url,
1139-
master_copy_path=master_copy,
1140-
created_at=created_at,
1141-
from_cache=True,
1147+
def create_master() -> None:
1148+
"""Create master copy atomically - only one worker does this."""
1149+
# Use sync GitSync for atomic init (only runs once per session)
1150+
sync_repo = GitSync(
1151+
url=remote_url,
1152+
path=master_copy,
1153+
remotes={
1154+
"origin": GitRemote(
1155+
name="origin",
1156+
push_url=remote_url,
1157+
fetch_url=remote_url,
1158+
),
1159+
},
11421160
)
1143-
return
1161+
sync_repo.obtain()
11441162

1145-
repo = AsyncGitSync(
1146-
url=remote_url,
1147-
path=master_copy,
1148-
remotes={
1149-
"origin": GitRemote(
1150-
name="origin",
1151-
push_url=remote_url,
1152-
fetch_url=remote_url,
1153-
),
1154-
},
1163+
# atomic_init returns True if this process did the init, False if waited
1164+
from_cache = not atomic_init(
1165+
master_copy,
1166+
create_master,
1167+
marker_name=".libvcs_master_initialized",
11551168
)
1156-
await repo.obtain()
1169+
1170+
# All workers get a unique copy from master (exclude marker file)
1171+
shutil.copytree(
1172+
master_copy,
1173+
new_checkout_path,
1174+
ignore=shutil.ignore_patterns(".libvcs_master_initialized"),
1175+
)
1176+
repo = AsyncGitSync(url=remote_url, path=new_checkout_path)
11571177
yield RepoFixtureResult(
11581178
repo=repo,
1159-
path=master_copy,
1179+
path=new_checkout_path,
11601180
remote_url=remote_url,
11611181
master_copy_path=master_copy,
11621182
created_at=created_at,
1163-
from_cache=False,
1183+
from_cache=from_cache,
11641184
)
11651185

11661186
@pytest_asyncio.fixture
@@ -1190,28 +1210,33 @@ async def async_hg_repo(
11901210
# Unified master copy shared with hg_repo
11911211
master_copy = remote_repos_path / "hg_repo_master"
11921212

1193-
if master_copy.exists():
1194-
shutil.copytree(master_copy, new_checkout_path)
1195-
repo = AsyncHgSync(url=remote_url, path=new_checkout_path)
1196-
yield RepoFixtureResult(
1197-
repo=repo,
1198-
path=new_checkout_path,
1199-
remote_url=remote_url,
1200-
master_copy_path=master_copy,
1201-
created_at=created_at,
1202-
from_cache=True,
1203-
)
1204-
return
1213+
def create_master() -> None:
1214+
"""Create master copy atomically - only one worker does this."""
1215+
# Use sync HgSync for atomic init (only runs once per session)
1216+
sync_repo = HgSync(url=remote_url, path=master_copy)
1217+
sync_repo.obtain()
1218+
1219+
# atomic_init returns True if this process did the init, False if waited
1220+
from_cache = not atomic_init(
1221+
master_copy,
1222+
create_master,
1223+
marker_name=".libvcs_master_initialized",
1224+
)
12051225

1206-
repo = AsyncHgSync(url=remote_url, path=master_copy)
1207-
await repo.obtain()
1226+
# All workers get a unique copy from master (exclude marker file)
1227+
shutil.copytree(
1228+
master_copy,
1229+
new_checkout_path,
1230+
ignore=shutil.ignore_patterns(".libvcs_master_initialized"),
1231+
)
1232+
repo = AsyncHgSync(url=remote_url, path=new_checkout_path)
12081233
yield RepoFixtureResult(
12091234
repo=repo,
1210-
path=master_copy,
1235+
path=new_checkout_path,
12111236
remote_url=remote_url,
12121237
master_copy_path=master_copy,
12131238
created_at=created_at,
1214-
from_cache=False,
1239+
from_cache=from_cache,
12151240
)
12161241

12171242
@pytest_asyncio.fixture
@@ -1241,28 +1266,33 @@ async def async_svn_repo(
12411266
# Unified master copy shared with svn_repo
12421267
master_copy = remote_repos_path / "svn_repo_master"
12431268

1244-
if master_copy.exists():
1245-
shutil.copytree(master_copy, new_checkout_path)
1246-
repo = AsyncSvnSync(url=remote_url, path=new_checkout_path)
1247-
yield RepoFixtureResult(
1248-
repo=repo,
1249-
path=new_checkout_path,
1250-
remote_url=remote_url,
1251-
master_copy_path=master_copy,
1252-
created_at=created_at,
1253-
from_cache=True,
1254-
)
1255-
return
1269+
def create_master() -> None:
1270+
"""Create master copy atomically - only one worker does this."""
1271+
# Use sync SvnSync for atomic init (only runs once per session)
1272+
sync_repo = SvnSync(url=remote_url, path=str(master_copy))
1273+
sync_repo.obtain()
1274+
1275+
# atomic_init returns True if this process did the init, False if waited
1276+
from_cache = not atomic_init(
1277+
master_copy,
1278+
create_master,
1279+
marker_name=".libvcs_master_initialized",
1280+
)
12561281

1257-
repo = AsyncSvnSync(url=remote_url, path=master_copy)
1258-
await repo.obtain()
1282+
# All workers get a unique copy from master (exclude marker file)
1283+
shutil.copytree(
1284+
master_copy,
1285+
new_checkout_path,
1286+
ignore=shutil.ignore_patterns(".libvcs_master_initialized"),
1287+
)
1288+
repo = AsyncSvnSync(url=remote_url, path=new_checkout_path)
12591289
yield RepoFixtureResult(
12601290
repo=repo,
1261-
path=master_copy,
1291+
path=new_checkout_path,
12621292
remote_url=remote_url,
12631293
master_copy_path=master_copy,
12641294
created_at=created_at,
1265-
from_cache=False,
1295+
from_cache=from_cache,
12661296
)
12671297

12681298

0 commit comments

Comments
 (0)