Replace blobless clone with size-limited partial clone

Rename CloneStrategy.BLOBLESS to PARTIAL and switch the git filter from
blob:none to blob:limit=$BLOB_SIZE_LIMIT (default 1m). Small blobs come
down with the initial clone, so enrichment avoids a round-trip per file
while still skipping oversized blobs. The limit is configurable via the
BLOB_SIZE_LIMIT env var.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

Author: J03D03

tests/enrichment/test_batch_processing.py

@@ -6,10 +6,10 @@
 
 
 class TestPickCloneStrategy:
-    def test_below_threshold_is_blobless(self):
+    def test_below_threshold_is_partial(self):
         commits = {"https://github.com/a/a": {"c1", "c2", "c3"}}
         assert _pick_clone_strategy(commits, threshold=10) == {
-            "https://github.com/a/a": CloneStrategy.BLOBLESS
+            "https://github.com/a/a": CloneStrategy.PARTIAL
         }
 
     def test_at_threshold_is_full(self):
@@ -24,7 +24,7 @@ def test_mixed_repos(self):
             "https://github.com/big/repo": {f"c{i}" for i in range(50)},
         }
         result = _pick_clone_strategy(commits, threshold=10)
-        assert result["https://github.com/small/repo"] is CloneStrategy.BLOBLESS
+        assert result["https://github.com/small/repo"] is CloneStrategy.PARTIAL
         assert result["https://github.com/big/repo"] is CloneStrategy.FULL
 
     def test_default_threshold_matches_config(self):
@@ -34,7 +34,7 @@ def test_default_threshold_matches_config(self):
         }
         result = _pick_clone_strategy(commits)
         assert result["https://github.com/a/a"] is CloneStrategy.FULL
-        assert result["https://github.com/b/b"] is CloneStrategy.BLOBLESS
+        assert result["https://github.com/b/b"] is CloneStrategy.PARTIAL
 
     def test_empty_input(self):
         assert _pick_clone_strategy({}) == {}

vfc_datasets/config.py

@@ -47,6 +47,10 @@ def _env_path(name: str, default: Path) -> Path:
 # Clone strategy: repos with >= this many commits to enrich get a full clone
 FULL_CLONE_THRESHOLD = _env_int("FULL_CLONE_THRESHOLD", 100, minimum=1)
 
+# Partial clone size filter. Git accepts suffixes (k/m/g). Blobs above this
+# size are omitted during clone and fetched lazily on access.
+BLOB_SIZE_LIMIT = os.getenv("BLOB_SIZE_LIMIT", "1m")
+
 # Dataset Caching
 USE_DATASET_CACHE = os.getenv("USE_DATASET_CACHE", "true").lower() in ("true", "yes")
 

vfc_datasets/transformations/enrichment/batch_processing.py

@@ -22,9 +22,9 @@ def _pick_clone_strategy(
     commits_by_url: dict[str, set[str]],
     threshold: int = FULL_CLONE_THRESHOLD,
 ) -> dict[str, CloneStrategy]:
-    """Pick FULL for repos with >= threshold commits to enrich, else BLOBLESS."""
+    """Pick FULL for repos with >= threshold commits to enrich, else PARTIAL."""
     return {
-        url: (CloneStrategy.FULL if len(cids) >= threshold else CloneStrategy.BLOBLESS)
+        url: (CloneStrategy.FULL if len(cids) >= threshold else CloneStrategy.PARTIAL)
         for url, cids in commits_by_url.items()
     }
 

vfc_datasets/utils/git/repository.py

@@ -1,7 +1,6 @@
 import enum
 import errno
 import logging
-import multiprocessing
 import shutil
 from collections.abc import Iterable
 from concurrent.futures import ThreadPoolExecutor, as_completed
@@ -11,7 +10,7 @@
 from git import GitCommandError, InvalidGitRepositoryError, Repo
 from tqdm.auto import tqdm
 
-from vfc_datasets.config import GIT_CLONE_TIMEOUT, MAX_CLONE_WORKERS
+from vfc_datasets.config import BLOB_SIZE_LIMIT, GIT_CLONE_TIMEOUT, MAX_CLONE_WORKERS
 from vfc_datasets.dataset_entry import DatasetEntry
 
 from .url import url_to_pathname
@@ -22,11 +21,11 @@
 class CloneStrategy(enum.Enum):
     """How much of a repository to fetch.
 
-    BLOBLESS: partial clone, on-demand blob fetch. Cheap for few commits/repo.
+    PARTIAL: partial clone filtered by BLOB_SIZE_LIMIT; large blobs fetched on demand.
     FULL: all objects local. Cheaper once commits/repo exceeds FULL_CLONE_THRESHOLD.
     """
 
-    BLOBLESS = "blobless"
+    PARTIAL = "partial"
     FULL = "full"
 
 
@@ -188,8 +187,8 @@ def _clone_new_repo(
         "-c",
         "http.lowSpeedTime=600",  # 10 min before timeout
     ]
-    if strategy is CloneStrategy.BLOBLESS:
-        clone_options.extend(["--filter", "blob:none"])
+    if strategy is CloneStrategy.PARTIAL:
+        clone_options.extend(["--filter", f"blob:limit={BLOB_SIZE_LIMIT}"])
 
     try:
         repo = Repo.clone_from(
@@ -225,7 +224,7 @@ def clone_repository(
     git_url: str,
     branch: str | None = None,
     timeout: int | None = None,
-    strategy: CloneStrategy = CloneStrategy.BLOBLESS,
+    strategy: CloneStrategy = CloneStrategy.PARTIAL,
 ) -> Repo | None:
     """Clone or reuse a git repository with a simplified interface."""
     git_url = (git_url or "").strip()
@@ -251,13 +250,13 @@ def clone_repositories(
     max_workers: int | None = None,
     branch: str | None = None,
     timeout: int | None = None,
-    strategy: CloneStrategy | dict[str, CloneStrategy] = CloneStrategy.BLOBLESS,
+    strategy: CloneStrategy | dict[str, CloneStrategy] = CloneStrategy.PARTIAL,
 ) -> dict[str, Repo | None]:
     """Clone all repositories from dataset entries using parallel processing.
 
     ``strategy`` may be a single CloneStrategy applied to every URL, or a dict
     mapping project_url -> CloneStrategy. URLs missing from the dict fall back
-    to BLOBLESS.
+    to PARTIAL.
     """
     project_urls = {entry.project_url for entry in entries if entry.project_url}
 
@@ -273,7 +272,7 @@ def clone_repositories(
 
     def _strategy_for(url: str) -> CloneStrategy:
         if isinstance(strategy, dict):
-            return strategy.get(url, CloneStrategy.BLOBLESS)
+            return strategy.get(url, CloneStrategy.PARTIAL)
         return strategy
 
     results = {}