Skip large diffs and limit clone concurrency

Author: J03D03

vfc_datasets/config.py

@@ -42,6 +42,7 @@ def _env_path(name: str, default: Path) -> Path:
 
 # Git Operations
 GIT_CLONE_TIMEOUT = _env_int("GIT_CLONE_TIMEOUT", 3600, minimum=1)  # 1 hour default
+MAX_CLONE_WORKERS = _env_int("MAX_CLONE_WORKERS", 4, minimum=1)
 
 # Clone strategy: repos with >= this many commits to enrich get a full clone
 FULL_CLONE_THRESHOLD = _env_int("FULL_CLONE_THRESHOLD", 100, minimum=1)

vfc_datasets/transformations/enrichment/add_commit_data_local.py

@@ -21,12 +21,15 @@ def _get_commit_info(repo: Repo, commit_id: str, max_diff_size: int) -> CommitDa
     """Get commit info from an open Repo object."""
     try:
         commit = repo.commit(commit_id)
-        diff_text = get_commit_diff(repo, commit_id)
 
-        # Check diff size limit
-        diff_data: str | None = diff_text
-        if len(diff_text) > max_diff_size:
-            diff_data = None
+        # Note: Using line count as a safe fast-path heuristic for max_diff_size (chars).
+        if commit.stats.total["lines"] > max_diff_size:
+            return None
+
+        diff_data: str | None = None
+        diff_text = get_commit_diff(repo, commit_id)
+        if len(diff_text) <= max_diff_size:
+            diff_data = diff_text
 
         return CommitData(
             message=str(commit.message),

vfc_datasets/transformations/enrichment/add_no_comment.py

@@ -232,13 +232,14 @@ def _process_diff_item(diff_item: Any, repo: Repo, include_unsupported: bool) ->
 
 
 def _get_diff_no_comments(
-    repo: Repo, commit_id: str, include_unsupported: bool = True
+    repo: Repo, commit_id: str, max_diff_size: int, include_unsupported: bool = True
 ) -> str | None:
     """Generate diff with comments stripped.
 
     Args:
         repo: Git repository object
         commit_id: Commit SHA to process
+        max_diff_size: Maximum diff size (heuristic)
         include_unsupported: If True, include original diff for unsupported files.
                             If False, skip unsupported files entirely.
     """
@@ -247,6 +248,10 @@ def _get_diff_no_comments(
         if not commit.parents:
             return None
 
+        # Note: Using line count as a safe fast-path heuristic for max_diff_size (chars).
+        if commit.stats.total["lines"] > max_diff_size:
+            return None
+
         diffs = commit.parents[0].diff(commit, create_patch=True)
         file_diffs: list[str] = []
 
@@ -261,15 +266,17 @@ def _get_diff_no_comments(
         return None
 
 
-def _process_batch(args: tuple[str, list[str], bool]) -> dict[str, str]:
+def _process_batch(args: tuple[str, list[str], int, bool]) -> dict[str, str]:
     """Process a batch of commits."""
-    repo_path, commit_ids, include_unsupported = args
+    repo_path, commit_ids, max_diff_size, include_unsupported = args
     results: dict[str, str] = {}
 
     try:
         with Repo(repo_path) as repository:
             for commit_id in commit_ids:
-                diff = _get_diff_no_comments(repository, commit_id, include_unsupported)
+                diff = _get_diff_no_comments(
+                    repository, commit_id, max_diff_size, include_unsupported
+                )
                 if diff is not None:
                     results[commit_id] = diff
     except Exception as e:
@@ -313,6 +320,6 @@ def strip_diff_comments(
         ),
         batch_fn=_process_batch,
         apply_fn=_apply_diff,
-        batch_extra_args=(include_unsupported,),
+        batch_extra_args=(MAX_DIFF_SIZE, include_unsupported),
         desc="Stripping comments",
     )

vfc_datasets/utils/git/repository.py

@@ -4,14 +4,14 @@
 import multiprocessing
 import shutil
 from collections.abc import Iterable
-from concurrent.futures import ProcessPoolExecutor, as_completed
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from pathlib import Path
 from urllib.parse import urlparse
 
 from git import GitCommandError, InvalidGitRepositoryError, Repo
 from tqdm.auto import tqdm
 
-from vfc_datasets.config import GIT_CLONE_TIMEOUT
+from vfc_datasets.config import GIT_CLONE_TIMEOUT, MAX_CLONE_WORKERS
 from vfc_datasets.dataset_entry import DatasetEntry
 
 from .url import url_to_pathname
@@ -269,7 +269,7 @@ def clone_repositories(
         timeout = GIT_CLONE_TIMEOUT
 
     if max_workers is None:
-        max_workers = min(multiprocessing.cpu_count(), 16, len(project_urls))
+        max_workers = min(MAX_CLONE_WORKERS, len(project_urls))
 
     def _strategy_for(url: str) -> CloneStrategy:
         if isinstance(strategy, dict):
@@ -279,7 +279,7 @@ def _strategy_for(url: str) -> CloneStrategy:
     results = {}
     failed_urls = []
 
-    with ProcessPoolExecutor(max_workers=max_workers) as executor:
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
         future_to_url = {
             executor.submit(
                 clone_repository,