feat(fetch): add fetch-mode to work around HuggingFace clone failures (bug 2047876)

src/taskgraph/run-task/fetch-content

@@ -794,13 +794,43 @@ def _github_submodule_required(repo: str, commit: str):
         return True
 
 
+GIT_FETCH_MODES = ("clone", "init_and_fetch")
+
+
+def _populate_git_dir(git_dir, repo, commit, fetch_mode, env):
+    if fetch_mode == "clone":
+        log(f"cloning {repo} to {git_dir}")
+        subprocess.run(["git", "clone", "-n", repo, str(git_dir)], check=True, env=env)
+        return commit
+
+    if fetch_mode == "init_and_fetch":
+        # https://bugzilla.mozilla.org/show_bug.cgi?id=2047876
+        log(f"initializing empty repository in {git_dir}")
+        git_dir.mkdir(parents=True, exist_ok=True)
+        subprocess.run(["git", "init", "--quiet", str(git_dir)], check=True)
+
+        log(f"adding remote origin {repo}")
+        subprocess.run(
+            ["git", "remote", "add", "origin", repo], cwd=str(git_dir), check=True
+        )
+
+        log(f"fetching {commit} from {repo}")
+        subprocess.run(
+            ["git", "fetch", "origin", commit], cwd=str(git_dir), check=True, env=env
+        )
+        return "FETCH_HEAD"
+
+    raise ValueError(f"unknown fetch mode: {fetch_mode}")
+
+
 def git_checkout_archive(
     dest_path: pathlib.Path,
     repo: str,
     commit: str,
     prefix=None,
     ssh_key=None,
     include_dot_git=False,
+    fetch_mode="clone",
 ):
     """Produce an archive of the files comprising a Git checkout."""
     dest_path.parent.mkdir(parents=True, exist_ok=True)
@@ -830,11 +860,6 @@ def git_checkout_archive(
 
         git_dir = temp_dir / prefix
 
-        # This could be faster with a shallow clone. However, Git requires a ref
-        # to initiate a clone. Since the commit-ish may not refer to a ref, we
-        # simply perform a full clone followed by a checkout.
-        print(f"cloning {repo} to {git_dir}")
-
         env = os.environ.copy()
         keypath = ""
         if ssh_key:
@@ -854,11 +879,11 @@ def git_checkout_archive(
 
             env = {"GIT_SSH_COMMAND": f"ssh -o 'StrictHostKeyChecking no' -i {keypath}"}
 
-        subprocess.run(["git", "clone", "-n", repo, str(git_dir)], check=True, env=env)
+        revision = _populate_git_dir(git_dir, repo, commit, fetch_mode, env)
 
         # Always use a detached head so that git prints out what it checked out.
         subprocess.run(
-            ["git", "checkout", "--detach", commit], cwd=str(git_dir), check=True
+            ["git", "checkout", "--detach", revision], cwd=str(git_dir), check=True
         )
 
         # When including the .git, we want --depth 1, but a direct clone would not
@@ -930,6 +955,7 @@ def command_git_checkout_archive(args):
             prefix=args.path_prefix,
             ssh_key=args.ssh_key_secret,
             include_dot_git=args.include_dot_git,
+            fetch_mode=args.fetch_mode,
         )
     except Exception:
         try:
@@ -1084,6 +1110,13 @@ def main():
     git_checkout.add_argument(
         "--include-dot-git", action="store_true", help="Include the .git directory"
     )
+    git_checkout.add_argument(
+        "--fetch-mode",
+        choices=GIT_FETCH_MODES,
+        default="clone",
+        help="How to populate the checkout: clone the repository, or init an "
+        "empty one and fetch only the requested commit",
+    )
 
     url = subparsers.add_parser("static-url", help="Download a static URL")
     url.set_defaults(func=command_static_url)

src/taskgraph/transforms/fetch.py

@@ -281,6 +281,11 @@ class GitFetchSchema(Schema, forbid_unknown_fields=False, kw_only=True):
     include_dot_git: Optional[bool] = None
     artifact_name: Optional[str] = None
     path_prefix: Optional[str] = None
+    # How to populate the checkout. "clone" (the default) clones the repo,
+    # "init_and_fetch" fetches only `revision` and leaves no branches or tags,
+    # for git servers that cannot serve a full clone. See
+    # https://bugzilla.mozilla.org/show_bug.cgi?id=2047876
+    fetch_mode: Optional[Literal["clone", "init_and_fetch"]] = None
     # ssh-key is a taskcluster secret path (e.g. project/civet/github-deploy-key)
     # In the secret dictionary, the key should be specified as
     #  "ssh_privkey": "-----BEGIN OPENSSH PRIVATE KEY-----\nkfksnb3jc..."
@@ -320,6 +325,11 @@ def create_git_fetch_task(config, name, fetch):
         args.append("--include-dot-git")
         digest_data.append(".git")
 
+    fetch_mode = fetch.get("fetch-mode")
+    if fetch_mode and fetch_mode != "clone":
+        args.extend(["--fetch-mode", fetch_mode])
+        digest_data.append(f"fetch-mode={fetch_mode}")
+
     return {
         "command": args,
         "artifact_name": artifact_name,

test/test_scripts_fetch_content.py

@@ -4,6 +4,7 @@
 import pathlib
 import shutil
 import stat
+import subprocess
 import sys
 import tarfile
 import urllib.request
@@ -15,6 +16,8 @@
 
 import taskgraph
 
+from .conftest import nowin
+
 
 @pytest.fixture(scope="module")
 def fetch_content_mod():
@@ -379,3 +382,138 @@ def test_merge_tree_readonly_dir_from_later_fetch(tmp_path, fetch_content_mod):
     assert (dest / "tests" / "a.txt").read_text() == "a"
     assert (dest / "tests" / "b.txt").read_text() == "b"
     assert stat.S_IMODE((dest / "tests").stat().st_mode) == 0o555
+
+
+@pytest.fixture
+def local_git_repo(tmp_path):
+    repo = tmp_path / "upstream"
+    repo.mkdir()
+
+    def run(*args):
+        subprocess.run(
+            ["git", "-C", str(repo), *args],
+            check=True,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+
+    run("init", "--quiet", "--initial-branch", "main")
+    run("config", "user.name", "Test")
+    run("config", "user.email", "test@example.com")
+
+    (repo / "first.txt").write_text("first\n")
+    run("add", "first.txt")
+    run("commit", "--quiet", "-m", "first")
+    first = subprocess.run(
+        ["git", "-C", str(repo), "rev-parse", "HEAD"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    run("tag", "v1")
+
+    (repo / "second.txt").write_text("second\n")
+    run("add", "second.txt")
+    run("commit", "--quiet", "-m", "second")
+
+    return repo, first
+
+
+def archive_names(fetch_content_mod, path):
+    with path.open("rb") as fh:
+        with fetch_content_mod.ZstdDecompressor().stream_reader(fh) as reader:
+            with tarfile.open(fileobj=reader, mode="r|") as tf:
+                return sorted(member.name for member in tf)
+
+
+@nowin
+def test_populate_git_dir_init_and_fetch(
+    fetch_content_mod, local_git_repo, tmp_path, mocker
+):
+    repo, first = local_git_repo
+    spy = mocker.spy(fetch_content_mod.subprocess, "run")
+    git_dir = tmp_path / "checkout"
+
+    revision = fetch_content_mod._populate_git_dir(
+        git_dir, f"file://{repo}", first, "init_and_fetch", os.environ.copy()
+    )
+
+    assert revision == "FETCH_HEAD"
+
+    argvs = [call.args[0] for call in spy.call_args_list]
+    assert ["git", "fetch", "origin", first] in argvs
+    assert not any(argv[:2] == ["git", "clone"] for argv in argvs)
+
+    assert (
+        subprocess.run(
+            ["git", "-C", str(git_dir), "cat-file", "-t", first],
+            check=True,
+            capture_output=True,
+            text=True,
+        ).stdout.strip()
+        == "commit"
+    )
+
+
+@nowin
+def test_populate_git_dir_clone(fetch_content_mod, local_git_repo, tmp_path, mocker):
+    repo, first = local_git_repo
+    spy = mocker.spy(fetch_content_mod.subprocess, "run")
+    git_dir = tmp_path / "checkout"
+
+    revision = fetch_content_mod._populate_git_dir(
+        git_dir, f"file://{repo}", first, "clone", os.environ.copy()
+    )
+
+    assert revision == first
+    assert [call.args[0] for call in spy.call_args_list] == [
+        ["git", "clone", "-n", f"file://{repo}", str(git_dir)]
+    ]
+
+
+def test_populate_git_dir_unknown_mode(fetch_content_mod, tmp_path):
+    with pytest.raises(ValueError):
+        fetch_content_mod._populate_git_dir(
+            tmp_path / "checkout", "file:///nonexistent", "abcdef", "nope", {}
+        )
+
+
+@nowin
+def test_git_checkout_archive_fetch_modes_agree(
+    fetch_content_mod, local_git_repo, tmp_path
+):
+    repo, first = local_git_repo
+    names = {}
+
+    for fetch_mode in fetch_content_mod.GIT_FETCH_MODES:
+        dest = tmp_path / f"{fetch_mode}.tar.zst"
+        fetch_content_mod.git_checkout_archive(
+            dest, f"file://{repo}", first, prefix="checkout", fetch_mode=fetch_mode
+        )
+        assert dest.exists()
+        names[fetch_mode] = archive_names(fetch_content_mod, dest)
+
+    assert names["clone"] == names["init_and_fetch"]
+    assert "checkout/first.txt" in names["clone"]
+    assert "checkout/second.txt" not in names["clone"]
+
+
+@nowin
+def test_git_checkout_archive_init_and_fetch_include_dot_git(
+    fetch_content_mod, local_git_repo, tmp_path
+):
+    repo, first = local_git_repo
+    dest = tmp_path / "with-dot-git.tar.zst"
+
+    fetch_content_mod.git_checkout_archive(
+        dest,
+        f"file://{repo}",
+        first,
+        prefix="checkout",
+        include_dot_git=True,
+        fetch_mode="init_and_fetch",
+    )
+
+    names = archive_names(fetch_content_mod, dest)
+    assert any(name.startswith("checkout/.git/") for name in names)
+    assert "checkout/first.txt" in names

test/test_transforms_fetch.py

@@ -31,6 +31,36 @@ def assert_static_url(task):
     assert task["attributes"]["fetch-artifact"] == "public/resource"
 
 
+GIT_REVISION = "0123456789abcdef0123456789abcdef01234567"
+
+
+def expected_git_command(*extra_args):
+    return [
+        "fetch-content",
+        "git-checkout-archive",
+        "--path-prefix",
+        "repo",
+        "https://example.com/repo",
+        GIT_REVISION,
+        "/builds/worker/artifacts/repo.tar.zst",
+        *extra_args,
+    ]
+
+
+def assert_git(task):
+    assert task["run"]["command"] == expected_git_command()
+
+
+def assert_git_fetch_mode_clone(task):
+    assert task["run"]["command"] == expected_git_command()
+
+
+def assert_git_fetch_mode_init_and_fetch(task):
+    assert task["run"]["command"] == expected_git_command(
+        "--fetch-mode", "init_and_fetch"
+    )
+
+
 @pytest.mark.parametrize(
     "task_input",
     (
@@ -48,6 +78,38 @@ def assert_static_url(task):
             },
             id="static-url",
         ),
+        pytest.param(
+            {
+                "fetch": {
+                    "type": "git",
+                    "repo": "https://example.com/repo",
+                    "revision": GIT_REVISION,
+                },
+            },
+            id="git",
+        ),
+        pytest.param(
+            {
+                "fetch": {
+                    "type": "git",
+                    "repo": "https://example.com/repo",
+                    "revision": GIT_REVISION,
+                    "fetch-mode": "clone",
+                },
+            },
+            id="git-fetch-mode-clone",
+        ),
+        pytest.param(
+            {
+                "fetch": {
+                    "type": "git",
+                    "repo": "https://example.com/repo",
+                    "revision": GIT_REVISION,
+                    "fetch-mode": "init_and_fetch",
+                },
+            },
+            id="git-fetch-mode-init-and-fetch",
+        ),
     ),
 )
 def test_transforms(request, run_transform, task_input):
@@ -62,3 +124,32 @@ def test_transforms(request, run_transform, task_input):
     param_id = request.node.callspec.id
     assertion_func = globals()[f"assert_{param_id.replace('-', '_')}"]
     assertion_func(task)
+
+
+@pytest.mark.parametrize(
+    "fetch_mode,expected_extra_digest",
+    (
+        pytest.param(None, [], id="unset"),
+        pytest.param("clone", [], id="clone"),
+        pytest.param(
+            "init_and_fetch", ["fetch-mode=init_and_fetch"], id="init-and-fetch"
+        ),
+    ),
+)
+def test_git_fetch_mode_digest_data(fetch_mode, expected_extra_digest):
+    fetch_config = {
+        "type": "git",
+        "repo": "https://example.com/repo",
+        "revision": GIT_REVISION,
+    }
+    if fetch_mode:
+        fetch_config["fetch-mode"] = fetch_mode
+
+    result = fetch.create_git_fetch_task(None, "fake-task-name", fetch_config)
+
+    assert result["digest_data"] == [
+        GIT_REVISION,
+        "repo",
+        "repo.tar.zst",
+        *expected_extra_digest,
+    ]