diff --git a/src/taskgraph/run-task/fetch-content b/src/taskgraph/run-task/fetch-content index 337d807fb..b6a800196 100755 --- a/src/taskgraph/run-task/fetch-content +++ b/src/taskgraph/run-task/fetch-content @@ -794,6 +794,35 @@ def _github_submodule_required(repo: str, commit: str): return True +GIT_FETCH_MODES = ("clone", "init_and_fetch") + + +def _populate_git_dir(git_dir, repo, commit, fetch_mode, env): + if fetch_mode == "clone": + log(f"cloning {repo} to {git_dir}") + subprocess.run(["git", "clone", "-n", repo, str(git_dir)], check=True, env=env) + return commit + + if fetch_mode == "init_and_fetch": + # https://bugzilla.mozilla.org/show_bug.cgi?id=2047876 + log(f"initializing empty repository in {git_dir}") + git_dir.mkdir(parents=True, exist_ok=True) + subprocess.run(["git", "init", "--quiet", str(git_dir)], check=True) + + log(f"adding remote origin {repo}") + subprocess.run( + ["git", "remote", "add", "origin", repo], cwd=str(git_dir), check=True + ) + + log(f"fetching {commit} from {repo}") + subprocess.run( + ["git", "fetch", "origin", commit], cwd=str(git_dir), check=True, env=env + ) + return "FETCH_HEAD" + + raise ValueError(f"unknown fetch mode: {fetch_mode}") + + def git_checkout_archive( dest_path: pathlib.Path, repo: str, @@ -801,6 +830,7 @@ def git_checkout_archive( prefix=None, ssh_key=None, include_dot_git=False, + fetch_mode="clone", ): """Produce an archive of the files comprising a Git checkout.""" dest_path.parent.mkdir(parents=True, exist_ok=True) @@ -830,11 +860,6 @@ def git_checkout_archive( git_dir = temp_dir / prefix - # This could be faster with a shallow clone. However, Git requires a ref - # to initiate a clone. Since the commit-ish may not refer to a ref, we - # simply perform a full clone followed by a checkout. - print(f"cloning {repo} to {git_dir}") - env = os.environ.copy() keypath = "" if ssh_key: @@ -854,11 +879,11 @@ def git_checkout_archive( env = {"GIT_SSH_COMMAND": f"ssh -o 'StrictHostKeyChecking no' -i {keypath}"} - subprocess.run(["git", "clone", "-n", repo, str(git_dir)], check=True, env=env) + revision = _populate_git_dir(git_dir, repo, commit, fetch_mode, env) # Always use a detached head so that git prints out what it checked out. subprocess.run( - ["git", "checkout", "--detach", commit], cwd=str(git_dir), check=True + ["git", "checkout", "--detach", revision], cwd=str(git_dir), check=True ) # When including the .git, we want --depth 1, but a direct clone would not @@ -930,6 +955,7 @@ def command_git_checkout_archive(args): prefix=args.path_prefix, ssh_key=args.ssh_key_secret, include_dot_git=args.include_dot_git, + fetch_mode=args.fetch_mode, ) except Exception: try: @@ -1084,6 +1110,13 @@ def main(): git_checkout.add_argument( "--include-dot-git", action="store_true", help="Include the .git directory" ) + git_checkout.add_argument( + "--fetch-mode", + choices=GIT_FETCH_MODES, + default="clone", + help="How to populate the checkout: clone the repository, or init an " + "empty one and fetch only the requested commit", + ) url = subparsers.add_parser("static-url", help="Download a static URL") url.set_defaults(func=command_static_url) diff --git a/src/taskgraph/transforms/fetch.py b/src/taskgraph/transforms/fetch.py index 2258cb85a..66a509b72 100644 --- a/src/taskgraph/transforms/fetch.py +++ b/src/taskgraph/transforms/fetch.py @@ -281,6 +281,11 @@ class GitFetchSchema(Schema, forbid_unknown_fields=False, kw_only=True): include_dot_git: Optional[bool] = None artifact_name: Optional[str] = None path_prefix: Optional[str] = None + # How to populate the checkout. "clone" (the default) clones the repo, + # "init_and_fetch" fetches only `revision` and leaves no branches or tags, + # for git servers that cannot serve a full clone. See + # https://bugzilla.mozilla.org/show_bug.cgi?id=2047876 + fetch_mode: Optional[Literal["clone", "init_and_fetch"]] = None # ssh-key is a taskcluster secret path (e.g. project/civet/github-deploy-key) # In the secret dictionary, the key should be specified as # "ssh_privkey": "-----BEGIN OPENSSH PRIVATE KEY-----\nkfksnb3jc..." @@ -320,6 +325,11 @@ def create_git_fetch_task(config, name, fetch): args.append("--include-dot-git") digest_data.append(".git") + fetch_mode = fetch.get("fetch-mode") + if fetch_mode and fetch_mode != "clone": + args.extend(["--fetch-mode", fetch_mode]) + digest_data.append(f"fetch-mode={fetch_mode}") + return { "command": args, "artifact_name": artifact_name, diff --git a/test/test_scripts_fetch_content.py b/test/test_scripts_fetch_content.py index a971c22c5..be52c677d 100644 --- a/test/test_scripts_fetch_content.py +++ b/test/test_scripts_fetch_content.py @@ -4,6 +4,7 @@ import pathlib import shutil import stat +import subprocess import sys import tarfile import urllib.request @@ -15,6 +16,8 @@ import taskgraph +from .conftest import nowin + @pytest.fixture(scope="module") def fetch_content_mod(): @@ -379,3 +382,138 @@ def test_merge_tree_readonly_dir_from_later_fetch(tmp_path, fetch_content_mod): assert (dest / "tests" / "a.txt").read_text() == "a" assert (dest / "tests" / "b.txt").read_text() == "b" assert stat.S_IMODE((dest / "tests").stat().st_mode) == 0o555 + + +@pytest.fixture +def local_git_repo(tmp_path): + repo = tmp_path / "upstream" + repo.mkdir() + + def run(*args): + subprocess.run( + ["git", "-C", str(repo), *args], + check=True, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + + run("init", "--quiet", "--initial-branch", "main") + run("config", "user.name", "Test") + run("config", "user.email", "test@example.com") + + (repo / "first.txt").write_text("first\n") + run("add", "first.txt") + run("commit", "--quiet", "-m", "first") + first = subprocess.run( + ["git", "-C", str(repo), "rev-parse", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + run("tag", "v1") + + (repo / "second.txt").write_text("second\n") + run("add", "second.txt") + run("commit", "--quiet", "-m", "second") + + return repo, first + + +def archive_names(fetch_content_mod, path): + with path.open("rb") as fh: + with fetch_content_mod.ZstdDecompressor().stream_reader(fh) as reader: + with tarfile.open(fileobj=reader, mode="r|") as tf: + return sorted(member.name for member in tf) + + +@nowin +def test_populate_git_dir_init_and_fetch( + fetch_content_mod, local_git_repo, tmp_path, mocker +): + repo, first = local_git_repo + spy = mocker.spy(fetch_content_mod.subprocess, "run") + git_dir = tmp_path / "checkout" + + revision = fetch_content_mod._populate_git_dir( + git_dir, f"file://{repo}", first, "init_and_fetch", os.environ.copy() + ) + + assert revision == "FETCH_HEAD" + + argvs = [call.args[0] for call in spy.call_args_list] + assert ["git", "fetch", "origin", first] in argvs + assert not any(argv[:2] == ["git", "clone"] for argv in argvs) + + assert ( + subprocess.run( + ["git", "-C", str(git_dir), "cat-file", "-t", first], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + == "commit" + ) + + +@nowin +def test_populate_git_dir_clone(fetch_content_mod, local_git_repo, tmp_path, mocker): + repo, first = local_git_repo + spy = mocker.spy(fetch_content_mod.subprocess, "run") + git_dir = tmp_path / "checkout" + + revision = fetch_content_mod._populate_git_dir( + git_dir, f"file://{repo}", first, "clone", os.environ.copy() + ) + + assert revision == first + assert [call.args[0] for call in spy.call_args_list] == [ + ["git", "clone", "-n", f"file://{repo}", str(git_dir)] + ] + + +def test_populate_git_dir_unknown_mode(fetch_content_mod, tmp_path): + with pytest.raises(ValueError): + fetch_content_mod._populate_git_dir( + tmp_path / "checkout", "file:///nonexistent", "abcdef", "nope", {} + ) + + +@nowin +def test_git_checkout_archive_fetch_modes_agree( + fetch_content_mod, local_git_repo, tmp_path +): + repo, first = local_git_repo + names = {} + + for fetch_mode in fetch_content_mod.GIT_FETCH_MODES: + dest = tmp_path / f"{fetch_mode}.tar.zst" + fetch_content_mod.git_checkout_archive( + dest, f"file://{repo}", first, prefix="checkout", fetch_mode=fetch_mode + ) + assert dest.exists() + names[fetch_mode] = archive_names(fetch_content_mod, dest) + + assert names["clone"] == names["init_and_fetch"] + assert "checkout/first.txt" in names["clone"] + assert "checkout/second.txt" not in names["clone"] + + +@nowin +def test_git_checkout_archive_init_and_fetch_include_dot_git( + fetch_content_mod, local_git_repo, tmp_path +): + repo, first = local_git_repo + dest = tmp_path / "with-dot-git.tar.zst" + + fetch_content_mod.git_checkout_archive( + dest, + f"file://{repo}", + first, + prefix="checkout", + include_dot_git=True, + fetch_mode="init_and_fetch", + ) + + names = archive_names(fetch_content_mod, dest) + assert any(name.startswith("checkout/.git/") for name in names) + assert "checkout/first.txt" in names diff --git a/test/test_transforms_fetch.py b/test/test_transforms_fetch.py index d5d93385c..fb4880b7b 100644 --- a/test/test_transforms_fetch.py +++ b/test/test_transforms_fetch.py @@ -31,6 +31,36 @@ def assert_static_url(task): assert task["attributes"]["fetch-artifact"] == "public/resource" +GIT_REVISION = "0123456789abcdef0123456789abcdef01234567" + + +def expected_git_command(*extra_args): + return [ + "fetch-content", + "git-checkout-archive", + "--path-prefix", + "repo", + "https://example.com/repo", + GIT_REVISION, + "/builds/worker/artifacts/repo.tar.zst", + *extra_args, + ] + + +def assert_git(task): + assert task["run"]["command"] == expected_git_command() + + +def assert_git_fetch_mode_clone(task): + assert task["run"]["command"] == expected_git_command() + + +def assert_git_fetch_mode_init_and_fetch(task): + assert task["run"]["command"] == expected_git_command( + "--fetch-mode", "init_and_fetch" + ) + + @pytest.mark.parametrize( "task_input", ( @@ -48,6 +78,38 @@ def assert_static_url(task): }, id="static-url", ), + pytest.param( + { + "fetch": { + "type": "git", + "repo": "https://example.com/repo", + "revision": GIT_REVISION, + }, + }, + id="git", + ), + pytest.param( + { + "fetch": { + "type": "git", + "repo": "https://example.com/repo", + "revision": GIT_REVISION, + "fetch-mode": "clone", + }, + }, + id="git-fetch-mode-clone", + ), + pytest.param( + { + "fetch": { + "type": "git", + "repo": "https://example.com/repo", + "revision": GIT_REVISION, + "fetch-mode": "init_and_fetch", + }, + }, + id="git-fetch-mode-init-and-fetch", + ), ), ) def test_transforms(request, run_transform, task_input): @@ -62,3 +124,32 @@ def test_transforms(request, run_transform, task_input): param_id = request.node.callspec.id assertion_func = globals()[f"assert_{param_id.replace('-', '_')}"] assertion_func(task) + + +@pytest.mark.parametrize( + "fetch_mode,expected_extra_digest", + ( + pytest.param(None, [], id="unset"), + pytest.param("clone", [], id="clone"), + pytest.param( + "init_and_fetch", ["fetch-mode=init_and_fetch"], id="init-and-fetch" + ), + ), +) +def test_git_fetch_mode_digest_data(fetch_mode, expected_extra_digest): + fetch_config = { + "type": "git", + "repo": "https://example.com/repo", + "revision": GIT_REVISION, + } + if fetch_mode: + fetch_config["fetch-mode"] = fetch_mode + + result = fetch.create_git_fetch_task(None, "fake-task-name", fetch_config) + + assert result["digest_data"] == [ + GIT_REVISION, + "repo", + "repo.tar.zst", + *expected_extra_digest, + ]