From 1a24a830be250ec28bb729e4707c5fc3a72ee09b Mon Sep 17 00:00:00 2001 From: Ruoqing He Date: Tue, 30 Jun 2026 08:56:33 +0000 Subject: [PATCH] ci: Enable per-crate selective pipeline generation Running the full pipeline for every change in a Cargo workspace retests crates a pull request never touched. Add a `--workspace-selective` parameter that, when set to `True`, emits one step per workspace member tagged with Buildkite's native `if_changed` property, so each crate's shared crate-scoped tests only run when files in that crate (or in a workspace-global path) change. Workspace members are discovered by parsing the workspace's `Cargo.toml` directly so the script does not need `cargo` on the host. The shared test description is supplied via `-t`; a crate may extend it with its own `.buildkite/test_description.json`, whose tests run only when files in that crate change. Signed-off-by: Ruoqing He --- .buildkite/autogenerate_pipeline.py | 208 ++++++++++++++++++++++++---- 1 file changed, 181 insertions(+), 27 deletions(-) diff --git a/.buildkite/autogenerate_pipeline.py b/.buildkite/autogenerate_pipeline.py index 26f51df..eb076d6 100755 --- a/.buildkite/autogenerate_pipeline.py +++ b/.buildkite/autogenerate_pipeline.py @@ -46,6 +46,12 @@ DOCKER_PLUGIN_CONFIG='{ }' TIMEOUTS_MIN='["style": 30]' ``` + +When invoked with `--workspace-selective`, the script emits one step per +workspace crate tagged with Buildkite's native `if_changed` property, so each +crate's shared crate-scoped tests only run when files in that crate (or in a +workspace-global path) change. Tests tagged with `"scope": "workspace"` +(e.g. `commit-format`) have no `if_changed` and always run. """ import yaml @@ -54,6 +60,8 @@ import os import sys import pathlib import copy +import glob +import tomllib from argparse import ArgumentParser, RawTextHelpFormatter from textwrap import dedent @@ -85,6 +93,13 @@ DEFAULT_AGENT_TAG_HYPERVISOR = os.getenv("DEFAULT_AGENT_TAG_HYPERVISOR", "kvm") BUILDKITE_PATH = pathlib.Path(__file__).parent.resolve() +# Per-crate file listing the platforms CI should run on. +PLATFORMS_FILE = ".platform" + +# Paths added to every shared crate-scoped step's `if_changed`, so a change to +# the workspace as a whole fires each crate's shared tests. +WORKSPACE_GLOBAL_PATHS = ("Cargo.toml", ".buildkite/**", "rust-vmm-ci/**") + class BuildkiteStep: """ @@ -232,16 +247,26 @@ class BuildkiteStep: conditional = input.get("conditional") timeout = input.get("timeout_in_minutes") queue = input.get("queue") + crate = input.get("crate") + crate_path = input.get("crate_path") # Mandatory keys. assert test_name, "Step is missing test name." platform_string = f"-{platform}" if platform else "" - self.step_config["label"] = f"{test_name}{platform_string}" + # When the step is crate-scoped, prefix the label with the crate to disambiguate. + crate_string = f"{crate}: " if crate else "" + self.step_config["label"] = f"{crate_string}{test_name}{platform_string}" assert command, "Step is missing command." if "{target_platform}" in command: assert platform, "Command requires platform, but platform is missing." command = command.replace("{target_platform}", platform) + if "{crate}" in command: + assert crate, "Command requires crate, but crate is missing." + command = command.replace("{crate}", crate) + if "{crate_path}" in command: + assert crate_path, "Command requires crate path, but crate path is missing." + command = command.replace("{crate_path}", crate_path) # Modify command and tag name for `riscv64` CI if platform == "riscv64": # Wrap command with '' to avoid escaping early by `ENTRYPOINT` @@ -282,6 +307,9 @@ class BuildkiteStep: "test_name", "queue", "hypervisor", + "crate", + "crate_path", + "scope", ] additional_keys = { k: v @@ -305,6 +333,38 @@ class BuildkiteConfig: def __init__(self): self.bk_config = None + @staticmethod + def _skip_test(test_name): + """Return whether `test_name` is excluded via the `TESTS_TO_SKIP` env.""" + return bool(TESTS_TO_SKIP) and test_name in json.loads(TESTS_TO_SKIP) + + def _append_test_steps( + self, test, platform_allowlist, crate=None, crate_path=None, if_changed=None + ): + """Append the steps for `test`, one per allowed platform, scoped to `crate`.""" + platforms = test.get("platform") + # The platform is optional. + if not platforms: + platforms = [None] + + for platform in platforms: + # Filter test enabled in platform_allowlist + if platform is not None and platform not in platform_allowlist: + # Skip disabled platform + continue + + step_input = copy.deepcopy(test) + step_input["platform"] = platform + step_input["crate"] = crate + step_input["crate_path"] = crate_path + if if_changed is not None: + step_input["if_changed"] = if_changed + if not step_input.get("hypervisor"): + step_input["hypervisor"] = DEFAULT_AGENT_TAG_HYPERVISOR + + step = BuildkiteStep() + self.bk_config["steps"].append(step.build(step_input)) + def build(self, input, platform_allowlist): """Build the final Buildkite configuration fron the json input.""" @@ -313,33 +373,58 @@ class BuildkiteConfig: assert tests, "Input is missing list of tests." for test in tests: - platforms = test.get("platform") - test_name = test.get("test_name") + if self._skip_test(test.get("test_name")): + continue + self._append_test_steps(test, platform_allowlist) - if TESTS_TO_SKIP: - tests_to_skip = json.loads(TESTS_TO_SKIP) - if test_name in tests_to_skip: + # Return the object's attributes and their values as a dictionary. + return self.bk_config + + def _append_crate_steps(self, test, crate, crate_path, if_changed=None): + """Append `test` scoped to `crate`, using the crate's own `.platform`.""" + allowlist = determine_allowlist(os.path.join(crate_path, PLATFORMS_FILE)) + self._append_test_steps( + test, + allowlist, + crate=crate, + crate_path=crate_path, + if_changed=if_changed, + ) + + def build_workspace_selective(self, input, dirs, root_allowlist): + """Build the workspace-selective configuration with `if_changed` per step.""" + + self.bk_config = {"steps": []} + tests = input.get("tests") + assert tests, "Input is missing list of tests." + + # Shared tests: workspace-scoped ones always run; crate-scoped ones run + # per crate and fire on changes to that crate or a workspace-global path. + for test in tests: + if self._skip_test(test.get("test_name")): + continue + if test.get("scope", "crate") == "workspace": + self._append_test_steps(test, root_allowlist) + continue + for crate in sorted(dirs): + crate_path = dirs[crate] + self._append_crate_steps( + test, + crate, + crate_path, + if_changed=[f"{crate_path}/**", *WORKSPACE_GLOBAL_PATHS], + ) + + # Per-crate extras fire only on changes inside that crate (decoupled + # from workspace-global paths so a root change does not retrigger them). + for crate in sorted(dirs): + crate_path = dirs[crate] + for test in crate_test_description(crate_path): + if self._skip_test(test.get("test_name")): continue - - # The platform is optional. When it is not specified, we don't add - # it to the step so that we can run the test in any environment. - if not platforms: - platforms = [None] - - for platform in platforms: - # Filter test enabled in platform_allowlist - if platform is not None and platform not in platform_allowlist: - # Skip disabled platform - continue - - step_input = copy.deepcopy(test) - step_input["platform"] = platform - if not step_input.get("hypervisor"): - step_input["hypervisor"] = DEFAULT_AGENT_TAG_HYPERVISOR - - step = BuildkiteStep() - step_output = step.build(step_input) - self.bk_config["steps"].append(step_output) + self._append_crate_steps( + test, crate, crate_path, if_changed=[f"{crate_path}/**"] + ) # Return the object's attributes and their values as a dictionary. return self.bk_config @@ -357,6 +442,43 @@ def determine_allowlist(config_file): return ["x86_64", "aarch64"] +def crate_test_description(crate_path): + """Return the crate's own extra tests from `/.buildkite/`, if any.""" + path = os.path.join(crate_path, ".buildkite", "test_description.json") + if not os.path.exists(path): + return [] + with open(path) as json_file: + return json.load(json_file).get("tests", []) + + +def workspace_members(): + """Return `{crate_name: relative_path}` for each workspace member. + + Parses the workspace's Cargo.toml manifests with the stdlib `tomllib` + module so this script does not require `cargo` on the host. + """ + with open("Cargo.toml", "rb") as f: + patterns = tomllib.load(f).get("workspace", {}).get("members", []) + + members = {} + for pattern in patterns: + # Expand glob patterns like `crates/*`; bare paths pass through. + matches = ( + sorted(glob.glob(pattern)) + if any(c in pattern for c in "*?[") + else [pattern] + ) + for member_dir in matches: + manifest = os.path.join(member_dir, "Cargo.toml") + if not os.path.isfile(manifest): + continue + with open(manifest, "rb") as f: + name = tomllib.load(f).get("package", {}).get("name") + if name: + members[name] = member_dir + return members + + def generate_pipeline(config_file, platform_allowlist): """Generate the pipeline yaml file from a json configuration file.""" @@ -369,6 +491,19 @@ def generate_pipeline(config_file, platform_allowlist): yaml.dump(output, sys.stdout, sort_keys=False) +def generate_workspace_selective_pipeline(config_file, root_allowlist): + """Generate a workspace-selective pipeline from a json configuration file.""" + + with open(config_file) as json_file: + json_cfg = json.load(json_file) + + dirs = workspace_members() + + config = BuildkiteConfig() + output = config.build_workspace_selective(json_cfg, dirs, root_allowlist) + yaml.dump(output, sys.stdout, sort_keys=False) + + if __name__ == "__main__": help_text = dedent(""" This script supports overriding the following configurations through @@ -408,6 +543,25 @@ if __name__ == "__main__": ), default=f"{os.getcwd()}/.platform", ) + # `--workspace-selective` takes a value rather than being a flag (no + # `action="store_true"`) because the infrastructure that invokes this + # script supports parameters with arguments only, not bare options. + parser.add_argument( + "--workspace-selective", + default="False", + metavar="BOOL", + help=( + "When 'True', generate a selective pipeline for a Cargo workspace,\n" + "emitting one step per crate gated by Buildkite's `if_changed`\n" + "property so each crate's shared tests only run when files in that\n" + "crate (or a workspace-global path) change. The per-crate test\n" + "description (using the {crate}/{crate_path} placeholders and\n" + "`scope`) is supplied via -t. Default: 'False'." + ), + ) args = parser.parse_args() platform_allowlist = determine_allowlist(args.platform_allowlist) - generate_pipeline(args.test_description, platform_allowlist) + if args.workspace_selective == "True": + generate_workspace_selective_pipeline(args.test_description, platform_allowlist) + else: + generate_pipeline(args.test_description, platform_allowlist)