Rewrite the ci.py script in Rust
#136864
Merged
Rewrite the ci.py script in Rust
#136864
Conversation
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
rustbot
added
A-rustc-dev-guide
|
The rustc-dev-guide subtree was changed. If this PR only touches the dev guide consider submitting a PR directly to rust-lang/rustc-dev-guide otherwise thank you for updating the dev guide with your changes. |
jieyouxu
reviewed
Feb 11, 2025
marcoieni
reviewed
Feb 11, 2025
|
There is a risk (arguably tiny) that may or may not be worth considering, which is that with a stable toolchain bump (through GHA's rustup I think?) that this somehow can't build anymore1 or fail at runtime due to I dunno if a rustup Footnotes |
marcoieni
reviewed
Feb 11, 2025
|
I wrote a test. Add this to Cargo.toml: [dev-dependencies]
insta = "1"test file: use std::process::{Command, Stdio};
const TEST_JOBS_YML_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/tests/jobs.yml");
#[test]
fn test_cargo_check() {
let output = Command::new("cargo")
.args(["run", "-q", "calculate-job-matrix", "--jobs-file", TEST_JOBS_YML_PATH])
.env("GITHUB_EVENT_NAME", "push")
.env("COMMIT_MESSAGE", "test commit message")
.env("GITHUB_REF", "refs/heads/auto")
.stdout(Stdio::piped())
.output()
.expect("Failed to execute command");
let stdout = String::from_utf8(output.stdout).unwrap();
let stderr = String::from_utf8(output.stderr).unwrap();
if !output.status.success() {
panic!("cargo run failed: {}\n{}", stdout, stderr);
}
insta::assert_snapshot!(stdout, @r#"
jobs=[{"name":"aarch64-gnu","full_name":"auto - aarch64-gnu","os":"ubuntu-22.04-arm","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","TOOLSTATE_PUBLISH":"1"},"free_disk":true},{"name":"x86_64-gnu-llvm-18-1","full_name":"auto - x86_64-gnu-llvm-18-1","os":"ubuntu-24.04","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","DOCKER_SCRIPT":"stage_2_test_set1.sh","IMAGE":"x86_64-gnu-llvm-18","READ_ONLY_SRC":"'0'","RUST_BACKTRACE":"1","TOOLSTATE_PUBLISH":"1"},"free_disk":true},{"name":"aarch64-apple","full_name":"auto - aarch64-apple","os":"macos-14","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","DEPLOY_BUCKET":"rust-lang-ci2","MACOSX_DEPLOYMENT_TARGET":"11.0","MACOSX_STD_DEPLOYMENT_TARGET":"11.0","NO_DEBUG_ASSERTIONS":"1","NO_LLVM_ASSERTIONS":"1","NO_OVERFLOW_CHECKS":"1","RUSTC_RETRY_LINKER_ON_SEGFAULT":"1","RUST_CONFIGURE_ARGS":"--enable-sanitizers --enable-profiler --set rust.jemalloc","SCRIPT":"./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin","SELECT_XCODE":"/Applications/Xcode_15.4.app","TOOLSTATE_PUBLISH":"1","USE_XCODE_CLANG":"1"}},{"name":"dist-i686-msvc","full_name":"auto - dist-i686-msvc","os":"windows-2022","env":{"ARTIFACTS_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZN24CBO55","AWS_REGION":"us-west-1","CACHES_AWS_ACCESS_KEY_ID":"AKIA46X5W6CZI5DHEBFL","CODEGEN_BACKENDS":"llvm,cranelift","DEPLOY_BUCKET":"rust-lang-ci2","DIST_REQUIRE_ALL_TOOLS":"1","RUST_CONFIGURE_ARGS":"--build=i686-pc-windows-msvc --host=i686-pc-windows-msvc --target=i686-pc-windows-msvc,i586-pc-windows-msvc --enable-full-tools --enable-profiler","SCRIPT":"python x.py dist bootstrap --include-default-paths","TOOLSTATE_PUBLISH":"1"}}]
run_type=auto
"#);
}diffs of main.rs: diff --git a/src/ci/citool/src/main.rs b/src/ci/citool/src/main.rs
index 9a22d1264fa..4c6591443a2 100644
--- a/src/ci/citool/src/main.rs
+++ b/src/ci/citool/src/main.rs
@@ -1,5 +1,5 @@
-use std::collections::HashMap;
-use std::path::Path;
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
use std::process::Command;
use anyhow::Context;
@@ -17,13 +17,13 @@ struct Job {
name: String,
/// GitHub runner on which the job should be executed
os: String,
- env: HashMap<String, Value>,
+ env: BTreeMap<String, Value>,
/// Should the job be only executed on a specific channel?
#[serde(default)]
only_on_channel: Option<String>,
/// Rest of attributes that will be passed through to GitHub actions
#[serde(flatten)]
- extra_keys: HashMap<String, Value>,
+ extra_keys: BTreeMap<String, Value>,
}
impl Job {
@@ -44,11 +44,11 @@ fn image(&self) -> String {
#[derive(serde::Deserialize, Debug)]
struct JobEnvironments {
#[serde(rename = "pr")]
- pr_env: HashMap<String, Value>,
+ pr_env: BTreeMap<String, Value>,
#[serde(rename = "try")]
- try_env: HashMap<String, Value>,
+ try_env: BTreeMap<String, Value>,
#[serde(rename = "auto")]
- auto_env: HashMap<String, Value>,
+ auto_env: BTreeMap<String, Value>,
}
#[derive(serde::Deserialize, Debug)]
@@ -71,7 +71,7 @@ fn find_auto_job_by_name(&self, name: &str) -> Option<Job> {
}
fn load_job_db(path: &Path) -> anyhow::Result<JobDatabase> {
- let db = std::fs::read_to_string(path)?;
+ let db = read_to_string(path)?;
let mut db: Value = serde_yaml::from_str(&db)?;
// We need to expand merge keys (<<), because serde_yaml can't deal with them
@@ -92,9 +92,9 @@ struct GithubActionsJob {
/// prefix (PR/try/auto).
full_name: String,
os: String,
- env: HashMap<String, String>,
+ env: BTreeMap<String, String>,
#[serde(flatten)]
- extra_keys: HashMap<String, serde_json::Value>,
+ extra_keys: BTreeMap<String, serde_json::Value>,
}
/// Type of workflow that is being executed on CI
@@ -175,7 +175,7 @@ fn skip_jobs(jobs: Vec<Job>, channel: &str) -> Vec<Job> {
.collect()
}
-fn to_string_map(map: &HashMap<String, Value>) -> HashMap<String, String> {
+fn to_string_map(map: &BTreeMap<String, Value>) -> BTreeMap<String, String> {
map.iter()
.map(|(key, value)| {
(
@@ -232,7 +232,7 @@ fn calculate_jobs(
let jobs = jobs
.into_iter()
.map(|job| {
- let mut env: HashMap<String, String> = to_string_map(base_env);
+ let mut env: BTreeMap<String, String> = to_string_map(base_env);
env.extend(to_string_map(&job.env));
let full_name = format!("{prefix} - {}", job.name);
@@ -314,7 +314,7 @@ fn run_workflow_locally(db: JobDatabase, job_type: JobType, name: String) -> any
};
let job = find_linux_job(&jobs, &name).with_context(|| format!("Cannot find job {name}"))?;
- let mut custom_env: HashMap<String, String> = HashMap::new();
+ let mut custom_env: BTreeMap<String, String> = BTreeMap::new();
// Replicate src/ci/scripts/setup-environment.sh
// Adds custom environment variables to the job
if name.starts_with("dist-") {
@@ -340,7 +340,10 @@ fn run_workflow_locally(db: JobDatabase, job_type: JobType, name: String) -> any
enum Args {
/// Calculate a list of jobs that should be executed on CI.
/// Should only be used on CI inside GitHub actions.
- CalculateJobMatrix,
+ CalculateJobMatrix {
+ #[clap(long)]
+ jobs_file: Option<PathBuf>,
+ },
/// Execute a given CI job locally.
#[clap(name = "run-local")]
RunJobLocally {
@@ -362,19 +365,29 @@ enum JobType {
fn main() -> anyhow::Result<()> {
let args = Args::parse();
- let db = load_job_db(Path::new(JOBS_YML_PATH)).context("Cannot load jobs.yml")?;
+ let default_jobs_file = Path::new(JOBS_YML_PATH);
+ let default_db = |jobs_path| load_job_db(jobs_path).context("Cannot load jobs.yml");
match args {
- Args::CalculateJobMatrix => {
+ Args::CalculateJobMatrix { jobs_file } => {
+ let jobs_path = jobs_file.as_deref().unwrap_or(default_jobs_file);
let gh_ctx = load_github_ctx()
.context("Cannot load environment variables from GitHub Actions")?;
- let channel = std::fs::read_to_string(Path::new(CI_DIRECTORY).join("channel"))
+ let channel = read_to_string(Path::new(CI_DIRECTORY).join("channel"))
.context("Cannot read channel file")?;
- calculate_job_matrix(db, gh_ctx, &channel).context("Failed to calculate job matrix")?;
+ calculate_job_matrix(default_db(jobs_path)?, gh_ctx, &channel)
+ .context("Failed to calculate job matrix")?;
+ }
+ Args::RunJobLocally { job_type, name } => {
+ run_workflow_locally(default_db(default_jobs_file)?, job_type, name)?
}
- Args::RunJobLocally { job_type, name } => run_workflow_locally(db, job_type, name)?,
}
Ok(())
}
+
+fn read_to_string<P: AsRef<Path>>(path: P) -> anyhow::Result<String> {
+ let error = format!("Cannot read file {:?}", path.as_ref());
+ std::fs::read_to_string(path).context(error)
+}tests/jobs.yml: runners:
- &base-job
env: { }
- &job-linux-4c
os: ubuntu-24.04
# Free some disk space to avoid running out of space during the build.
free_disk: true
<<: *base-job
- &job-linux-16c
os: ubuntu-22.04-16core-64gb
<<: *base-job
- &job-macos-m1
os: macos-14
<<: *base-job
- &job-windows
os: windows-2022
<<: *base-job
- &job-aarch64-linux
# Free some disk space to avoid running out of space during the build.
free_disk: true
os: ubuntu-22.04-arm
<<: *base-job
envs:
env-x86_64-apple-tests: &env-x86_64-apple-tests
SCRIPT: ./x.py --stage 2 test --skip tests/ui --skip tests/rustdoc -- --exact
RUST_CONFIGURE_ARGS: --build=x86_64-apple-darwin --enable-sanitizers --enable-profiler --set rust.jemalloc
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
# Ensure that host tooling is tested on our minimum supported macOS version.
MACOSX_DEPLOYMENT_TARGET: 10.12
MACOSX_STD_DEPLOYMENT_TARGET: 10.12
SELECT_XCODE: /Applications/Xcode_15.2.app
NO_LLVM_ASSERTIONS: 1
NO_DEBUG_ASSERTIONS: 1
NO_OVERFLOW_CHECKS: 1
production:
&production
DEPLOY_BUCKET: rust-lang-ci2
# AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
# AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
# rotate them in a single branch while keeping the old key in another
# branch, which wouldn't be possible if the key was named with the kind
# (caches, artifacts...).
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
AWS_REGION: us-west-1
TOOLSTATE_PUBLISH: 1
try:
<<: *production
# The following env var activates faster `try` builds in `opt-dist` by, e.g.
# - building only the more commonly useful components (we rarely need e.g. rust-docs in try
# builds)
# - not running `opt-dist`'s post-optimization smoke tests on the resulting toolchain
#
# If you *want* these to happen however, temporarily comment it before triggering a try build.
DIST_TRY_BUILD: 1
auto:
<<: *production
pr:
PR_CI_JOB: 1
# Jobs that run on each push to a pull request (PR)
# These jobs automatically inherit envs.pr, to avoid repeating
# it in each job definition.
pr:
- name: mingw-check
<<: *job-linux-4c
- name: mingw-check-tidy
continue_on_error: true
<<: *job-linux-4c
# Jobs that run when you perform a try build (@bors try)
# These jobs automatically inherit envs.try, to avoid repeating
# it in each job definition.
try:
- name: dist-x86_64-linux
env:
CODEGEN_BACKENDS: llvm,cranelift
<<: *job-linux-16c
# Main CI jobs that have to be green to merge a commit into master
# These jobs automatically inherit envs.auto, to avoid repeating
# it in each job definition.
auto:
- name: aarch64-gnu
<<: *job-aarch64-linux
# The x86_64-gnu-llvm-18 job is split into multiple jobs to run tests in parallel.
# x86_64-gnu-llvm-18-1 skips tests that run in x86_64-gnu-llvm-18-{2,3}.
- name: x86_64-gnu-llvm-18-1
env:
RUST_BACKTRACE: 1
READ_ONLY_SRC: "0"
IMAGE: x86_64-gnu-llvm-18
DOCKER_SCRIPT: stage_2_test_set1.sh
<<: *job-linux-4c
####################
# macOS Builders #
####################
- name: aarch64-apple
env:
SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin
RUST_CONFIGURE_ARGS: >-
--enable-sanitizers
--enable-profiler
--set rust.jemalloc
RUSTC_RETRY_LINKER_ON_SEGFAULT: 1
SELECT_XCODE: /Applications/Xcode_15.4.app
USE_XCODE_CLANG: 1
# Aarch64 tooling only needs to support macOS 11.0 and up as nothing else
# supports the hardware, so only need to test it there.
MACOSX_DEPLOYMENT_TARGET: 11.0
MACOSX_STD_DEPLOYMENT_TARGET: 11.0
NO_LLVM_ASSERTIONS: 1
NO_DEBUG_ASSERTIONS: 1
NO_OVERFLOW_CHECKS: 1
<<: *job-macos-m1
######################
# Windows Builders #
######################
- name: dist-i686-msvc
env:
RUST_CONFIGURE_ARGS: >-
--build=i686-pc-windows-msvc
--host=i686-pc-windows-msvc
--target=i686-pc-windows-msvc,i586-pc-windows-msvc
--enable-full-tools
--enable-profiler
SCRIPT: python x.py dist bootstrap --include-default-paths
DIST_REQUIRE_ALL_TOOLS: 1
CODEGEN_BACKENDS: llvm,cranelift
<<: *job-windowsWhat do you think? |
It should be possible to pin the toolchain, but it would also mean that we'd need to download a new Rust toolchain on every workflow execution. Not a big deal, but still would be nice to avoid I guess. The same situation held also for the previous Python script, updates to the Ubuntu base image could have broken it. But we manually control which runner image we use, so unless you consider Rust to be severely less stable than Python in this regard, I think it should be fine. It will be a bit more brittle because dependencies will have to be downloaded from crates.io, which can fail sometimes, but if crates.io fails, the rest of the CI workflow would fail anyway.. |
marcoieni
reviewed
Feb 11, 2025
marcoieni
reviewed
Feb 11, 2025
marcoieni
reviewed
Feb 11, 2025
|
Great point with |
Of course, I'm not saying the python version didn't have this issue :P |
|
LGTM. You can r=me when you feel it's ready 👍 |
|
Thanks! I added running tests on CI. I'll do a few tests with try builds. @bors try |
bors
added a commit
to rust-lang-ci/rust
that referenced
this pull request
Feb 11, 2025
Rewrite the `ci.py` script in Rust It would seem that I would learn by now that any script written in Python will become unmaintainable sooner or later, but alas.. r? `@marcoieni`
|
💔 Test failed - checks-actions |
bors
added
S-waiting-on-author
|
@bors try |
bors
added a commit
to rust-lang-ci/rust
that referenced
this pull request
Feb 11, 2025
Rewrite the `ci.py` script in Rust It would seem that I would learn by now that any script written in Python will become unmaintainable sooner or later, but alas.. r? `@marcoieni` try-job: aarch64-gnu try-job: dist-x86_64-linux-alt
|
💔 Test failed - checks-actions |
This comment has been minimized.
This comment has been minimized.
|
Fixed handling of environment variables and reading of the @bors try |
bors
added a commit
to rust-lang-ci/rust
that referenced
this pull request
Feb 17, 2025
Rewrite the `ci.py` script in Rust It would seem that I would learn by now that any script written in Python will become unmaintainable sooner or later, but alas.. r? `@marcoieni` try-job: aarch64-gnu try-job: dist-x86_64-linux-alt try-job: x86_64-msvc-ext2 Fixes: rust-lang#137013
To avoid including a newline at the end, which broke `only_on_channel` comparison.
|
A job failed! Check out the build log: (web) (plain) Click to see the possible cause of the failure (guessed by this bot) |
|
@bors try |
bors
added a commit
to rust-lang-ci/rust
that referenced
this pull request
Feb 17, 2025
Rewrite the `ci.py` script in Rust It would seem that I would learn by now that any script written in Python will become unmaintainable sooner or later, but alas.. r? `@marcoieni` try-job: aarch64-gnu try-job: dist-x86_64-linux-alt try-job: x86_64-msvc-ext2 Fixes: rust-lang#137013
|
☀️ Try build successful - checks-actions |
|
Ok now I'm confident that the script works as it should. @rustbot ready |
bors
added a commit
to rust-lang-ci/rust
that referenced
this pull request
Feb 19, 2025
[WIP] Postprocess bootstrap metrics into GitHub summary Based on rust-lang#136864. r? `@ghost`
|
@marcoieni Once you have a bit of time, could you please recheck if the latest changes look fine, and if so, reapprove? I didn't want to r=you without you taking another look. I have like five branches stacked on top of this 😂 So wanted to get it through the queue. Thanks! |
|
@bors r+ rollup=never |
bors
added
S-waiting-on-bors
|
@bors p=4 (scheduling) |
|
☀️ Test successful - checks-actions |
|
Finished benchmarking commit (4b696e6): comparison URL. Overall result: no relevant changes - no action needed@rustbot label: -perf-regression Instruction countThis benchmark run did not return any relevant results for this metric. Max RSS (memory usage)Results (primary -1.7%)This is a less reliable metric that may be of interest but was not used to determine the overall result at the top of this comment.
CyclesThis benchmark run did not return any relevant results for this metric. Binary sizeThis benchmark run did not return any relevant results for this metric. Bootstrap: 773.253s -> 774.29s (0.13%) |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
It would seem that I would learn by now that any script written in Python will become unmaintainable sooner or later, but alas..
r? @marcoieni
try-job: aarch64-gnu
try-job: dist-x86_64-linux-alt
try-job: x86_64-msvc-ext2
Fixes: #137013