Skip to content

Commit efc49e4

Browse files
committed
Add support for tidy linting via external tools for non-rust files
This change adds the flag `--check-extras` to `tidy`. It accepts a comma separated list of any of the options: - py (test everything applicable for python files) - py:lint (lint python files using `ruff`) - py:fmt (check formatting for python files using `black`) - shell or shell:lint (lint shell files using `shellcheck`) Specific files to check can also be specified via positional args. Examples: - `./x test tidy --check-extras=shell,py` - `./x test tidy --check-extras=py:fmt -- src/bootstrap/bootstrap.py` - `./x test tidy --check-extras=shell -- src/ci/*.sh` - Python formatting can be applied with bless: `./x test tidy --ckeck-extras=py:fmt --bless` `ruff` and `black` need to be installed via pip; this tool manages these within a virtual environment at `build/venv`. `shellcheck` needs to be installed on the system already.
1 parent 7a5d2d0 commit efc49e4

File tree

14 files changed

+662
-6
lines changed

14 files changed

+662
-6
lines changed

src/bootstrap/builder/tests.rs

+2
Original file line numberDiff line numberDiff line change
@@ -587,6 +587,7 @@ mod dist {
587587
run: None,
588588
only_modified: false,
589589
skip: vec![],
590+
extra_checks: None,
590591
};
591592

592593
let build = Build::new(config);
@@ -658,6 +659,7 @@ mod dist {
658659
pass: None,
659660
run: None,
660661
only_modified: false,
662+
extra_checks: None,
661663
};
662664
// Make sure rustfmt binary not being found isn't an error.
663665
config.channel = "beta".to_string();

src/bootstrap/flags.rs

+11
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,10 @@ pub enum Subcommand {
336336
/// whether to automatically update stderr/stdout files
337337
bless: bool,
338338
#[arg(long)]
339+
/// comma-separated list of other files types to check (accepts py, py:lint,
340+
/// py:fmt, shell)
341+
extra_checks: Option<String>,
342+
#[arg(long)]
339343
/// rerun tests even if the inputs are unchanged
340344
force_rerun: bool,
341345
#[arg(long)]
@@ -473,6 +477,13 @@ impl Subcommand {
473477
}
474478
}
475479

480+
pub fn extra_checks(&self) -> Option<&str> {
481+
match *self {
482+
Subcommand::Test { ref extra_checks, .. } => extra_checks.as_ref().map(String::as_str),
483+
_ => None,
484+
}
485+
}
486+
476487
pub fn only_modified(&self) -> bool {
477488
match *self {
478489
Subcommand::Test { only_modified, .. } => only_modified,

src/bootstrap/test.rs

+9
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
//! our CI.
55
66
use std::env;
7+
use std::ffi::OsStr;
78
use std::ffi::OsString;
89
use std::fs;
910
use std::iter;
@@ -1094,6 +1095,14 @@ impl Step for Tidy {
10941095
if builder.config.cmd.bless() {
10951096
cmd.arg("--bless");
10961097
}
1098+
if let Some(s) = builder.config.cmd.extra_checks() {
1099+
cmd.arg(format!("--extra-checks={s}"));
1100+
}
1101+
let mut args = std::env::args_os();
1102+
if let Some(_) = args.find(|arg| arg == OsStr::new("--")) {
1103+
cmd.arg("--");
1104+
cmd.args(args);
1105+
}
10971106

10981107
if builder.config.channel == "dev" || builder.config.channel == "nightly" {
10991108
builder.info("fmt check");

src/ci/docker/host-x86_64/mingw-check-tidy/Dockerfile

+4-2
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,13 @@ COPY scripts/sccache.sh /scripts/
2626
RUN sh /scripts/sccache.sh
2727

2828
COPY host-x86_64/mingw-check/reuse-requirements.txt /tmp/
29-
RUN pip3 install --no-deps --no-cache-dir --require-hashes -r /tmp/reuse-requirements.txt
29+
RUN pip3 install --no-deps --no-cache-dir --require-hashes -r /tmp/reuse-requirements.txt \
30+
&& pip3 install virtualenv
3031

3132
COPY host-x86_64/mingw-check/validate-toolstate.sh /scripts/
3233
COPY host-x86_64/mingw-check/validate-error-codes.sh /scripts/
3334

3435
# NOTE: intentionally uses python2 for x.py so we can test it still works.
3536
# validate-toolstate only runs in our CI, so it's ok for it to only support python3.
36-
ENV SCRIPT python2.7 ../x.py test --stage 0 src/tools/tidy tidyselftest
37+
ENV SCRIPT TIDY_PRINT_DIFF=1 python2.7 ../x.py test \
38+
--stage 0 src/tools/tidy tidyselftest --extra-checks=py:lint

src/etc/completions/x.py.fish

+1
Original file line numberDiff line numberDiff line change
@@ -227,6 +227,7 @@ complete -c x.py -n "__fish_seen_subcommand_from doc" -s h -l help -d 'Print hel
227227
complete -c x.py -n "__fish_seen_subcommand_from test" -l skip -d 'skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times' -r
228228
complete -c x.py -n "__fish_seen_subcommand_from test" -l test-args -d 'extra arguments to be passed for the test tool being used (e.g. libtest, compiletest or rustdoc)' -r
229229
complete -c x.py -n "__fish_seen_subcommand_from test" -l rustc-args -d 'extra options to pass the compiler when running tests' -r
230+
complete -c x.py -n "__fish_seen_subcommand_from test" -l extra-checks -d 'comma-separated list of other files types to check (accepts py, py:lint, py:fmt, shell)' -r
230231
complete -c x.py -n "__fish_seen_subcommand_from test" -l compare-mode -d 'mode describing what file the actual ui output will be compared to' -r
231232
complete -c x.py -n "__fish_seen_subcommand_from test" -l pass -d 'force {check,build,run}-pass tests to this mode' -r
232233
complete -c x.py -n "__fish_seen_subcommand_from test" -l run -d 'whether to execute run-* tests' -r

src/etc/completions/x.py.ps1

+1
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,7 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock {
299299
[CompletionResult]::new('--skip', 'skip', [CompletionResultType]::ParameterName, 'skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times')
300300
[CompletionResult]::new('--test-args', 'test-args', [CompletionResultType]::ParameterName, 'extra arguments to be passed for the test tool being used (e.g. libtest, compiletest or rustdoc)')
301301
[CompletionResult]::new('--rustc-args', 'rustc-args', [CompletionResultType]::ParameterName, 'extra options to pass the compiler when running tests')
302+
[CompletionResult]::new('--extra-checks', 'extra-checks', [CompletionResultType]::ParameterName, 'comma-separated list of other files types to check (accepts py, py:lint, py:fmt, shell)')
302303
[CompletionResult]::new('--compare-mode', 'compare-mode', [CompletionResultType]::ParameterName, 'mode describing what file the actual ui output will be compared to')
303304
[CompletionResult]::new('--pass', 'pass', [CompletionResultType]::ParameterName, 'force {check,build,run}-pass tests to this mode')
304305
[CompletionResult]::new('--run', 'run', [CompletionResultType]::ParameterName, 'whether to execute run-* tests')

src/etc/completions/x.py.sh

+5-1
Original file line numberDiff line numberDiff line change
@@ -1569,7 +1569,7 @@ _x.py() {
15691569
return 0
15701570
;;
15711571
x.py__test)
1572-
opts="-v -i -j -h --no-fail-fast --skip --test-args --rustc-args --no-doc --doc --bless --force-rerun --only-modified --compare-mode --pass --run --rustfix-coverage --verbose --incremental --config --build-dir --build --host --target --exclude --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..."
1572+
opts="-v -i -j -h --no-fail-fast --skip --test-args --rustc-args --no-doc --doc --bless --extra-checks --force-rerun --only-modified --compare-mode --pass --run --rustfix-coverage --verbose --incremental --config --build-dir --build --host --target --exclude --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..."
15731573
if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then
15741574
COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") )
15751575
return 0
@@ -1587,6 +1587,10 @@ _x.py() {
15871587
COMPREPLY=($(compgen -f "${cur}"))
15881588
return 0
15891589
;;
1590+
--extra-checks)
1591+
COMPREPLY=($(compgen -f "${cur}"))
1592+
return 0
1593+
;;
15901594
--compare-mode)
15911595
COMPREPLY=($(compgen -f "${cur}"))
15921596
return 0

src/tools/tidy/config/black.toml

+15
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
[tool.black]
2+
# Ignore all submodules
3+
extend-exclude = """(\
4+
src/doc/nomicon|\
5+
src/tools/cargo/|\
6+
src/doc/reference/|\
7+
src/doc/book/|\
8+
src/doc/rust-by-example/|\
9+
library/stdarch/|\
10+
src/doc/rustc-dev-guide/|\
11+
src/doc/edition-guide/|\
12+
src/llvm-project/|\
13+
src/doc/embedded-book/|\
14+
library/backtrace/
15+
)"""

src/tools/tidy/config/requirements.in

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# requirements.in This is the source file for our pinned version requirements
2+
# file "requirements.txt" To regenerate that file, pip-tools is required
3+
# (`python -m pip install pip-tools`). Once installed, run: `pip-compile
4+
# --generate-hashes src/tools/tidy/config/requirements.in`
5+
#
6+
# Note: this generation step should be run with the oldest supported python
7+
# version (currently 3.7) to ensure backward compatibility
8+
9+
black==23.3.0
10+
ruff==0.0.272
+117
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
#
2+
# This file is autogenerated by pip-compile with Python 3.11
3+
# by the following command:
4+
#
5+
# pip-compile --generate-hashes src/tools/tidy/config/requirements.in
6+
#
7+
black==23.3.0 \
8+
--hash=sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5 \
9+
--hash=sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915 \
10+
--hash=sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326 \
11+
--hash=sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940 \
12+
--hash=sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b \
13+
--hash=sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30 \
14+
--hash=sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c \
15+
--hash=sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c \
16+
--hash=sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab \
17+
--hash=sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27 \
18+
--hash=sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2 \
19+
--hash=sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961 \
20+
--hash=sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9 \
21+
--hash=sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb \
22+
--hash=sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70 \
23+
--hash=sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331 \
24+
--hash=sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2 \
25+
--hash=sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266 \
26+
--hash=sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d \
27+
--hash=sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6 \
28+
--hash=sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b \
29+
--hash=sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925 \
30+
--hash=sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8 \
31+
--hash=sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4 \
32+
--hash=sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3
33+
# via -r src/tools/tidy/config/requirements.in
34+
click==8.1.3 \
35+
--hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \
36+
--hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48
37+
# via black
38+
importlib-metadata==6.7.0 \
39+
--hash=sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4 \
40+
--hash=sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5
41+
# via click
42+
mypy-extensions==1.0.0 \
43+
--hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \
44+
--hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782
45+
# via black
46+
packaging==23.1 \
47+
--hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \
48+
--hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f
49+
# via black
50+
pathspec==0.11.1 \
51+
--hash=sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687 \
52+
--hash=sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293
53+
# via black
54+
platformdirs==3.6.0 \
55+
--hash=sha256:57e28820ca8094678b807ff529196506d7a21e17156cb1cddb3e74cebce54640 \
56+
--hash=sha256:ffa199e3fbab8365778c4a10e1fbf1b9cd50707de826eb304b50e57ec0cc8d38
57+
# via black
58+
ruff==0.0.272 \
59+
--hash=sha256:06b8ee4eb8711ab119db51028dd9f5384b44728c23586424fd6e241a5b9c4a3b \
60+
--hash=sha256:1609b864a8d7ee75a8c07578bdea0a7db75a144404e75ef3162e0042bfdc100d \
61+
--hash=sha256:19643d448f76b1eb8a764719072e9c885968971bfba872e14e7257e08bc2f2b7 \
62+
--hash=sha256:273a01dc8c3c4fd4c2af7ea7a67c8d39bb09bce466e640dd170034da75d14cab \
63+
--hash=sha256:27b2ea68d2aa69fff1b20b67636b1e3e22a6a39e476c880da1282c3e4bf6ee5a \
64+
--hash=sha256:48eccf225615e106341a641f826b15224b8a4240b84269ead62f0afd6d7e2d95 \
65+
--hash=sha256:677284430ac539bb23421a2b431b4ebc588097ef3ef918d0e0a8d8ed31fea216 \
66+
--hash=sha256:691d72a00a99707a4e0b2846690961157aef7b17b6b884f6b4420a9f25cd39b5 \
67+
--hash=sha256:86bc788245361a8148ff98667da938a01e1606b28a45e50ac977b09d3ad2c538 \
68+
--hash=sha256:905ff8f3d6206ad56fcd70674453527b9011c8b0dc73ead27618426feff6908e \
69+
--hash=sha256:9c4bfb75456a8e1efe14c52fcefb89cfb8f2a0d31ed8d804b82c6cf2dc29c42c \
70+
--hash=sha256:a37ec80e238ead2969b746d7d1b6b0d31aa799498e9ba4281ab505b93e1f4b28 \
71+
--hash=sha256:ae9b57546e118660175d45d264b87e9b4c19405c75b587b6e4d21e6a17bf4fdf \
72+
--hash=sha256:bd2bbe337a3f84958f796c77820d55ac2db1e6753f39d1d1baed44e07f13f96d \
73+
--hash=sha256:d5a208f8ef0e51d4746930589f54f9f92f84bb69a7d15b1de34ce80a7681bc00 \
74+
--hash=sha256:dc406e5d756d932da95f3af082814d2467943631a587339ee65e5a4f4fbe83eb \
75+
--hash=sha256:ee76b4f05fcfff37bd6ac209d1370520d509ea70b5a637bdf0a04d0c99e13dff
76+
# via -r src/tools/tidy/config/requirements.in
77+
tomli==2.0.1 \
78+
--hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
79+
--hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
80+
# via black
81+
typed-ast==1.5.4 \
82+
--hash=sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2 \
83+
--hash=sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1 \
84+
--hash=sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6 \
85+
--hash=sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62 \
86+
--hash=sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac \
87+
--hash=sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d \
88+
--hash=sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc \
89+
--hash=sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2 \
90+
--hash=sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97 \
91+
--hash=sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35 \
92+
--hash=sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6 \
93+
--hash=sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1 \
94+
--hash=sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4 \
95+
--hash=sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c \
96+
--hash=sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e \
97+
--hash=sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec \
98+
--hash=sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f \
99+
--hash=sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72 \
100+
--hash=sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47 \
101+
--hash=sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72 \
102+
--hash=sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe \
103+
--hash=sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6 \
104+
--hash=sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3 \
105+
--hash=sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66
106+
# via black
107+
typing-extensions==4.6.3 \
108+
--hash=sha256:88a4153d8505aabbb4e13aacb7c486c2b4a33ca3b3f807914a9b4c844c471c26 \
109+
--hash=sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5
110+
# via
111+
# black
112+
# importlib-metadata
113+
# platformdirs
114+
zipp==3.15.0 \
115+
--hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \
116+
--hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556
117+
# via importlib-metadata

src/tools/tidy/config/ruff.toml

+41
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Configuration for ruff python linter, run as part of tidy external tools
2+
3+
# B (bugbear), E (pycodestyle, standard), EXE (executables) F (flakes, standard)
4+
# ERM for error messages would be beneficial at some point
5+
select = ["B", "E", "EXE", "F"]
6+
7+
ignore = [
8+
"E501", # line-too-long
9+
"F403", # undefined-local-with-import-star
10+
"F405", # undefined-local-with-import-star-usage
11+
]
12+
13+
# lowest possible for ruff
14+
target-version = "py37"
15+
16+
# Ignore all submodules
17+
extend-exclude = [
18+
"src/doc/nomicon/",
19+
"src/tools/cargo/",
20+
"src/doc/reference/",
21+
"src/doc/book/",
22+
"src/doc/rust-by-example/",
23+
"library/stdarch/",
24+
"src/doc/rustc-dev-guide/",
25+
"src/doc/edition-guide/",
26+
"src/llvm-project/",
27+
"src/doc/embedded-book/",
28+
"library/backtrace/",
29+
# Hack: CI runs from a subdirectory under the main checkout
30+
"../src/doc/nomicon/",
31+
"../src/tools/cargo/",
32+
"../src/doc/reference/",
33+
"../src/doc/book/",
34+
"../src/doc/rust-by-example/",
35+
"../library/stdarch/",
36+
"../src/doc/rustc-dev-guide/",
37+
"../src/doc/edition-guide/",
38+
"../src/llvm-project/",
39+
"../src/doc/embedded-book/",
40+
"../library/backtrace/",
41+
]

0 commit comments

Comments
 (0)