Skip to content

Commit 4dbd62a

Browse files
borsbors
committed
Auto merge of #718 - Mark-Simulacrum:shrink-html-report, r=Mark-Simulacrum
zstd-compress archives Archives are smaller, faster to compress, and faster to decompress. Installing zstd is pretty easy on most platforms as well, on my couple systems the default system tar is able to read .tar.zst files just fine too. It's a win for the duration of the compression though; zstd compressing the all.tar tarball takes ~12 seconds locally vs. 2 minutes with gzip (at default settings), and zstd produces a ~500MB smaller compressed result (1.2 GB -> 705MB).
2 parents 6bf2017 + f773968 commit 4dbd62a

14 files changed

+91
-60
lines changed

‎Cargo.lock

+29
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

+1
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ docsrs-metadata = { git = "https://github.com/rust-lang/docs.rs/" }
1919
dotenv = "0.15"
2020
failure = "0.1.3"
2121
flate2 = "1"
22+
zstd = "0.13.0"
2223
http = "0.2"
2324
hyper = "0.14"
2425
lazy_static = "1.0"

‎src/report/archives.rs

+23-24
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ use crate::experiments::Experiment;
88
use crate::prelude::*;
99
use crate::report::{compare, Comparison, ReportWriter};
1010
use crate::results::{EncodedLog, EncodingType, ReadResults};
11-
use flate2::{write::GzEncoder, Compression};
1211
use indexmap::IndexMap;
1312
use tar::{Builder as TarBuilder, Header as TarHeader};
1413
use tempfile::tempfile;
@@ -156,7 +155,7 @@ fn write_all_archive<DB: ReadResults, W: ReportWriter>(
156155
// writes to S3 (requiring buffer management etc) while avoiding keeping the blob entirely
157156
// in memory.
158157
let backing = tempfile()?;
159-
let mut all = TarBuilder::new(GzEncoder::new(backing, Compression::default()));
158+
let mut all = TarBuilder::new(zstd::stream::Encoder::new(backing, 0)?);
160159
for entry in iterate(db, ex, crates, config) {
161160
let entry = entry?;
162161
let mut header = entry.header();
@@ -180,9 +179,9 @@ fn write_all_archive<DB: ReadResults, W: ReportWriter>(
180179
view = &buffer[..];
181180
}
182181
match dest.write_bytes(
183-
"logs-archives/all.tar.gz",
182+
"logs-archives/all.tar.zst",
184183
view,
185-
&"application/gzip".parse().unwrap(),
184+
&"application/zstd".parse().unwrap(),
186185
EncodingType::Plain,
187186
) {
188187
Ok(()) => break,
@@ -192,7 +191,7 @@ fn write_all_archive<DB: ReadResults, W: ReportWriter>(
192191
} else {
193192
std::thread::sleep(std::time::Duration::from_secs(2));
194193
warn!(
195-
"retry ({}/{}) writing logs-archives/all.tar.gz ({} bytes) (error: {:?})",
194+
"retry ({}/{}) writing logs-archives/all.tar.zst ({} bytes) (error: {:?})",
196195
i,
197196
RETRIES,
198197
view.len(),
@@ -206,7 +205,7 @@ fn write_all_archive<DB: ReadResults, W: ReportWriter>(
206205

207206
Ok(Archive {
208207
name: "All the crates".to_string(),
209-
path: "logs-archives/all.tar.gz".to_string(),
208+
path: "logs-archives/all.tar.zst".to_string(),
210209
})
211210
}
212211

@@ -229,22 +228,22 @@ pub fn write_logs_archives<DB: ReadResults, W: ReportWriter>(
229228

230229
by_comparison
231230
.entry(entry.comparison)
232-
.or_insert_with(|| TarBuilder::new(GzEncoder::new(Vec::new(), Compression::default())))
231+
.or_insert_with(|| TarBuilder::new(zstd::stream::Encoder::new(Vec::new(), 3).unwrap()))
233232
.append_data(&mut entry.header(), &entry.path, &entry.log_bytes[..])?;
234233
}
235234

236235
for (comparison, archive) in by_comparison.drain(..) {
237236
let data = archive.into_inner()?.finish()?;
238237
dest.write_bytes(
239-
format!("logs-archives/{comparison}.tar.gz"),
238+
format!("logs-archives/{comparison}.tar.zst"),
240239
&data,
241-
&"application/gzip".parse().unwrap(),
240+
&"application/zstd".parse().unwrap(),
242241
EncodingType::Plain,
243242
)?;
244243

245244
archives.push(Archive {
246245
name: format!("{comparison} crates"),
247-
path: format!("logs-archives/{comparison}.tar.gz"),
246+
path: format!("logs-archives/{comparison}.tar.zst"),
248247
});
249248
}
250249

@@ -261,11 +260,11 @@ mod tests {
261260
use crate::prelude::*;
262261
use crate::report::DummyWriter;
263262
use crate::results::{DatabaseDB, EncodingType, FailureReason, TestResult, WriteResults};
264-
use flate2::read::GzDecoder;
265263
use mime::Mime;
266264
use rustwide::logging::LogStorage;
267265
use std::io::Read;
268266
use tar::Archive;
267+
use zstd::stream::Decoder;
269268

270269
#[test]
271270
fn test_logs_archives_generation() {
@@ -355,20 +354,20 @@ mod tests {
355354
assert_eq!(
356355
&archives_paths,
357356
&[
358-
"logs-archives/all.tar.gz",
359-
"logs-archives/regressed.tar.gz",
360-
"logs-archives/test-pass.tar.gz",
357+
"logs-archives/all.tar.zst",
358+
"logs-archives/regressed.tar.zst",
359+
"logs-archives/test-pass.tar.zst",
361360
]
362361
);
363362

364363
// Load the content of all the archives
365-
let mime: Mime = "application/gzip".parse().unwrap();
366-
let all_content = writer.get("logs-archives/all.tar.gz", &mime);
367-
let mut all = Archive::new(GzDecoder::new(all_content.as_slice()));
368-
let regressed_content = writer.get("logs-archives/regressed.tar.gz", &mime);
369-
let mut regressed = Archive::new(GzDecoder::new(regressed_content.as_slice()));
370-
let test_pass_content = writer.get("logs-archives/test-pass.tar.gz", &mime);
371-
let mut test_pass = Archive::new(GzDecoder::new(test_pass_content.as_slice()));
364+
let mime: Mime = "application/zstd".parse().unwrap();
365+
let all_content = writer.get("logs-archives/all.tar.zst", &mime);
366+
let mut all = Archive::new(Decoder::new(all_content.as_slice()).unwrap());
367+
let regressed_content = writer.get("logs-archives/regressed.tar.zst", &mime);
368+
let mut regressed = Archive::new(Decoder::new(regressed_content.as_slice()).unwrap());
369+
let test_pass_content = writer.get("logs-archives/test-pass.tar.zst", &mime);
370+
let mut test_pass = Archive::new(Decoder::new(test_pass_content.as_slice()).unwrap());
372371

373372
macro_rules! check_content {
374373
($archive:ident: { $($file:expr => $match:expr,)* }) => {{
@@ -401,21 +400,21 @@ mod tests {
401400
}}
402401
}
403402

404-
// Check all.tar.gz
403+
// Check all.tar.zst
405404
check_content!(all: {
406405
format!("regressed/{}/{}.txt", crate1.id(), ex.toolchains[0]) => "tc1 crate1",
407406
format!("regressed/{}/{}.txt", crate1.id(), ex.toolchains[1]) => "tc2 crate1",
408407
format!("test-pass/{}/{}.txt", crate2.id(), ex.toolchains[0]) => "tc1 crate2",
409408
format!("test-pass/{}/{}.txt", crate2.id(), ex.toolchains[1]) => "tc2 crate2",
410409
});
411410

412-
// Check regressed.tar.gz
411+
// Check regressed.tar.zst
413412
check_content!(regressed: {
414413
format!("regressed/{}/{}.txt", crate1.id(), ex.toolchains[0]) => "tc1 crate1",
415414
format!("regressed/{}/{}.txt", crate1.id(), ex.toolchains[1]) => "tc2 crate1",
416415
});
417416

418-
// Check test-pass.tar.gz
417+
// Check test-pass.tar.zst
419418
check_content!(test_pass: {
420419
format!("test-pass/{}/{}.txt", crate2.id(), ex.toolchains[0]) => "tc1 crate2",
421420
format!("test-pass/{}/{}.txt", crate2.id(), ex.toolchains[1]) => "tc2 crate2",

‎src/report/mod.rs

+3-3
Original file line numberDiff line numberDiff line change
@@ -243,9 +243,9 @@ pub fn generate_report<DB: ReadResults>(
243243
})
244244
});
245245
// Convert errors to Nones
246-
let mut crate_results = crate_results.map(|r| r.ok()).collect::<Vec<_>>();
247-
let crate2 = crate_results.pop().unwrap();
248-
let crate1 = crate_results.pop().unwrap();
246+
let mut crate_results = crate_results.map(|r| r.ok());
247+
let crate1 = crate_results.next().unwrap();
248+
let crate2 = crate_results.next().unwrap();
249249
let comp = compare(
250250
config,
251251
krate,

‎src/results/mod.rs

+7-5
Original file line numberDiff line numberDiff line change
@@ -125,18 +125,20 @@ macro_rules! test_result_enum {
125125

126126
fn from_str(input: &str) -> Fallible<Self> {
127127
// if there is more than one ':' we assume it's part of a failure reason serialization
128-
let parts: Vec<&str> = input.splitn(2, ':').collect();
128+
let mut parts = input.splitn(2, ':');
129+
let part1 = parts.next().unwrap();
130+
let part2 = parts.next();
129131

130-
if parts.len() == 1 {
131-
match parts[0] {
132+
if part2.is_none() {
133+
match part1 {
132134
$($with_reason_repr => Ok($name::$with_reason_name($reason::Unknown)),)*
133135
$($reasonless_repr => Ok($name::$reasonless_name),)*
134136
other => Err(TestResultParseError::UnknownResult(other.into()).into()),
135137
}
136138
} else {
137-
match parts[0] {
139+
match part1 {
138140
$($reasonless_repr => Err(TestResultParseError::UnexpectedFailureReason.into()),)*
139-
$($with_reason_repr => Ok($name::$with_reason_name(parts[1].parse()?)),)*
141+
$($with_reason_repr => Ok($name::$with_reason_name(part2.unwrap().parse()?)),)*
140142
other => Err(TestResultParseError::UnknownResult(other.into()).into()),
141143
}
142144
}

‎templates/report/downloads.html

+1-1
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414

1515
<div class="category">
1616
<div class="header header-background toggle" data-toggle="#downloads-archives">
17-
Build logs (tar.gz)
17+
Build logs (tar.zst)
1818
</div>
1919
<div class="crates" id="downloads-archives">
2020
{% for archive in available_archives %}

‎tests/minicrater/blacklist/downloads.html.context.expected.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,15 @@
22
"available_archives": [
33
{
44
"name": "All the crates",
5-
"path": "logs-archives/all.tar.gz"
5+
"path": "logs-archives/all.tar.zst"
66
},
77
{
88
"name": "test-pass crates",
9-
"path": "logs-archives/test-pass.tar.gz"
9+
"path": "logs-archives/test-pass.tar.zst"
1010
},
1111
{
1212
"name": "test-skipped crates",
13-
"path": "logs-archives/test-skipped.tar.gz"
13+
"path": "logs-archives/test-skipped.tar.zst"
1414
}
1515
],
1616
"crates_count": 3,

‎tests/minicrater/clippy/downloads.html.context.expected.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,15 @@
22
"available_archives": [
33
{
44
"name": "All the crates",
5-
"path": "logs-archives/all.tar.gz"
5+
"path": "logs-archives/all.tar.zst"
66
},
77
{
88
"name": "test-pass crates",
9-
"path": "logs-archives/test-pass.tar.gz"
9+
"path": "logs-archives/test-pass.tar.zst"
1010
},
1111
{
1212
"name": "regressed crates",
13-
"path": "logs-archives/regressed.tar.gz"
13+
"path": "logs-archives/regressed.tar.zst"
1414
}
1515
],
1616
"crates_count": 2,

‎tests/minicrater/doc/downloads.html.context.expected.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,15 @@
22
"available_archives": [
33
{
44
"name": "All the crates",
5-
"path": "logs-archives/all.tar.gz"
5+
"path": "logs-archives/all.tar.zst"
66
},
77
{
88
"name": "test-pass crates",
9-
"path": "logs-archives/test-pass.tar.gz"
9+
"path": "logs-archives/test-pass.tar.zst"
1010
},
1111
{
1212
"name": "build-fail crates",
13-
"path": "logs-archives/build-fail.tar.gz"
13+
"path": "logs-archives/build-fail.tar.zst"
1414
}
1515
],
1616
"crates_count": 2,

‎tests/minicrater/full/downloads.html.context.expected.json

+7-7
Original file line numberDiff line numberDiff line change
@@ -2,31 +2,31 @@
22
"available_archives": [
33
{
44
"name": "All the crates",
5-
"path": "logs-archives/all.tar.gz"
5+
"path": "logs-archives/all.tar.zst"
66
},
77
{
88
"name": "regressed crates",
9-
"path": "logs-archives/regressed.tar.gz"
9+
"path": "logs-archives/regressed.tar.zst"
1010
},
1111
{
1212
"name": "fixed crates",
13-
"path": "logs-archives/fixed.tar.gz"
13+
"path": "logs-archives/fixed.tar.zst"
1414
},
1515
{
1616
"name": "broken crates",
17-
"path": "logs-archives/broken.tar.gz"
17+
"path": "logs-archives/broken.tar.zst"
1818
},
1919
{
2020
"name": "build-fail crates",
21-
"path": "logs-archives/build-fail.tar.gz"
21+
"path": "logs-archives/build-fail.tar.zst"
2222
},
2323
{
2424
"name": "test-pass crates",
25-
"path": "logs-archives/test-pass.tar.gz"
25+
"path": "logs-archives/test-pass.tar.zst"
2626
},
2727
{
2828
"name": "test-fail crates",
29-
"path": "logs-archives/test-fail.tar.gz"
29+
"path": "logs-archives/test-fail.tar.zst"
3030
}
3131
],
3232
"crates_count": 17,

‎tests/minicrater/ignore-blacklist/downloads.html.context.expected.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,15 @@
22
"available_archives": [
33
{
44
"name": "All the crates",
5-
"path": "logs-archives/all.tar.gz"
5+
"path": "logs-archives/all.tar.zst"
66
},
77
{
88
"name": "test-pass crates",
9-
"path": "logs-archives/test-pass.tar.gz"
9+
"path": "logs-archives/test-pass.tar.zst"
1010
},
1111
{
1212
"name": "test-fail crates",
13-
"path": "logs-archives/test-fail.tar.gz"
13+
"path": "logs-archives/test-fail.tar.zst"
1414
}
1515
],
1616
"crates_count": 3,

‎tests/minicrater/missing-repo/downloads.html.context.expected.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@
22
"available_archives": [
33
{
44
"name": "All the crates",
5-
"path": "logs-archives/all.tar.gz"
5+
"path": "logs-archives/all.tar.zst"
66
},
77
{
88
"name": "broken crates",
9-
"path": "logs-archives/broken.tar.gz"
9+
"path": "logs-archives/broken.tar.zst"
1010
}
1111
],
1212
"crates_count": 1,

‎tests/minicrater/resource-exhaustion/downloads.html.context.expected.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,15 @@
22
"available_archives": [
33
{
44
"name": "All the crates",
5-
"path": "logs-archives/all.tar.gz"
5+
"path": "logs-archives/all.tar.zst"
66
},
77
{
88
"name": "test-pass crates",
9-
"path": "logs-archives/test-pass.tar.gz"
9+
"path": "logs-archives/test-pass.tar.zst"
1010
},
1111
{
1212
"name": "spurious-fixed crates",
13-
"path": "logs-archives/spurious-fixed.tar.gz"
13+
"path": "logs-archives/spurious-fixed.tar.zst"
1414
}
1515
],
1616
"crates_count": 2,

0 commit comments

Comments
 (0)