treewide: Replace in-process serial mutex with nextest config

`cargo test` only runs test in parallel within one test module. [`cargo
nextest`] can execute tests in parallel across crates, but in doing so
executes each test in its own process (supposedly for reduced
flakiness). This means our existing in-process mutexes provided by the
`serial_test` crate don't work when running under nextest.

We write our own `#[serial]` procedural macro which, in addition to
adding the same existing in-process mutex from the `serial_test` crate,
also renames tests such that they can be identified by the nextest
filterset DSL. (Although it has a limited ability to parse `cfg` attrs,
there is no facility for inspecting arbitrary attrs attached to the test
or anything like that; otherwise we could just use the existing
`#[serial]` and somehow specify it in the filterset.) This allows
creating [test groups] which don't allow parallel execution within them.

We then get the benefits of parallel test execution across crates; so
for the bulk of the time, 1 test in the mysql group is running and 1
test in the postgres group is running, getting us to (in a simple test
on an M3 Max) 181% CPU usage and a commensurate 81% speedup in clock
time.

```sh
$ time TZ=UTC RUN_SLOW_TESTS=1 cargo nextest run \
    -E 'not package(readyset-clustertest)'
...
     Summary [ 393.304s] 2667 tests run: 2667 passed, 84 skipped
620.70s user 207.41s system 181% cpu 7:35.10 total

$ time TZ=UTC RUN_SLOW_TESTS=1 cargo test --all \
    --exclude readyset-clustertest
...
710.37s user 154.68s system 104% cpu 13:44.92 total
```

[`cargo nextest`]: https://nexte.st/
[test groups]: https://nexte.st/docs/configuration/test-groups/

Change-Id: I75a6b762bb18b4ebe7927cfe60eb7cefea038ee8
Reviewed-on: https://gerrit.readyset.name/c/readyset/+/8507
Tested-by: Buildkite CI
Reviewed-by: Johnathan Davis <[email protected]>

Author: mvzink

.config/nextest.toml

@@ -0,0 +1,26 @@
+[profile.default]
+# This one test is absurdly long locally even compared to other tests marked slow, and not very
+# valuable unless you're working on related code. In which case, run with --ignore-default-filter.
+default-filter = 'not test(test_large_packet_write)'
+
+[test-groups]
+serial = { max-threads = 1 }
+serial-postgres = { max-threads = 1 }
+serial-postgres13 = { max-threads = 1 }
+serial-mysql = { max-threads = 1 }
+
+[[profile.default.overrides]]
+filter = 'test(/_serial$/)'
+test-group = 'serial'
+
+[[profile.default.overrides]]
+filter = 'test(/_serial_postgres$/)'
+test-group = 'serial-postgres'
+
+[[profile.default.overrides]]
+filter = 'test(/_serial_postgres13$/)'
+test-group = 'serial-postgres13'
+
+[[profile.default.overrides]]
+filter = 'test(/_serial_mysql$/)'
+test-group = 'serial-mysql'

Cargo.lock

@@ -56,7 +56,7 @@ members = [
     "tinylb",
     "test-utils",
     "timestamp-service",
-    "tournament-kway"
+    "tournament-kway",
 ]
 resolver = "2"
 
@@ -113,7 +113,13 @@ csv = "1.3"
 dashmap = "6.1.0"
 deadpool-postgres = "0.14.0"
 derive_builder = "0.20.2"
-derive_more = { version = "1.0.0", features = ["from", "into", "try_into", "deref", "display"] }
+derive_more = { version = "1.0.0", features = [
+    "from",
+    "into",
+    "try_into",
+    "deref",
+    "display",
+] }
 diff = "0.1.13"
 enum-display-derive = "0.1.1"
 enum-kinds = "0.5.1"
@@ -160,7 +166,9 @@ native-tls = "0.2.11"
 ndarray = "0.16.1"
 nom = "7.1.3"
 nom_locate = "4.2.0"
-notify = { version = "7.0.0", default-features = false, features = ["macos_kqueue"] }
+notify = { version = "7.0.0", default-features = false, features = [
+    "macos_kqueue",
+] }
 num-integer = "0.1.46"
 num_cpus = "1.16.0"
 opentelemetry = "0.21.0"
@@ -189,7 +197,10 @@ regex = "1.10.4"
 reqwest = "0.11.27"
 rlimit = "0.10.1"
 rmp-serde = "1.2.0"
-rocksdb = { version = "0.22.0", default-features = false, features = ["lz4", "jemalloc"] }
+rocksdb = { version = "0.22.0", default-features = false, features = [
+    "lz4",
+    "jemalloc",
+] }
 rust_decimal = "1.26"
 rusty-fork = "0.3.0"
 rustyline = "14.0"
@@ -228,7 +239,7 @@ tikv-jemalloc-sys = "0.5.4"
 tikv-jemallocator = "0.5.4"
 time = "0.3"
 timekeeper = { version = "0.3.2", default-features = false }
-tokio = { version = "1.37",  features = ["full"] }
+tokio = { version = "1.37", features = ["full"] }
 tokio-native-tls = "0.3.1"
 tokio-retry = "0.3"
 tokio-scoped = "0.2.0"
@@ -259,12 +270,12 @@ zipf = "7.0.1"
 jobserver = "0.1.32"
 
 [profile.release]
-debug=true
-lto="thin"
+debug = true
+lto = "thin"
 
 [profile.release-fat-lto]
 inherits = "release"
-lto="fat"
+lto = "fat"
 
 [profile.release-dist]
 # configs for distro release packages (i.e. deb, rpm, etc.)

Cargo.toml

@@ -24,6 +24,6 @@ readyset-data = { path = "../readyset-data/" }
 test-strategy = { workspace = true }
 proptest = { workspace = true }
 mysql_async = { workspace = true }
-tokio = { workspace = true, features = ["full"] }
 serial_test = { workspace = true }
+tokio = { workspace = true, features = ["full"] }
 test-utils = { path = "../test-utils" }

data-generator/Cargo.toml

@@ -6,8 +6,8 @@ use mysql_async::Value;
 use nom_sql::{Dialect, DialectDisplay, SqlType};
 use proptest::prop_assume;
 use rand::{rngs::SmallRng, SeedableRng};
-use serial_test::serial;
 use test_strategy::proptest;
+use test_utils::serial;
 use test_utils::slow;
 
 async fn mysql_connection() -> mysql_async::Conn {
@@ -28,8 +28,8 @@ async fn mysql_connection() -> mysql_async::Conn {
     .unwrap()
 }
 
+#[serial(mysql)]
 #[proptest]
-#[serial]
 #[slow]
 fn value_of_type_always_valid(
     #[any(generate_arrays = false, dialect = Some(Dialect::MySQL))] ty: SqlType,
@@ -56,8 +56,8 @@ fn value_of_type_always_valid(
     .unwrap();
 }
 
+#[serial(mysql)]
 #[proptest]
-#[serial]
 #[slow]
 fn unique_value_of_type_always_valid(
     #[any(generate_arrays = false, dialect = Some(Dialect::MySQL))] ty: SqlType,
@@ -87,8 +87,8 @@ fn unique_value_of_type_always_valid(
     .unwrap();
 }
 
+#[serial(mysql)]
 #[proptest]
-#[serial]
 #[slow]
 fn random_value_of_type_always_valid(
     #[any(generate_arrays = false, dialect = Some(Dialect::MySQL))] ty: SqlType,

data-generator/tests/mysql.rs

@@ -8,7 +8,7 @@ edition = "2021"
 [dependencies]
 anyhow = { workspace = true }
 chrono = { workspace = true }
-clap = { workspace = true, features = ["derive","env"] }
+clap = { workspace = true, features = ["derive", "env"] }
 derive_more = { workspace = true }
 futures-util = { workspace = true }
 itertools = { workspace = true }
@@ -42,3 +42,4 @@ mysql_async.workspace = true
 serial_test = { workspace = true }
 tokio = { workspace = true, features = ["full"] }
 tokio-postgres.workspace = true
+test-utils = { path = "../test-utils" }

query-generator/Cargo.toml

@@ -4,8 +4,8 @@ use mysql_async::prelude::Queryable;
 use mysql_async::{OptsBuilder, Params};
 use nom_sql::{CreateTableStatement, Dialect, DialectDisplay};
 use query_generator::{GeneratorState, QuerySeed};
-use serial_test::serial;
 use test_strategy::proptest;
+use test_utils::serial;
 
 async fn mysql_connection() -> mysql_async::Conn {
     let db_name = env::var("MYSQL_DB").unwrap_or_else(|_| "test".to_owned());
@@ -42,8 +42,8 @@ async fn mysql_connection() -> mysql_async::Conn {
     conn
 }
 
+#[serial(mysql)]
 #[proptest]
-#[serial]
 #[ignore = "Currently failing"]
 fn queries_work_in_mysql(seed: QuerySeed) {
     let rt = tokio::runtime::Builder::new_multi_thread()

query-generator/tests/mysql.rs

@@ -14,7 +14,7 @@ categories = ["api-bindings", "database"]
 [dependencies]
 anyhow = { workspace = true }
 async-bincode = { workspace = true }
-clap = { workspace = true, features = ["derive","env"] }
+clap = { workspace = true, features = ["derive", "env"] }
 fail = { workspace = true }
 thiserror = { workspace = true }
 hyper = { workspace = true, features = ["stream", "client", "http2"] }
@@ -35,7 +35,14 @@ time = { workspace = true, features = ["local-offset"] }
 tower-service = { workspace = true }
 tower-layer = { workspace = true }
 tokio-tower = { workspace = true }
-tower = { workspace = true, features = ["limit", "balance", "buffer", "discover", "util", "timeout"] }
+tower = { workspace = true, features = [
+    "limit",
+    "balance",
+    "buffer",
+    "discover",
+    "util",
+    "timeout",
+] }
 tracing = { workspace = true, features = ["release_max_level_debug"] }
 tracing-futures = { workspace = true }
 slab = { workspace = true }
@@ -53,12 +60,21 @@ cloudflare-zlib = { workspace = true, features = ["arm-always"] }
 smallvec = { workspace = true }
 rocksdb.workspace = true
 
-tokio-postgres = { workspace = true, features = ["with-chrono-0_4", "with-eui48-1", "with-uuid-0_8", "with-serde_json-1", "with-bit-vec-0_6"] }
+tokio-postgres = { workspace = true, features = [
+    "with-chrono-0_4",
+    "with-eui48-1",
+    "with-uuid-0_8",
+    "with-serde_json-1",
+    "with-bit-vec-0_6",
+] }
 metrics = { workspace = true }
 metrics-util = { workspace = true }
 itertools = { workspace = true }
 bytes = { workspace = true }
-rust_decimal = { workspace = true, features = ["db-tokio-postgres", "serde-str"] }
+rust_decimal = { workspace = true, features = [
+    "db-tokio-postgres",
+    "serde-str",
+] }
 eui48 = { workspace = true }
 uuid = { workspace = true, features = ["v4"] }
 bit-vec = { workspace = true, features = ["serde"] }
@@ -92,6 +108,7 @@ replication-offset = { path = "../replication-offset" }
 [dev-dependencies]
 serial_test = { workspace = true }
 tempfile = { workspace = true }
+test-utils = { path = "../test-utils" }
 
 [features]
 failure_injection = ["fail/failpoints"]

readyset-client/Cargo.toml

@@ -984,7 +984,7 @@ mod tests {
     use rand::{thread_rng, Rng};
     use readyset_data::Dialect;
     use reqwest::Url;
-    use serial_test::serial;
+    use test_utils::serial;
 
     use super::*;
     use crate::consensus::CacheDDLRequest;

readyset-client/src/consensus/consul.rs

@@ -31,14 +31,14 @@ readyset-tracing = { path = "../readyset-tracing" }
 readyset-errors = { path = "../readyset-errors" }
 
 [dev-dependencies]
-serial_test = { workspace = true }
-criterion = { workspace = true, features=["async_tokio"]}
+criterion = { workspace = true, features = ["async_tokio"] }
 itertools = { workspace = true }
 readyset-client-metrics = { path = "../readyset-client-metrics" }
-test-utils = { path = "../test-utils" }
 readyset-clustertest-macros = { path = "./macros" }
 readyset-tracing = { path = "../readyset-tracing" }
 rust_decimal = { workspace = true }
+serial_test = { workspace = true }
+test-utils = { path = "../test-utils" }
 
 [build-dependencies]
 anyhow = { workspace = true }

readyset-clustertest/Cargo.toml

@@ -1,10 +1,11 @@
 extern crate proc_macro;
 use proc_macro::TokenStream;
 use quote::quote;
-use syn::{parse_macro_input, parse_quote, ItemFn};
+use syn::{parse_macro_input, parse_quote, Ident, ItemFn};
 
 #[proc_macro_attribute]
-pub fn clustertest(_attr: TokenStream, input: TokenStream) -> TokenStream {
+pub fn clustertest(args: TokenStream, input: TokenStream) -> TokenStream {
+    let group = parse_macro_input!(args as Option<Ident>);
     let input_fn = parse_macro_input!(input as ItemFn);
 
     let fn_block = *input_fn.block;
@@ -25,7 +26,7 @@ pub fn clustertest(_attr: TokenStream, input: TokenStream) -> TokenStream {
 
     let result = quote! {
         #[tokio::test(flavor = "multi_thread")]
-        #[serial]
+        #[serial(#group)]
         #test_with_tracing
     };
     result.into()