refactor: split integration tests from CLI (part 1) (#22308)

This PR separates integration tests from CLI tests into a new project
named `cli_tests`. This is a prerequisite for an integration test runner
that can work with either the CLI binary in the current project, or one
that is built ahead of time.

## Background

Rust does not have the concept of artifact dependencies yet
(rust-lang/cargo#9096). Because of this, the
only way we can ensure a binary is built before running associated tests
is by hanging tests off the crate with the binary itself.

Unfortunately this means that to run those tests, you _must_ build the
binary and in the case of the deno executable that might be a 10 minute
wait in release mode.

## Implementation

To allow for tests to run with and without the requirement that the
binary is up-to-date, we split the integration tests into a project of
their own. As these tests would not require the binary to build itself
before being run as-is, we add a stub integration `[[test]]` target in
the `cli` project that invokes these tests using `cargo test`.

The stub test runner we add has `harness = false` so that we can get
access to a `main` function. This `main` function's sole job is to
`execvp` the command `cargo test -p deno_cli`, effectively "calling"
another cargo target.

This ensures that the deno executable is always correctly rebuilt before
running the stub test runner from `cli`, and gets us closer to be able
to run the entire integration test suite on arbitrary deno executables
(and therefore split the build into multiple phases).

The new `cli_tests` project lives within `cli` to avoid a large PR. In
later PRs, the test data will be split from the `cli` project. As there
are a few thousand files, it'll be better to do this as a completely
separate PR to avoid noise.

Author: mmastrac

.github/workflows/ci.generate.ts

@@ -794,7 +794,7 @@ const ci = {
             // Run unit then integration tests. Skip doc tests here
             // since they are sometimes very slow on Mac.
             "cargo test --locked --lib",
-            "cargo test --locked --test '*'",
+            "cargo test --locked --tests",
           ].join("\n"),
           env: { CARGO_PROFILE_DEV_DEBUG: 0 },
         },

.github/workflows/ci.yml

@@ -482,7 +482,7 @@ jobs:
           (startsWith(github.ref, 'refs/tags/') || matrix.os != 'linux'))
         run: |-
           cargo test --locked --lib
-          cargo test --locked --test '*'
+          cargo test --locked --tests
         env:
           CARGO_PROFILE_DEV_DEBUG: 0
       - name: Test (release)

Cargo.lock

@@ -6,6 +6,7 @@ members = [
   "bench_util",
   "cli",
   "cli/napi/sym",
+  "cli/tests",
   "ext/broadcast_channel",
   "ext/cache",
   "ext/canvas",

Cargo.toml

@@ -4,6 +4,7 @@
 name = "deno"
 version = "1.40.4"
 authors.workspace = true
+autotests = false
 default-run = "deno"
 edition.workspace = true
 exclude = ["tests/testdata/npm/registry/*"]
@@ -16,6 +17,11 @@ name = "deno"
 path = "main.rs"
 doc = false
 
+[[test]]
+name = "integration"
+path = "tests/integration_tests_runner.rs"
+harness = false
+
 [[bench]]
 name = "deno_bench"
 harness = false
@@ -149,19 +155,8 @@ nix.workspace = true
 
 [dev-dependencies]
 deno_bench_util.workspace = true
-deno_core = { workspace = true, features = ["include_js_files_for_snapshotting", "unsafe_use_unprotected_platform"] }
-fastwebsockets = { workspace = true, features = ["upgrade", "unstable-split"] }
-flaky_test = "=0.1.0"
-http.workspace = true
-http-body-util.workspace = true
-hyper.workspace = true
-hyper-util.workspace = true
-once_cell.workspace = true
-os_pipe.workspace = true
 pretty_assertions.workspace = true
 test_util.workspace = true
-trust-dns-client = "=0.22.0"
-trust-dns-server = "=0.22.1"
 
 [package.metadata.winres]
 # This section defines the metadata that appears in the deno.exe PE header.

cli/Cargo.toml

@@ -0,0 +1,51 @@
+# Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+
+[package]
+name = "cli_tests"
+version = "0.0.0"
+authors.workspace = true
+autotests = false
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[lib]
+path = "lib.rs"
+
+[features]
+run = []
+
+[[test]]
+name = "integration_tests"
+path = "integration_tests.rs"
+required-features = ["run"]
+
+[dev-dependencies]
+bytes.workspace = true
+deno_ast.workspace = true
+deno_bench_util.workspace = true
+deno_core = { workspace = true, features = ["include_js_files_for_snapshotting", "unsafe_use_unprotected_platform"] }
+deno_fetch.workspace = true
+deno_lockfile.workspace = true
+deno_tls.workspace = true
+fastwebsockets = { workspace = true, features = ["upgrade", "unstable-split"] }
+flaky_test = "=0.1.0"
+http.workspace = true
+http-body-util.workspace = true
+hyper.workspace = true
+hyper-util.workspace = true
+once_cell.workspace = true
+os_pipe.workspace = true
+pretty_assertions.workspace = true
+serde.workspace = true
+serde_repr.workspace = true
+test_util.workspace = true
+tokio.workspace = true
+tokio-util.workspace = true
+tower-lsp.workspace = true
+trust-dns-client = "=0.22.0"
+trust-dns-server = "=0.22.1"
+url.workspace = true
+
+[target.'cfg(unix)'.dev-dependencies]
+nix.workspace = true

cli/tests/Cargo.toml

@@ -1,14 +1,14 @@
 // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
 
-use deno_runtime::deno_net::ops_tls::TlsStream;
-use deno_runtime::deno_tls::rustls;
-use deno_runtime::deno_tls::rustls_pemfile;
-use lsp_types::Url;
+use deno_tls::rustls;
+use deno_tls::rustls_pemfile;
+use deno_tls::rustls_tokio_stream::TlsStream;
 use std::io::BufReader;
 use std::io::Cursor;
 use std::io::Read;
 use std::sync::Arc;
 use test_util as util;
+use url::Url;
 use util::testdata_path;
 use util::TestContext;
 

cli/tests/integration/cert_tests.rs

@@ -6,7 +6,7 @@ use deno_core::error::AnyError;
 use deno_core::serde_json;
 use deno_core::serde_json::json;
 use deno_core::url;
-use deno_runtime::deno_fetch::reqwest;
+use deno_fetch::reqwest;
 use fastwebsockets::FragmentCollector;
 use fastwebsockets::Frame;
 use fastwebsockets::WebSocket;

cli/tests/integration/inspector_tests.rs

@@ -7,7 +7,7 @@ use test_util as util;
 
 util::unit_test_factory!(
   js_unit_test,
-  "tests/unit",
+  "../tests/unit",
   "*.ts",
   [
     abort_controller_test,

cli/tests/integration/js_unit_tests.rs

@@ -8,7 +8,7 @@ use util::env_vars_for_npm_tests;
 
 util::unit_test_factory!(
   node_unit_test,
-  "tests/unit_node",
+  "../tests/unit_node",
   "**/*_test.ts",
   [
     _fs_access_test = _fs / _fs_access_test,

cli/tests/integration/node_unit_tests.rs

@@ -3,7 +3,7 @@
 use bytes::Bytes;
 use deno_core::serde_json::json;
 use deno_core::url;
-use deno_runtime::deno_fetch::reqwest;
+use deno_fetch::reqwest;
 use pretty_assertions::assert_eq;
 use std::io::Read;
 use std::io::Write;

cli/tests/integration/run_tests.rs

@@ -3,4 +3,5 @@
 // The tests exist in a sub folder instead of as separate files in
 // this directory so that cargo doesn't compile each file as a new crate.
 
+#[cfg(test)]
 mod integration;

cli/tests/integration_tests.rs

@@ -0,0 +1,18 @@
+// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+pub fn main() {
+  let mut args = vec!["cargo", "test", "-p", "cli_tests", "--features", "run"];
+
+  if !cfg!(debug_assertions) {
+    args.push("--release");
+  }
+
+  args.push("--");
+
+  // If any args were passed to this process, pass them through to the child
+  let orig_args = std::env::args().skip(1).collect::<Vec<_>>();
+  let orig_args: Vec<&str> =
+    orig_args.iter().map(|x| x.as_ref()).collect::<Vec<_>>();
+  args.extend(orig_args);
+
+  test_util::spawn::exec_replace("cargo", &args).unwrap();
+}

cli/tests/integration_tests_runner.rs

@@ -0,0 +1 @@
+// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.

cli/tests/lib.rs

@@ -19,6 +19,7 @@ deno_native_certs = "0.2.0"
 once_cell.workspace = true
 rustls = { workspace = true, features = ["dangerous_configuration"] }
 rustls-pemfile.workspace = true
+rustls-tokio-stream.workspace = true
 rustls-webpki.workspace = true
 serde.workspace = true
 webpki-roots.workspace = true

ext/tls/Cargo.toml

@@ -3,6 +3,7 @@
 pub use deno_native_certs;
 pub use rustls;
 pub use rustls_pemfile;
+pub use rustls_tokio_stream;
 pub use webpki;
 pub use webpki_roots;
 

ext/tls/lib.rs

@@ -53,6 +53,7 @@ tempfile.workspace = true
 termcolor.workspace = true
 tokio.workspace = true
 url.workspace = true
+win32job = "2"
 
 [target.'cfg(windows)'.dependencies]
 winapi = { workspace = true, features = ["consoleapi", "synchapi", "handleapi", "namedpipeapi", "winbase", "winerror"] }

test_util/Cargo.toml

@@ -32,6 +32,7 @@ pub mod lsp;
 mod npm;
 pub mod pty;
 pub mod servers;
+pub mod spawn;
 
 pub use builders::DenoChild;
 pub use builders::TestCommandBuilder;