Auto merge of rust-lang#7906 - ehuss:macos-10.15, r=alexcrichton

Switch azure to macOS 10.15.

Switches CI to the macOS 10.15 image.  Since 32-bit support is no longer available, this changes how cross-compile testing works.  I decided to use `x86_64-apple-ios` as a cross target, since it can easily build/link on macOS.  `cargo run` won't work without a simulator, so some of the tests are restructured to check if `cargo run` is allowed.  If you do have a simulator, it should Just Work.  CI doesn't seem to be configured with a simulator installed, and I didn't bother to look if that would be possible (the simulators tend to be several gigabytes in size).

An alternative approach would be to use wasm as a cross target, which is also fairly easy to support.  But wasm is a sufficiently different target that it can cause some issues in some tests, and is a bit harder to run as an executable.

This also adds some more help text on how to configure cross-compile tests.

Rustup is now installed on macOS by default, so no need to install it.  Unfortunately self-updates are not allowed, but hopefully that won't be an issue.

Closes rust-lang#7821

Author: bors

CONTRIBUTING.md

@@ -115,10 +115,8 @@ may also need to install additional tools for the target. For example, on Ubuntu
 you should install the `gcc-multilib` package.
 
 If you can't install an alternate target, you can set the
-`CFG_DISABLE_CROSS_TESTS=1` environment variable to disable these tests.
-Unfortunately 32-bit support on macOS is going away, so you may not be able to
-run these tests on macOS. The Windows cross tests only support the MSVC
-toolchain.
+`CFG_DISABLE_CROSS_TESTS=1` environment variable to disable these tests. The
+Windows cross tests only support the MSVC toolchain.
 
 Some of the nightly tests require the `rustc-dev` component installed. This
 component includes the compiler as a library. This may already be installed

azure-pipelines.yml

@@ -26,12 +26,12 @@ jobs:
 
 - job: macOS
   pool:
-    vmImage: macos-10.13
+    vmImage: macos-10.15
   steps:
     - template: ci/azure-test-all.yml
   variables:
     TOOLCHAIN: stable
-    OTHER_TARGET: i686-apple-darwin
+    OTHER_TARGET: x86_64-apple-ios
 
 - job: Windows
   pool:

ci/azure-install-rust.yml

@@ -1,15 +1,4 @@
 steps:
-  - bash: |
-      set -e
-      if command -v rustup; then
-        echo `command -v rustup` `rustup -V` already installed
-        rustup self update
-      else
-        curl -sSL https://sh.rustup.rs | sh -s -- -y --default-toolchain="$TOOLCHAIN" --profile=minimal
-        echo "##vso[task.prependpath]$HOME/.cargo/bin"
-      fi
-    displayName: Install rustup
-
   - bash: |
       set -e
       rustup set profile minimal

crates/cargo-test-support/src/cross_compile.rs

@@ -9,52 +9,70 @@
 //!
 //! These tests are all disabled on rust-lang/rust's CI, but run in Cargo's CI.
 
-use crate::{basic_bin_manifest, main_file, project};
+use crate::{basic_manifest, main_file, project};
+use cargo::util::ProcessError;
+use cargo::CargoResult;
 use std::env;
-use std::process::Command;
+use std::fmt::Write;
+use std::process::{Command, Output};
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::sync::Once;
 
+/// Whether or not the resulting cross binaries can run on the host.
+static CAN_RUN_ON_HOST: AtomicBool = AtomicBool::new(false);
+
 pub fn disabled() -> bool {
-    // First, disable if `./configure` requested so.
+    // First, disable if requested.
     match env::var("CFG_DISABLE_CROSS_TESTS") {
         Ok(ref s) if *s == "1" => return true,
         _ => {}
     }
 
-    // Right now, the Windows bots cannot cross compile due to the Mingw setup,
-    // so we disable ourselves on all but macOS/Linux setups where the rustc
-    // install script ensures we have both architectures.
+    // Cross tests are only tested to work on macos, linux, and MSVC windows.
     if !(cfg!(target_os = "macos") || cfg!(target_os = "linux") || cfg!(target_env = "msvc")) {
         return true;
     }
 
     // It's not particularly common to have a cross-compilation setup, so
     // try to detect that before we fail a bunch of tests through no fault
     // of the user.
-    static CAN_RUN_CROSS_TESTS: AtomicBool = AtomicBool::new(false);
+    static CAN_BUILD_CROSS_TESTS: AtomicBool = AtomicBool::new(false);
     static CHECK: Once = Once::new();
 
     let cross_target = alternate();
 
-    CHECK.call_once(|| {
+    let run_cross_test = || -> CargoResult<Output> {
         let p = project()
             .at("cross_test")
-            .file("Cargo.toml", &basic_bin_manifest("cross_test"))
-            .file("src/cross_test.rs", &main_file(r#""testing!""#, &[]))
+            .file("Cargo.toml", &basic_manifest("cross_test", "1.0.0"))
+            .file("src/main.rs", &main_file(r#""testing!""#, &[]))
             .build();
 
-        let result = p
+        let build_result = p
             .cargo("build --target")
             .arg(&cross_target)
             .exec_with_output();
 
+        if build_result.is_ok() {
+            CAN_BUILD_CROSS_TESTS.store(true, Ordering::SeqCst);
+        }
+
+        let result = p
+            .cargo("run --target")
+            .arg(&cross_target)
+            .exec_with_output();
+
         if result.is_ok() {
-            CAN_RUN_CROSS_TESTS.store(true, Ordering::SeqCst);
+            CAN_RUN_ON_HOST.store(true, Ordering::SeqCst);
         }
+        build_result
+    };
+
+    CHECK.call_once(|| {
+        drop(run_cross_test());
     });
 
-    if CAN_RUN_CROSS_TESTS.load(Ordering::SeqCst) {
+    if CAN_BUILD_CROSS_TESTS.load(Ordering::SeqCst) {
         // We were able to compile a simple project, so the user has the
         // necessary `std::` bits installed. Therefore, tests should not
         // be disabled.
@@ -75,74 +93,134 @@ pub fn disabled() -> bool {
     }
 
     // We are responsible for warning the user, which we do by panicking.
-    let rustup_available = Command::new("rustup").output().is_ok();
-
-    let linux_help = if cfg!(target_os = "linux") {
+    let mut message = format!(
         "
+Cannot cross compile to {}.
 
-You may need to install runtime libraries for your Linux distribution as well."
-            .to_string()
-    } else {
-        "".to_string()
-    };
+This failure can be safely ignored. If you would prefer to not see this
+failure, you can set the environment variable CFG_DISABLE_CROSS_TESTS to \"1\".
 
-    let rustup_help = if rustup_available {
-        format!(
+Alternatively, you can install the necessary libraries to enable cross
+compilation tests. Cross compilation tests depend on your host platform.
+",
+        cross_target
+    );
+
+    if cfg!(target_os = "linux") {
+        message.push_str(
             "
+Linux cross tests target i686-unknown-linux-gnu, which requires the ability to
+build and run 32-bit targets. This requires the 32-bit libraries to be
+installed. For example, on Ubuntu, run `sudo apt install gcc-multilib` to
+install the necessary libraries.
+",
+        );
+    } else if cfg!(target_os = "macos") {
+        message.push_str(
+            "
+macOS cross tests target x86_64-apple-ios, which requires the iOS SDK to be
+installed. This should be included with Xcode automatically. If you are using
+the Xcode command line tools, you'll need to install the full Xcode app (from
+the Apple App Store), and switch to it with this command:
 
-Alternatively, you can install the necessary libraries for cross-compilation with
+    sudo xcode-select --switch /Applications/Xcode.app/Contents/Developer
 
-    rustup target add {}{}",
-            cross_target, linux_help
-        )
+Some cross-tests want to *run* the executables on the host. These tests will
+be ignored if this is not possible. On macOS, this means you need an iOS
+simulator installed to run these tests. To install a simulator, open Xcode, go
+to preferences > Components, and download the latest iOS simulator.
+",
+        );
+    } else if cfg!(target_os = "windows") {
+        message.push_str(
+            "
+Windows cross tests target i686-pc-windows-msvc, which requires the ability
+to build and run 32-bit targets. This should work automatically if you have
+properly installed Visual Studio build tools.
+",
+        );
     } else {
-        "".to_string()
-    };
+        // The check at the top should prevent this.
+        panic!("platform should have been skipped");
+    }
 
-    panic!(
-        "Cannot cross compile to {}.
+    let rustup_available = Command::new("rustup").output().is_ok();
+    if rustup_available {
+        write!(
+            message,
+            "
+Make sure that the appropriate `rustc` target is installed with rustup:
 
-This failure can be safely ignored. If you would prefer to not see this
-failure, you can set the environment variable CFG_DISABLE_CROSS_TESTS to \"1\".{}
+    rustup target add {}
 ",
-        cross_target, rustup_help
-    );
+            cross_target
+        )
+        .unwrap();
+    } else {
+        write!(
+            message,
+            "
+rustup does not appear to be installed. Make sure that the appropriate
+`rustc` target is installed for the target `{}`.
+",
+            cross_target
+        )
+        .unwrap();
+    }
+
+    // Show the actual error message.
+    match run_cross_test() {
+        Ok(_) => message.push_str("\nUh oh, second run succeeded?\n"),
+        Err(err) => match err.downcast_ref::<ProcessError>() {
+            Some(proc_err) => write!(message, "\nTest error: {}\n", proc_err).unwrap(),
+            None => write!(message, "\nUnexpected non-process error: {}\n", err).unwrap(),
+        },
+    }
+
+    panic!(message);
 }
 
-pub fn alternate() -> String {
-    let platform = match env::consts::OS {
-        "linux" => "unknown-linux-gnu",
-        "macos" => "apple-darwin",
-        "windows" => "pc-windows-msvc",
-        _ => unreachable!(),
-    };
-    let arch = match env::consts::ARCH {
-        "x86" => "x86_64",
-        "x86_64" => "i686",
-        _ => unreachable!(),
-    };
-    format!("{}-{}", arch, platform)
+/// The alternate target-triple to build with.
+///
+/// Only use this function on tests that check `cross_compile::disabled`.
+pub fn alternate() -> &'static str {
+    if cfg!(target_os = "macos") {
+        "x86_64-apple-ios"
+    } else if cfg!(target_os = "linux") {
+        "i686-unknown-linux-gnu"
+    } else if cfg!(all(target_os = "windows", target_env = "msvc")) {
+        "i686-pc-windows-msvc"
+    } else {
+        panic!("This test should be gated on cross_compile::disabled.");
+    }
 }
 
 pub fn alternate_arch() -> &'static str {
-    match env::consts::ARCH {
-        "x86" => "x86_64",
-        "x86_64" => "x86",
-        _ => unreachable!(),
+    if cfg!(target_os = "macos") {
+        "x86_64"
+    } else {
+        "x86"
     }
 }
 
-pub fn host() -> String {
-    let platform = match env::consts::OS {
-        "linux" => "unknown-linux-gnu",
-        "macos" => "apple-darwin",
-        "windows" => "pc-windows-msvc",
-        _ => unreachable!(),
-    };
-    let arch = match env::consts::ARCH {
-        "x86" => "i686",
-        "x86_64" => "x86_64",
-        _ => unreachable!(),
-    };
-    format!("{}-{}", arch, platform)
+/// Whether or not the host can run cross-compiled executables.
+pub fn can_run_on_host() -> bool {
+    if disabled() {
+        return false;
+    }
+    // macos is currently configured to cross compile to x86_64-apple-ios
+    // which requires a simulator to run. Azure's CI image appears to have the
+    // SDK installed, but are not configured to launch iOS images with a
+    // simulator.
+    if cfg!(target_os = "macos") {
+        if CAN_RUN_ON_HOST.load(Ordering::SeqCst) {
+            return true;
+        } else {
+            println!("Note: Cannot run on host, skipping.");
+            return false;
+        }
+    } else {
+        assert!(CAN_RUN_ON_HOST.load(Ordering::SeqCst));
+        return true;
+    }
 }

crates/cargo-test-support/src/lib.rs

@@ -1741,6 +1741,10 @@ fn _process(t: &OsStr) -> cargo::util::ProcessBuilder {
         .env_remove("GIT_COMMITTER_NAME")
         .env_remove("GIT_COMMITTER_EMAIL")
         .env_remove("MSYSTEM"); // assume cmd.exe everywhere on windows
+    if cfg!(target_os = "macos") {
+        // Work-around a bug in macOS 10.15, see `link_or_copy` for details.
+        p.env("__CARGO_COPY_DONT_LINK_DO_NOT_USE_THIS", "1");
+    }
     p
 }
 

src/cargo/util/paths.rs

@@ -384,7 +384,20 @@ fn _link_or_copy(src: &Path, dst: &Path) -> CargoResult<()> {
         };
         symlink(src, dst)
     } else {
-        fs::hard_link(src, dst)
+        if env::var_os("__CARGO_COPY_DONT_LINK_DO_NOT_USE_THIS").is_some() {
+            // This is a work-around for a bug in macOS 10.15. When running on
+            // APFS, there seems to be a strange race condition with
+            // Gatekeeper where it will forcefully kill a process launched via
+            // `cargo run` with SIGKILL. Copying seems to avoid the problem.
+            // This shouldn't affect anyone except Cargo's test suite because
+            // it is very rare, and only seems to happen under heavy load and
+            // rapidly creating lots of executables and running them.
+            // See https://github.com/rust-lang/cargo/issues/7821 for the
+            // gory details.
+            fs::copy(src, dst).map(|_| ())
+        } else {
+            fs::hard_link(src, dst)
+        }
     };
     link_result
         .or_else(|err| {

tests/testsuite/build_script.rs

@@ -3554,6 +3554,8 @@ fn rename_with_link_search_path() {
 }
 
 #[cargo_test]
+// Don't have a cdylib cross target on macos.
+#[cfg_attr(target_os = "macos", ignore)]
 fn rename_with_link_search_path_cross() {
     if cross_compile::disabled() {
         return;