Fixes to exit tests.

This PR supersedes #603, #613, and #614. Exit tests remain an experimental feature.

 ## Clarify that 8-bit exit codes aren't a problem on macOS and Windows. (#603)

The documentation for the experimental exit tests feature currently says that on POSIX-like systems, only the low 8 bits of a process' exit code are preserved. This would be true if we used `wait()`, `wait4()`, etc. and `WEXITSTATUS()`, but we use `waitid()` instead which is [supposed to](https://pubs.opengroup.org/onlinepubs/9699919799/functions/exit.html) preserve the full exit code. It does so on Darwin, but not on Linux; Windows doesn't use `waitid()` but does report the full exit code.

Now, we're not currently building for any other POSIX-like systems that support processes (WASI/Wasm doesn't count here), so I've left in some weasel words and added a canary unit test. It will let us know if/when we add a platform that where `waitid()` doesn't preserve all the bits of the exit code, and we can amend the documentation in that case.

 ## Implement an equality operator for ExitCondition. (#613)

This PR implements `==` and `===` for `ExitCondition`, part of the experimental exit tests feature. These operators are necessary in order to allow for exit tests to support more complex matching by trailing closure (e.g. to support inspecting `stdout`.) Because `.failure` is a fuzzy case, `==` fuzzy-matches while `===` exactly matches. `Hashable` conformance is unavailable.

Example usage:

```swift
let lhs: ExitCondition = .failure
let rhs: ExitCondition = .signal(SIGTERM)
print(lhs == rhs) // prints "true"
print(lhs === rhs) // prints "false"
```

 ## Allow throwing an error from an exit test's body. (#614)

This PR amends the signatures of the exit test macros (`#expect(exitsWith:) {}` and `try #require(exitsWith:) {}`) to allow bodies to throw errors. If they do, they are treated as uncaught errors and the child process terminates abnormally (in the same way it does if an error is thrown from the main function of a Swift program.)

Author: grynspan

Sources/Testing/ExitTests/ExitCondition.swift

@@ -17,6 +17,17 @@ private import _TestingInternals
 /// ``expect(exitsWith:_:sourceLocation:performing:)`` or
 /// ``require(exitsWith:_:sourceLocation:performing:)`` to configure which exit
 /// statuses should be considered successful.
+///
+/// Two instances of this type can be compared; if either instance is equal to
+/// ``failure``, it will compare equal to any instance except ``success``. To
+/// check if two instances are exactly equal, use the `===` operator:
+///
+/// ```swift
+/// let lhs: ExitCondition = .failure
+/// let rhs: ExitCondition = .signal(SIGINT)
+/// print(lhs == rhs) // prints "true"
+/// print(lhs === rhs) // prints "false"
+/// ```
 @_spi(Experimental)
 #if SWT_NO_EXIT_TESTS
 @available(*, unavailable, message: "Exit tests are not available on this platform.")
@@ -44,9 +55,9 @@ public enum ExitCondition: Sendable {
   /// | Linux | [`<stdlib.h>`](https://sourceware.org/glibc/manual/latest/html_node/Exit-Status.html), `<sysexits.h>` |
   /// | Windows | [`<stdlib.h>`](https://learn.microsoft.com/en-us/cpp/c-runtime-library/exit-success-exit-failure) |
   ///
-  /// On POSIX-like systems including macOS and Linux, only the low unsigned 8
-  /// bits (0&ndash;255) of the exit code are reliably preserved and reported to
-  /// a parent process.
+  /// On macOS and Windows, the full exit code reported by the process is
+  /// yielded to the parent process. Linux and other POSIX-like systems may only
+  /// reliably report the low unsigned 8 bits (0&ndash;255) of the exit code.
   case exitCode(_ exitCode: CInt)
 
   /// The process terminated with the given signal.
@@ -62,43 +73,60 @@ public enum ExitCondition: Sendable {
   /// | macOS | [`<signal.h>`](https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man3/signal.3.html) |
   /// | Linux | [`<signal.h>`](https://sourceware.org/glibc/manual/latest/html_node/Standard-Signals.html) |
   /// | Windows | [`<signal.h>`](https://learn.microsoft.com/en-us/cpp/c-runtime-library/signal-constants) |
+  ///
+  /// On Windows, by default, the C runtime will terminate a process with exit
+  /// code `-3` if a raised signal is not handled, exactly as if `exit(-3)` were
+  /// called. As a result, this case is unavailable on that platform. Developers
+  /// should use ``failure`` instead when testing signal handling on Windows.
 #if os(Windows)
   @available(*, unavailable, message: "On Windows, use .failure instead.")
 #endif
   case signal(_ signal: CInt)
 }
 
-// MARK: -
+// MARK: - Equatable, Hashable
 
 #if SWT_NO_EXIT_TESTS
 @available(*, unavailable, message: "Exit tests are not available on this platform.")
 #endif
-extension ExitCondition {
-  /// Check whether this instance matches another.
-  ///
-  /// - Parameters:
-  ///   - other: The other instance to compare against.
-  ///
-  /// - Returns: Whether or not this instance is equal to, or at least covers,
-  ///   the other instance.
-  func matches(_ other: ExitCondition) -> Bool {
-    return switch (self, other) {
-    case (.failure, .failure):
-      true
+extension ExitCondition: Equatable {
+  public static func ==(lhs: Self, rhs: Self) -> Bool {
+    return switch (lhs, rhs) {
     case let (.failure, .exitCode(exitCode)), let (.exitCode(exitCode), .failure):
       exitCode != EXIT_SUCCESS
+#if !os(Windows)
+    case (.failure, .signal), (.signal, .failure):
+      // All terminating signals are considered failures.
+      true
+#endif
+    default:
+      lhs === rhs
+    }
+  }
+
+  public static func ===(lhs: Self, rhs: Self) -> Bool {
+    return switch (lhs, rhs) {
+    case (.failure, .failure):
+      true
     case let (.exitCode(lhs), .exitCode(rhs)):
       lhs == rhs
 #if !os(Windows)
     case let (.signal(lhs), .signal(rhs)):
       lhs == rhs
-    case (.signal, .failure), (.failure, .signal):
-      // All terminating signals are considered failures.
-      true
-    case (.signal, .exitCode), (.exitCode, .signal):
-      // Signals do not match exit codes.
-      false
 #endif
+    default:
+      false
     }
   }
+
+  public static func !==(lhs: Self, rhs: Self) -> Bool {
+    !(lhs === rhs)
+  }
+}
+
+@available(*, unavailable, message: "ExitCondition does not conform to Hashable.")
+extension ExitCondition: Hashable {
+  public func hash(into hasher: inout Hasher) {
+    fatalError("Unsupported")
+  }
 }

Sources/Testing/ExitTests/ExitTest.swift

@@ -21,7 +21,7 @@ public struct ExitTest: Sendable {
   public var expectedExitCondition: ExitCondition
 
   /// The body closure of the exit test.
-  fileprivate var body: @Sendable () async -> Void
+  fileprivate var body: @Sendable () async throws -> Void
 
   /// The source location of the exit test.
   ///
@@ -37,12 +37,16 @@ public struct ExitTest: Sendable {
   /// terminate the process in a way that causes the corresponding expectation
   /// to fail.
   public func callAsFunction() async -> Never {
-    await body()
+    do {
+      try await body()
+    } catch {
+      _errorInMain(error)
+    }
 
     // Run some glue code that terminates the process with an exit condition
     // that does not match the expected one. If the exit test's body doesn't
     // terminate, we'll manually call exit() and cause the test to fail.
-    let expectingFailure = expectedExitCondition.matches(.failure)
+    let expectingFailure = expectedExitCondition == .failure
     exit(expectingFailure ? EXIT_SUCCESS : EXIT_FAILURE)
   }
 }
@@ -63,7 +67,7 @@ public protocol __ExitTestContainer {
   static var __sourceLocation: SourceLocation { get }
 
   /// The body function of the exit test.
-  static var __body: @Sendable () async -> Void { get }
+  static var __body: @Sendable () async throws -> Void { get }
 }
 
 extension ExitTest {
@@ -118,7 +122,7 @@ extension ExitTest {
 /// convention.
 func callExitTest(
   exitsWith expectedExitCondition: ExitCondition,
-  performing body: @escaping @Sendable () async -> Void,
+  performing body: @escaping @Sendable () async throws -> Void,
   expression: __Expression,
   comments: @autoclosure () -> [Comment],
   isRequired: Bool,
@@ -150,7 +154,7 @@ func callExitTest(
   }
 
   return __checkValue(
-    expectedExitCondition.matches(actualExitCondition),
+    expectedExitCondition == actualExitCondition,
     expression: expression,
     expressionWithCapturedRuntimeValues: expression.capturingRuntimeValues(actualExitCondition),
     mismatchedExitConditionDescription: String(describingForTest: expectedExitCondition),

Sources/Testing/Expectations/Expectation+Macro.swift

@@ -440,7 +440,9 @@ public macro require(
 /// a clean environment for execution, it is not called within the context of
 /// the original test. If `expression` does not terminate the child process, the
 /// process is terminated automatically as if the main function of the child
-/// process were allowed to return naturally.
+/// process were allowed to return naturally. If an error is thrown from
+/// `expression`, it is handed as if the error were thrown from `main()` and the
+/// process is terminated.
 ///
 /// Once the child process terminates, the parent process resumes and compares
 /// its exit status against `exitCondition`. If they match, the exit test has
@@ -488,8 +490,8 @@ public macro require(
 ///     issues should be attributed.
 ///   - expression: The expression to be evaluated.
 ///
-/// - Throws: An instance of ``ExpectationFailedError`` if `condition` evaluates
-///   to `false`.
+/// - Throws: An instance of ``ExpectationFailedError`` if the exit condition of
+///   the child process does not equal `expectedExitCondition`.
 ///
 /// Use this overload of `#require()` when an expression will cause the current
 /// process to terminate and the nature of that termination will determine if
@@ -515,7 +517,9 @@ public macro require(
 /// a clean environment for execution, it is not called within the context of
 /// the original test. If `expression` does not terminate the child process, the
 /// process is terminated automatically as if the main function of the child
-/// process were allowed to return naturally.
+/// process were allowed to return naturally. If an error is thrown from
+/// `expression`, it is handed as if the error were thrown from `main()` and the
+/// process is terminated.
 ///
 /// Once the child process terminates, the parent process resumes and compares
 /// its exit status against `exitCondition`. If they match, the exit test has
@@ -550,5 +554,5 @@ public macro require(
   exitsWith expectedExitCondition: ExitCondition,
   _ comment: @autoclosure () -> Comment? = nil,
   sourceLocation: SourceLocation = #_sourceLocation,
-  performing expression: @convention(thin) () async -> Void
+  performing expression: @convention(thin) () async throws -> Void
 ) = #externalMacro(module: "TestingMacros", type: "ExitTestRequireMacro")

Sources/Testing/Expectations/ExpectationChecking+Macro.swift

@@ -1103,15 +1103,15 @@ public func __checkClosureCall<R>(
 @_spi(Experimental)
 public func __checkClosureCall(
   exitsWith expectedExitCondition: ExitCondition,
-  performing body: @convention(thin) () async -> Void,
+  performing body: @convention(thin) () async throws -> Void,
   expression: __Expression,
   comments: @autoclosure () -> [Comment],
   isRequired: Bool,
   sourceLocation: SourceLocation
 ) async -> Result<Void, any Error> {
   await callExitTest(
     exitsWith: expectedExitCondition,
-    performing: { await body() },
+    performing: { try await body() },
     expression: expression,
     comments: comments(),
     isRequired: isRequired,

Sources/TestingMacros/ConditionMacro.swift

@@ -362,15 +362,23 @@ extension ExitTestConditionMacro {
       static var __sourceLocation: Testing.SourceLocation {
         \(createSourceLocationExpr(of: macro, context: context))
       }
-      static var __body: @Sendable () async -> Void {
+      static var __body: @Sendable () async throws -> Void {
         \(bodyArgumentExpr.trimmed)
       }
       static var __expectedExitCondition: Testing.ExitCondition {
         \(arguments[expectedExitConditionIndex].expression.trimmed)
       }
     }
     """
-    arguments[trailingClosureIndex].expression = "{ \(enumDecl) }"
+
+    // Explicitly include a closure signature to work around a compiler bug
+    // type-checking thin throwing functions after macro expansion.
+    // SEE: rdar://133979438
+    arguments[trailingClosureIndex].expression = """
+    { () async throws in
+      \(enumDecl)
+    }
+    """
 
     // Replace the exit test body (as an argument to the macro) with a stub
     // closure that hosts the type we created above.

Tests/TestingTests/ExitTestTests.swift

@@ -32,6 +32,9 @@ private import _TestingInternals
       await Task.yield()
       exit(123)
     }
+    await #expect(exitsWith: .failure) {
+      throw MyError()
+    }
 #if !os(Windows)
     await #expect(exitsWith: .signal(SIGKILL)) {
       _ = kill(getpid(), SIGKILL)
@@ -197,6 +200,42 @@ private import _TestingInternals
       }.run(configuration: configuration)
     }
   }
+
+#if !os(Linux)
+  @Test("Exit test reports > 8 bits of the exit code")
+  func fullWidthExitCode() async {
+    // On macOS and Linux, we use waitid() which per POSIX should report the
+    // full exit code, not just the low 8 bits. This behaviour is not
+    // well-documented and while Darwin correctly reports the full value, Linux
+    // does not (at least as of this writing) and other POSIX-like systems may
+    // also have issues. This test serves as a canary when adding new platforms
+    // that we need to document the difference.
+    //
+    // Windows does not have the 8-bit exit code restriction and always reports
+    // the full CInt value back to the testing library.
+    await #expect(exitsWith: .exitCode(512)) {
+      exit(512)
+    }
+  }
+#endif
+
+  @Test("Exit condition exact matching (===)")
+  func exitConditionMatching() {
+    #expect(ExitCondition.success === .success)
+    #expect(ExitCondition.success === .exitCode(EXIT_SUCCESS))
+    #expect(ExitCondition.success !== .exitCode(EXIT_FAILURE))
+
+    #expect(ExitCondition.failure === .failure)
+
+    #expect(ExitCondition.exitCode(EXIT_FAILURE &+ 1) !== .exitCode(EXIT_FAILURE))
+
+#if !os(Windows)
+    #expect(ExitCondition.success !== .exitCode(EXIT_FAILURE))
+    #expect(ExitCondition.success !== .signal(SIGINT))
+    #expect(ExitCondition.signal(SIGINT) === .signal(SIGINT))
+    #expect(ExitCondition.signal(SIGTERM) !== .signal(SIGINT))
+#endif
+  }
 }
 
 // MARK: - Fixtures

Tests/TestingTests/Support/FileHandleTests.swift

@@ -8,7 +8,7 @@
 // See https://swift.org/CONTRIBUTORS.txt for Swift project authors
 //
 
-@testable import Testing
+@testable @_spi(Experimental) import Testing
 private import _TestingInternals
 
 #if !SWT_NO_FILE_IO
@@ -63,6 +63,16 @@ struct FileHandleTests {
     }
   }
 
+#if !SWT_NO_EXIT_TESTS
+  @Test("Writing requires contiguous storage")
+  func writeIsContiguous() async {
+    await #expect(exitsWith: .failure) {
+      let fileHandle = try FileHandle.null(mode: "wb")
+      try fileHandle.write([1, 2, 3, 4, 5].lazy.filter { $0 == 1 })
+    }
+  }
+#endif
+
   @Test("Can read from a file")
   func canRead() throws {
     let bytes: [UInt8] = (0 ..< 8192).map { _ in