testsuite: Supress stderr when running integration tests

This fixes a problem where test output was leaking into GitHub Actions
results (#8419), making the output unnecessarily verbose and harder to read.

This makes the test output much cleaner while still preserving all the
diagnostic information when tests fail.

Fixes #8419

Author: mpickering

cabal-install/cabal-install.cabal

@@ -425,6 +425,8 @@ test-suite integration-tests2
     , process
     , tasty >= 1.2.3 && <1.6
     , tasty-hunit >= 0.10
+    , tasty-expected-failure
+    , silently
     , tagged
 
 test-suite long-tests

cabal-install/tests/IntegrationTests2.hs

@@ -89,8 +89,13 @@ import System.Process (callProcess)
 
 import Data.Tagged (Tagged (..))
 import Test.Tasty
-import Test.Tasty.HUnit
+import qualified Test.Tasty.HUnit as T ( testCase )
+import Test.Tasty.HUnit hiding ( testCase )
 import Test.Tasty.Options
+import Test.Tasty.Runners
+import Test.Tasty.ExpectedFailure
+
+import System.IO.Silently
 
 import qualified Data.ByteString as BS
 import Data.Maybe (fromJust)
@@ -113,11 +118,28 @@ main = do
   defaultMainWithIngredients
     (defaultIngredients ++ [includingOptions projectConfigOptionDescriptions])
     ( withProjectConfig $ \config ->
-        testGroup
-          "Integration tests (internal)"
-          (tests config)
+          testGroup
+            "Integration tests (internal)"
+            (tests config)
     )
 
+-- Tests are run silently, unless they fail. Firstly because it is annoying to
+-- see lots of stderr from your unit tests. Secondly becaus this output
+-- leaks into the result of github actions (#8419)
+silentTest :: TestTree -> TestTree
+silentTest = wrapTest silentHelper
+  where
+    silentHelper t = do
+      (out, res) <- hCapture [stderr] t
+
+      return $ if not (resultSuccessful res)
+        then res { resultDescription = resultDescription res <> "\nCaptured output:\n" <> out }
+        else res
+
+testCase :: String -> Assertion -> TestTree
+testCase desc action = silentTest (T.testCase desc action)
+
+
 tests :: ProjectConfig -> [TestTree]
 tests config =
   -- TODO: tests for: