Setup CI for Apple Platforms

.builder/actions/crt-ci-prep.py

@@ -0,0 +1,9 @@
+import Builder
+
+class CrtCiPrep(Builder.Action):
+    def run(self, env):
+        env.shell.setenv("AWS_TESTING_STS_ROLE_ARN", env.shell.get_secret("aws-c-auth-testing/sts-role-arn"))
+        actions = [
+            Builder.SetupCrossCICrtEnvironment()
+        ]
+        return Builder.Script(actions, name='crt-ci-prep')

.builder/actions/xcode-tests.py

@@ -0,0 +1,16 @@
+import Builder
+import os
+import sys
+
+class XCodeTests(Builder.Action):
+    def run(self, env):
+        destination = env.shell.getenv("XCODE_DESTINATION")
+        commands =[
+            'xcodebuild',
+            '-scheme',
+            'aws-crt-swift-Package',
+            'test',
+            '-destination',
+            "platform={}".format(destination)
+        ]
+        env.shell.exec(commands, check=True)

.github/workflows/ci.yml

@@ -39,34 +39,87 @@ jobs:
           aws s3 cp --debug s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
           ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-swift-5-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }}
   osx:
-    runs-on: macos-12
+    runs-on: ${{ matrix.runner }}
     env:
       DEVELOPER_DIR: /Applications/Xcode.app
+      XCODE_DESTINATION: 'OS X'
+      NSUnbufferedIO: YES
+    strategy:
+      fail-fast: false
+      matrix:
+        # This matrix runs tests on Mac, on oldest & newest supported Xcodes
+        runner:
+          - macos-12
+          - macos-13
+          - macos-14
+          - macos-13-xlarge
+          - macos-14-xlarge
     steps:
       - name: Build ${{ env.PACKAGE_NAME }} + consumers
         run: |
           python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
           chmod a+x builder
           ./builder build -p ${{ env.PACKAGE_NAME }}
 
-  # TODO: Run tests for devices
   devices:
-    runs-on: macos-12
+    runs-on: ${{ matrix.runner }}
     env:
-      DEVELOPER_DIR: /Applications/Xcode.app
+      DEVELOPER_DIR: /Applications/${{ matrix.xcode }}.app/Contents/Developer
+      XCODE_DESTINATION: ${{matrix.target.destination}}
+      NSUnbufferedIO: YES
     strategy:
       fail-fast: false
       matrix:
+        # This matrix runs tests on iOS, tvOS & watchOS, on oldest & newest supported Xcodes
+        runner:
+          - macos-12
+          - macos-13
+          - macos-14
+          - macos-13-xlarge
+          - macos-14-xlarge
         target:
-          - ios
-          - tvos
-          - watchos
+          [{ os: ios, destination: 'iOS Simulator,OS=16.1,name=iPhone 14'},
+           { os: ios, destination: 'iOS Simulator,OS=17.2,name=iPhone 15'},
+           { os: tvos, destination: 'tvOS Simulator,OS=16.1,name=Apple TV 4K (3rd generation) (at 1080p)'},
+           { os: tvos, destination: 'tvOS Simulator,OS=17.2,name=Apple TV 4K (3rd generation) (at 1080p)'},
+           { os: watchos, destination: 'watchOS Simulator,OS=10.2,name=Apple Watch SE (40mm) (2nd generation)'},
+           { os: watchos, destination: 'watchOS Simulator,OS=9.1,name=Apple Watch Series 5 (40mm)'}]
+        xcode:
+          - Xcode_14.1
+          - Xcode_15.2
+        exclude:
+          # Don't run old macOS with new Xcode
+          - runner: macos-13-xlarge
+            xcode: Xcode_15.2
+          - runner: macos-12
+            xcode: Xcode_15.2
+          - runner: macos-13
+            xcode: Xcode_15.2
+          # Don't run new macOS with old Xcode
+          - runner: macos-14-xlarge
+            xcode: Xcode_14.1
+          - runner: macos-14
+            xcode: Xcode_14.1
+          # Don't run old simulators with new Xcode
+          - target: { os: tvos, destination: 'tvOS Simulator,OS=16.1,name=Apple TV 4K (3rd generation) (at 1080p)'}
+            xcode: Xcode_15.2
+          - target: { os: ios, destination: 'iOS Simulator,OS=16.1,name=iPhone 14'}
+            xcode: Xcode_15.2
+          # Don't run new simulators with old Xcode
+          - target: { os: tvos, destination: 'tvOS Simulator,OS=17.2,name=Apple TV 4K (3rd generation) (at 1080p)'}
+            xcode: Xcode_14.1
+          - target: { os: ios, destination: 'iOS Simulator,OS=17.2,name=iPhone 15'}
+            xcode: Xcode_14.1
+          - target: { os: watchos, destination: 'watchOS Simulator,OS=10.2,name=Apple Watch SE (40mm) (2nd generation)'}
+            xcode: Xcode_14.1
+          - target: { os: watchos, destination: 'watchOS Simulator,OS=9.1,name=Apple Watch Series 5 (40mm)'}
+            xcode: Xcode_15.2
     steps:
       - name: Build ${{ env.PACKAGE_NAME }} + consumers
         run: |
           python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
           chmod a+x builder
-          ./builder build -p ${{ env.PACKAGE_NAME }} --target=${{ matrix.target }}-armv8
+          ./builder build -p ${{ env.PACKAGE_NAME }} --target=${{ matrix.target.os }}-armv8
 
   check-submodules:
     runs-on: ubuntu-22.04 # latest

Test/AwsCommonRuntimeKitTests/XCBaseTestCase.swift

@@ -10,7 +10,10 @@ class XCBaseTestCase: XCTestCase {
 
     override func setUp() {
         super.setUp()
-        Logger.initialize(pipe: stdout, level: .trace)
+        // XCode currently lacks a way to enable logs exclusively for failed tests only.
+        // To prevent log spamming, we use `error` log level to only print error message.
+        // We should update this once a more efficient log processing method becomes available.
+        Logger.initialize(pipe: stdout, level: .error)
 
         // Override the allocator with tracing allocator
         allocator = tracingAllocator.rawValue
@@ -35,20 +38,40 @@ extension XCTestCase {
     }
 
     func skipIfiOS() throws {
-        if #available(iOS 10, *) {
+        #if os(iOS)
             throw XCTSkip("Skipping test on iOS")
-        }
+        #endif
     }
 
     func skipifmacOS() throws {
-        if #available(macOS 10.14, *) {
+        #if os(macOS)
             throw XCTSkip("Skipping test on macOS")
-        }
+        #endif
     }
 
     func skipIfLinux() throws {
         #if os(Linux)
             throw XCTSkip("Skipping test on linux")
         #endif
     }
+
+    func skipIfwatchOS() throws {
+        #if os(watchOS)
+            throw XCTSkip("Skipping test on watchOS")
+        #endif
+    }
+
+    func skipIftvOS() throws {
+        #if os(tvOS)
+            throw XCTSkip("Skipping test on tvOS")
+        #endif
+    }
+
+    /// Return the environment variable value, or Skip the test if env var is not set.
+    func getEnvironmentVarOrSkipTest(environmentVarName name: String) throws -> String {
+        guard let result = ProcessInfo.processInfo.environment[name] else {
+            throw XCTSkip("Skipping test because environment is not configured properly.")
+        }
+        return result
+    }
 }

builder.json

@@ -6,8 +6,10 @@
     "swift --version",
     "swift build -c release"
   ],
+  "run_tests": true,
   "test_steps": [
-    "swift test"
+    "crt-ci-prep",
+    "xcode-tests"
   ],
   "hosts": {
     "al2012": {
@@ -38,7 +40,13 @@
     },
     "macos": {
       "+test_steps": [
-       "./integ_test.sh"
+        "./integ_test.sh"
+      ]
+    },
+    "linux":{
+      "!test_steps":[
+        "crt-ci-prep",
+        "swift test"
       ]
     }
   }

integ_test.sh

@@ -1,3 +1,4 @@
 #!/bin/bash
 curl -L -o /tmp/http_client_test.py https://raw.githubusercontent.com/awslabs/aws-c-http/main/integration-testing/http_client_test.py
-python3 /tmp/http_client_test.py .build/x86_64-apple-macosx/release/Elasticurl
+ARCH=$(uname -a | sed 's/.* \([^ ]*\)$/\1/')
+python3 /tmp/http_client_test.py .build/$ARCH-apple-macosx/release/Elasticurl