Bug 1688178 - Big Sur/M1 updates: xcframeworks, using Xcode clang, lipo

.circleci/config.yml

@@ -527,6 +527,10 @@ jobs:
       xcode: "12.3.0"
     steps:
       - checkout
+      - run:
+          name: Install new carthage
+          command: |
+            brew upgrade carthage
       - run:
           name: Set Ruby Version
           command: echo 'chruby ruby-2.6.6' >> ~/.bash_profile
@@ -553,7 +557,11 @@ jobs:
           name: Setup build environment
           command: |
             rustup target add aarch64-apple-ios x86_64-apple-ios
+            cargo install cargo-lipo
+
+            # Bootstrap dependencies
             bin/bootstrap.sh
+
             # See https://circleci.com/docs/2.0/testing-ios/#pre-starting-the-simulator
             xcrun instruments -w "iPhone 11 (14" || true
             # Store build type for use in cache key
@@ -618,29 +626,9 @@ jobs:
             else
               bash bin/build-carthage.sh Glean
             fi
-      - run:
-          name: "Create Carthage bin-only project specification"
-          command: |
-            ZIP_URL=https://circleci.com/api/v1.1/project/github/mozilla/glean/$CIRCLE_BUILD_NUM/artifacts/0/dist/Glean.framework.zip
-            echo "{\"0.0.1\":\"$ZIP_URL\"}" > mozilla.glean.json
-            # Store the build number for retrieval in a later step.
-            echo "$CIRCLE_BUILD_NUM" > ios-build-num.txt
       - store_artifacts:
           path: Glean.framework.zip
           destination: dist/Glean.framework.zip
-      - store_artifacts:
-          path: mozilla.glean.json
-          destination: dist/mozilla.glean.json
-      - persist_to_workspace:
-          root: .
-          paths:
-            - ios-build-num.txt
-      - run:
-          name: "Carthage binary snapshot URL"
-          command: |
-            JSON_URL=https://circleci.com/api/v1.1/project/github/mozilla/glean/$CIRCLE_BUILD_NUM/artifacts/0/dist/mozilla.glean.json
-            echo "Add the following line to your Cartfile:"
-            echo "binary \"$JSON_URL\" ~> 0.0.1-snapshot # mozilla/glean@$CIRCLE_SHA1"
       - persist_to_workspace:
           root: .
           paths:
@@ -652,33 +640,33 @@ jobs:
     steps:
       - checkout
       - skip-if-doc-only
+      - run:
+          name: Install new carthage
+          command: |
+            brew upgrade carthage
       - install-rustup
       - setup-rust-toolchain
       - run:
           name: Setup build environment
           command: |
             rustup target add aarch64-apple-ios x86_64-apple-ios
+            cargo install cargo-lipo
+
             # List available devices -- allows us to see what's there
             xcrun instruments -w list || true
             # See https://circleci.com/docs/2.0/testing-ios/#pre-starting-the-simulator
             xcrun instruments -w "iPhone 11 (14" || true
-      - attach_workspace:
-          at: .
       - run:
-          name: Use binary build of Glean
+          name: Use current commit of Glean
           command: |
-            # Retrieve the previous build number
-            IOS_BUILD_NUM=$(< ios-build-num.txt)
             GLEAN_PATH="$(pwd)"
             CARTFILE_PATH="${GLEAN_PATH}/samples/ios/app/Cartfile"
-            # The previous step generated a binary file and the corresponding JSON manifest
-            JSON_URL="https://circleci.com/api/v1.1/project/github/mozilla/glean/${IOS_BUILD_NUM}/artifacts/0/dist/mozilla.glean.json"
 
             echo "Current Cartfile:"
             cat "${CARTFILE_PATH}"
             echo "================="
             echo "New Cartfile:"
-            sed -i.bak "/mozilla\/glean/s#.*#binary \"$JSON_URL\" ~> 0.0.1-SNAPSHOT#" "$CARTFILE_PATH"
+            sed -i.bak "/mozilla\/glean/s#.*#github \"mozilla/glean\" \"${CIRCLE_SHA1}\"#" "$CARTFILE_PATH"
             cat "${CARTFILE_PATH}"
       - run:
           name: Bootstrap dependencies
@@ -1128,8 +1116,6 @@ workflows:
             branches:
               only: main
       - iOS integration test:
-          requires:
-            - iOS build and test
           filters:
             branches:
               only: main
@@ -1180,7 +1166,7 @@ workflows:
               ignore: main
       - iOS integration test:
           requires:
-            - iOS build and test
+            - hold
           filters:
             branches:
               ignore: main

.circleci/jazzy.yml

@@ -0,0 +1,13 @@
+---
+sdk: iphone
+module: Glean
+author_url: https://mozilla.github.com/glean
+github_url: https://github.com/mozilla/glean
+readme: README.iOS.md
+xcodebuild_arguments:
+    - "-workspace"
+    - "./glean-core/ios/Glean.xcodeproj/project.xcworkspace"
+    - "-scheme"
+    - "Glean"
+    - "-destination"
+    - "platform=iOS Simulator,name=iPhone 11"

bin/bootstrap.sh

@@ -4,8 +4,4 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-WORKSPACE_ROOT="$( cd "$(dirname "$0")/.." ; pwd -P )"
-XCODE_XCCONFIG_FILE="$WORKSPACE_ROOT/xcconfig/xcode-12-fix-carthage-lipo.xcconfig"
-export XCODE_XCCONFIG_FILE
-
-carthage bootstrap --platform iOS --color auto --cache-builds
+carthage bootstrap --platform iOS --color auto --cache-builds --use-xcframeworks

bin/build-swift-docs.sh

@@ -9,13 +9,11 @@ set -eo pipefail
 # Build Swift with one command
 # Requires jazzy from https://github.com/realm/jazzy
 
+WORKSPACE_ROOT="$( cd "$(dirname "$0")/.." ; pwd -P )"
+cd "$WORKSPACE_ROOT"
+
 jazzy --version
 jazzy \
-    --clean \
-    --output build/docs/swift \
-    --sdk iphone \
-    --module Glean \
-    --xcodebuild-arguments -workspace,./glean-core/ios/Glean.xcodeproj/project.xcworkspace,-scheme,Glean,-destination,"generic/platform=iOS" \
-    --author_url https://mozilla.github.com/glean \
-    --github_url https://github.com/mozilla/glean \
-    --readme README.iOS.md
+  --clean \
+  --config "$WORKSPACE_ROOT/.circleci/jazzy.yml" \
+  --output "$WORKSPACE_ROOT/build/docs/swift"

build-scripts/xc-universal-binary.sh

@@ -1,55 +1,54 @@
 #!/usr/bin/env bash
-set -euvx
 
 # This should be invoked from inside xcode, not manually
-if [ "$#" -ne 4 ]
+if [ "$#" -ne 2 ]
 then
     echo "Usage (note: only call inside xcode!):"
     echo "Args: $*"
-    echo "path/to/build-scripts/xc-universal-binary.sh <STATIC_LIB_NAME> <FFI_TARGET> <GLEAN_ROOT_PATH> <buildvariant>"
+    echo "path/to/build-scripts/xc-universal-binary.sh <FFI_TARGET> <GLEAN_ROOT_PATH>"
     exit 1
 fi
-# e.g. libglean_ffi.a
-STATIC_LIB_NAME=$1
+
 # what to pass to cargo build -p, e.g. glean_ffi
-FFI_TARGET=$2
+FFI_TARGET=$1
 # path to app services root
-GLEAN_ROOT=$3
-# buildvariant from our xcconfigs
-BUILDVARIANT=$4
-
-RELFLAG=
-RELDIR="debug"
-if [[ "$BUILDVARIANT" != "debug" ]]; then
-    RELFLAG=--release
-    RELDIR=release
+GLEAN_ROOT=$2
+
+if [ -d "$HOME/.cargo/bin" ]; then
+  export PATH="$HOME/.cargo/bin:$PATH"
 fi
 
-TARGETDIR=$GLEAN_ROOT/target
-
-# We can't use cargo lipo because we can't link to universal libraries :(
-# https://github.com/rust-lang/rust/issues/55235
-IOS_TRIPLES=("x86_64-apple-ios" "aarch64-apple-ios")
-for i in "${!IOS_TRIPLES[@]}"; do
-    env -i PATH="$PATH" \
-    $HOME/.cargo/bin/cargo build -p $FFI_TARGET --lib $RELFLAG --target ${IOS_TRIPLES[$i]}
-done
-
-UNIVERSAL_BINARY=$TARGETDIR/universal/$RELDIR/$STATIC_LIB_NAME
-NEED_LIPO=
-
-# if the universal binary doesnt exist, or if it's older than the static libs,
-# we need to run `lipo` again.
-if [[ ! -f "$UNIVERSAL_BINARY" ]]; then
-    NEED_LIPO=1
-elif [[ $(stat -f "%m" $TARGETDIR/x86_64-apple-ios/$RELDIR/$STATIC_LIB_NAME) -gt $(stat -f "%m" $UNIVERSAL_BINARY) ]]; then
-    NEED_LIPO=1
-elif [[ $(stat -f "%m" $TARGETDIR/aarch64-apple-ios/$RELDIR/$STATIC_LIB_NAME) -gt $(stat -f "%m" $UNIVERSAL_BINARY) ]]; then
-    NEED_LIPO=1
+if ! command -v cargo-lipo 2>/dev/null >/dev/null;
+then
+    echo "$(basename $0) failed."
+    echo "Requires cargo-lipo to build universal library."
+    echo "Install it with:"
+    echo
+    echo "   cargo install cargo-lipo"
+    exit 1
 fi
-if [[ "$NEED_LIPO" = "1" ]]; then
-    mkdir -p $TARGETDIR/universal/$RELDIR
-    lipo -create -output "$UNIVERSAL_BINARY" \
-        $TARGETDIR/x86_64-apple-ios/$RELDIR/$STATIC_LIB_NAME \
-        $TARGETDIR/aarch64-apple-ios/$RELDIR/$STATIC_LIB_NAME
+
+# Ease testing of this script by assuming something about the environment.
+if [ -z "$ACTION" ]; then
+  export ACTION=build
 fi
+
+# Always build both architectures on x86_64.
+export ARCHS="arm64 x86_64"
+
+set -euvx
+
+if [[ -n "${DEVELOPER_SDK_DIR:-}" ]]; then
+  # Assume we're in Xcode, which means we're probably cross-compiling.
+  # In this case, we need to add an extra library search path for build scripts and proc-macros,
+  # which run on the host instead of the target.
+  # (macOS Big Sur does not have linkable libraries in /usr/lib/.)
+  export LIBRARY_PATH="${DEVELOPER_SDK_DIR}/MacOSX.sdk/usr/lib:${LIBRARY_PATH:-}"
+fi
+
+# Force correct target for dependencies compiled with `cc`.
+# Required for M1 MacBooks (Arm target).
+# Without this some dependencies might be compiled for the wrong target.
+export CFLAGS_x86_64_apple_ios="-target x86_64-apple-ios"
+
+cargo lipo --xcode-integ --manifest-path "$GLEAN_ROOT/Cargo.toml" --package "$FFI_TARGET"

docs/dev/ios/setup-ios-build-environment.md

@@ -2,7 +2,7 @@
 
 ## Prepare your build environment
 
-1. Install Xcode 11.0
+1. Install Xcode 12.4
 2. Install [Carthage](https://github.com/Carthage/Carthage): `brew install carthage`
 3. Ensure you have Python 3 installed: `brew install python`
 4. Install linting and formatting tools: `brew install swiftlint`
@@ -24,14 +24,18 @@ devices and iOS emulators, the following targets need to be installed:
 rustup target add aarch64-apple-ios x86_64-apple-ios
 ```
 
+Install helper tools:
+
+```
+cargo install cargo-lipo
+```
+
 ## Building
 
 This should be relatively straightforward and painless:
 
 1. Ensure your repository is up-to-date.
-
 2. Ensure Rust is up-to-date by running `rustup update`.
-
 3. Run a build using the command `make build-swift`
     * To run tests use `make test-swift`