Updating the dependencies and build pipelines to support Arm64 macOS (#1267)

* Updating the protobuf dependency and build pipelines for ARM64 mac

* Moving to use macOS-latest

* Creating a mechanism to generate packages

* Converting GeneratePackage to a Variable

* Explicit Generate Package check

Author: vrdmr

azure-pipelines.yml

@@ -7,7 +7,7 @@ trigger:
 
 variables:
     patchBuildNumberForDev: $(Build.BuildNumber)
-    PROD_V4_WORKER_PY : 'python/prodV4/worker.py'
+    PROD_V4_WORKER_PY: 'python/prodV4/worker.py'
 
 jobs:
 - job: Build_WINDOWS_X64
@@ -98,7 +98,7 @@ jobs:
       artifactName: '$(pythonVersion)_LINUX_X64'
 - job: Build_OSX_X64
   pool:
-    vmImage: 'macOS-12'
+    vmImage: 'macOS-latest'
   strategy:
     matrix:
       Python37V4:
@@ -122,10 +122,30 @@ jobs:
       pythonVersion: '$(pythonVersion)'
       workerPath: '$(workerPath)'
       artifactName: '$(pythonVersion)_OSX_X64'
+- job: Build_OSX_ARM64
+  pool:
+    vmImage: 'macOS-latest'
+  strategy:
+    matrix:
+      Python39V4:
+        pythonVersion: '3.9'
+        workerPath: $(PROD_V4_WORKER_PY)
+      Python310V4:
+        pythonVersion: '3.10'
+        workerPath: $(PROD_V4_WORKER_PY)
+      Python311V4:
+        pythonVersion: '3.11'
+        workerPath: $(PROD_V4_WORKER_PY)
+  steps:
+  - template: pack/templates/macos_64_env_gen.yml
+    parameters:
+      pythonVersion: '$(pythonVersion)'
+      workerPath: '$(workerPath)'
+      artifactName: '$(pythonVersion)_OSX_ARM64'
 
 - job: PackageWorkers
-  dependsOn: ['Build_WINDOWS_X64', 'Build_WINDOWS_X86', 'Build_LINUX_X64', 'Build_OSX_X64']
-  condition: or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), eq(variables['Build.SourceBranch'], 'refs/heads/dev'))
+  dependsOn: ['Build_WINDOWS_X64', 'Build_WINDOWS_X86', 'Build_LINUX_X64', 'Build_OSX_X64', 'Build_OSX_ARM64']
+  condition: or(startsWith(variables['Build.SourceBranch'], 'refs/heads/release/'), eq(variables['Build.SourceBranch'], 'refs/heads/dev'), eq(variables['GeneratePackage'], True))
   pool:
       name: '1ES-Hosted-AzFunc'
       demands:

pack/Microsoft.Azure.Functions.V4.PythonWorker.nuspec

@@ -22,14 +22,17 @@
     <file src="..\3.9_WINDOWS_X86\**" target="tools\3.9\WINDOWS\X86" />
     <file src="..\3.9_LINUX_X64\**" target="tools\3.9\LINUX\X64" />
     <file src="..\3.9_OSX_X64\**" target="tools\3.9\OSX\X64" />
+    <file src="..\3.9_OSX_X64\**" target="tools\3.9\OSX\Arm64" />
     <file src="..\3.10_WINDOWS_X64\**" target="tools\3.10\WINDOWS\X64" />
     <file src="..\3.10_WINDOWS_X86\**" target="tools\3.10\WINDOWS\X86" />
     <file src="..\3.10_LINUX_X64\**" target="tools\3.10\LINUX\X64" />
     <file src="..\3.10_OSX_X64\**" target="tools\3.10\OSX\X64" />
+    <file src="..\3.10_OSX_X64\**" target="tools\3.10\OSX\Arm64" />
     <file src="..\3.11_WINDOWS_X64\**" target="tools\3.11\WINDOWS\X64" />
     <file src="..\3.11_WINDOWS_X86\**" target="tools\3.11\WINDOWS\X86" />
     <file src="..\3.11_LINUX_X64\**" target="tools\3.11\LINUX\X64" />
     <file src="..\3.11_OSX_X64\**" target="tools\3.11\OSX\X64" />
+    <file src="..\3.11_OSX_X64\**" target="tools\3.11\OSX\Arm64" />
     <file src="..\python\prodV4\worker.config.json" target="tools" />
     <file src="Microsoft.Azure.Functions.PythonWorker.targets" target="build" />
     <file src="..\_manifest\manifest.json" target="SBOM\manifest.json" />

pack/scripts/mac_arm64_deps.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+python -m venv .env
+source .env/bin/activate
+python -m pip install --upgrade pip==23.0
+
+python -m pip install .
+
+python -m pip install . --no-compile --target "$BUILD_SOURCESDIRECTORY/deps"

pack/templates/macos_64_env_gen.yml

@@ -0,0 +1,32 @@
+parameters:
+  pythonVersion: ''
+  artifactName: ''
+  workerPath: ''
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: ${{ parameters.pythonVersion }}
+    addToPath: true
+- task: ShellScript@2
+  inputs:
+    disableAutoCwd: true
+    scriptPath: 'pack/scripts/mac_arm64_deps.sh'
+- task: CopyFiles@2
+  inputs:
+    contents: |
+      ${{ parameters.workerPath }}
+    targetFolder: '$(Build.ArtifactStagingDirectory)'
+    flattenFolders: true
+- task: CopyFiles@2
+  inputs:
+    sourceFolder: '$(Build.SourcesDirectory)/deps'
+    contents: |
+      **
+      !grpc_tools/**/*
+      !grpcio_tools*/*
+    targetFolder: '$(Build.ArtifactStagingDirectory)'
+- task: PublishBuildArtifacts@1
+  inputs:
+    pathtoPublish: '$(Build.ArtifactStagingDirectory)'
+    artifactName: ${{ parameters.artifactName }}

python/prodV4/worker.config.json

@@ -1,10 +1,10 @@
 {
     "description":{
         "language":"python",
-        "defaultRuntimeVersion":"3.9",
+        "defaultRuntimeVersion":"3.10",
         "supportedOperatingSystems":["LINUX", "OSX", "WINDOWS"],
         "supportedRuntimeVersions":["3.7", "3.8", "3.9", "3.10", "3.11"],
-        "supportedArchitectures":["X64", "X86"],
+        "supportedArchitectures":["X64", "X86", "Arm64"],
         "extensions":[".py"],
         "defaultExecutablePath":"python",
         "defaultWorkerPath":"%FUNCTIONS_WORKER_RUNTIME_VERSION%/{os}/{architecture}/worker.py",

setup.py

@@ -110,14 +110,14 @@
     "python-dateutil~=2.8.2"
 ]
 
-if sys.version_info[:3] < (3, 11, 0):
-    INSTALL_REQUIRES.append("protobuf~=3.19.3")
-    INSTALL_REQUIRES.append("grpcio-tools~=1.43.0")
-    INSTALL_REQUIRES.append("grpcio~=1.43.0")
+if sys.version_info[:3] < (3, 9, 0):
+    INSTALL_REQUIRES.extend(
+        ("protobuf~=3.19.3", "grpcio-tools~=1.43.0", "grpcio~=1.43.0")
+    )
 else:
-    INSTALL_REQUIRES.append("protobuf~=4.22.0")
-    INSTALL_REQUIRES.append("grpcio-tools~=1.51.3")
-    INSTALL_REQUIRES.append("grpcio~=1.51.3")
+    INSTALL_REQUIRES.extend(
+        ("protobuf~=4.22.0", "grpcio-tools~=1.54.2", "grpcio~=1.54.2")
+    )
 
 EXTRA_REQUIRES = {
     "dev": [