Skip to content

Commit dbc7a19

Browse files
snnnsnnn
authored
Update win-ci-pipeline.yml: enable xnnpack tests (#16244)
1. Enable xnnpack test 2. Change TSA database name from onnxruntime_master to onnxruntime_main. This is a leftover of renaming the "master" branch to "main" 3. Add two static analysis jobs for WinML and DML 4. Rename the machine pool "aiinfra-dml-winbuild" to "onnxruntime-Win2019-GPU-dml-A10", so that the internal and public ADO instances use the same machine pool name. 5. Move Windows GPU CI build pipeline from "onnxruntime-Win2022-GPU-T4" to "onnxruntime-Win2022-GPU-A10" machine pool, because we do not have enough T4 GPUs.
1 parent 9be1332 commit dbc7a19

File tree

14 files changed

+130
-71
lines changed

14 files changed

+130
-71
lines changed

‎.gdn/.gdntsa

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,3 @@
11
{
2-
"codebaseName": "onnxruntime_master"
2+
"codebaseName": "onnxruntime_main"
33
}

‎.github/workflows/lint.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@ name: Lint
33
on:
44
push:
55
branches:
6-
- master
76
- main
7+
- rel-*
88
pull_request:
99

1010
jobs:

‎.github/workflows/linux.yml

+5-1
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,14 @@ name: Linux_CI
22
on:
33
push:
44
branches:
5-
- master
65
- main
6+
- rel-*
77
pull_request:
88

9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
11+
cancel-in-progress: true
12+
913
jobs:
1014
Onnxruntime-TVM:
1115
runs-on: ubuntu-latest

‎.github/workflows/sca.yml

+84-2
Original file line numberDiff line numberDiff line change
@@ -3,18 +3,19 @@ on:
33
push:
44
branches:
55
- main
6+
- rel-*
67
pull_request:
78

89
concurrency:
9-
group: ${{ github.head_ref || github.run_id }}
10+
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
1011
cancel-in-progress: true
1112

1213
env:
1314
AZCOPY_AUTO_LOGIN_TYPE: MSI
1415
AZCOPY_MSI_CLIENT_ID: 63b63039-6328-442f-954b-5a64d124e5b4
1516

1617
jobs:
17-
Onnxruntime-SCA:
18+
Onnxruntime-SCA-training-CUDA:
1819
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
1920
steps:
2021
- uses: actions/checkout@v3
@@ -33,6 +34,11 @@ jobs:
3334
run: azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" cuda_sdk
3435

3536

37+
- name: Delete build folder
38+
run: |
39+
if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b }
40+
&tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x64 -install_prefix D:\b\Debug\installed -build_config Debug
41+
3642
# The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter.
3743
- name: Build code
3844
env:
@@ -49,3 +55,79 @@ jobs:
4955
with:
5056
sarif_file: ${{ github.workspace }}\output\MergeResult.sarif
5157
category: VS_SCA
58+
59+
# No python
60+
Onnxruntime-SCA-win32-WINML-x64:
61+
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
62+
steps:
63+
- uses: actions/checkout@v3
64+
with:
65+
submodules: false
66+
- uses: actions/setup-python@v3
67+
with:
68+
python-version: '3.11.x'
69+
architecture: 'x64'
70+
71+
- uses: actions/setup-node@v3
72+
with:
73+
node-version: 18
74+
75+
- name: Delete build folder
76+
run: |
77+
if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b }
78+
&tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x64 -install_prefix D:\b\Debug\installed -build_config Debug
79+
80+
# The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter.
81+
- name: Build code
82+
env:
83+
CAExcludePath: 'C:\Program Files;D:\b;${{ github.workspace }}\cmake'
84+
run: python tools\ci_build\build.py --enable_training --build_java --compile_no_warning_as_error --config Debug --build_dir D:\b --skip_submodule_sync --build_csharp --update --build --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --cmake_extra_defines onnxruntime_USE_CUSTOM_STATIC_ANALYSIS_RULES=ON --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON --cmake_extra_defines onnxruntime_REDIRECT_STATIC_ANALYSIS_OUTPUTS_TO_FILE=ON --ms_experimental --use_dml --use_winml --disable_rtti --enable_wcos --build_shared_lib
85+
86+
- name: Generate sarif
87+
working-directory: D:\b
88+
run: npx @microsoft/sarif-multitool merge *.sarif --recurse --output-directory=${{ github.workspace }}\output --output-file=MergeResult.sarif --merge-runs && dir ${{ github.workspace }}\output
89+
90+
- name: Upload SARIF to GitHub
91+
uses: github/codeql-action/upload-sarif@v2
92+
continue-on-error: true
93+
with:
94+
sarif_file: ${{ github.workspace }}\output\MergeResult.sarif
95+
category: VS_SCA_WIN32_WINML_X64
96+
97+
# No java, No python
98+
Onnxruntime-SCA-win32-WINML-x86:
99+
runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"]
100+
steps:
101+
- uses: actions/checkout@v3
102+
with:
103+
submodules: false
104+
- uses: actions/setup-python@v3
105+
with:
106+
python-version: '3.11.x'
107+
architecture: 'x86'
108+
109+
- uses: actions/setup-node@v3
110+
with:
111+
node-version: 18
112+
113+
- name: Delete build folder
114+
run: |
115+
if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b }
116+
&tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x86 -install_prefix D:\b\Debug\installed -build_config Debug
117+
118+
# The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter.
119+
- name: Build code
120+
env:
121+
CAExcludePath: 'C:\Program Files;D:\b;${{ github.workspace }}\cmake'
122+
run: python tools\ci_build\build.py --enable_training --compile_no_warning_as_error --config Debug --build_dir D:\b --skip_submodule_sync --build_csharp --update --build --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --cmake_extra_defines onnxruntime_USE_CUSTOM_STATIC_ANALYSIS_RULES=ON --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON --cmake_extra_defines onnxruntime_REDIRECT_STATIC_ANALYSIS_OUTPUTS_TO_FILE=ON --ms_experimental --use_dml --use_winml --disable_rtti --enable_wcos --build_shared_lib
123+
124+
- name: Generate sarif
125+
working-directory: D:\b
126+
run: npx @microsoft/sarif-multitool merge *.sarif --recurse --output-directory=${{ github.workspace }}\output --output-file=MergeResult.sarif --merge-runs && dir ${{ github.workspace }}\output
127+
128+
- name: Upload SARIF to GitHub
129+
uses: github/codeql-action/upload-sarif@v2
130+
continue-on-error: true
131+
with:
132+
sarif_file: ${{ github.workspace }}\output\MergeResult.sarif
133+
category: VS_SCA_WIN32_WINML_X86

‎.github/workflows/windows.yml

+5-1
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,14 @@ name: Windows_CI
22
on:
33
push:
44
branches:
5-
- master
65
- main
6+
- rel-*
77
pull_request:
88

9+
concurrency:
10+
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
11+
cancel-in-progress: true
12+
913
jobs:
1014
Onnxruntime-TVM:
1115
runs-on: windows-latest

‎tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -794,7 +794,7 @@ stages:
794794

795795
- template: nuget/templates/dml-vs-2022.yml
796796
parameters:
797-
AgentPool : 'aiinfra-dml-winbuild'
797+
AgentPool : 'onnxruntime-Win2019-GPU-dml-A10'
798798
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
799799
ArtifactName: 'drop-nuget-dml'
800800
StageName: 'Windows_CI_GPU_DML_Dev'
@@ -817,7 +817,7 @@ stages:
817817
818818
- template: nuget/templates/dml-vs-2022.yml
819819
parameters:
820-
AgentPool : 'aiinfra-dml-winbuild'
820+
AgentPool : 'onnxruntime-Win2019-GPU-dml-A10'
821821
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
822822
ArtifactName: 'drop-win-dml-x86-zip'
823823
StageName: 'Windows_CI_GPU_DML_Dev_x86'
@@ -840,7 +840,7 @@ stages:
840840
841841
- template: nuget/templates/dml-vs-2022.yml
842842
parameters:
843-
AgentPool : 'aiinfra-dml-winbuild'
843+
AgentPool : 'onnxruntime-Win2019-GPU-dml-A10'
844844
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
845845
ArtifactName: 'drop-win-dml-arm64-zip'
846846
StageName: 'Windows_CI_GPU_DML_Dev_arm64'

‎tools/ci_build/github/azure-pipelines/nuget/templates/dml-vs-2022.yml

+8-5
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ stages:
4848
ALLOW_RELEASED_ONNX_OPSET_ONLY: ${{ parameters.AllowReleasedOpsetOnly }}
4949
${{ if eq(parameters.EnableLto, true) }}:
5050
build_py_lto_flag: --enable_lto
51-
${{ if or(eq(parameters.AgentPool, 'aiinfra-dml-winbuild'), eq(parameters.AgentPool, 'onnxruntime-Win-CPU-2019')) }}:
51+
${{ if or(eq(parameters.AgentPool, 'onnxruntime-Win2019-GPU-dml-A10'), eq(parameters.AgentPool, 'onnxruntime-Win-CPU-2019')) }}:
5252
vsVersion: '2019'
5353

5454
steps:
@@ -72,7 +72,13 @@ stages:
7272
7373
- task: NodeTool@0
7474
inputs:
75-
versionSpec: '16.x'
75+
versionSpec: '18.x'
76+
77+
- task: onebranch.pipeline.tsaoptions@1
78+
displayName: 'OneBranch TSAOptions'
79+
inputs:
80+
tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json'
81+
appendSourceBranchName: false
7682

7783
- task: UsePythonVersion@0
7884
inputs:
@@ -272,9 +278,6 @@ stages:
272278
- template: ../../templates/compliance.yml
273279
parameters :
274280
msbuildPlatform: ${{ parameters.sln_platform }}
275-
# parameter couldn't be compared with string, so I have to use boolean.
276-
${{ if eq(variables.vsVersion, '2019') }}:
277-
vs2022: false
278281

279282
- template: ../../templates/component-governance-component-detection-steps.yml
280283
parameters :

‎tools/ci_build/github/azure-pipelines/templates/compliance.yml

+4-44
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,6 @@ parameters:
44
type: string
55
default: x64
66

7-
- name: vs2022
8-
displayName: Visual Studio Version
9-
type: string
10-
default: true
11-
127
steps:
138
- task: CredScan@2
149
displayName: 'Run CredScan'
@@ -23,45 +18,10 @@ steps:
2318
arguments: 'analyze $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.dll --recurse --verbose'
2419
continueOnError: true
2520

26-
- task: DeleteFiles@1
27-
displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo'
28-
inputs:
29-
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
30-
Contents: |
31-
**/*.obj
32-
**/*.pdb
33-
**/*.dll
34-
35-
#Manually set msBuildCommandline so that we can also set CAExcludePath
36-
- task: securedevelopmentteam.vss-secure-development-tools.build-task-prefast.SDLNativeRules@2
37-
displayName: 'Run the PREfast SDL Native Rules for MSBuild'
38-
inputs:
39-
userProvideBuildInfo: msBuildInfo
40-
msBuildArchitecture: x64
41-
${{ if eq(parameters.vs2022, false)}}:
42-
msBuildVersion: 16.0
43-
msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform="${{parameters.msbuildPlatform}}" /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64'
44-
${{ else }}:
45-
msBuildVersion: 17.0
46-
msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform="${{parameters.msbuildPlatform}}" /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64'
47-
continueOnError: true
48-
49-
- task: securedevelopmentteam.vss-secure-development-tools.build-task-report.SdtReport@1
50-
displayName: 'Create Security Analysis Report'
51-
inputs:
52-
BinSkim: true
53-
BinSkimBreakOn: WarningAbove
54-
CredScan: true
55-
SDLNativeRules: true
56-
57-
- task: securedevelopmentteam.vss-secure-development-tools.build-task-publishsecurityanalysislogs.PublishSecurityAnalysisLogs@2
58-
displayName: 'Publish Security Analysis Logs'
59-
continueOnError: true
60-
61-
- task: securedevelopmentteam.vss-secure-development-tools.build-task-uploadtotsa.TSAUpload@1
21+
- task: TSAUpload@2
22+
displayName: 'TSA upload'
6223
condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
63-
displayName: 'TSA Upload'
6424
inputs:
65-
tsaVersion: TsaV2
66-
codeBaseName: 'onnxruntime_master'
25+
GdnPublishTsaOnboard: false
26+
GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
6727
continueOnError: true

‎tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml

+1
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ jobs:
6464
setVcvars: true
6565
ALLOW_RELEASED_ONNX_OPSET_ONLY: '0'
6666
DocUpdateNeeded: false # Set to true during document generation if there are diffs
67+
NVIDIA_TF32_OVERRIDE: '0'
6768
skipComponentGovernanceDetection: true
6869
DEPS_CACHE_DIR: $(Agent.TempDirectory)/deps_ccache
6970
ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache

‎tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml

+4-4
Original file line numberDiff line numberDiff line change
@@ -303,31 +303,31 @@ stages:
303303

304304
- template: py-win-gpu.yml
305305
parameters:
306-
MACHINE_POOL: 'aiinfra-dml-winbuild'
306+
MACHINE_POOL: 'onnxruntime-Win2019-GPU-dml-A10'
307307
PYTHON_VERSION: '3.8'
308308
EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
309309
ENV_SETUP_SCRIPT: setup_env.bat
310310
EP_NAME: directml
311311

312312
- template: py-win-gpu.yml
313313
parameters:
314-
MACHINE_POOL: 'aiinfra-dml-winbuild'
314+
MACHINE_POOL: 'onnxruntime-Win2019-GPU-dml-A10'
315315
PYTHON_VERSION: '3.9'
316316
EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
317317
ENV_SETUP_SCRIPT: setup_env.bat
318318
EP_NAME: directml
319319

320320
- template: py-win-gpu.yml
321321
parameters:
322-
MACHINE_POOL: 'aiinfra-dml-winbuild'
322+
MACHINE_POOL: 'onnxruntime-Win2019-GPU-dml-A10'
323323
PYTHON_VERSION: '3.10'
324324
EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
325325
ENV_SETUP_SCRIPT: setup_env.bat
326326
EP_NAME: directml
327327

328328
- template: py-win-gpu.yml
329329
parameters:
330-
MACHINE_POOL: 'aiinfra-dml-winbuild'
330+
MACHINE_POOL: 'onnxruntime-Win2019-GPU-dml-A10'
331331
PYTHON_VERSION: '3.11'
332332
EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
333333
ENV_SETUP_SCRIPT: setup_env.bat

‎tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml

+1-1
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ jobs:
2828
clean: all
2929
pool: ${{ parameters.MACHINE_POOL }}
3030
variables:
31-
${{if or(eq(parameters.MACHINE_POOL, 'onnxruntime-gpu-winbuild-T4'), eq(parameters.MACHINE_POOL, 'aiinfra-dml-winbuild'))}}:
31+
${{if or(eq(parameters.MACHINE_POOL, 'onnxruntime-gpu-winbuild-T4'), eq(parameters.MACHINE_POOL, 'onnxruntime-Win2019-GPU-dml-A10'))}}:
3232
VSGenerator: 'Visual Studio 16 2019'
3333
${{else}}:
3434
VSGenerator: 'Visual Studio 17 2022'

‎tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml

+2-2
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ stages:
8181
msbuildPlatform: x64
8282
isX86: false
8383
job_name_suffix: x64_release
84-
RunOnnxRuntimeTests: false
84+
RunOnnxRuntimeTests: true
8585
RunStaticCodeAnalysis: false
8686
isTraining: false
8787
ORT_EP_NAME: XNNPACK
@@ -211,4 +211,4 @@ stages:
211211
ORT_EP_NAME: CPU
212212
GenerateDocumentation: false
213213
WITH_CACHE: true
214-
MachinePool: 'onnxruntime-Win-CPU-2022'
214+
MachinePool: 'onnxruntime-Win-CPU-2022'

0 commit comments

Comments
 (0)