288 lines
8.9 KiB
YAML
288 lines
8.9 KiB
YAML
trigger: none
|
|
|
|
pr:
|
|
autoCancel: true
|
|
drafts: false
|
|
branches:
|
|
include:
|
|
- master
|
|
paths:
|
|
include:
|
|
- neural_compressor
|
|
- test
|
|
- setup.py
|
|
- requirements.txt
|
|
- .azure-pipelines/ut-basic.yml
|
|
- .azure-pipelines/template/docker-template.yml
|
|
- .azure-pipelines/scripts/ut
|
|
- .azure-pipelines/scripts/fwk_version.sh
|
|
- .azure-pipelines/scripts/install_nc.sh
|
|
exclude:
|
|
- test/3x
|
|
- neural_compressor/common
|
|
- neural_compressor/torch
|
|
- neural_compressor/tensorflow
|
|
- neural_compressor/onnxrt
|
|
- neural_compressor/transformers
|
|
- neural_compressor/evaluation
|
|
- .azure-pipelines/scripts/ut/3x
|
|
|
|
pool: ICX-16C
|
|
|
|
variables:
|
|
IMAGE_NAME: "neural-compressor"
|
|
IMAGE_TAG: "py310"
|
|
UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir
|
|
DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir
|
|
ARTIFACT_NAME: "UT_coverage_report"
|
|
REPO: $(Build.Repository.Uri)
|
|
|
|
stages:
|
|
- stage: Adaptor
|
|
displayName: Unit Test FWKs adaptor
|
|
dependsOn: []
|
|
jobs:
|
|
- job:
|
|
displayName: Test FWKs adaptor
|
|
steps:
|
|
- template: template/ut-template.yml
|
|
parameters:
|
|
dockerConfigName: "commonDockerConfig"
|
|
utScriptFileName: "run_basic_adaptor"
|
|
uploadPath: $(UPLOAD_PATH)
|
|
utArtifact: "ut_adaptor"
|
|
|
|
- stage: API
|
|
displayName: Unit Test User facing API
|
|
dependsOn: []
|
|
jobs:
|
|
- job:
|
|
displayName: Test User facing API
|
|
steps:
|
|
- template: template/ut-template.yml
|
|
parameters:
|
|
dockerConfigName: "commonDockerConfig"
|
|
utScriptFileName: "run_basic_api"
|
|
uploadPath: $(UPLOAD_PATH)
|
|
utArtifact: "ut_api"
|
|
|
|
- stage: Pruning
|
|
displayName: Unit Test Pruning
|
|
dependsOn: []
|
|
jobs:
|
|
- job:
|
|
displayName: Test PyTorch Pruning
|
|
steps:
|
|
- template: template/ut-template.yml
|
|
parameters:
|
|
dockerConfigName: "commonDockerConfig"
|
|
utScriptFileName: "run_basic_pt_pruning"
|
|
uploadPath: $(UPLOAD_PATH)
|
|
utArtifact: "ut_pt-pruning"
|
|
- job:
|
|
displayName: Test TensorFlow Pruning
|
|
steps:
|
|
- template: template/ut-template.yml
|
|
parameters:
|
|
dockerConfigName: "commonDockerConfig"
|
|
utScriptFileName: "run_basic_tf_pruning"
|
|
uploadPath: $(UPLOAD_PATH)
|
|
utArtifact: "ut_tf-pruning"
|
|
|
|
- stage: TFNewAPI
|
|
displayName: Unit Test TF newAPI
|
|
dependsOn: []
|
|
jobs:
|
|
- job:
|
|
displayName: Test TF newAPI
|
|
steps:
|
|
- template: template/ut-template.yml
|
|
parameters:
|
|
dockerConfigName: "commonDockerConfig"
|
|
utScriptFileName: "run_basic_adaptor_tfnewapi"
|
|
uploadPath: $(UPLOAD_PATH)
|
|
utArtifact: "ut_tfnewapi"
|
|
|
|
- stage: ITEX
|
|
displayName: Unit Test ITEX
|
|
dependsOn: []
|
|
jobs:
|
|
- job:
|
|
displayName: Test ITEX
|
|
steps:
|
|
- template: template/ut-template.yml
|
|
parameters:
|
|
dockerConfigName: "commonDockerConfig"
|
|
utScriptFileName: "run_basic_itex"
|
|
uploadPath: $(UPLOAD_PATH)
|
|
utArtifact: "ut_itex"
|
|
|
|
- stage: Others
|
|
displayName: Unit Test other basic case
|
|
dependsOn: []
|
|
jobs:
|
|
- job:
|
|
displayName: Test other basic case
|
|
steps:
|
|
- template: template/ut-template.yml
|
|
parameters:
|
|
dockerConfigName: "commonDockerConfig"
|
|
utScriptFileName: "run_basic_others"
|
|
uploadPath: $(UPLOAD_PATH)
|
|
utArtifact: "ut_others"
|
|
|
|
- stage: Adaptor_base
|
|
displayName: Unit Test FWKs adaptor baseline
|
|
dependsOn: []
|
|
jobs:
|
|
- job:
|
|
displayName: Test FWKs adaptor baseline
|
|
steps:
|
|
- template: template/ut-template.yml
|
|
parameters:
|
|
dockerConfigName: "gitCloneDockerConfig"
|
|
utScriptFileName: "run_basic_adaptor"
|
|
uploadPath: $(UPLOAD_PATH)
|
|
utArtifact: "ut-base_adaptor"
|
|
repo: $(REPO)
|
|
|
|
- stage: API_base
|
|
displayName: Unit Test User facing API baseline
|
|
dependsOn: []
|
|
jobs:
|
|
- job:
|
|
displayName: Test User facing API baseline
|
|
steps:
|
|
- template: template/ut-template.yml
|
|
parameters:
|
|
dockerConfigName: "gitCloneDockerConfig"
|
|
utScriptFileName: "run_basic_api"
|
|
uploadPath: $(UPLOAD_PATH)
|
|
utArtifact: "ut-base_api"
|
|
repo: $(REPO)
|
|
|
|
- stage: Pruning_base
|
|
displayName: Unit Test Pruning baseline
|
|
dependsOn: []
|
|
jobs:
|
|
- job:
|
|
displayName: Test PyTorch Pruning baseline
|
|
steps:
|
|
- template: template/ut-template.yml
|
|
parameters:
|
|
dockerConfigName: "gitCloneDockerConfig"
|
|
utScriptFileName: "run_basic_pt_pruning"
|
|
uploadPath: $(UPLOAD_PATH)
|
|
utArtifact: "ut-base_pt-pruning"
|
|
repo: $(REPO)
|
|
- job:
|
|
displayName: Test TensorFlow Pruning baseline
|
|
steps:
|
|
- template: template/ut-template.yml
|
|
parameters:
|
|
dockerConfigName: "gitCloneDockerConfig"
|
|
utScriptFileName: "run_basic_tf_pruning"
|
|
uploadPath: $(UPLOAD_PATH)
|
|
utArtifact: "ut-base_tf-pruning"
|
|
repo: $(REPO)
|
|
|
|
- stage: TFNewAPI_base
|
|
displayName: Unit Test TF newAPI baseline
|
|
dependsOn: []
|
|
jobs:
|
|
- job:
|
|
displayName: Test TF newAPI baseline
|
|
steps:
|
|
- template: template/ut-template.yml
|
|
parameters:
|
|
dockerConfigName: "gitCloneDockerConfig"
|
|
utScriptFileName: "run_basic_adaptor_tfnewapi"
|
|
uploadPath: $(UPLOAD_PATH)
|
|
utArtifact: "ut-base_tfnewapi"
|
|
repo: $(REPO)
|
|
|
|
- stage: ITEX_base
|
|
displayName: Unit Test ITEX baseline
|
|
dependsOn: []
|
|
jobs:
|
|
- job:
|
|
displayName: Test ITEX baseline
|
|
steps:
|
|
- template: template/ut-template.yml
|
|
parameters:
|
|
dockerConfigName: "gitCloneDockerConfig"
|
|
utScriptFileName: "run_basic_itex"
|
|
uploadPath: $(UPLOAD_PATH)
|
|
utArtifact: "ut-base_itex"
|
|
repo: $(REPO)
|
|
|
|
- stage: Others_base
|
|
displayName: Unit Test other cases baseline
|
|
dependsOn: []
|
|
jobs:
|
|
- job:
|
|
displayName: Test other cases baseline
|
|
steps:
|
|
- template: template/ut-template.yml
|
|
parameters:
|
|
dockerConfigName: "gitCloneDockerConfig"
|
|
utScriptFileName: "run_basic_others"
|
|
uploadPath: $(UPLOAD_PATH)
|
|
utArtifact: "ut-base_others"
|
|
repo: $(REPO)
|
|
|
|
- stage: Coverage
|
|
displayName: "Coverage Compare"
|
|
pool:
|
|
vmImage: "ubuntu-latest"
|
|
dependsOn: [Adaptor, API, Pruning, TFNewAPI, ITEX, Others, Adaptor_base, API_base, Pruning_base, TFNewAPI_base, ITEX_base, Others_base]
|
|
jobs:
|
|
- job: CollectDatafiles
|
|
steps:
|
|
- script: |
|
|
if [[ ! $(docker images | grep -i ${IMAGE_NAME}:${IMAGE_TAG}) ]]; then
|
|
docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
|
|
fi
|
|
docker images | grep -i ${IMAGE_NAME}
|
|
if [[ $? -ne 0 ]]; then
|
|
echo "NO Such Repo"
|
|
exit 1
|
|
fi
|
|
displayName: "Build develop docker image"
|
|
|
|
- task: DownloadPipelineArtifact@2
|
|
inputs:
|
|
artifact:
|
|
patterns: '*_coverage/.coverage.*'
|
|
path: $(DOWNLOAD_PATH)
|
|
|
|
- script: |
|
|
echo "--- create container ---"
|
|
docker run -d -it --name="collectLogs" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash
|
|
echo "--- docker ps ---"
|
|
docker ps
|
|
echo "--- collect logs ---"
|
|
docker exec collectLogs /bin/bash +x -c "cd /neural-compressor/.azure-pipelines/scripts \
|
|
&& bash install_nc.sh \
|
|
&& bash ut/collect_log.sh"
|
|
displayName: "Collect UT Coverage"
|
|
|
|
- task: PublishCodeCoverageResults@2
|
|
inputs:
|
|
summaryFileLocation: $(Build.SourcesDirectory)/log_dir/coverage_PR/coverage.xml
|
|
|
|
- task: PublishPipelineArtifact@1
|
|
condition: succeededOrFailed()
|
|
inputs:
|
|
targetPath: $(UPLOAD_PATH)
|
|
artifact: $(ARTIFACT_NAME)
|
|
publishLocation: "pipeline"
|
|
|
|
- task: Bash@3
|
|
condition: always()
|
|
inputs:
|
|
targetType: "inline"
|
|
script: |
|
|
docker exec collectLogs bash -c "rm -fr /neural-compressor/* && rm -fr /neural-compressor/.* || true"
|
|
displayName: "Docker clean up"
|