azure-pipelines.yml

trigger:
  - 1.2.x
  - release/*

pr:
  - 1.2.x
  - release/*

pool:
    vmImage: 'Ubuntu-latest'

# set the build name
name: $[ variables['branchName'] ]

# run tests and full conda/tox build matrix every night at 2am
schedules:
- cron: "0 2 * * 1-5"
  displayName: Nightly full build
  branches:
    include:
    - 1.2.x

resources:
  repositories:
    - repository: pytools
      type: github
      endpoint: BCG-Gamma
      name: BCG-Gamma/pytools
      ref: 1.2.x

variables:
  ${{ if not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')) }}:
    branchName: $[ replace(variables['Build.SourceBranch'], 'refs/heads/', '') ]
  ${{ if startsWith(variables['Build.SourceBranch'], 'refs/pull/') }}:
    branchName: $[ replace(variables['System.PullRequest.SourceBranch'], 'refs/heads/', '') ]
  source_is_release_branch: $[ startsWith(variables['branchName'], 'release/') ]
  source_is_develop_branch: $[ startsWith(variables['branchName'], 'dev/') ]
  is_scheduled: $[ eq(variables['Build.Reason'], 'Schedule') ]
  project_name: pytools
  project_root: $(project_name)
  package_name: gamma-pytools
  sshPublicKey: $(sshPublicKey_pytools)

stages:
  # check code quality first to fail fast (isort, flake8, black)
  - stage: code_quality_checks
    displayName: 'Code quality'
    jobs:
      - job:
        displayName: 'isort'
        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: '3.7.*'
            displayName: 'use Python 3.7'
          - script: |
              python -m pip install isort==5.5.4
              python -m isort --check --diff .
            displayName: 'Run isort'
      - job:
        displayName: 'black'
        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: '3.7.*'
            displayName: 'use Python 3.7'
          - script: |
              python -m pip install black==20.8b1
              python -m black --check .
            displayName: 'Run black'
      - job:
        displayName: 'flake8'
        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: '3.7.*'
            displayName: 'use Python 3.7'
          - script: |
              python -m pip install flake8==3.9.0 flake8-comprehensions flake8-import-order
              python -m flake8 --config tox.ini -v .
            displayName: 'Run flake8'

  # detect whether the build config (pyproject.toml) was changed -> then we must run a build test
  - stage: detect_build_config_changes
    displayName: 'Pyproject.toml build config'

    jobs:

      - job: checkout_and_diff
        displayName: 'detect changes'
        steps:
          - checkout: self

          - task: Bash@3
            name: diff
            inputs:
              targetType: 'inline'
              script: |
                echo Repo: $(Build.DefinitionName)
                cd $(System.DefaultWorkingDirectory)
                files_changed=$(git diff $(Build.SourceVersion)^! --name-only)
                echo "Files changed since last commit: ${files_changed}"
                n_files_changed=$(git diff $(Build.SourceVersion)^! --name-only | grep -i -E 'meta\.yaml|pyproject\.toml|azure-pipelines\.yml|tox\.ini|make\.py'  | wc -l | xargs)
                if [ ${n_files_changed} -gt 0 ]
                then
                build_changed=1
                echo "build config has been changed"
                else
                build_changed=0
                echo "build config is unchanged";
                fi
                # set +x to prevent Azure from randomly appending a \' of the ## statement
                set +x
                echo "##vso[task.setvariable variable=conda_build_config_changed;isOutput=true]$build_changed"


  - stage:
    displayName: 'Unit tests'
    dependsOn: 'detect_build_config_changes'
    variables:
      conda_build_config_changed: $[ stageDependencies.detect_build_config_changes.checkout_and_diff.outputs['diff.conda_build_config_changed'] ]

    jobs:
    - job:
      displayName: 'pytest @ develop environment'
      condition: ne(variables.source_is_release_branch, 'True')

      pool:
          vmImage: 'ubuntu-latest'

      steps:
        - task: UsePythonVersion@0
          inputs:
            versionSpec: '3.7.*'
          displayName: 'use Python 3.7'

        - checkout: self

        - script: dir $(Build.SourcesDirectory)

        - task: Bash@3
          inputs:
            targetType: 'inline'
            script: |
              set -eux
              eval "$(conda shell.bash hook)"
              export PYTHONPATH=$(System.DefaultWorkingDirectory)/src/
              export RUN_PACKAGE_VERSION_TEST=$(project_name)
              conda env create
              conda activate $(project_name)-develop
              pytest \
                 --cov $(project_name) \
                 --cov-config "tox.ini" \
                 --cov-report=xml:coverage.xml --cov-report=html:htmlcov \
                 --junitxml pytest.xml \
                 . -s
          displayName: 'pytest'

        - task: PublishTestResults@2
          condition: succeededOrFailed()
          inputs:
            testResultsFiles: '$(System.DefaultWorkingDirectory)/*.xml'
            searchFolder: '$(System.DefaultWorkingDirectory)/'
            testRunTitle: 'Publish test results'

        - task: PublishCodeCoverageResults@1
          inputs:
            codeCoverageTool: Cobertura
            summaryFileLocation: '$(System.DefaultWorkingDirectory)/coverage.xml'
            reportDirectory: '$(System.DefaultWorkingDirectory)/htmlcov'

  # conda env & tox build test
  # testing matrix of python & sklearn versions
  # - for release branch: 3 build tests
  # - otherwise: 1 build test
  - stage: conda_tox_build
    displayName: 'Conda/tox build & test'
    dependsOn: 'detect_build_config_changes'
    variables:
      conda_build_config_changed: $[ stageDependencies.detect_build_config_changes.checkout_and_diff.outputs['diff.conda_build_config_changed'] ]

    jobs:
    - job:
      displayName: 'single'
      condition: >
        and(
        ne(variables.source_is_release_branch, 'True'),
        ne(variables.source_is_develop_branch, 'True'),
        ne(variables.is_scheduled, 'True'),
        ne(stageDependencies.detect_build_config_changes.checkout_and_diff.outputs['diff.conda_build_config_changed'], '0')
        )

      pool:
          vmImage: 'ubuntu-latest'
      strategy:
        matrix:
          maximum_dependencies_conda:
            FACET_V_PYTHON_BUILD: '=3.8.*'
            BUILD_SYSTEM: 'conda'
            PKG_DEPENDENCIES: 'max'
          maximum_dependencies_tox:
            FACET_V_PYTHON_BUILD: '=3.8.*'
            BUILD_SYSTEM: 'tox'
            PKG_DEPENDENCIES: 'max'

      steps:
        - task: UsePythonVersion@0
          inputs:
            versionSpec: '$(FACET_V_PYTHON_BUILD)'
          displayName: 'Use Python $(FACET_V_PYTHON_BUILD)'

        - checkout: pytools
        - checkout: self

        - script: dir $(Build.SourcesDirectory)

        - script: |
            conda install -y -c anaconda conda-build~=3.21 conda-verify toml=0.10.* flit=3.0.* packaging~=20.9
          displayName: 'Install conda-build, flit, toml'
          condition: eq(variables['BUILD_SYSTEM'], 'conda')

        - script: |
            python -m pip install "toml==0.10.*"
            python -m pip install "flit==3.0.*"
            flit --version
            python -m pip install "tox==3.20.*"
            tox --version
          displayName: 'Install tox, flit & toml'
          condition: eq(variables['BUILD_SYSTEM'], 'tox')

        - task: Bash@3
          inputs:
            targetType: 'inline'
            script: |
              set -eux
              if [ "$BUILD_SYSTEM" = "conda" ] ; then eval "$(conda shell.bash hook)" ; fi
              export RUN_PACKAGE_VERSION_TEST=$(project_name)

              cd $(Build.SourcesDirectory)/$(project_root)
              ./make.py $(project_name) $(BUILD_SYSTEM) $(PKG_DEPENDENCIES)
          displayName: "build & test"

        - task: CopyFiles@2
          inputs:
            sourceFolder: $(System.DefaultWorkingDirectory)/$(project_root)/dist
            targetFolder: $(Build.ArtifactStagingDirectory)

        - task: PublishBuildArtifacts@1
          inputs:
            pathtoPublish: $(Build.ArtifactStagingDirectory)
            artifactName: $(BUILD_SYSTEM)_$(PKG_DEPENDENCIES)
            publishLocation: Container

    - job:
      displayName: 'matrix'
      condition: >
        or(
        eq(variables.source_is_develop_branch, 'True'),
        eq(variables.source_is_release_branch, 'True'),
        eq(variables.is_scheduled, 'True')
        )

      pool:
        vmImage: 'ubuntu-latest'
      strategy:
        matrix:
          default_dependencies_conda:
            FACET_V_PYTHON_BUILD: '=3.7.*'
            BUILD_SYSTEM: 'conda'
            PKG_DEPENDENCIES: 'default'
          minimum_dependencies_conda:
            FACET_V_PYTHON_BUILD: '=3.6.*'
            BUILD_SYSTEM: 'conda'
            PKG_DEPENDENCIES: 'min'
          maximum_dependencies_conda:
            FACET_V_PYTHON_BUILD: '=3.8.*'
            BUILD_SYSTEM: 'conda'
            PKG_DEPENDENCIES: 'max'
          default_dependencies_tox:
            FACET_V_PYTHON_BUILD: '=3.7.*'
            BUILD_SYSTEM: 'tox'
            PKG_DEPENDENCIES: 'default'
          minimum_dependencies_tox:
            FACET_V_PYTHON_BUILD: '=3.6.*'
            BUILD_SYSTEM: 'tox'
            PKG_DEPENDENCIES: 'min'
          maximum_dependencies_tox:
            FACET_V_PYTHON_BUILD: '=3.8.*'
            BUILD_SYSTEM: 'tox'
            PKG_DEPENDENCIES: 'max'


      steps:
        - task: UsePythonVersion@0
          inputs:
            versionSpec: '$(FACET_V_PYTHON_BUILD)'
          displayName: 'Use Python $(FACET_V_PYTHON_BUILD)'

        - checkout: pytools
        - checkout: self

        - script: dir $(Build.SourcesDirectory)

        - script: |
            conda install -y -c anaconda conda-build~=3.21 conda-verify toml=0.10.* flit=3.0.* packaging~=20.9
          displayName: 'Install conda-build, flit, toml'
          condition: eq(variables['BUILD_SYSTEM'], 'conda')

        - script: |
            python -m pip install "toml==0.10.*"
            python -m pip install "flit==3.0.*"
            flit --version
            python -m pip install "tox==3.20.*"
            tox --version
          displayName: 'Install tox, flit & toml'
          condition: eq(variables['BUILD_SYSTEM'], 'tox')

        - task: Bash@3
          inputs:
            targetType: 'inline'
            script: |
              set -eux
              if [ "$BUILD_SYSTEM" = "conda" ] ; then eval "$(conda shell.bash hook)" ; fi
              export RUN_PACKAGE_VERSION_TEST=$(project_name)

              cd $(Build.SourcesDirectory)/$(project_root)
              ./make.py $(project_name) $(BUILD_SYSTEM) $(PKG_DEPENDENCIES)
          displayName: "build & test"

        - task: CopyFiles@2
          inputs:
            sourceFolder: $(System.DefaultWorkingDirectory)/$(project_root)/dist
            targetFolder: $(Build.ArtifactStagingDirectory)

        - task: PublishBuildArtifacts@1
          inputs:
            pathtoPublish: $(Build.ArtifactStagingDirectory)
            artifactName: $(BUILD_SYSTEM)_$(PKG_DEPENDENCIES)
            publishLocation: Container

  # apply veracode static code analysis during nightly build
  - stage: veracode_check
    displayName: 'Veracode check'

    jobs:
      - job:
        displayName: 'Veracode check'
        condition: eq(variables['Build.Reason'], 'Schedule')

        steps:
          - task: Bash@3
            inputs:
              targetType: 'inline'
              script: |
                set -eux
                eval "$(conda shell.bash hook)"
                cd $(System.DefaultWorkingDirectory)
                mkdir static_scan
                git archive --format=zip --output static_scan/archive.zip HEAD

          - task: Veracode@3
            inputs:
              ConnectionDetailsSelection: 'Endpoint'
              AnalysisService: 'veracode'
              veracodeAppProfile: 'FACET'
              version: '$(Build.BuildID)'
              filepath: '$(System.DefaultWorkingDirectory)/static_scan'
              sandboxName: '$(project_name)'
              createSandBox: false
              createProfile: false
              failTheBuildIfVeracodeScanDidNotInitiate: false
              scanStatusCheckInterval: '60'
              importResults: false
              failBuildOnPolicyFail: false

  - stage: check_release
    displayName: 'Check Release Version'
    dependsOn: conda_tox_build

    jobs:
      - job:
        displayName: 'Check Release'
        condition: >
          or(
          eq(variables.source_is_release_branch, 'True'),
          eq(variables.source_is_develop_branch, 'True')
          )

        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: '3.7.*'
            displayName: 'use Python 3.7'

          - checkout: pytools
          - checkout: self

          - task: Bash@3
            env:
              BRANCH_NAME: $(branchName)
            inputs:
              targetType: inline
              condition: ne(variables.branchName, 'develop')
              script: |
                set -eux
                python -m pip install toml~=0.10.2 packaging~=20.9
                cd $(System.DefaultWorkingDirectory)/pytools
                python <<EOF
                from os import environ
                from make import ToxBuilder

                branch_env = "BRANCH_NAME"
                assert branch_env in environ, f"ENV variable '{branch_env}' with git branch name missing."
                branch = environ[branch_env]
                print(f"Checking package version consistency with branch: {branch}")
                if branch == "develop":
                  exit(0)
                assert (branch.startswith("release/") or branch.startswith("dev/")
                ), "This check should only run on versioned branches – check pipeline."

                branch_version = branch.split("/", maxsplit=1)[1]
                package_version = ToxBuilder("$(project_name)", "default").package_version

                assert (
                    package_version == branch_version
                ), f"Package version '{package_version}' does not match '{branch_version}' from branch."

                print("Check passed.")

                EOF

            displayName: "Check version consistency"


  # release on merges into release/*:
  # - add release tag
  # - create GitHub release with changelog
  # - upload conda and pip packages as artifacts to GitHub
  - stage:
    displayName: 'Release'
    dependsOn: check_release
    variables:
    - group: artifact_publication

    jobs:
      - job:
        displayName: 'Release'
        condition: >
          or(
          eq(variables.source_is_release_branch, 'True'),
          eq(variables.source_is_develop_branch, 'True')
          )

        steps:
          - task: UsePythonVersion@0
            inputs:
              versionSpec: '3.7.*'
            displayName: 'use Python 3.7'

          - checkout: pytools
          - checkout: self

          - task: Bash@3
            inputs:
              targetType: inline
              script: |
                set -eux
                echo "Getting version"
                pip install packaging~=20.9
                package_path=$(System.DefaultWorkingDirectory)/$(project_root)/src/$(project_name)
                export PYTHONPATH=$(System.DefaultWorkingDirectory)/pytools/sphinx/base
                version=$(python -c "import make_util; print(make_util.get_package_version(package_path='$package_path'))")
                echo "Current version: $version"
                echo "Detecting pre-release ('dev' or 'rc' in version)"
                prerelease=False
                [[ $version == *dev* ]] && prerelease=True && echo "Development release identified"
                [[ $version == *rc* ]] && prerelease=True && echo "Pre-release identified"
                # set +x to prevent Azure from randomly appending a \' of the ## statement
                set +x
                echo "##vso[task.setvariable variable=current_version]$version"
                echo "##vso[task.setvariable variable=is_prerelease]$prerelease"
            displayName: "Get package version"

          - task: DownloadBuildArtifacts@0
            inputs:
              artifactName: tox_default

          - task: DownloadBuildArtifacts@0
            inputs:
              artifactName: conda_default

          - script: |
              set -eux
              cd $(System.DefaultWorkingDirectory)
              eval "$(conda shell.bash hook)"
              conda install -y anaconda-client
              anaconda login --username "${CONDA_USERNAME}" --password "${CONDA_PASSWORD}"
              anaconda upload --user BCG_Gamma --force $(System.ArtifactsDirectory)/conda_default/conda/noarch/$(package_name)-*.tar.bz2
              anaconda logout
              # set +x to prevent Azure from randomly appending a \' of the ## statement
              set +x
              echo "##vso[task.setvariable variable=conda_published]True"
            displayName: 'Publish to Anaconda'

            condition: eq(variables['source_is_release_branch'], 'True')
            env:
              CONDA_PASSWORD: $(anaconda_pw)
              CONDA_USERNAME: $(anaconda_user)

          - script: |
              set -eux
              cd $(System.DefaultWorkingDirectory)/$(project_root)
              pip install flit
              flit install -s
              flit publish
              # set +x to prevent Azure from randomly appending a \' of the ## statement
              set +x
              echo "##vso[task.setvariable variable=pypi_published]True"
            displayName: 'Publish to PyPi'
            condition: eq(variables['source_is_release_branch'], 'True')
            env:
              FLIT_PASSWORD: $(pypi_pw)
              FLIT_USERNAME: $(pypi_user)

          - task: GitHubRelease@1
            condition: >
              and(
                eq(variables['source_is_release_branch'], 'True'),
                succeededOrFailed(),
                or(
                  eq(variables['conda_published'], 'True'),
                  eq(variables['pypi_published'], 'True')
                )
              )
            inputs:
              gitHubConnection: github-release
              repositoryName: $(Build.Repository.Name)
              action: create
              target: $(Build.SourceVersion)
              tagSource: userSpecifiedTag
              tag: $(current_version)
              title: $(project_name) $(current_version)
              releaseNotesSource: inline
              releaseNotesInline: |
                This is the $(current_version) release of $(package_name).

                You can upgrade your current pip installation via

                    pip install --upgrade $(package_name)

                Your conda package can be upgraded by running

                    conda install -c conda-forge -c bcg_gamma $(package_name)

              isDraft: false
              isPrerelease: $(is_prerelease)
              assets: |
                $(System.ArtifactsDirectory)/tox_default/tox/$(package_name)-*.tar.gz
                $(System.ArtifactsDirectory)/conda_default/conda/noarch/$(package_name)-*.tar.bz2

  # render docs and publish to GitHub Pages
  - stage:
    displayName: 'Docs'

    variables:
    - group: github_ssh

    jobs:
    - job:
      displayName: 'Build and publish docs'
      condition: >
        or(
        eq(variables.source_is_release_branch, 'True'),
        eq(variables.source_is_develop_branch, 'True')
        )


      pool:
          vmImage: 'ubuntu-latest'

      steps:
        - task: UsePythonVersion@0
          inputs:
            versionSpec: '3.7.*'
          displayName: 'use Python 3.7'

        - task: InstallSSHKey@0
          inputs:
            knownHostsEntry: $(knownHostsEntry)
            sshPublicKey: $(sshPublicKey)
            sshKeySecureFile: 'deploy_docs_$(project_name)'
          displayName: 'Install the deploy SSH key'

        - checkout: pytools
        - checkout: self

        - script: dir $(Build.SourcesDirectory)

        - task: Bash@3
          inputs:
            targetType: 'inline'
            script: |
              set -eux
              cd $(System.DefaultWorkingDirectory)/$(project_root)
              git checkout --track origin/github-pages
              mkdir -p docs
              sudo apt-get install tree
              echo "Current docs contents:"
              tree docs
              mkdir $(Build.ArtifactStagingDirectory)/old_docs
              cp -r docs $(Build.ArtifactStagingDirectory)/old_docs
          displayName: 'Save current docs version'

        - task: Bash@3
          inputs:
            targetType: 'inline'
            script: |
              set -eux
              eval "$(conda shell.bash hook)"
              cd $(System.DefaultWorkingDirectory)/$(project_root)
              echo "Checking out $(branchName)"
              git checkout $(branchName)
              git status
              export PYTHONPATH=$(System.DefaultWorkingDirectory)/$(project_root)/src/
              conda env create -f environment.yml
              conda activate $(project_name)-develop
              cd $(System.DefaultWorkingDirectory)/$(project_root)/
              python sphinx/make.py html
              echo "Current docs contents:"
              tree docs
          displayName: 'Build new docs version'

        - task: Bash@3
          inputs:
            targetType: 'inline'
            script: |
              set -eux
              eval "$(conda shell.bash hook)"
              cp -r $(Build.ArtifactStagingDirectory)/old_docs/docs .
              echo "Current docs contents:"
              tree docs
              mkdir -p $(System.DefaultWorkingDirectory)/$(project_root)/sphinx/build/
              cp -R docs/docs-version $(System.DefaultWorkingDirectory)/$(project_root)/sphinx/build/
              echo "Building sphinx docs"
              conda activate $(project_name)-develop
              cd $(System.DefaultWorkingDirectory)/$(project_root)
              python sphinx/make.py prepare_docs_deployment
              echo "Current docs contents:"
              tree docs
              mkdir $(Build.ArtifactStagingDirectory)/new_docs
              mv docs $(Build.ArtifactStagingDirectory)/new_docs
          displayName: 'Update saved docs'

        - task: Bash@3
          condition: eq(variables['source_is_release_branch'], 'True')
          inputs:
            targetType: 'inline'
            script: |
              set -eux
              cd $(System.DefaultWorkingDirectory)/$(project_root)
              echo "Adjusting git credentials"
              git config --global user.name "Azure Pipelines"
              git config --global user.email "azuredevops@microsoft.com"
              git config --global url.ssh://git@github.com/.insteadOf https://github.com/
              git checkout github-pages
              cp -r $(Build.ArtifactStagingDirectory)/new_docs/docs .
              git status
              git add docs
              echo "Staged docs HTML build"
              git status
              git commit -m "Publish GitHub Pages [skip ci]"
              echo "Committed to local branch github-pages"
              git push --set-upstream origin github-pages
          displayName: 'Publish docs'