From 6639aa779685785a05fa1c7f6c796fbf2b1ca008 Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Fri, 11 Oct 2024 15:12:03 -0700 Subject: [PATCH 1/4] Changes required for sdk-repo based emitter pipelines --- .../stages/archetype-typespec-emitter.yml | 452 ++++++++++++++++++ .../steps/create-authenticated-npmrc.yml | 23 + .../Helpers/CommandInvocation-Helpers.ps1 | 26 +- eng/common/scripts/New-RegenerateMatrix.ps1 | 11 +- eng/common/scripts/common.ps1 | 1 + .../typespec/New-EmitterPackageJson.ps1 | 6 +- 6 files changed, 510 insertions(+), 9 deletions(-) create mode 100644 eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml create mode 100644 eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml diff --git a/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml b/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml new file mode 100644 index 00000000000..f50b94688cf --- /dev/null +++ b/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml @@ -0,0 +1,452 @@ +parameters: +# Path to the emitter package. This is used as the base path for script invocations. +- name: EmitterPackagePath + type: string + +# Pool to use for the pipeline stages +- name: Pool + type: object + +# Whether to build alpha versions of the packages. This is passed as a flag to the build script. +- name: BuildPrereleaseVersion + type: boolean + default: true + +# Whether to use the `next` version of TypeSpec. This is passed as a flag to the initialize script. +- name: UseTypeSpecNext + type: boolean + default: false + +# Custom steps to run after the repository is cloned but before other job steps. +- name: InitializationSteps + type: stepList + default: [] + +# Indicates the build matrix to use for post-build autorest validation +- name: TestMatrix + type: object + default: {} + +# Whether to run the publish and regeneration stages. If false, only the build stage will run. +- name: ShouldPublish + type: boolean + default: false + +# List of packages to publish. Each package is an object with the following properties: +# name: The name of the package. This is used to determine the name of the file to publish. +# type: The type of package. Currently supported values are 'npm' and 'nuget'. +# file: The path to the file to publish. This is relative to the packages directory in the build artifacts directory. +- name: Packages + type: object + default: [] + +# Whether to publish to the internal feed. +- name: PublishInternal + type: boolean + default: true + +# Indicates if the Publish stage should depend on the Test stage +- name: PublishDependsOnTest + type: boolean + default: false + +# Whether to publish to the internal feed. +- name: ShouldRegenrate + type: boolean + default: false + +# Number of jobs to generate. This is the maximum number of jobs that will be generated. The actual number of jobs will be reduced if it would result in fewer than MinimumPerJob packages per job. +- name: RegenerationJobCount + type: number + default: 10 + +# Minimum number of packages to generate per job. +- name: MinimumPerJob + type: number + default: 10 + +# Indicates if regenration matrix should only contain folders with typespec files +- name: OnlyGenerateTypespec + type: boolean + default: false + +stages: + +# Build stage +# Responsible for building the autorest generator and typespec emitter packages +# Produces the artifact `build_artifacts` which contains the following: +# package-versions.json: Contains a map of package name to version for the packages that were built +# overrides.json: Contains npm package version overrides for the emitter and generator +# packages/: Contains the packages to publish +# lock-files/: Contains package.json and package-lock.json files for use in the test stage to ensure tests are run against the correct dependency versions +- stage: Build + pool: ${{ parameters.Pool }} + jobs: + - job: Build + steps: + # Validate parameters and fail early if invalid + - ${{ each package in parameters.Packages }}: + - ${{ if notIn(package.type, 'npm', 'nuget') }}: + - script: | + echo "Package ${{ package.name }} has unsupported type: ${{ package.type }}" + exit 1 + displayName: 'Unsupported package type' + condition: always() + + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - ${{ parameters.InitializationSteps }} + + - task: PowerShell@2 + displayName: 'Run initialize script' + inputs: + pwsh: true + filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Initialize.ps1 + arguments: -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} + workingDirectory: ${{ parameters.EmitterPackagePath }} + + - task: PowerShell@2 + displayName: 'Run build script' + name: ci_build + inputs: + pwsh: true + filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Build-Packages.ps1 + arguments: > + -BuildNumber "$(Build.BuildNumber)" + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + -PublishInternal:$${{ parameters.PublishInternal }} + -Prerelease:$${{ parameters.BuildPrereleaseVersion }} + + - pwsh: | + $sourceBranch = '$(Build.SourceBranch)' + $buildReason = '$(Build.Reason)' + $buildNumber = '$(Build.BuildNumber)' + + if ($buildReason -eq 'Schedule') { + $branchName = 'validate-typespec-scheduled' + } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { + $branchName = "validate-typespec-pr-$($Matches[1])" + } else { + $branchName = "validate-typespec-$buildNumber" + } + + Write-Host "Setting variable 'branchName' to '$branchName'" + Write-Host "##vso[task.setvariable variable=branchName;isOutput=true]$branchName" + displayName: Set branch name + name: set_branch_name + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: build_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + +# Publish stage +# Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json` +# Produces the artifact `publish_artifacts` which contains the following: +# emitter-package.json: Created using the package json from the build step. +# emitter-package-lock.json: Created by calling `npm install` using `emitter-package.json` +- ${{ if parameters.ShouldPublish }}: + - stage: Publish + dependsOn: + - Build + - ${{ if and(parameters.PublishDependsOnTest, ne(length(parameters.TestMatrix), 0)) }}: + - Test + variables: + buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts + pool: ${{ parameters.Pool }} + jobs: + - job: Publish + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - download: current + artifact: build_artifacts + displayName: Download build artifacts + + # Create authenticated .npmrc file for publishing + - ${{ if eq(parameters.PublishInternal, 'true') }}: + - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml + parameters: + npmrcPath: $(buildArtifactsPath)/packages/.npmrc + registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/ + - ${{ else }}: + - pwsh: | + "//registry.npmjs.org/:_authToken=$(azure-sdk-npm-token)" | Out-File '.npmrc' + displayName: Authenticate .npmrc for npmjs.org + workingDirectory: $(buildArtifactsPath)/packages + + # per package, publishing using appropriate tool + - ${{ each package in parameters.Packages }}: + - ${{ if eq(package.type, 'npm') }}: + - pwsh: | + $file = Resolve-Path "${{ package.file }}" + Write-Host "npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages" + npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages + displayName: Publish ${{ package.name }} + workingDirectory: $(buildArtifactsPath)/packages + - ${{ elseif eq(package.type, 'nuget') }}: + - task: NuGetCommand@2 + displayName: Publish ${{ package.name }} + inputs: + command: 'push' + packagesToPush: $(buildArtifactsPath)/packages/${{ package.file }} + # Nuget packages are always published to the same internal feed https://dev.azure.com/azure-sdk/public/_packaging?_a=feed&feed=azure-sdk-for-net + nuGetFeedType: 'internal' + publishVstsFeed: '29ec6040-b234-4e31-b139-33dc4287b756/fa8c16a3-dbe0-4de2-a297-03065ec1ba3f' + + - task: PowerShell@2 + displayName: Create emitter-package.json + inputs: + pwsh: true + filePath: ./eng/common/scripts/typespec/New-EmitterPackageJson.ps1 + arguments: > + -PackageJsonPath '$(buildArtifactsPath)/lock-files/package.json' + -OverridesPath '$(buildArtifactsPath)/overrides.json' + -OutputDirectory '$(Build.ArtifactStagingDirectory)' + workingDirectory: $(Build.SourcesDirectory) + + - task: PowerShell@2 + displayName: Create emitter-package-lock.json + inputs: + pwsh: true + filePath: ./eng/common/scripts/typespec/New-EmitterPackageLock.ps1 + arguments: > + -EmitterPackageJsonPath '$(Build.ArtifactStagingDirectory)/emitter-package.json' + -OutputDirectory '$(Build.ArtifactStagingDirectory)' + workingDirectory: $(Build.SourcesDirectory) + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: publish_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + +# Regenerate stage +# Responsible for regenerating the SDK code using the emitter package and the generation matrix. +- ${{ if and(parameters.ShouldPublish, parameters.ShouldRegenrate) }}: + - stage: Regenerate + dependsOn: + - Build + - Publish + variables: + pullRequestTargetBranch: 'main' + publishArtifactsPath: $(Pipeline.Workspace)/publish_artifacts + branchName: $[stageDependencies.Build.Build.outputs['set_branch_name.branchName']] + pool: ${{ parameters.Pool }} + jobs: + - job: Initialize + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - "/*" + - "!SessionRecords" + + - download: current + displayName: Download pipeline artifacts + + - pwsh: | + Write-Host "Copying emitter-package.json to $(Build.SourcesDirectory)/eng" + Copy-Item $(publishArtifactsPath)/emitter-package.json $(Build.SourcesDirectory)/eng/ -Force + + Write-Host "Copying emitter-package-lock.json to $(Build.SourcesDirectory)/eng" + Copy-Item $(publishArtifactsPath)/emitter-package-lock.json $(Build.SourcesDirectory)/eng/ -Force + displayName: Copy emitter-package json files + + - ${{ parameters.InitializationSteps }} + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoOwner: azure-sdk + TargetRepoName: $(Build.Repository.Name) + BaseRepoBranch: $(branchName) + CommitMsg: Initialize repository for autorest build $(Build.BuildNumber) + WorkingDirectory: $(Build.SourcesDirectory) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + # To accomodate scheduled runs and retries, we want to overwrite any existing changes on the branch + PushArgs: --force + + - task: PowerShell@2 + displayName: Get generation job matrix + name: generate_matrix + inputs: + pwsh: true + filePath: ./eng/common/scripts/New-RegenerateMatrix.ps1 + arguments: > + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + -OutputVariableName matrix + -JobCount ${{ parameters.RegenerationJobCount }} + -MinimumPerJob ${{ parameters.MinimumPerJob }} + -OnlyTypespec ${{ parameters.OnlyGenerateTypespec }} + workingDirectory: $(Build.SourcesDirectory) + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: matrix_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + + - job: Generate + dependsOn: Initialize + strategy: + matrix: $[dependencies.Initialize.outputs['generate_matrix.matrix']] + variables: + matrixArtifactsPath: $(Pipeline.Workspace)/matrix_artifacts + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - "/*" + - "!SessionRecords" + + - download: current + displayName: Download pipeline artifacts + + - ${{ parameters.InitializationSteps }} + + - task: PowerShell@2 + displayName: Call regeneration script + inputs: + pwsh: true + filePath: ./eng/common/scripts/Update-GeneratedSdks.ps1 + arguments: > + -PackageDirectoriesFile "$(matrixArtifactsPath)/$(DirectoryList)" + workingDirectory: $(Build.SourcesDirectory) + continueOnError: true + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoOwner: azure-sdk + TargetRepoName: $(Build.Repository.Name) + BaseRepoBranch: $(branchName) + CommitMsg: Update SDK code $(JobKey) + WorkingDirectory: $(Build.SourcesDirectory) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + + - job: Create_PR + displayName: Create PR + dependsOn: + - Generate + variables: + generateJobResult: $[dependencies.Generate.result] + emitterVersion: $[stageDependencies.Build.Build.outputs['ci_build.emitterVersion']] + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - pwsh: | + $generateJobResult = '$(generateJobResult)' + $emitterVersion = '$(emitterVersion)' + $collectionUri = '$(System.CollectionUri)' + $project = '$(System.TeamProject)' + $definitionName = '$(Build.DefinitionName)' + $repoUrl = '$(Build.Repository.Uri)' + $repoName = '$(Build.Repository.Name)' + $sourceBranch = '$(Build.SourceBranch)' + $reason = '$(Build.Reason)' + $buildId = '$(Build.BuildId)' + $buildNumber = '$(Build.BuildNumber)' + $preRelease = '${{ parameters.BuildPrereleaseVersion }}' -eq 'true' + + $prBody = "Generated by $definitionName build [$buildNumber]($collectionUri/$project/_build/results?buildId=$buildId)
" + + if ($sourceBranch -match "^refs/heads/(.+)$") { + $prBody += "Triggered from branch: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" + } elseif ($sourceBranch -match "^refs/tags/(.+)$") { + $prBody += "Triggered from tag: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" + } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { + $prBody += "Triggered from pull request: $repoUrl/pull/$($Matches[1])" + } else { + $prBody += "Triggered from [$sourceBranch]($repoUrl/tree/$sourceBranch)" + } + + if ($reason -eq 'Schedule') { + $prTitle = "Scheduled code regeneration test" + } else { + if ($preRelease) { + $prTitle = "Update typespec emitter version to prerelease $emitterVersion" + } else { + $prTitle = "Update typespec emitter version to $emitterVersion" + } + + if ($generateJobResult -ne 'Succeeded') { + $prTitle = "Failed: $prTitle" + } + } + + Write-Host "Setting variable 'PullRequestTitle' to '$prTitle'" + Write-Host "##vso[task.setvariable variable=PullRequestTitle]$prTitle" + + Write-Host "Setting variable 'PullRequestBody' to '$prBody'" + Write-Host "##vso[task.setvariable variable=PullRequestBody]$prBody" + + $repoNameParts = $repoName.Split('/') + if ($repoNameParts.Length -eq 2) { + Write-Host "Setting variable 'RepoOwner' to '$($repoNameParts[0])'" + Write-Host "##vso[task.setvariable variable=RepoOwner]$($repoNameParts[0])" + + Write-Host "Setting variable 'RepoName' to '$($repoNameParts[1])'" + Write-Host "##vso[task.setvariable variable=RepoName]$($repoNameParts[1])" + } else { + Write-Error "Build.Repository.Name not in the expected {Owner}/{Name} format" + } + displayName: Get PR title and body + + - task: PowerShell@2 + displayName: Create pull request + inputs: + pwsh: true + filePath: ./eng/common/scripts/Submit-PullRequest.ps1 + arguments: > + -RepoOwner '$(RepoOwner)' + -RepoName '$(RepoName)' + -BaseBranch '$(pullRequestTargetBranch)' + -PROwner 'azure-sdk' + -PRBranch '$(branchName)' + -AuthToken '$(azuresdk-github-pat)' + -PRTitle '$(PullRequestTitle)' + -PRBody '$(PullRequestBody)' + -OpenAsDraft $true + -PRLabels 'Do Not Merge' + workingDirectory: $(Build.SourcesDirectory) + +# Test stage +# Responsible for running unit and functional tests using a matrix passed in as the parameter `TestMatrix`. +# Will only run if the parameter `TestMatrix` is not empty. +# The contents of the artficact `build_artifacts` are available under the path `$(buildArtifactsPath)`. +- ${{ if ne(length(parameters.TestMatrix), 0) }}: + - stage: Test + dependsOn: Build + pool: ${{ parameters.Pool }} + jobs: + - job: Test + strategy: + matrix: ${{ parameters.TestMatrix }} + variables: + buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - download: current + artifact: build_artifacts + displayName: Download build artifacts + + - ${{ parameters.InitializationSteps }} + + - task: PowerShell@2 + displayName: 'Run initialize script' + inputs: + pwsh: true + filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Initialize.ps1 + arguments: -BuildArtifactsPath '$(buildArtifactsPath)' + + - task: PowerShell@2 + displayName: 'Run test script' + inputs: + pwsh: true + filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Test-Packages.ps1 + arguments: > + $(TestArguments) + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: test_artifacts_$(System.JobName) + artifactPath: $(Build.ArtifactStagingDirectory) diff --git a/eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml b/eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml new file mode 100644 index 00000000000..4b6f08359bd --- /dev/null +++ b/eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml @@ -0,0 +1,23 @@ +parameters: + - name: npmrcPath + type: string + - name: registryUrl + type: string + +steps: +- pwsh: | + Write-Host "Creating .npmrc file ${{ parameters.npmrcPath }} for registry ${{ parameters.registryUrl }}" + $parentFolder = Split-Path -Path '${{ parameters.npmrcPath }}' -Parent + + if (!(Test-Path $parentFolder)) { + Write-Host "Creating folder $parentFolder" + New-Item -Path $parentFolder -ItemType Directory | Out-Null + } + + $content = "registry=${{ parameters.registryUrl }}`n`nalways-auth=true" + $content | Out-File '${{ parameters.npmrcPath }}' + displayName: 'Create .npmrc' +- task: npmAuthenticate@0 + displayName: Authenticate .npmrc + inputs: + workingFile: ${{ parameters.npmrcPath }} diff --git a/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 b/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 index 5dc0c8c7da1..0b9f810b83a 100644 --- a/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 +++ b/eng/common/scripts/Helpers/CommandInvocation-Helpers.ps1 @@ -1,5 +1,14 @@ -function Invoke-LoggedCommand($Command, $ExecutePath, [switch]$GroupOutput) +function Invoke-LoggedCommand { + [CmdletBinding()] + param + ( + [string] $Command, + [string] $ExecutePath, + [switch] $GroupOutput, + [int[]] $AllowedExitCodes = @(0) + ) + $pipelineBuild = !!$env:TF_BUILD $startTime = Get-Date @@ -22,7 +31,7 @@ function Invoke-LoggedCommand($Command, $ExecutePath, [switch]$GroupOutput) Write-Host "##[endgroup]" } - if($LastExitCode -ne 0) + if($LastExitCode -notin $AllowedExitCodes) { if($pipelineBuild) { Write-Error "##[error]Command failed to execute ($duration): $Command`n" @@ -40,3 +49,16 @@ function Invoke-LoggedCommand($Command, $ExecutePath, [switch]$GroupOutput) } } } + +function Set-ConsoleEncoding +{ + [CmdletBinding()] + param + ( + [string] $Encoding = 'utf-8' + ) + + $outputEncoding = [System.Text.Encoding]::GetEncoding($Encoding) + [Console]::OutputEncoding = $outputEncoding + [Console]::InputEncoding = $outputEncoding +} diff --git a/eng/common/scripts/New-RegenerateMatrix.ps1 b/eng/common/scripts/New-RegenerateMatrix.ps1 index 1df97420c25..9176c23e4f5 100644 --- a/eng/common/scripts/New-RegenerateMatrix.ps1 +++ b/eng/common/scripts/New-RegenerateMatrix.ps1 @@ -14,7 +14,7 @@ param ( [int]$MinimumPerJob = 10, [Parameter()] - [string]$OnlyTypespec + [string]$OnlyTypeSpec ) . (Join-Path $PSScriptRoot common.ps1) @@ -57,14 +57,13 @@ function Split-Items([array]$Items) { New-Item -ItemType Directory -Path $OutputDirectory -Force | Out-Null if (Test-Path "Function:$GetDirectoriesForGenerationFn") { - $directoriesForGeneration = &$GetDirectoriesForGenerationFn + $directoriesForGeneration = &$GetDirectoriesForGenerationFn -OnlyTypeSpec $OnlyTypespec } else { $directoriesForGeneration = Get-ChildItem "$RepoRoot/sdk" -Directory | Get-ChildItem -Directory -} - -if ($OnlyTypespec) { - $directoriesForGeneration = $directoriesForGeneration | Where-Object { Test-Path "$_/tsp-location.yaml" } + if ($OnlyTypespec) { + $directoriesForGeneration = $directoriesForGeneration | Where-Object { Test-Path "$_/tsp-location.yaml" } + } } [array]$packageDirectories = $directoriesForGeneration diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 831b4719f88..a8c38d0a012 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -16,6 +16,7 @@ $EngScriptsDir = Join-Path $EngDir "scripts" . (Join-Path $EngCommonScriptsDir artifact-metadata-parsing.ps1) . (Join-Path $EngCommonScriptsDir "Helpers" git-helpers.ps1) . (Join-Path $EngCommonScriptsDir "Helpers" Package-Helpers.ps1) +. (Join-Path $EngCommonScriptsDir "Helpers" CommandInvocation-Helpers.ps1) # Setting expected from common languages settings $Language = "Unknown" diff --git a/eng/common/scripts/typespec/New-EmitterPackageJson.ps1 b/eng/common/scripts/typespec/New-EmitterPackageJson.ps1 index d9e45038b87..24c6c50a966 100644 --- a/eng/common/scripts/typespec/New-EmitterPackageJson.ps1 +++ b/eng/common/scripts/typespec/New-EmitterPackageJson.ps1 @@ -1,3 +1,4 @@ +#Requires -Version 7.0 [CmdletBinding()] param ( [parameter(Mandatory = $true)] @@ -35,9 +36,12 @@ if ($OverridesPath) { $devDependencies = [ordered]@{} -foreach ($package in $packageJson.peerDependencies.Keys | Sort-Object) { +$possiblyPinnedPackages = $packageJson['azure-sdk/emitter-package-json-pinning'] ?? $packageJson.peerDependencies.Keys; + +foreach ($package in $possiblyPinnedPackages | Sort-Object) { $pinnedVersion = $packageJson.devDependencies[$package] if ($pinnedVersion -and -not $overrides[$package]) { + #We have a dev pinned version that isn't overridden by the overrides.json file Write-Host "Pinning $package to $pinnedVersion" $devDependencies[$package] = $pinnedVersion } From f47ba1475fa2fe2957980a88d3aaea376164f96a Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Tue, 15 Oct 2024 14:32:18 -0700 Subject: [PATCH 2/4] Rename template --- .../stages/archetype-typespec-emitter.yml | 82 +++++++++---------- 1 file changed, 39 insertions(+), 43 deletions(-) diff --git a/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml b/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml index f50b94688cf..c3e6bc3ac86 100644 --- a/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml +++ b/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml @@ -40,8 +40,8 @@ parameters: type: object default: [] -# Whether to publish to the internal feed. -- name: PublishInternal +# Whether to publish to npmjs.org. +- name: PublishPublic type: boolean default: true @@ -50,7 +50,7 @@ parameters: type: boolean default: false -# Whether to publish to the internal feed. +# Whether to regenerate sdk clients using the new emitter. - name: ShouldRegenrate type: boolean default: false @@ -84,15 +84,6 @@ stages: jobs: - job: Build steps: - # Validate parameters and fail early if invalid - - ${{ each package in parameters.Packages }}: - - ${{ if notIn(package.type, 'npm', 'nuget') }}: - - script: | - echo "Package ${{ package.name }} has unsupported type: ${{ package.type }}" - exit 1 - displayName: 'Unsupported package type' - condition: always() - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - ${{ parameters.InitializationSteps }} @@ -114,7 +105,7 @@ stages: arguments: > -BuildNumber "$(Build.BuildNumber)" -OutputDirectory "$(Build.ArtifactStagingDirectory)" - -PublishInternal:$${{ parameters.PublishInternal }} + -TargetNpmJsFeed:$${{ parameters.PublishPublic }} -Prerelease:$${{ parameters.BuildPrereleaseVersion }} - pwsh: | @@ -163,36 +154,41 @@ stages: artifact: build_artifacts displayName: Download build artifacts - # Create authenticated .npmrc file for publishing - - ${{ if eq(parameters.PublishInternal, 'true') }}: - - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml - parameters: - npmrcPath: $(buildArtifactsPath)/packages/.npmrc - registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/ - - ${{ else }}: - - pwsh: | - "//registry.npmjs.org/:_authToken=$(azure-sdk-npm-token)" | Out-File '.npmrc' - displayName: Authenticate .npmrc for npmjs.org - workingDirectory: $(buildArtifactsPath)/packages - - # per package, publishing using appropriate tool - - ${{ each package in parameters.Packages }}: - - ${{ if eq(package.type, 'npm') }}: - - pwsh: | - $file = Resolve-Path "${{ package.file }}" - Write-Host "npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages" - npm publish $file --verbose --access public --prefix $(buildArtifactsPath)/packages - displayName: Publish ${{ package.name }} - workingDirectory: $(buildArtifactsPath)/packages - - ${{ elseif eq(package.type, 'nuget') }}: - - task: NuGetCommand@2 - displayName: Publish ${{ package.name }} - inputs: - command: 'push' - packagesToPush: $(buildArtifactsPath)/packages/${{ package.file }} - # Nuget packages are always published to the same internal feed https://dev.azure.com/azure-sdk/public/_packaging?_a=feed&feed=azure-sdk-for-net - nuGetFeedType: 'internal' - publishVstsFeed: '29ec6040-b234-4e31-b139-33dc4287b756/fa8c16a3-dbe0-4de2-a297-03065ec1ba3f' + # Create authenticated .npmrc file for publishing to devops + - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml + parameters: + npmrcPath: $(buildArtifactsPath)/packages/.npmrc + registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/ + + # publish to devops feed + - pwsh: | + $packageFiles = Get-ChildItem -Path . -Filter '*.tgz' + foreach ($file in $packageFiles.Name) { + Write-Host "npm publish $file --verbose --access public" + npm publish $file --verbose --access public + } + displayName: Publish npm to Azure DevOps feed + workingDirectory: $(buildArtifactsPath)/packages + + - ${{ if parameters.PublishPublic }}: + # publish to npmjs.org using ESRP + - task: EsrpRelease@7 + inputs: + displayName: 'Publish to npmjs.org' + ConnectedServiceName: 'Azure SDK Engineering System' + ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' + KeyVaultName: 'AzureSDKEngKeyVault' + AuthCertName: 'azure-sdk-esrp-release-auth-certificate' + SignCertName: 'azure-sdk-esrp-release-sign-certificate' + Intent: 'PackageDistribution' + ContentType: 'npm' + FolderLocation: $(buildArtifactsPath)/packages + Owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + Approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + ServiceEndpointUrl: 'https://api.esrp.microsoft.com' + MainPublisher: 'ESRPRELPACMANTEST' + DomainTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' + productstate: ${{parameters.Tag}} - task: PowerShell@2 displayName: Create emitter-package.json From 14a5b406db9a014684dcc355e400de4037c41376 Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Wed, 16 Oct 2024 13:22:53 -0700 Subject: [PATCH 3/4] Convert emitter archetype to pipeline template --- .../stages/archetype-typespec-emitter.yml | 745 +++++++++--------- 1 file changed, 369 insertions(+), 376 deletions(-) diff --git a/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml b/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml index c3e6bc3ac86..99848e9eedc 100644 --- a/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml +++ b/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml @@ -1,11 +1,11 @@ parameters: -# Path to the emitter package. This is used as the base path for script invocations. -- name: EmitterPackagePath - type: string - # Pool to use for the pipeline stages - name: Pool type: object + default: + name: $(LINUXPOOL) + image: $(LINUXVMIMAGE) + os: linux # Whether to build alpha versions of the packages. This is passed as a flag to the build script. - name: BuildPrereleaseVersion @@ -32,14 +32,6 @@ parameters: type: boolean default: false -# List of packages to publish. Each package is an object with the following properties: -# name: The name of the package. This is used to determine the name of the file to publish. -# type: The type of package. Currently supported values are 'npm' and 'nuget'. -# file: The path to the file to publish. This is relative to the packages directory in the build artifacts directory. -- name: Packages - type: object - default: [] - # Whether to publish to npmjs.org. - name: PublishPublic type: boolean @@ -70,379 +62,380 @@ parameters: type: boolean default: false -stages: - -# Build stage -# Responsible for building the autorest generator and typespec emitter packages -# Produces the artifact `build_artifacts` which contains the following: -# package-versions.json: Contains a map of package name to version for the packages that were built -# overrides.json: Contains npm package version overrides for the emitter and generator -# packages/: Contains the packages to publish -# lock-files/: Contains package.json and package-lock.json files for use in the test stage to ensure tests are run against the correct dependency versions -- stage: Build - pool: ${{ parameters.Pool }} - jobs: - - job: Build - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - - - ${{ parameters.InitializationSteps }} - - - task: PowerShell@2 - displayName: 'Run initialize script' - inputs: - pwsh: true - filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Initialize.ps1 - arguments: -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} - workingDirectory: ${{ parameters.EmitterPackagePath }} - - - task: PowerShell@2 - displayName: 'Run build script' - name: ci_build - inputs: - pwsh: true - filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Build-Packages.ps1 - arguments: > - -BuildNumber "$(Build.BuildNumber)" - -OutputDirectory "$(Build.ArtifactStagingDirectory)" - -TargetNpmJsFeed:$${{ parameters.PublishPublic }} - -Prerelease:$${{ parameters.BuildPrereleaseVersion }} - - - pwsh: | - $sourceBranch = '$(Build.SourceBranch)' - $buildReason = '$(Build.Reason)' - $buildNumber = '$(Build.BuildNumber)' - - if ($buildReason -eq 'Schedule') { - $branchName = 'validate-typespec-scheduled' - } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { - $branchName = "validate-typespec-pr-$($Matches[1])" - } else { - $branchName = "validate-typespec-$buildNumber" - } - - Write-Host "Setting variable 'branchName' to '$branchName'" - Write-Host "##vso[task.setvariable variable=branchName;isOutput=true]$branchName" - displayName: Set branch name - name: set_branch_name - - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - artifactName: build_artifacts - artifactPath: $(Build.ArtifactStagingDirectory) - -# Publish stage -# Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json` -# Produces the artifact `publish_artifacts` which contains the following: -# emitter-package.json: Created using the package json from the build step. -# emitter-package-lock.json: Created by calling `npm install` using `emitter-package.json` -- ${{ if parameters.ShouldPublish }}: - - stage: Publish - dependsOn: - - Build - - ${{ if and(parameters.PublishDependsOnTest, ne(length(parameters.TestMatrix), 0)) }}: - - Test - variables: - buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts - pool: ${{ parameters.Pool }} - jobs: - - job: Publish - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - - - download: current - artifact: build_artifacts - displayName: Download build artifacts - - # Create authenticated .npmrc file for publishing to devops - - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml - parameters: - npmrcPath: $(buildArtifactsPath)/packages/.npmrc - registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/ - - # publish to devops feed - - pwsh: | - $packageFiles = Get-ChildItem -Path . -Filter '*.tgz' - foreach ($file in $packageFiles.Name) { - Write-Host "npm publish $file --verbose --access public" - npm publish $file --verbose --access public - } - displayName: Publish npm to Azure DevOps feed - workingDirectory: $(buildArtifactsPath)/packages - - - ${{ if parameters.PublishPublic }}: - # publish to npmjs.org using ESRP - - task: EsrpRelease@7 - inputs: - displayName: 'Publish to npmjs.org' - ConnectedServiceName: 'Azure SDK Engineering System' - ClientId: '5f81938c-2544-4f1f-9251-dd9de5b8a81b' - KeyVaultName: 'AzureSDKEngKeyVault' - AuthCertName: 'azure-sdk-esrp-release-auth-certificate' - SignCertName: 'azure-sdk-esrp-release-sign-certificate' - Intent: 'PackageDistribution' - ContentType: 'npm' - FolderLocation: $(buildArtifactsPath)/packages - Owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} - Approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} - ServiceEndpointUrl: 'https://api.esrp.microsoft.com' - MainPublisher: 'ESRPRELPACMANTEST' - DomainTenantId: '72f988bf-86f1-41af-91ab-2d7cd011db47' - productstate: ${{parameters.Tag}} +extends: + template: /eng/pipelines/templates/stages/1es-redirect.yml + parameters: + stages: + + # Build stage + # Responsible for building the autorest generator and typespec emitter packages + # Produces the artifact `build_artifacts` which contains the following: + # package-versions.json: Contains a map of package name to version for the packages that were built + # overrides.json: Contains npm package version overrides for the emitter and generator + # packages/: Contains the packages to publish + # lock-files/: Contains package.json and package-lock.json files for use in the test stage to ensure tests are run against the correct dependency versions + - stage: Build + pool: ${{ parameters.Pool }} + jobs: + - job: Build + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - ${{ parameters.InitializationSteps }} - task: PowerShell@2 - displayName: Create emitter-package.json + displayName: 'Run initialize script' inputs: pwsh: true - filePath: ./eng/common/scripts/typespec/New-EmitterPackageJson.ps1 - arguments: > - -PackageJsonPath '$(buildArtifactsPath)/lock-files/package.json' - -OverridesPath '$(buildArtifactsPath)/overrides.json' - -OutputDirectory '$(Build.ArtifactStagingDirectory)' - workingDirectory: $(Build.SourcesDirectory) + filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Initialize-WorkingDirectory.ps1 + arguments: -UseTypeSpecNext:$${{ parameters.UseTypeSpecNext }} - task: PowerShell@2 - displayName: Create emitter-package-lock.json + displayName: 'Run build script' + name: ci_build inputs: pwsh: true - filePath: ./eng/common/scripts/typespec/New-EmitterPackageLock.ps1 + filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Build-Emitter.ps1 arguments: > - -EmitterPackageJsonPath '$(Build.ArtifactStagingDirectory)/emitter-package.json' - -OutputDirectory '$(Build.ArtifactStagingDirectory)' - workingDirectory: $(Build.SourcesDirectory) - - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - artifactName: publish_artifacts - artifactPath: $(Build.ArtifactStagingDirectory) - -# Regenerate stage -# Responsible for regenerating the SDK code using the emitter package and the generation matrix. -- ${{ if and(parameters.ShouldPublish, parameters.ShouldRegenrate) }}: - - stage: Regenerate - dependsOn: - - Build - - Publish - variables: - pullRequestTargetBranch: 'main' - publishArtifactsPath: $(Pipeline.Workspace)/publish_artifacts - branchName: $[stageDependencies.Build.Build.outputs['set_branch_name.branchName']] - pool: ${{ parameters.Pool }} - jobs: - - job: Initialize - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - "/*" - - "!SessionRecords" - - - download: current - displayName: Download pipeline artifacts - - - pwsh: | - Write-Host "Copying emitter-package.json to $(Build.SourcesDirectory)/eng" - Copy-Item $(publishArtifactsPath)/emitter-package.json $(Build.SourcesDirectory)/eng/ -Force - - Write-Host "Copying emitter-package-lock.json to $(Build.SourcesDirectory)/eng" - Copy-Item $(publishArtifactsPath)/emitter-package-lock.json $(Build.SourcesDirectory)/eng/ -Force - displayName: Copy emitter-package json files - - - ${{ parameters.InitializationSteps }} - - - template: /eng/common/pipelines/templates/steps/git-push-changes.yml - parameters: - BaseRepoOwner: azure-sdk - TargetRepoName: $(Build.Repository.Name) - BaseRepoBranch: $(branchName) - CommitMsg: Initialize repository for autorest build $(Build.BuildNumber) - WorkingDirectory: $(Build.SourcesDirectory) - ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts - # To accomodate scheduled runs and retries, we want to overwrite any existing changes on the branch - PushArgs: --force - - - task: PowerShell@2 - displayName: Get generation job matrix - name: generate_matrix - inputs: - pwsh: true - filePath: ./eng/common/scripts/New-RegenerateMatrix.ps1 - arguments: > - -OutputDirectory "$(Build.ArtifactStagingDirectory)" - -OutputVariableName matrix - -JobCount ${{ parameters.RegenerationJobCount }} - -MinimumPerJob ${{ parameters.MinimumPerJob }} - -OnlyTypespec ${{ parameters.OnlyGenerateTypespec }} - workingDirectory: $(Build.SourcesDirectory) - - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - artifactName: matrix_artifacts - artifactPath: $(Build.ArtifactStagingDirectory) - - - job: Generate - dependsOn: Initialize - strategy: - matrix: $[dependencies.Initialize.outputs['generate_matrix.matrix']] - variables: - matrixArtifactsPath: $(Pipeline.Workspace)/matrix_artifacts - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - "/*" - - "!SessionRecords" - - - download: current - displayName: Download pipeline artifacts - - - ${{ parameters.InitializationSteps }} - - - task: PowerShell@2 - displayName: Call regeneration script - inputs: - pwsh: true - filePath: ./eng/common/scripts/Update-GeneratedSdks.ps1 - arguments: > - -PackageDirectoriesFile "$(matrixArtifactsPath)/$(DirectoryList)" - workingDirectory: $(Build.SourcesDirectory) - continueOnError: true - - - template: /eng/common/pipelines/templates/steps/git-push-changes.yml - parameters: - BaseRepoOwner: azure-sdk - TargetRepoName: $(Build.Repository.Name) - BaseRepoBranch: $(branchName) - CommitMsg: Update SDK code $(JobKey) - WorkingDirectory: $(Build.SourcesDirectory) - ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts - - - job: Create_PR - displayName: Create PR - dependsOn: - - Generate - variables: - generateJobResult: $[dependencies.Generate.result] - emitterVersion: $[stageDependencies.Build.Build.outputs['ci_build.emitterVersion']] - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - - - pwsh: | - $generateJobResult = '$(generateJobResult)' - $emitterVersion = '$(emitterVersion)' - $collectionUri = '$(System.CollectionUri)' - $project = '$(System.TeamProject)' - $definitionName = '$(Build.DefinitionName)' - $repoUrl = '$(Build.Repository.Uri)' - $repoName = '$(Build.Repository.Name)' - $sourceBranch = '$(Build.SourceBranch)' - $reason = '$(Build.Reason)' - $buildId = '$(Build.BuildId)' - $buildNumber = '$(Build.BuildNumber)' - $preRelease = '${{ parameters.BuildPrereleaseVersion }}' -eq 'true' - - $prBody = "Generated by $definitionName build [$buildNumber]($collectionUri/$project/_build/results?buildId=$buildId)
" - - if ($sourceBranch -match "^refs/heads/(.+)$") { - $prBody += "Triggered from branch: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" - } elseif ($sourceBranch -match "^refs/tags/(.+)$") { - $prBody += "Triggered from tag: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" - } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { - $prBody += "Triggered from pull request: $repoUrl/pull/$($Matches[1])" - } else { - $prBody += "Triggered from [$sourceBranch]($repoUrl/tree/$sourceBranch)" - } - - if ($reason -eq 'Schedule') { - $prTitle = "Scheduled code regeneration test" - } else { - if ($preRelease) { - $prTitle = "Update typespec emitter version to prerelease $emitterVersion" + -BuildNumber "$(Build.BuildNumber)" + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + -TargetNpmJsFeed:$${{ parameters.PublishPublic }} + -Prerelease:$${{ parameters.BuildPrereleaseVersion }} + + - pwsh: | + $sourceBranch = '$(Build.SourceBranch)' + $buildReason = '$(Build.Reason)' + $buildNumber = '$(Build.BuildNumber)' + + if ($buildReason -eq 'Schedule') { + $branchName = 'validate-typespec-scheduled' + } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { + $branchName = "validate-typespec-pr-$($Matches[1])" } else { - $prTitle = "Update typespec emitter version to $emitterVersion" + $branchName = "validate-typespec-$buildNumber" } - if ($generateJobResult -ne 'Succeeded') { - $prTitle = "Failed: $prTitle" - } - } - - Write-Host "Setting variable 'PullRequestTitle' to '$prTitle'" - Write-Host "##vso[task.setvariable variable=PullRequestTitle]$prTitle" - - Write-Host "Setting variable 'PullRequestBody' to '$prBody'" - Write-Host "##vso[task.setvariable variable=PullRequestBody]$prBody" - - $repoNameParts = $repoName.Split('/') - if ($repoNameParts.Length -eq 2) { - Write-Host "Setting variable 'RepoOwner' to '$($repoNameParts[0])'" - Write-Host "##vso[task.setvariable variable=RepoOwner]$($repoNameParts[0])" - - Write-Host "Setting variable 'RepoName' to '$($repoNameParts[1])'" - Write-Host "##vso[task.setvariable variable=RepoName]$($repoNameParts[1])" - } else { - Write-Error "Build.Repository.Name not in the expected {Owner}/{Name} format" - } - displayName: Get PR title and body - - - task: PowerShell@2 - displayName: Create pull request - inputs: - pwsh: true - filePath: ./eng/common/scripts/Submit-PullRequest.ps1 - arguments: > - -RepoOwner '$(RepoOwner)' - -RepoName '$(RepoName)' - -BaseBranch '$(pullRequestTargetBranch)' - -PROwner 'azure-sdk' - -PRBranch '$(branchName)' - -AuthToken '$(azuresdk-github-pat)' - -PRTitle '$(PullRequestTitle)' - -PRBody '$(PullRequestBody)' - -OpenAsDraft $true - -PRLabels 'Do Not Merge' - workingDirectory: $(Build.SourcesDirectory) - -# Test stage -# Responsible for running unit and functional tests using a matrix passed in as the parameter `TestMatrix`. -# Will only run if the parameter `TestMatrix` is not empty. -# The contents of the artficact `build_artifacts` are available under the path `$(buildArtifactsPath)`. -- ${{ if ne(length(parameters.TestMatrix), 0) }}: - - stage: Test - dependsOn: Build - pool: ${{ parameters.Pool }} - jobs: - - job: Test - strategy: - matrix: ${{ parameters.TestMatrix }} - variables: - buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - - - download: current - artifact: build_artifacts - displayName: Download build artifacts - - - ${{ parameters.InitializationSteps }} - - - task: PowerShell@2 - displayName: 'Run initialize script' - inputs: - pwsh: true - filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Initialize.ps1 - arguments: -BuildArtifactsPath '$(buildArtifactsPath)' - - - task: PowerShell@2 - displayName: 'Run test script' - inputs: - pwsh: true - filePath: ${{ parameters.EmitterPackagePath }}/eng/scripts/Test-Packages.ps1 - arguments: > - $(TestArguments) - -OutputDirectory "$(Build.ArtifactStagingDirectory)" - - - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml - parameters: - artifactName: test_artifacts_$(System.JobName) - artifactPath: $(Build.ArtifactStagingDirectory) + Write-Host "Setting variable 'branchName' to '$branchName'" + Write-Host "##vso[task.setvariable variable=branchName;isOutput=true]$branchName" + displayName: Set branch name + name: set_branch_name + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: build_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + + # Publish stage + # Responsible for publishing the packages in `build_artifacts/packages` and producing `emitter-package-lock.json` + # Produces the artifact `publish_artifacts` which contains the following: + # emitter-package.json: Created using the package json from the build step. + # emitter-package-lock.json: Created by calling `npm install` using `emitter-package.json` + - ${{ if parameters.ShouldPublish }}: + - stage: Publish + dependsOn: + - Build + - ${{ if and(parameters.PublishDependsOnTest, ne(length(parameters.TestMatrix), 0)) }}: + - Test + variables: + buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts + pool: ${{ parameters.Pool }} + jobs: + - job: Publish + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - download: current + artifact: build_artifacts + displayName: Download build artifacts + + # Create authenticated .npmrc file for publishing to devops + - template: /eng/common/pipelines/templates/steps/create-authenticated-npmrc.yml + parameters: + npmrcPath: $(buildArtifactsPath)/packages/.npmrc + registryUrl: https://pkgs.dev.azure.com/azure-sdk/public/_packaging/azure-sdk-for-js-test-autorest/npm/registry/ + + # publish to devops feed + - pwsh: | + $packageFiles = Get-ChildItem -Path . -Filter '*.tgz' + foreach ($file in $packageFiles.Name) { + Write-Host "npm publish $file --verbose --access public" + npm publish $file --verbose --access public + } + displayName: Publish to DevOps feed + workingDirectory: $(buildArtifactsPath)/packages + + - ${{ if parameters.PublishPublic }}: + # publish to npmjs.org using ESRP + - task: EsrpRelease@7 + inputs: + displayName: Publish to npmjs.org + ConnectedServiceName: Azure SDK Engineering System + ClientId: 5f81938c-2544-4f1f-9251-dd9de5b8a81b + KeyVaultName: AzureSDKEngKeyVault + AuthCertName: azure-sdk-esrp-release-auth-certificate + SignCertName: azure-sdk-esrp-release-sign-certificate + Intent: PackageDistribution + ContentType: npm + FolderLocation: $(buildArtifactsPath)/packages + Owners: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + Approvers: ${{ coalesce(variables['Build.RequestedForEmail'], 'azuresdk@microsoft.com') }} + ServiceEndpointUrl: https://api.esrp.microsoft.com + MainPublisher: ESRPRELPACMANTEST + DomainTenantId: 72f988bf-86f1-41af-91ab-2d7cd011db47 + + - task: PowerShell@2 + displayName: Create emitter-package.json + inputs: + pwsh: true + filePath: ./eng/common/scripts/typespec/New-EmitterPackageJson.ps1 + arguments: > + -PackageJsonPath '$(buildArtifactsPath)/lock-files/package.json' + -OverridesPath '$(buildArtifactsPath)/overrides.json' + -OutputDirectory '$(Build.ArtifactStagingDirectory)' + workingDirectory: $(Build.SourcesDirectory) + + - task: PowerShell@2 + displayName: Create emitter-package-lock.json + inputs: + pwsh: true + filePath: ./eng/common/scripts/typespec/New-EmitterPackageLock.ps1 + arguments: > + -EmitterPackageJsonPath '$(Build.ArtifactStagingDirectory)/emitter-package.json' + -OutputDirectory '$(Build.ArtifactStagingDirectory)' + workingDirectory: $(Build.SourcesDirectory) + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: publish_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + + # Regenerate stage + # Responsible for regenerating the SDK code using the emitter package and the generation matrix. + - ${{ if and(parameters.ShouldPublish, parameters.ShouldRegenrate) }}: + - stage: Regenerate + dependsOn: + - Build + - Publish + variables: + pullRequestTargetBranch: 'main' + publishArtifactsPath: $(Pipeline.Workspace)/publish_artifacts + branchName: $[stageDependencies.Build.Build.outputs['set_branch_name.branchName']] + pool: ${{ parameters.Pool }} + jobs: + - job: Initialize + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - "/*" + - "!SessionRecords" + + - download: current + displayName: Download pipeline artifacts + + - pwsh: | + Write-Host "Copying emitter-package.json to $(Build.SourcesDirectory)/eng" + Copy-Item $(publishArtifactsPath)/emitter-package.json $(Build.SourcesDirectory)/eng/ -Force + + Write-Host "Copying emitter-package-lock.json to $(Build.SourcesDirectory)/eng" + Copy-Item $(publishArtifactsPath)/emitter-package-lock.json $(Build.SourcesDirectory)/eng/ -Force + displayName: Copy emitter-package json files + + - ${{ parameters.InitializationSteps }} + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoOwner: azure-sdk + TargetRepoName: $(Build.Repository.Name) + BaseRepoBranch: $(branchName) + CommitMsg: Initialize repository for autorest build $(Build.BuildNumber) + WorkingDirectory: $(Build.SourcesDirectory) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + # To accomodate scheduled runs and retries, we want to overwrite any existing changes on the branch + PushArgs: --force + + - task: PowerShell@2 + displayName: Get generation job matrix + name: generate_matrix + inputs: + pwsh: true + filePath: ./eng/common/scripts/New-RegenerateMatrix.ps1 + arguments: > + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + -OutputVariableName matrix + -JobCount ${{ parameters.RegenerationJobCount }} + -MinimumPerJob ${{ parameters.MinimumPerJob }} + -OnlyTypespec ${{ parameters.OnlyGenerateTypespec }} + workingDirectory: $(Build.SourcesDirectory) + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: matrix_artifacts + artifactPath: $(Build.ArtifactStagingDirectory) + + - job: Generate + dependsOn: Initialize + strategy: + matrix: $[dependencies.Initialize.outputs['generate_matrix.matrix']] + variables: + matrixArtifactsPath: $(Pipeline.Workspace)/matrix_artifacts + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - "/*" + - "!SessionRecords" + + - download: current + displayName: Download pipeline artifacts + + - ${{ parameters.InitializationSteps }} + + - task: PowerShell@2 + displayName: Call regeneration script + inputs: + pwsh: true + filePath: ./eng/common/scripts/Update-GeneratedSdks.ps1 + arguments: > + -PackageDirectoriesFile "$(matrixArtifactsPath)/$(DirectoryList)" + workingDirectory: $(Build.SourcesDirectory) + continueOnError: true + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoOwner: azure-sdk + TargetRepoName: $(Build.Repository.Name) + BaseRepoBranch: $(branchName) + CommitMsg: Update SDK code $(JobKey) + WorkingDirectory: $(Build.SourcesDirectory) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + + - job: Create_PR + displayName: Create PR + dependsOn: + - Generate + variables: + generateJobResult: $[dependencies.Generate.result] + emitterVersion: $[stageDependencies.Build.Build.outputs['ci_build.emitterVersion']] + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - pwsh: | + $generateJobResult = '$(generateJobResult)' + $emitterVersion = '$(emitterVersion)' + $collectionUri = '$(System.CollectionUri)' + $project = '$(System.TeamProject)' + $definitionName = '$(Build.DefinitionName)' + $repoUrl = '$(Build.Repository.Uri)' + $repoName = '$(Build.Repository.Name)' + $sourceBranch = '$(Build.SourceBranch)' + $reason = '$(Build.Reason)' + $buildId = '$(Build.BuildId)' + $buildNumber = '$(Build.BuildNumber)' + $preRelease = '${{ parameters.BuildPrereleaseVersion }}' -eq 'true' + + $prBody = "Generated by $definitionName build [$buildNumber]($collectionUri/$project/_build/results?buildId=$buildId)
" + + if ($sourceBranch -match "^refs/heads/(.+)$") { + $prBody += "Triggered from branch: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" + } elseif ($sourceBranch -match "^refs/tags/(.+)$") { + $prBody += "Triggered from tag: [$($Matches[1])]($repoUrl/tree/$sourceBranch)" + } elseif ($sourceBranch -match "^refs/pull/(\d+)/(head|merge)$") { + $prBody += "Triggered from pull request: $repoUrl/pull/$($Matches[1])" + } else { + $prBody += "Triggered from [$sourceBranch]($repoUrl/tree/$sourceBranch)" + } + + if ($reason -eq 'Schedule') { + $prTitle = "Scheduled code regeneration test" + } else { + if ($preRelease) { + $prTitle = "Update typespec emitter version to prerelease $emitterVersion" + } else { + $prTitle = "Update typespec emitter version to $emitterVersion" + } + + if ($generateJobResult -ne 'Succeeded') { + $prTitle = "Failed: $prTitle" + } + } + + Write-Host "Setting variable 'PullRequestTitle' to '$prTitle'" + Write-Host "##vso[task.setvariable variable=PullRequestTitle]$prTitle" + + Write-Host "Setting variable 'PullRequestBody' to '$prBody'" + Write-Host "##vso[task.setvariable variable=PullRequestBody]$prBody" + + $repoNameParts = $repoName.Split('/') + if ($repoNameParts.Length -eq 2) { + Write-Host "Setting variable 'RepoOwner' to '$($repoNameParts[0])'" + Write-Host "##vso[task.setvariable variable=RepoOwner]$($repoNameParts[0])" + + Write-Host "Setting variable 'RepoName' to '$($repoNameParts[1])'" + Write-Host "##vso[task.setvariable variable=RepoName]$($repoNameParts[1])" + } else { + Write-Error "Build.Repository.Name not in the expected {Owner}/{Name} format" + } + displayName: Get PR title and body + + - task: PowerShell@2 + displayName: Create pull request + inputs: + pwsh: true + filePath: ./eng/common/scripts/Submit-PullRequest.ps1 + arguments: > + -RepoOwner '$(RepoOwner)' + -RepoName '$(RepoName)' + -BaseBranch '$(pullRequestTargetBranch)' + -PROwner 'azure-sdk' + -PRBranch '$(branchName)' + -AuthToken '$(azuresdk-github-pat)' + -PRTitle '$(PullRequestTitle)' + -PRBody '$(PullRequestBody)' + -OpenAsDraft $true + -PRLabels 'Do Not Merge' + workingDirectory: $(Build.SourcesDirectory) + + # Test stage + # Responsible for running unit and functional tests using a matrix passed in as the parameter `TestMatrix`. + # Will only run if the parameter `TestMatrix` is not empty. + # The contents of the artficact `build_artifacts` are available under the path `$(buildArtifactsPath)`. + - ${{ if ne(length(parameters.TestMatrix), 0) }}: + - stage: Test + dependsOn: Build + pool: ${{ parameters.Pool }} + jobs: + - job: Test + strategy: + matrix: ${{ parameters.TestMatrix }} + variables: + buildArtifactsPath: $(Pipeline.Workspace)/build_artifacts + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + + - download: current + artifact: build_artifacts + displayName: Download build artifacts + + - ${{ parameters.InitializationSteps }} + + - task: PowerShell@2 + displayName: 'Run initialize script' + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Initialize-WorkingDirectory.ps1 + arguments: -BuildArtifactsPath '$(buildArtifactsPath)' + + - task: PowerShell@2 + displayName: 'Run test script' + inputs: + pwsh: true + filePath: $(Build.SourcesDirectory)/eng/scripts/typespec/Test-Emitter.ps1 + arguments: > + $(TestArguments) + -OutputDirectory "$(Build.ArtifactStagingDirectory)" + + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + artifactName: test_artifacts_$(System.JobName) + artifactPath: $(Build.ArtifactStagingDirectory) From 085a69c6561d3f3b7861db3a4e0a6cdbc5a16e98 Mon Sep 17 00:00:00 2001 From: Patrick Hallisey Date: Wed, 16 Oct 2024 13:26:46 -0700 Subject: [PATCH 4/4] Move the archetype template out of the stages folder --- .../templates/{stages => }/archetype-typespec-emitter.yml | 1 - 1 file changed, 1 deletion(-) rename eng/common/pipelines/templates/{stages => }/archetype-typespec-emitter.yml (99%) diff --git a/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml b/eng/common/pipelines/templates/archetype-typespec-emitter.yml similarity index 99% rename from eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml rename to eng/common/pipelines/templates/archetype-typespec-emitter.yml index 99848e9eedc..71d16c908fc 100644 --- a/eng/common/pipelines/templates/stages/archetype-typespec-emitter.yml +++ b/eng/common/pipelines/templates/archetype-typespec-emitter.yml @@ -66,7 +66,6 @@ extends: template: /eng/pipelines/templates/stages/1es-redirect.yml parameters: stages: - # Build stage # Responsible for building the autorest generator and typespec emitter packages # Produces the artifact `build_artifacts` which contains the following: