base-job.yml

parameters:
  buildConfig: ''
  osGroup: ''
  archType: ''
  osSubgroup: ''
  platform: ''
  crossBuild: false
  crossrootfsDir: ''
  timeoutInMinutes: 120
  condition: true
  shouldContinueOnError: false
  container: ''
  buildSteps: []
  dependsOn: []
  dependsOnGlobalBuild: false
  dependOnEvaluatePaths: false
  globalBuildSuffix: ''
  variables: []
  name: ''
  displayName: ''
  runtimeVariant: ''
  pool: ''
  pgoType: ''

  packageDistroList:
  - image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-debpkg-20220504035737-cfdd435
    packageType: deb
    packagingArgs: /p:BuildDebPackage=true
  - image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-8-rpmpkg-20210714125410-daa5116
    packageType: rpm
    packagingArgs: /p:BuildRpmPackage=true

  isOfficialBuild: false
  buildFullPlatformManifest: false

  liveRuntimeBuildConfig: ''
  liveLibrariesBuildConfig: ''
  runtimeFlavor: 'coreclr'
  platforms: []

jobs:
- job: ${{ format('installer_{0}_{1}_{2}_{3}_{4}_', parameters.pgoType, parameters.runtimeFlavor, parameters.runtimeVariant, coalesce(parameters.name, parameters.platform), parameters.buildConfig) }}
  displayName: ${{ format('{0} Installer Build and Test {1} {2} {3} {4}', parameters.pgoType, parameters.runtimeFlavor, parameters.runtimeVariant, coalesce(parameters.name, parameters.platform), parameters.buildConfig) }}

  condition: and(succeeded(), ${{ parameters.condition }})
  pool: ${{ parameters.pool }}
  timeoutInMinutes: ${{ parameters.timeoutInMinutes }}

  # Do not attempt to clean workspace on Linux: the agent might not be able to remove the files
  # because they may be owned by "root" due to the way this job uses Docker. We do our own cleanup
  # in this case as a prepare step.
  ${{ if ne(parameters.osGroup, 'Linux') }}:
    workspace:
      clean: all

  variables:
  - ${{ each variable in parameters.variables }}:
    - ${{ variable }}

  - name: OfficialBuildArg
    value: ''

  - name: SkipTests
    value: ${{ or(
      not(in(parameters.archType, 'x64', 'x86')),
      eq(parameters.runtimeFlavor, 'mono'),
      eq(parameters.isOfficialBuild, true),
      eq(parameters.crossBuild, true),
      eq(parameters.pgoType, 'PGO')) }}

  - name: BuildAction
    value: -test

  - ${{ if eq(variables.SkipTests, true) }}:
    - name: BuildAction
      value: ''

  - name: SignType
    value: test

  - name: pgoInstrumentArg
    value: ''
  - ${{ if eq(parameters.pgoType, 'PGO' )}}:
    - name: pgoInstrumentArg
      value: '-pgoinstrument '

  # Set up non-PR build from internal project
  - ${{ if eq(parameters.isOfficialBuild, true) }}:
    - name: SignType
      value: $[ coalesce(variables.OfficialSignType, 'real') ]
    - name: OfficialBuildArg
      value: /p:OfficialBuildId=$(Build.BuildNumber)

  - name: buildCommandSourcesDirectory
    ${{ if not(in(parameters.osGroup, 'Linux', 'FreeBSD')) }}:
      value: '$(Build.SourcesDirectory)/'
    # This job runs within Docker containers, so Build.SourcesDirectory is not accurate.
    ${{ if in(parameters.osGroup, 'Linux', 'FreeBSD') }}:
      value: '/root/runtime/'

  - ${{ if eq(variables['System.TeamProject'], 'internal') }}:
    - group: AzureDevOps-Artifact-Feeds-Pats

  ###
  ### Platform-specific variable setup
  ###

  - ${{ if eq(parameters.osGroup, 'windows') }}:

    - name: CommonMSBuildArgs
      value: >-
        /p:TargetArchitecture=${{ parameters.archType }}
        /p:PortableBuild=true
        /p:SkipTests=$(SkipTests)
        /p:RuntimeFlavor=${{ parameters.runtimeFlavor }}
        $(OfficialBuildArg)
    - name: MsbuildSigningArguments
      value: >-
        /p:CertificateId=400
        /p:DotNetSignType=$(SignType)
    - name: TargetArchitecture
      value: ${{ parameters.archType }}

    - name: BaseJobBuildCommand
      value: >-
        build.cmd -subset host+packs -ci
        $(BuildAction)
        -configuration $(_BuildConfig)
        $(pgoInstrumentArg)
        $(LiveOverridePathArgs)
        $(CommonMSBuildArgs)
        $(MsbuildSigningArguments)

  - ${{ if eq(parameters.osGroup, 'OSX') }}:

    - name: CommonMSBuildArgs
      value: >-
        /p:PortableBuild=true
        /p:SkipTests=$(SkipTests)
        /p:RuntimeFlavor=${{ parameters.runtimeFlavor }}
        /p:TargetArchitecture=${{ parameters.archType }}
        /p:CrossBuild=${{ parameters.crossBuild }}

    - name: BaseJobBuildCommand
      value: >-
        $(Build.SourcesDirectory)/build.sh -ci
        $(BuildAction)
        -configuration $(_BuildConfig)
        -arch ${{ parameters.archType }}
        $(LiveOverridePathArgs)
        $(CommonMSBuildArgs)
        $(OfficialBuildArg)

  - ${{ if in(parameters.osGroup, 'iOS', 'tvOS', 'Android', 'Browser') }}:

    - name: CommonMSBuildArgs
      value: >-
        /p:PortableBuild=true
        /p:SkipTests=$(SkipTests)

    - name: BaseJobBuildCommand
      value: >-
        $(Build.SourcesDirectory)/build.sh -subset packs -ci
        $(BuildAction)
        -configuration $(_BuildConfig)
        -os ${{ parameters.osGroup }}
        -arch ${{ parameters.archType }}
        /p:StripSymbols=true
        $(LiveOverridePathArgs)
        $(CommonMSBuildArgs)
        $(OfficialBuildArg)

  - ${{ if in(parameters.osGroup, 'Linux', 'FreeBSD') }}:

    # Preserve the NuGet authentication env vars into the Docker container.
    # The 'NuGetAuthenticate' build step may have set these.
    - name: PreserveNuGetAuthDockerArgs
      value: >-
        -e VSS_NUGET_URI_PREFIXES
        -e VSS_NUGET_ACCESSTOKEN

    - ${{ if ne(parameters.container, '') }}:
      - name: RunArguments
        value: >-
          docker run --privileged --rm
          -v "$(Build.SourcesDirectory):/root/runtime"
          -w="/root/runtime"
          $(PreserveNuGetAuthDockerArgs)
          -e ROOTFS_DIR=${{ parameters.crossrootfsDir }}
          ${{ parameters.container }}

    - name: BuildScript
      value: ./build.sh
    - name: MSBuildScript
      value: /root/runtime/eng/common/msbuild.sh

    - ${{ if eq(parameters.isOfficialBuild, true) }}:
      - name: BuildScript
        value: ./eng/install-nuget-credprovider-then-build.sh --subset host+packs
      - name: MSBuildScript
        value: /root/runtime/eng/install-nuget-credprovider-then-msbuild.sh

    - name: CommonMSBuildArgs
      value: >-
        /p:Configuration=$(_BuildConfig)
        /p:TargetOS=${{ parameters.osGroup }}
        /p:TargetArchitecture=${{ parameters.archType }}
        /p:RuntimeFlavor=${{ parameters.runtimeFlavor }}
        $(OfficialBuildArg)

    - name: _PortableBuild
      value: ${{ eq(parameters.osSubgroup, '') }}

    - ${{ if and(eq(parameters.osSubgroup, '_musl'), eq(parameters.osGroup, 'Linux')) }}:
      # Set output RID manually: musl isn't properly detected. Make sure to also convert linux to
      # lowercase for RID format. (Detection normally converts, but we're preventing it.)
      - name: OutputRidArg
        value: /p:OutputRid=linux-musl-${{ parameters.archType }}
      - name: RuntimeOSArg
        value: /p:RuntimeOS=linux-musl
      - name: _PortableBuild
        value: true

    - name: BuildArguments
      value: >-
        -subset host+packs -ci
        $(BuildAction)
        /p:CrossBuild=${{ parameters.crossBuild }}
        /p:PortableBuild=$(_PortableBuild)
        /p:SkipTests=$(SkipTests)
        $(pgoInstrumentArg)
        $(LiveOverridePathArgs)
        $(CommonMSBuildArgs)
        $(OutputRidArg)
        $(RuntimeOSArg)

    - name: PublishArguments
      value: >-
        /p:PortableBuild=$(_PortableBuild)
        $(CommonMSBuildArgs)
        $(OutputRidArg)
        /bl:msbuild.publish.binlog

    - name: DockerRunMSBuild
      value: >-
        docker run
        -v $(Build.SourcesDirectory):/root/runtime
        -w=/root/runtime
        $(PreserveNuGetAuthDockerArgs)

    - name: installersSubsetArg
      value: --subset packs.installers

    - name: BaseJobBuildCommand
      value: |
        set -x
        df -h
        docker info
        $(RunArguments) $(BuildScript) $(BuildArguments)

  ###
  ### Common Live build override variable setup
  ###

  - name: LiveOverridePathArgs
    value: >-
      $(RuntimeArtifactsArgs)
      $(LibrariesConfigurationArg)

  - name: RuntimeArtifactsArgs
    value: ''
  - name: LibrariesConfigurationArg
    value: ''

  - name: RuntimeDownloadPath
    value: ''
  - name: LibrariesDownloadPath
    value: ''

  - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
    - name: liveRuntimeLegName
      value: ${{ format('{0}{1}_{2}_{3}',
        parameters.osGroup,
        parameters.osSubgroup,
        parameters.archType,
        parameters.liveRuntimeBuildConfig) }}
    - name: RuntimeDownloadPath
      value: 'artifacts/transport/${{ parameters.runtimeFlavor }}'
    - name: RuntimeArtifactsArgs
      value: >-
        /p:RuntimeArtifactsPath=$(buildCommandSourcesDirectory)$(RuntimeDownloadPath)
        /p:RuntimeConfiguration=${{ parameters.liveRuntimeBuildConfig }}
    - name: RuntimeArtifactName
      value: $(runtimeFlavorName)Product_${{ parameters.pgoType }}_${{ parameters.runtimeVariant }}_$(liveRuntimeLegName)

  - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
    - name: liveLibrariesLegName
      value: ${{ format('{0}{1}_{2}_{3}',
        parameters.osGroup,
        parameters.osSubgroup,
        parameters.archType,
        parameters.liveLibrariesBuildConfig) }}
    - name: LibrariesDownloadPath
      value: 'artifacts'
    - name: LibrariesArtifactName
      value: libraries_bin_$(liveLibrariesLegName)
    - name: LibrariesConfigurationArg
      value: ' /p:LibrariesConfiguration=${{ parameters.liveLibrariesBuildConfig }}'

  dependsOn:
  - ${{ if eq(parameters.dependOnEvaluatePaths, true) }}:
    - evaluate_paths
  - ${{ parameters.dependsOn }}
  - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
    - ${{ format('{0}_{1}_product_build_{2}{3}_{4}_{5}{6}',
        parameters.runtimeFlavor,
        parameters.runtimeVariant,
        parameters.osGroup,
        parameters.osSubgroup,
        parameters.archType,
        parameters.liveRuntimeBuildConfig,
        parameters.pgoType) }}
  - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
    - libraries_build_${{ format('{0}{1}_{2}_{3}',
        parameters.osGroup,
        parameters.osSubgroup,
        parameters.archType,
        parameters.liveLibrariesBuildConfig) }}

  steps:

  - ${{ if ne(parameters.container, '') }}:
    # Builds don't set user ID, so files might be owned by root and unable to be cleaned up by AzDO.
    # Clean up the build dirs ourselves in another Docker container to avoid failures.
    # Using hosted agents is tracked by https://github.com/dotnet/runtime/issues/3416
    - script: |
        set -x
        docker run --rm \
          -v "$(Agent.BuildDirectory):/root/build" \
          -w /root/build \
          ${{ parameters.container }} \
          bash -c '
            rm -v -rf a b s'
        mkdir "$(Agent.BuildDirectory)/s"
      displayName: Clean up old artifacts owned by root

  - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
    - task: NuGetAuthenticate@1

    - ${{ if eq(parameters.osGroup, 'windows') }}:
      # NuGet's http cache lasts 30 minutes. If we're on a static machine, this may interfere with
      # auto-update PRs by preventing the CI build from fetching the new version. Delete the cache.
      - powershell: Remove-Item -Recurse -ErrorAction Ignore "$env:LocalAppData\NuGet\v3-cache"
        displayName: Clear NuGet http cache (if exists)

      - task: MicroBuildSigningPlugin@2
        displayName: Install MicroBuild plugin for Signing
        inputs:
          signType: $(SignType)
          zipSources: false
          feedSource: https://dnceng.pkgs.visualstudio.com/_packaging/MicroBuildToolset/nuget/v3/index.json
        continueOnError: false
        condition: and(succeeded(), in(variables['SignType'], 'real', 'test'), eq(${{ parameters.isOfficialBuild }}, true))

  - checkout: self
    clean: true
    fetchDepth: $(checkoutFetchDepth)

  - ${{ if ne(parameters.liveRuntimeBuildConfig, '') }}:
    - template: /eng/pipelines/common/download-artifact-step.yml
      parameters:
        unpackFolder: $(Build.SourcesDirectory)/$(RuntimeDownloadPath)
        artifactFileName: '$(RuntimeArtifactName)$(archiveExtension)'
        artifactName: '$(RuntimeArtifactName)'
        displayName: '$(runtimeFlavorName) artifacts'

  - ${{ if ne(parameters.liveLibrariesBuildConfig, '') }}:
    - template: /eng/pipelines/common/download-artifact-step.yml
      parameters:
        unpackFolder: $(Build.SourcesDirectory)/$(LibrariesDownloadPath)
        artifactFileName: '$(LibrariesArtifactName)$(archiveExtension)'
        artifactName: '$(LibrariesArtifactName)'
        displayName: 'Libraries artifacts'
        cleanUnpackFolder: false

  - ${{ if in(parameters.osGroup, 'OSX', 'MacCatalyst', 'iOS', 'iOSSimulator', 'tvOS', 'tvOSSimulator') }}:
    - script: $(Build.SourcesDirectory)/eng/install-native-dependencies.sh ${{ parameters.osGroup }} ${{ parameters.archType }} azDO
      displayName: Install Build Dependencies

    - script: |
        du -sh $(Build.SourcesDirectory)/*
        df -h
      displayName: Disk Usage before Build

  - ${{ if ne(variables['System.TeamProject'], 'public') }}:
    - ${{ if ne(parameters.osGroup, 'windows') }}:
      - task: Bash@3
        displayName: Setup Private Feeds Credentials
        inputs:
          filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
          arguments: $(Build.SourcesDirectory)/NuGet.config $Token
        env:
          Token: $(dn-bot-dnceng-artifact-feeds-rw)
    - ${{ if eq(parameters.osGroup, 'windows') }}:
      - task: PowerShell@2
        displayName: Setup Private Feeds Credentials
        inputs:
          filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.ps1
          arguments: -ConfigFile $(Build.SourcesDirectory)/NuGet.config -Password $Env:Token
        env:
          Token: $(dn-bot-dnceng-artifact-feeds-rw)

  # Build the default subset non-MacOS platforms
  - ${{ if ne(parameters.osGroup, 'OSX') }}:
    - script: $(BaseJobBuildCommand)
      displayName: Build
      continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }}

  # Build corehost, sign and add entitlements to MacOS binaries
  - ${{ if eq(parameters.osGroup, 'OSX') }}:
    - script: $(BaseJobBuildCommand) -subset host.native
      displayName: Build CoreHost
      continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }}

    - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
      - template: /eng/pipelines/common/macos-sign-with-entitlements.yml
        parameters:
          filesToSign:
          - name: dotnet
            path: $(Build.SourcesDirectory)/artifacts/bin/osx-${{ parameters.archType }}.$(_BuildConfig)/corehost
            entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist
          - name: apphost
            path: $(Build.SourcesDirectory)/artifacts/bin/osx-${{ parameters.archType }}.$(_BuildConfig)/corehost
            entitlementsFile: $(Build.SourcesDirectory)/eng/pipelines/common/entitlements.plist

    - script: $(BaseJobBuildCommand) -subset host.pkg+host.tools+host.tests+packs
      displayName: Build and Package
      continueOnError: ${{ and(eq(variables.SkipTests, false), eq(parameters.shouldContinueOnError, true)) }}

  - ${{ if in(parameters.osGroup, 'OSX', 'iOS', 'tvOS') }}:
    - script: |
        du -sh $(Build.SourcesDirectory)/*
        df -h
      displayName: Disk Usage after Build

  # Only in glibc leg, we produce RPMs and Debs
  - ${{ if and(eq(parameters.runtimeFlavor, 'coreclr'), or(eq(parameters.platform, 'Linux_x64'), eq(parameters.platform, 'Linux_arm64')), eq(parameters.osSubgroup, ''), eq(parameters.pgoType, ''))}}:
    - ${{ each packageBuild in parameters.packageDistroList }}:
      # This leg's RID matches the build image. Build its distro-dependent packages, as well as
      # the distro-independent installers. (There's no particular reason to build the distro-
      # independent installers on this leg, but we need to do it somewhere.)
      # Currently, Linux_arm64 supports 'rpm' type only.
      - ${{ if or(not(eq(parameters.platform, 'Linux_arm64')), eq(packageBuild.packageType, 'rpm')) }}:
        - template: steps/build-linux-package.yml
          parameters:
            packageType: ${{ packageBuild.packageType }}
            image: ${{ packageBuild.image }}
            packageStepDescription: Runtime Deps, Runtime, Framework Packs installers
            subsetArg: $(installersSubsetArg)
            packagingArgs: ${{ packageBuild.packagingArgs }}

  - ${{ if ne(parameters.container, '') }}:
    # Files may be owned by root because builds don't set user ID. Later build steps run 'find' in
    # the source tree, which fails due to permissions in the 'NetCore*-Int-Pool' queues. This step
    # prevents the failure by using chown to clean up our source tree.
    - script: |
        set -x
        docker run --rm \
          -v "$(Agent.BuildDirectory):/root/build" \
          -w /root/build \
          ${{ parameters.container }} \
          bash -c "chown -R $(id -u):$(id -g) *"
      displayName: Update file ownership from root to build agent account
      continueOnError: true
      condition: succeededOrFailed()

  - template: steps/upload-job-artifacts.yml
    parameters:
      name: ${{ coalesce(parameters.name, parameters.platform) }}
      runtimeFlavor: ${{ parameters.runtimeFlavor }}
      runtimeVariant: ${{ parameters.runtimeVariant }}
      skipTests: ${{ variables.SkipTests }}
      isOfficialBuild: ${{ eq(parameters.isOfficialBuild, true) }}
      pgoType: ${{ parameters.pgoType }}

  - ${{ if ne(parameters.osGroup, 'windows') }}:
    - script: set -x && df -h
      displayName: Check remaining storage space
      condition: always()
      continueOnError: true

    # Force clean up machine in case any docker images are left behind
    - ${{ if ne(parameters.container, '') }}:
      - script: docker system prune -af && df -h
        displayName: Run Docker clean up
        condition: succeededOrFailed()