[main] Update dependencies from dotnet/arcade

eng/Version.Details.xml

@@ -41,9 +41,9 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.23567.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="9.0.0-beta.23601.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>b6801170e6ca8bf12bdff1f10d410c5e01660681</Sha>
+      <Sha>38aa367e014493c6f6ebfe94d9029bea01d072c4</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
     <Dependency Name="Microsoft.DiaSymReader.Pdb2Pdb" Version="1.1.0-beta2-19575-01">
@@ -54,9 +54,9 @@
       <Uri>https://github.com/dotnet/symreader-converter</Uri>
       <Sha>c5ba7c88f92e2dde156c324a8c8edc04d9fa4fe0</Sha>
     </Dependency>
-    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.23567.1">
+    <Dependency Name="Microsoft.DotNet.XliffTasks" Version="9.0.0-beta.23601.1">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>b6801170e6ca8bf12bdff1f10d410c5e01660681</Sha>
+      <Sha>38aa367e014493c6f6ebfe94d9029bea01d072c4</Sha>
     </Dependency>
   </ToolsetDependencies>
 </Dependencies>

eng/common/darc-init.ps1

@@ -1,6 +1,6 @@
 param (
     $darcVersion = $null,
-    $versionEndpoint = 'https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16',
+    $versionEndpoint = 'https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16',
     $verbosity = 'minimal',
     $toolpath = $null
 )

eng/common/darc-init.sh

@@ -2,7 +2,7 @@
 
 source="${BASH_SOURCE[0]}"
 darcVersion=''
-versionEndpoint='https://maestro-prod.westus2.cloudapp.azure.com/api/assets/darc-version?api-version=2019-01-16'
+versionEndpoint='https://maestro.dot.net/api/assets/darc-version?api-version=2019-01-16'
 verbosity='minimal'
 
 while [[ $# > 0 ]]; do

eng/common/post-build/add-build-to-channel.ps1

@@ -2,7 +2,7 @@ param(
   [Parameter(Mandatory=$true)][int] $BuildId,
   [Parameter(Mandatory=$true)][int] $ChannelId,
   [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
-  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net',
   [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
 )
 

eng/common/post-build/publish-using-darc.ps1

@@ -3,7 +3,7 @@ param(
   [Parameter(Mandatory=$true)][int] $PublishingInfraVersion,
   [Parameter(Mandatory=$true)][string] $AzdoToken,
   [Parameter(Mandatory=$true)][string] $MaestroToken,
-  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net',
   [Parameter(Mandatory=$true)][string] $WaitPublishingFinish,
   [Parameter(Mandatory=$false)][string] $ArtifactsPublishingAdditionalParameters,
   [Parameter(Mandatory=$false)][string] $SymbolPublishingAdditionalParameters

eng/common/post-build/redact-logs.ps1

@@ -0,0 +1,81 @@
+[CmdletBinding(PositionalBinding=$False)]
+param(
+  [Parameter(Mandatory=$true, Position=0)][string] $InputPath,
+  [Parameter(Mandatory=$true)][string] $BinlogToolVersion,
+  [Parameter(Mandatory=$false)][string] $DotnetPath,
+  [Parameter(Mandatory=$false)][string] $PackageFeed = 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public/nuget/v3/index.json',
+  # File with strings to redact - separated by newlines.
+  #  For comments start the line with '# ' - such lines are ignored 
+  [Parameter(Mandatory=$false)][string] $TokensFilePath,
+  [Parameter(ValueFromRemainingArguments=$true)][String[]]$TokensToRedact
+)
+
+try {
+  . $PSScriptRoot\post-build-utils.ps1
+
+  $packageName = 'binlogtool'
+
+  $dotnet = $DotnetPath
+
+  if (!$dotnet) {
+    $dotnetRoot = InitializeDotNetCli -install:$true
+    $dotnet = "$dotnetRoot\dotnet.exe"
+  }
+
+  $toolList = & "$dotnet" tool list -g
+
+  if ($toolList -like "*$packageName*") {
+    & "$dotnet" tool uninstall $packageName -g
+  }
+
+  $toolPath  = "$PSScriptRoot\..\..\..\.tools"
+  $verbosity = 'minimal'
+
+  New-Item -ItemType Directory -Force -Path $toolPath
+
+  Push-Location -Path $toolPath
+
+  try {
+    Write-Host "Installing Binlog redactor CLI..."
+    Write-Host "'$dotnet' new tool-manifest"
+    & "$dotnet" new tool-manifest
+    Write-Host "'$dotnet' tool install $packageName --local --add-source '$PackageFeed' -v $verbosity --version $BinlogToolVersion"
+    & "$dotnet" tool install $packageName --local --add-source "$PackageFeed" -v $verbosity --version $BinlogToolVersion
+
+    if (Test-Path $TokensFilePath) {
+        Write-Host "Adding additional sensitive data for redaction from file: " $TokensFilePath
+        $TokensToRedact += Get-Content -Path $TokensFilePath | Foreach {$_.Trim()} | Where { $_ -notmatch "^# " }
+    }
+
+    $optionalParams = [System.Collections.ArrayList]::new()
+
+    Foreach ($p in $TokensToRedact)
+    {
+      if($p -match '^\$\(.*\)$')
+      {
+        Write-Host ("Ignoring token {0} as it is probably unexpanded AzDO variable"  -f $p)
+      }          
+      elseif($p)
+      {
+        $optionalParams.Add("-p:" + $p) | Out-Null
+      }
+    }
+
+    & $dotnet binlogtool redact --input:$InputPath --recurse --in-place `
+      @optionalParams
+
+    if ($LastExitCode -ne 0) {
+      Write-PipelineTelemetryError -Category 'Redactor' -Type 'warning' -Message "Problems using Redactor tool (exit code: $LastExitCode). But ignoring them now."
+    }
+  }
+  finally {
+    Pop-Location
+  }
+
+  Write-Host 'done.'
+} 
+catch {
+  Write-Host $_
+  Write-PipelineTelemetryError -Category 'Redactor' -Message "There was an error while trying to redact logs. Error: $_"
+  ExitWithExitCode 1
+}

eng/common/post-build/trigger-subscriptions.ps1

@@ -2,7 +2,7 @@ param(
   [Parameter(Mandatory=$true)][string] $SourceRepo,
   [Parameter(Mandatory=$true)][int] $ChannelId,
   [Parameter(Mandatory=$true)][string] $MaestroApiAccessToken,
-  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro-prod.westus2.cloudapp.azure.com',
+  [Parameter(Mandatory=$false)][string] $MaestroApiEndPoint = 'https://maestro.dot.net',
   [Parameter(Mandatory=$false)][string] $MaestroApiVersion = '2019-01-16'
 )
 

eng/common/templates/job/publish-build-assets.yml

@@ -48,8 +48,8 @@ jobs:
     - group: AzureDevOps-Artifact-Feeds-Pats
     - name: runCodesignValidationInjection
       value: false
-    - ${{ if eq(parameters.publishAssetsImmediately, 'true') }}:
-      - template: /eng/common/templates/post-build/common-variables.yml
+    # unconditional - needed for logs publishing (redactor tool version)
+    - template: /eng/common/templates/post-build/common-variables.yml
 
   pool:
     # We don't use the collection uri here because it might vary (.visualstudio.com vs. dev.azure.com)
@@ -81,7 +81,7 @@ jobs:
         arguments: -task PublishBuildAssets -restore -msbuildEngine dotnet
           /p:ManifestsPath='$(Build.StagingDirectory)/Download/AssetManifests'
           /p:BuildAssetRegistryToken=$(MaestroAccessToken)
-          /p:MaestroApiEndpoint=https://maestro-prod.westus2.cloudapp.azure.com
+          /p:MaestroApiEndpoint=https://maestro.dot.net
           /p:PublishUsingPipelines=${{ parameters.publishUsingPipelines }}
           /p:OfficialBuildId=$(Build.BuildNumber)
       condition: ${{ parameters.condition }}

eng/common/templates/post-build/common-variables.yml

@@ -7,7 +7,7 @@ variables:
 
   # Default Maestro++ API Endpoint and API Version
   - name: MaestroApiEndPoint
-    value: "https://maestro-prod.westus2.cloudapp.azure.com"
+    value: "https://maestro.dot.net"
   - name: MaestroApiAccessToken
     value: $(MaestroAccessToken)
   - name: MaestroApiVersion
@@ -17,6 +17,8 @@ variables:
     value: 3.0.0
   - name: SymbolToolVersion
     value: 1.0.1
+  - name: BinlogToolVersion
+    value: 1.0.8
 
   - name: runCodesignValidationInjection
     value: false

eng/common/templates/post-build/post-build.yml

@@ -187,6 +187,7 @@ stages:
           parameters:
             StageLabel: 'Validation'
             JobLabel: 'Signing'
+            BinlogToolVersion: $(BinlogToolVersion)
 
     - job:
       displayName: SourceLink Validation

eng/common/templates/steps/publish-logs.yml

@@ -1,6 +1,9 @@
 parameters:
   StageLabel: ''
   JobLabel: ''
+  CustomSensitiveDataList: ''
+  # A default - in case value from eng/common/templates/post-build/common-variables.yml is not passed
+  BinlogToolVersion: '1.0.8'
 
 steps:
 - task: Powershell@2
@@ -12,7 +15,30 @@ steps:
       Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
   continueOnError: true
   condition: always()
-
+
+- task: PowerShell@2
+  displayName: Redact Logs
+  inputs:
+    filePath: $(Build.SourcesDirectory)/eng/common/post-build/redact-logs.ps1
+    # For now this needs to have explicit list of all sensitive data. Taken from eng/publishing/v3/publish.yml
+    # Sensitive data can as well be added to $(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt'
+    #  If the file exists - sensitive data for redaction will be sourced from it
+    #  (single entry per line, lines starting with '# ' are considered comments and skipped)
+    arguments: -InputPath '$(Build.SourcesDirectory)/PostBuildLogs' 
+      -BinlogToolVersion ${{parameters.BinlogToolVersion}}
+      -TokensFilePath '$(Build.SourcesDirectory)/eng/BinlogSecretsRedactionFile.txt'
+      '$(publishing-dnceng-devdiv-code-r-build-re)'
+      '$(MaestroAccessToken)'
+      '$(dn-bot-all-orgs-artifact-feeds-rw)'
+      '$(akams-client-id)'
+      '$(akams-client-secret)'
+      '$(microsoft-symbol-server-pat)'
+      '$(symweb-symbol-server-pat)'
+      '$(dn-bot-all-orgs-build-rw-code-rw)'
+      ${{parameters.CustomSensitiveDataList}}
+  continueOnError: true
+  condition: always()
+
 - task: PublishBuildArtifacts@1
   displayName: Publish Logs
   inputs:

global.json

@@ -32,6 +32,6 @@
     "dotnet": "8.0.100"
   },
   "msbuild-sdks": {
-    "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23567.1"
+    "Microsoft.DotNet.Arcade.Sdk": "9.0.0-beta.23601.1"
   }
 }