Skip to content

Commit a22e3ab

Browse files
mthalmanmthalman
committed
Support OfficialBuildId for SB
1 parent 96aa638 commit a22e3ab

15 files changed

+294
-94
lines changed

‎eng/pipelines/templates/jobs/vmr-build.yml

Lines changed: 83 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,18 @@ parameters:
6363
type: boolean
6464
default: false
6565

66+
# Name of previous job(s) (from the same template as this) whose output will be used to build this job
67+
# The SDK from its artifacts is copied to $(sourcesPath)/.dotnet
68+
- name: reuseBuildArtifactsFrom
69+
type: object
70+
default: ''
71+
72+
# Names of jobs that this job depends on. This only provides a job dependency and does not automatically
73+
# download artifacts from the other job. Use reuseBuildArtifactsFrom for that.
74+
- name: dependsOn
75+
type: object
76+
default: ''
77+
6678
#### SOURCE-ONLY parameters ####
6779

6880
# Instead of building the VMR directly, exports the sources into a tarball and builds from that
@@ -83,12 +95,6 @@ parameters:
8395
type: boolean
8496
default: false
8597

86-
# Name of previous job(s) (from the same template as this) whose output will be used to build this job
87-
# The SDK from its artifacts is copied to $(sourcesPath)/.dotnet
88-
- name: reuseBuildArtifactsFrom
89-
type: object
90-
default: ''
91-
9298
# Allow downloading artifacts from the internet during the build
9399
- name: runOnline
94100
type: boolean
@@ -108,6 +114,16 @@ parameters:
108114
type: boolean
109115
default: false
110116

117+
# Skip the build step (this would be used when wanting to run tests only)
118+
- name: skipBuild
119+
type: boolean
120+
default: false
121+
122+
# Custom steps to run before running tests
123+
- name: testInitSteps
124+
type: stepList
125+
default: []
126+
111127
#### sdk parameters ####
112128

113129
- name: isBuiltFromVmr
@@ -138,7 +154,7 @@ jobs:
138154
image: ${{ parameters.container.image }}
139155
options: $(defaultContainerOptions)
140156

141-
${{ if ne(parameters.reuseBuildArtifactsFrom, '') }}:
157+
${{ if or(ne(parameters.reuseBuildArtifactsFrom, ''), ne(parameters.dependsOn, '')) }}:
142158
${{ if eq(parameters.buildPass, '') }}:
143159
# For PR builds, skip the stage 2 build if the stage 1 build fails.
144160
# Otherwise, run the stage 2 build even if the stage 1 build fails so that we can get a complete assessment of the build status.
@@ -149,7 +165,11 @@ jobs:
149165
condition: succeededOrFailed()
150166
${{ else }}:
151167
condition: succeeded()
152-
dependsOn: ${{ parameters.reuseBuildArtifactsFrom }}
168+
dependsOn:
169+
- ${{ if ne(parameters.reuseBuildArtifactsFrom, '') }}:
170+
- ${{ parameters.reuseBuildArtifactsFrom }}
171+
- ${{ if ne(parameters.dependsOn, '') }}:
172+
- ${{ parameters.dependsOn }}
153173
variables:
154174
- ${{ if eq(variables['System.TeamProject'], 'internal') }}:
155175
- group: AzureDevOps-Artifact-Feeds-Pats
@@ -344,20 +364,15 @@ jobs:
344364
- ${{ if ne(parameters.reuseBuildArtifactsFrom,'') }}:
345365
- ${{ each reuseBuildArtifacts in parameters.reuseBuildArtifactsFrom }}:
346366
- ${{ if eq(parameters.buildSourceOnly, true) }}:
347-
- download: current
348-
artifact: ${{ reuseBuildArtifacts }}_Artifacts
349-
patterns: |
350-
**/Private.SourceBuilt.Artifacts.*.tar.gz
351-
**/dotnet-sdk-*.tar.gz
352-
**/*.nupkg
353-
displayName: Download Previous Build (${{ reuseBuildArtifacts }})
354-
355-
- task: CopyFiles@2
356-
displayName: Copy Previous Build (${{ reuseBuildArtifacts }} - Source Build artifacts)
357-
inputs:
358-
SourceFolder: $(Pipeline.Workspace)/${{ reuseBuildArtifacts }}_Artifacts/assets/Release
359-
Contents: '*.tar.gz'
360-
TargetFolder: $(sourcesPath)/prereqs/packages/archive/
367+
- template: ../steps/download-artifacts.yml@self
368+
parameters:
369+
artifactDescription: Previous Build (${{ reuseBuildArtifacts }} - Source Build artifacts)
370+
artifactName: ${{ reuseBuildArtifacts }}_Artifacts
371+
downloadFilePatterns: |
372+
**/Private.SourceBuilt.Artifacts.*.tar.gz
373+
**/dotnet-sdk-*.tar.gz
374+
copyDestination: $(sourcesPath)/prereqs/packages/archive/
375+
flattenDirs: true
361376

362377
- ${{ else }}:
363378
- task: DownloadPipelineArtifact@2
@@ -501,61 +516,62 @@ jobs:
501516
displayName: Prep the Build
502517
workingDirectory: $(sourcesPath)
503518
504-
- script: |
505-
set -ex
506-
df -h
519+
- ${{ if ne(parameters.skipBuild, 'true') }}:
507520

508-
customEnvVars=""
509-
customPreBuildArgs=""
510-
customBuildArgs="$(baseArguments) $(signArguments) $(_SignDiagnosticFilesArgs)"
511-
extraBuildProperties="$(baseProperties) $(targetProperties) $(signProperties) $(buildPassProperties) ${{ parameters.extraProperties }}"
521+
- script: |
522+
set -ex
523+
df -h
512524
513-
if [[ '${{ parameters.runOnline }}' == 'True' ]]; then
514-
customBuildArgs="$customBuildArgs --online"
515-
else
516-
customPreBuildArgs="$customPreBuildArgs sudo unshare -n"
517-
fi
525+
customEnvVars=""
526+
customPreBuildArgs=""
527+
customBuildArgs="$(baseArguments) $(signArguments) $(_SignDiagnosticFilesArgs)"
528+
extraBuildProperties="$(baseProperties) $(targetProperties) $(signProperties) $(buildPassProperties) ${{ parameters.extraProperties }}"
518529
519-
if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
520-
customBuildArgs="$customBuildArgs --poison"
521-
fi
530+
if [[ '${{ parameters.runOnline }}' == 'True' ]]; then
531+
customBuildArgs="$customBuildArgs --online"
532+
else
533+
customPreBuildArgs="$customPreBuildArgs sudo unshare -n"
534+
fi
522535
523-
if [[ '${{ parameters.buildFromArchive }}' == 'True' ]]; then
524-
customBuildArgs="$customBuildArgs --source-repository https://github.com/dotnet/dotnet"
525-
customBuildArgs="$customBuildArgs --source-version $(git -C "$(vmrPath)" rev-parse HEAD)"
526-
fi
536+
if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then
537+
customBuildArgs="$customBuildArgs --poison"
538+
fi
527539
528-
if [[ '${{ parameters.buildSourceOnly }}' == 'True' ]]; then
529-
customBuildArgs="$customBuildArgs --source-only"
530-
extraBuildProperties="$extraBuildProperties /p:ReportSbrpUsage=true"
531-
fi
540+
if [[ '${{ parameters.buildFromArchive }}' == 'True' ]]; then
541+
customBuildArgs="$customBuildArgs --source-repository https://github.com/dotnet/dotnet"
542+
customBuildArgs="$customBuildArgs --source-version $(git -C "$(vmrPath)" rev-parse HEAD)"
543+
fi
532544
533-
if [[ '${{ parameters.useMonoRuntime }}' == 'True' ]]; then
534-
customBuildArgs="$customBuildArgs --use-mono-runtime"
535-
fi
545+
if [[ '${{ parameters.buildSourceOnly }}' == 'True' ]]; then
546+
customBuildArgs="$customBuildArgs --source-only"
547+
extraBuildProperties="$extraBuildProperties /p:ReportSbrpUsage=true"
548+
fi
536549
537-
if [[ -n "${{ parameters.targetRid }}" ]]; then
538-
customBuildArgs="$customBuildArgs --target-rid ${{ parameters.targetRid }}"
539-
fi
550+
if [[ '${{ parameters.useMonoRuntime }}' == 'True' ]]; then
551+
customBuildArgs="$customBuildArgs --use-mono-runtime"
552+
fi
540553
541-
if [[ -n "${{ parameters.crossRootFs }}" ]]; then
542-
customEnvVars="$customEnvVars ROOTFS_DIR=${{ parameters.crossRootFs}}"
543-
if [[ '${{ parameters.targetArchitecture }}' != 'wasm' ]]; then
544-
extraBuildProperties="$extraBuildProperties /p:CrossBuild=true"
554+
if [[ -n "${{ parameters.targetRid }}" ]]; then
555+
customBuildArgs="$customBuildArgs --target-rid ${{ parameters.targetRid }}"
556+
fi
557+
558+
if [[ -n "${{ parameters.crossRootFs }}" ]]; then
559+
customEnvVars="$customEnvVars ROOTFS_DIR=${{ parameters.crossRootFs}}"
560+
if [[ '${{ parameters.targetArchitecture }}' != 'wasm' ]]; then
561+
extraBuildProperties="$extraBuildProperties /p:CrossBuild=true"
562+
fi
545563
fi
546-
fi
547564
548-
buildArgs="$(additionalBuildArgs) $customBuildArgs $extraBuildProperties"
549-
buildArgs=$(echo $buildArgs | xargs) # Remove extra spaces
565+
buildArgs="$(additionalBuildArgs) $customBuildArgs $extraBuildProperties"
550566
551-
for envVar in $customEnvVars; do
552-
customEnvVarsWithBashSyntax="$customEnvVarsWithBashSyntax export $envVar;"
553-
done
567+
for envVar in $customEnvVars; do
568+
customEnvVarsWithBashSyntax="$customEnvVarsWithBashSyntax export $envVar;"
569+
done
554570
555-
eval $customEnvVarsWithBashSyntax
556-
$customPreBuildArgs ./build.sh $buildArgs
557-
displayName: Build
558-
workingDirectory: $(sourcesPath)
571+
eval $customEnvVarsWithBashSyntax
572+
$customPreBuildArgs ./build.sh $buildArgs
573+
displayName: Build
574+
workingDirectory: $(sourcesPath)
559575
560576
- ${{ if ne(parameters.runOnline, 'True' )}}:
561577
- script: |
@@ -566,6 +582,8 @@ jobs:
566582
567583
# Only run tests if enabled
568584
- ${{ if eq(parameters.runTests, 'True') }}:
585+
- ${{ parameters.testInitSteps }}
586+
569587
# Setup the NuGet sources used by the tests to use private feeds. This is necessary when testing internal-only product
570588
# builds where the packages are only available in the private feeds. This allows the tests to restore from those feeds.
571589
- ${{ if eq(variables['System.TeamProject'], 'internal') }}:

‎eng/pipelines/templates/stages/vmr-verticals.yml

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -995,3 +995,77 @@ stages:
995995
- Windows_x64
996996
- Windows_x86
997997
- Windows_arm64
998+
999+
# Source Build Leg
1000+
- template: ../jobs/vmr-build.yml
1001+
parameters:
1002+
buildName: ${{ format('{0}_SourceBuild_Online_MsftSdk', variables.centOSStreamName) }}
1003+
isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
1004+
vmrBranch: ${{ variables.VmrBranch }}
1005+
targetArchitecture: x64
1006+
pool: ${{ parameters.pool_Linux }}
1007+
container:
1008+
name: ${{ variables.centOSStreamContainerName }}
1009+
image: ${{ variables.centOSStreamContainerImage }}
1010+
buildSourceOnly: true
1011+
runOnline: true
1012+
runTests: false
1013+
extraProperties: "/p:EnableDevBuildAsDefaultForSourceOnly=true"
1014+
1015+
# Source Build Validation
1016+
- template: ../jobs/vmr-build.yml
1017+
parameters:
1018+
buildName: ${{ format('{0}_SourceBuild_Online_MsftSdk', variables.centOSStreamName) }}_Validation
1019+
isBuiltFromVmr: ${{ parameters.isBuiltFromVmr }}
1020+
vmrBranch: ${{ variables.VmrBranch }}
1021+
targetArchitecture: x64
1022+
pool: ${{ parameters.pool_Linux }}
1023+
container:
1024+
name: ${{ variables.centOSStreamContainerName }}
1025+
image: ${{ variables.centOSStreamContainerImage }}
1026+
buildSourceOnly: true
1027+
runOnline: true
1028+
runTests: true
1029+
skipBuild: true
1030+
extraProperties: >
1031+
/p:DotNetSourceOnlyTestOnly=true
1032+
/p:ExtraRestoreSourcePath=$(Pipeline.Workspace)/msft-pkgs
1033+
/p:SourceBuildTestsCustomSourceBuiltPackagesPath=$(Pipeline.Workspace)/msft-pkgs
1034+
/p:SourceBuildTestsCustomSourceBuiltPackagesPathIsInclusive=true
1035+
dependsOn:
1036+
- AzureLinux_x64_Cross_x64
1037+
- Browser_Shortstack_wasm
1038+
- Windows_x64
1039+
- ${{ format('{0}_SourceBuild_Online_MsftSdk_x64', variables.centOSStreamName) }}
1040+
testInitSteps:
1041+
# Ensure the artifacts staging directory exists so that even if no files get placed there, it won't fail
1042+
# when publishing the artifacts.
1043+
- script: mkdir -p $(artifactsStagingDir)
1044+
displayName: Create Artifacts Staging Directory
1045+
- template: ../steps/download-artifacts.yml
1046+
parameters:
1047+
artifactDescription: Microsoft Linux Packages
1048+
artifactName: AzureLinux_x64_Cross_x64_Artifacts
1049+
downloadFilePatterns: packages/Release/Shipping/**
1050+
copyDestination: $(Pipeline.Workspace)/msft-pkgs
1051+
flattenDirs: true
1052+
- template: ../steps/download-artifacts.yml
1053+
parameters:
1054+
artifactDescription: Microsoft WASM Packages
1055+
artifactName: Browser_Shortstack_wasm_Artifacts
1056+
downloadFilePatterns: packages/Release/Shipping/**
1057+
copyDestination: $(Pipeline.Workspace)/msft-pkgs
1058+
flattenDirs: true
1059+
- template: ../steps/download-artifacts.yml
1060+
parameters:
1061+
artifactDescription: Microsoft Windows Packages
1062+
artifactName: Windows_x64_Artifacts
1063+
downloadFilePatterns: packages/Release/Shipping/**
1064+
copyDestination: $(Pipeline.Workspace)/msft-pkgs
1065+
flattenDirs: true
1066+
- template: ../steps/download-artifacts.yml
1067+
parameters:
1068+
artifactDescription: Source Build Assets
1069+
artifactName: ${{ format('{0}_SourceBuild_Online_MsftSdk_x64', variables.centOSStreamName) }}_Artifacts
1070+
downloadFilePatterns: assets/Release/**/*.tar.gz
1071+
copyDestination: $(Build.SourcesDirectory)/artifacts

‎eng/pipelines/templates/steps/download-artifacts.yml

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
parameters:
2+
- name: artifactDescription
3+
type: string
4+
- name: artifactName
5+
type: string
6+
- name: downloadFilePatterns
7+
type: string
8+
- name: copyDestination
9+
type: string
10+
- name: flattenDirs
11+
type: boolean
12+
default: false
13+
14+
steps:
15+
- download: current
16+
artifact: ${{ parameters.artifactName }}
17+
patterns: ${{ parameters.downloadFilePatterns }}
18+
displayName: Download ${{ parameters.artifactDescription }}
19+
- task: CopyFiles@2
20+
displayName: Copy ${{ parameters.artifactDescription }}
21+
inputs:
22+
SourceFolder: $(Pipeline.Workspace)/${{ parameters.artifactName }}
23+
Contents: '**'
24+
FlattenFolders: ${{ parameters.flattenDirs }}
25+
TargetFolder: ${{ parameters.copyDestination }}

‎src/Layout/Directory.Build.props

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,5 +72,4 @@
7272
<PropertyGroup>
7373
<SdkPkgSourcesRootDirectory>$(MSBuildThisFileDirectory)pkg\</SdkPkgSourcesRootDirectory>
7474
</PropertyGroup>
75-
7675
</Project>

‎src/Layout/redist/targets/RestoreLayout.targets

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212

1313
<!-- Blob storage directories are not stabilized, so these must refer to a package that does not stabilize -->
1414
<!-- In unified build, the layout does match, so use the runtime package versions rather than the VS redist versions -->
15-
<AspNetCoreBlobVersion>$(MicrosoftAspNetCoreAppRefInternalPackageVersion)</AspNetCoreBlobVersion>
15+
<AspNetCoreBlobVersion>$(MicrosoftAspNetCoreAppRuntimePackageVersion)</AspNetCoreBlobVersion>
1616
<NetRuntimeBlobVersion>$(MicrosoftNETCorePlatformsPackageVersion)</NetRuntimeBlobVersion>
1717
<WindowsDesktopBlobVersion>$(VSRedistCommonWindowsDesktopSharedFrameworkx64100PackageVersion)</WindowsDesktopBlobVersion>
1818
<WindowsDesktopBlobVersion Condition="'$(DotNetBuildOrchestrator)' == 'true'">$(MicrosoftWindowsDesktopAppRuntimePackageVersion)</WindowsDesktopBlobVersion>

‎src/SourceBuild/content/repo-projects/Directory.Build.props

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@
1515
<RunEachTargetSeparately Condition="'$(BuildInParallel)' == 'true' and '$(ContinuousIntegrationBuild)' != 'true'">true</RunEachTargetSeparately>
1616
</PropertyGroup>
1717

18-
<!-- TODO: Remove this import when the default build behavior for source-only builds changes to dev.
19-
https://github.com/dotnet/source-build/issues/4855 -->
20-
<Import Project="$(GitInfoRepoPropsFile)" Condition="Exists('$(GitInfoRepoPropsFile)') and '$(DotNetBuildSourceOnly)' == 'true'" />
18+
<!-- TODO: Remove this import and the EnableDevBuildAsDefaultForSourceOnly when dev builds are the default for source only.
19+
https://github.com/dotnet/source-build/issues/4922 -->
20+
<Import Project="$(GitInfoRepoPropsFile)" Condition="Exists('$(GitInfoRepoPropsFile)') and '$(DotNetBuildSourceOnly)' == 'true' and '$(EnableDevBuildAsDefaultForSourceOnly)' != 'true'" />
2121

2222
<PropertyGroup>
2323
<!-- Fake, to satisfy the SDK. -->
@@ -82,10 +82,10 @@
8282
<ArcadeFalseBoolBuildArg>false</ArcadeFalseBoolBuildArg>
8383
</PropertyGroup>
8484

85-
<!-- Add 100 to the revision number for non-source-only builds to avoid clashing with the existing msft official builds.
86-
Source-only builds don't update the revision as the assets don't get published to the same locations as the msft build and
87-
still use the OfficialBuildId from repositories, not the pipeline. -->
88-
<PropertyGroup Condition="'$(OfficialBuildId)' != '' and '$(DotNetBuildSourceOnly)' != 'true'">
85+
<!-- Add 100 to the revision number to avoid clashing with the existing msft official builds.
86+
This is necessary for source-only builds as well so that the version numbers are identical between the two builds
87+
since testing of source-build only requires some packages from the msft build. -->
88+
<PropertyGroup Condition="'$(OfficialBuildId)' != '' and ('$(DotNetBuildSourceOnly)' != 'true' or '$(EnableDevBuildAsDefaultForSourceOnly)' == 'true')">
8989
<OfficialBuildId>$(OfficialBuildId.Split('.')[0]).$([MSBuild]::Add($(OfficialBuildId.Split('.')[1]), 100))</OfficialBuildId>
9090
</PropertyGroup>
9191

@@ -104,9 +104,9 @@
104104

105105
<!-- Common args used for building and testing. -->
106106
<PropertyGroup>
107-
<!-- TODO: Remove the DotNetBuildSourceOnly condition when the default build behavior for source-only builds changes to dev.
108-
https://github.com/dotnet/source-build/issues/4855 -->
109-
<CommonArgs Condition="'$(ContinuousIntegrationBuild)' == 'true' or '$(DotNetBuildSourceOnly)' == 'true'">$(FlagParameterPrefix)ci</CommonArgs>
107+
<!-- TODO: Remove this import and the EnableDevBuildAsDefaultForSourceOnly when dev builds are the default for source only.
108+
https://github.com/dotnet/source-build/issues/4922 -->
109+
<CommonArgs Condition="'$(ContinuousIntegrationBuild)' == 'true' or ('$(DotNetBuildSourceOnly)' == 'true' and '$(EnableDevBuildAsDefaultForSourceOnly)' != 'true')">$(FlagParameterPrefix)ci</CommonArgs>
110110

111111
<!-- Pass down configuration properties -->
112112
<CommonArgs>$(CommonArgs) $(FlagParameterPrefix)configuration $(Configuration)</CommonArgs>
@@ -173,9 +173,9 @@
173173
Condition="'$(UseBootstrapArcade)' != 'true'" />
174174

175175
<!-- We pass '-ci', but also apply ci mode via env var for edge cases. (E.g. misbehaving inner builds.).
176-
TODO: Remove the DotNetBuildSourceOnly condition when the default build behavior for source-only builds changes to dev.
177-
https://github.com/dotnet/source-build/issues/4855 -->
178-
<EnvironmentVariables Condition="'$(ContinuousIntegrationBuild)' == 'true' or '$(DotNetBuildSourceOnly)' == 'true'" Include="ContinuousIntegrationBuild=true" />
176+
TODO: Remove this import and the EnableDevBuildAsDefaultForSourceOnly when dev builds are the default for source only.
177+
https://github.com/dotnet/source-build/issues/4922 -->
178+
<EnvironmentVariables Condition="'$(ContinuousIntegrationBuild)' == 'true' or ('$(DotNetBuildSourceOnly)' == 'true' and '$(EnableDevBuildAsDefaultForSourceOnly)' != 'true')" Include="ContinuousIntegrationBuild=true" />
179179

180180
<!-- Turn off node reuse for source build because repos use conflicting versions
181181
of compilers which cause assembly load errors.

0 commit comments

Comments
 (0)