From cd036fc6491ac9cacaa9ab2b1a11bb1870ec3c19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Wed, 10 Nov 2021 13:03:24 -0800 Subject: [PATCH 001/282] README.md - update versions to 1.1.1 --- README.md | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 1867c86e..7643bebb 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ For projects hosted by [GitHub](http://github.com) or [GitHub Enterprise](https: ```xml - + ``` @@ -58,7 +58,7 @@ For projects hosted by [Azure Repos](https://azure.microsoft.com/en-us/services/ ```xml - + ``` @@ -69,7 +69,7 @@ For projects hosted by on-prem [Azure DevOps Server](https://azure.microsoft.com ```xml - + ``` @@ -89,11 +89,11 @@ For projects hosted by [GitLab](https://gitlab.com) reference [Microsoft.SourceL ```xml - + ``` -Starting with version 1.1.0-beta-21470-02, Microsoft.SourceLink.GitLab assumes GitLab version 12.0+ by default. +Starting with version 1.1.1, Microsoft.SourceLink.GitLab assumes GitLab version 12.0+ by default. If your project is hosted by GitLab older than version 12.0 you must specify `SourceLinkGitLabHost` item group in addition to the package reference: ```xml @@ -113,7 +113,7 @@ For projects in git repositories hosted on [Bitbucket.org](https://bitbucket.org ```xml - + ``` @@ -128,23 +128,23 @@ If your project is hosted by Bitbucket Server or Bitbucket Data Center older tha The item group `SourceLinkBitbucketGitHost` specifies the domain of the Bitbucket host and the version of Bitbucket. The version is important since URL format for accessing files changes with version 4.7. By default Source Link assumes new format (version 4.7+). -### gitweb (pre-release) +### gitweb For projects hosted on-prem via [gitweb](https://git-scm.com/docs/gitweb) reference [Microsoft.SourceLink.GitWeb](https://www.nuget.org/packages/Microsoft.SourceLink.GitWeb) package: ```xml - + ``` -### gitea (pre-release) +### gitea For projects hosted on-prem via [gitea](https://gitea.io) reference [Microsoft.SourceLink.Gitea](https://www.nuget.org/packages/Microsoft.SourceLink.Gitea) package: ```xml - + ``` @@ -160,9 +160,9 @@ To add Source Link support to your native project add package references corresp ```xml - - - + + + ``` From 4f4f58596278e9ef72d68d7ea80b634e552c9c8f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 15 Nov 2021 13:38:02 +0000 Subject: [PATCH 002/282] Update dependencies from https://github.com/dotnet/arcade build 20211109.3 (#770) [main] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- eng/common/msbuild.ps1 | 1 + global.json | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index c00e7404..71f00d18 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - a7c57abb74deaee6dac921dd68f9c3c58059ebfb + fecf65bedcee9036b8ba9d8d7feef5413f294914 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/msbuild.ps1 b/eng/common/msbuild.ps1 index eea19cd8..f041e5dd 100644 --- a/eng/common/msbuild.ps1 +++ b/eng/common/msbuild.ps1 @@ -6,6 +6,7 @@ Param( [switch] $ci, [switch] $prepareMachine, [switch] $excludePrereleaseVS, + [string] $msbuildEngine = $null, [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs ) diff --git a/global.json b/global.json index 66616be4..1b6c589f 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "6.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21555.2" + "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21559.3" } } From 9c298b01ce0f161622bef0bc0c433a758d0dd3ac Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 29 Nov 2021 13:43:37 +0000 Subject: [PATCH 003/282] Update dependencies from https://github.com/dotnet/arcade build 20211126.4 (#773) [main] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 +- eng/common/build.sh | 4 -- eng/common/darc-init.sh | 2 +- eng/common/native/init-compiler.sh | 11 ++-- eng/common/sdk-task.ps1 | 3 - eng/common/sdl/execute-all-sdl-tools.ps1 | 2 +- eng/common/templates/job/job.yml | 1 + eng/common/tools.ps1 | 76 +++++++++++------------- eng/common/tools.sh | 67 ++++++++++----------- global.json | 2 +- 10 files changed, 78 insertions(+), 94 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 71f00d18..4d49a69d 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - fecf65bedcee9036b8ba9d8d7feef5413f294914 + 427c05909067bb2e484116ae2239456bb45adb85 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/build.sh b/eng/common/build.sh index bc07a1c6..55b298f1 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -187,10 +187,6 @@ function InitializeCustomToolset { } function Build { - - if [[ "$ci" == true ]]; then - TryLogClientIpAddress - fi InitializeToolset InitializeCustomToolset diff --git a/eng/common/darc-init.sh b/eng/common/darc-init.sh index 39abdbec..84c1d0cc 100755 --- a/eng/common/darc-init.sh +++ b/eng/common/darc-init.sh @@ -53,7 +53,7 @@ fi function InstallDarcCli { local darc_cli_package_name="microsoft.dotnet.darc" - InitializeDotNetCli + InitializeDotNetCli true local dotnet_root=$_InitializeDotNetCli if [ -z "$toolpath" ]; then diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index 8c944f30..e361e03f 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -2,6 +2,7 @@ # # This file detects the C/C++ compiler and exports it to the CC/CXX environment variables # +# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here! if [[ "$#" -lt 3 ]]; then echo "Usage..." @@ -111,12 +112,10 @@ if [[ -z "$CC" ]]; then exit 1 fi -if [[ "$compiler" == "clang" ]]; then - if command -v "lld$desired_version" > /dev/null; then - # Only lld version >= 9 can be considered stable - if [[ "$majorVersion" -ge 9 ]]; then - LDFLAGS="-fuse-ld=lld" - fi +# Only lld version >= 9 can be considered stable +if [[ "$compiler" == "clang" && "$majorVersion" -ge 9 ]]; then + if "$CC" -fuse-ld=lld -Wl,--version >/dev/null 2>&1; then + LDFLAGS="-fuse-ld=lld" fi fi diff --git a/eng/common/sdk-task.ps1 b/eng/common/sdk-task.ps1 index 7ab9baac..b1bca63a 100644 --- a/eng/common/sdk-task.ps1 +++ b/eng/common/sdk-task.ps1 @@ -83,9 +83,6 @@ try { } if ($restore) { - if ($ci) { - Try-LogClientIpAddress - } Build 'Restore' } diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 1157151f..e5bef8eb 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -124,7 +124,7 @@ try { Exec-BlockVerbosely { & $(Join-Path $PSScriptRoot 'run-sdl.ps1') ` -GuardianCliLocation $guardianCliLocation ` - -WorkingDirectory $workingDirectory ` + -WorkingDirectory $SourceDirectory ` -UpdateBaseline $UpdateBaseline ` -GdnFolder $gdnFolder } diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 37dceb1b..7678b94c 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -114,6 +114,7 @@ jobs: continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + - ${{ if and(eq(parameters.runAsPublic, 'false'), eq(variables['System.TeamProject'], 'internal')) }}: - task: NuGetAuthenticate@0 - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}: diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 44484289..f1e1cb53 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -163,9 +163,6 @@ function InitializeDotNetCli([bool]$install, [bool]$createSdkLocationFile) { # Disable telemetry on CI. if ($ci) { $env:DOTNET_CLI_TELEMETRY_OPTOUT=1 - - # In case of network error, try to log the current IP for reference - Try-LogClientIpAddress } # Source Build uses DotNetCoreSdkDir variable @@ -301,32 +298,45 @@ function InstallDotNet([string] $dotnetRoot, if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles } if ($noPath) { $installParameters.NoPath = $True } - try { - & $installScript @installParameters - } - catch { - if ($runtimeSourceFeed -or $runtimeSourceFeedKey) { - Write-Host "Failed to install dotnet from public location. Trying from '$runtimeSourceFeed'" - if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed } + $variations = @() + $variations += @($installParameters) - if ($runtimeSourceFeedKey) { - $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey) - $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes) - $installParameters.FeedCredential = $decodedString - } + $dotnetBuilds = $installParameters.Clone() + $dotnetbuilds.AzureFeed = "https://dotnetbuilds.azureedge.net/public" + $variations += @($dotnetBuilds) - try { - & $installScript @installParameters - } - catch { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from custom location '$runtimeSourceFeed'." - ExitWithExitCode 1 - } + if ($runtimeSourceFeed) { + $runtimeSource = $installParameters.Clone() + $runtimeSource.AzureFeed = $runtimeSourceFeed + if ($runtimeSourceFeedKey) { + $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey) + $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes) + $runtimeSource.FeedCredential = $decodedString + } + $variations += @($runtimeSource) + } + + $installSuccess = $false + foreach ($variation in $variations) { + if ($variation | Get-Member AzureFeed) { + $location = $variation.AzureFeed } else { - Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from public location." - ExitWithExitCode 1 + $location = "public location"; + } + Write-Host "Attempting to install dotnet from $location." + try { + & $installScript @variation + $installSuccess = $true + break + } + catch { + Write-Host "Failed to install dotnet from $location." } } + if (-not $installSuccess) { + Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from any of the specified locations." + ExitWithExitCode 1 + } } # @@ -882,24 +892,6 @@ if (!$disableConfigureToolsetImport) { } } -function Try-LogClientIpAddress() -{ - Write-Host "Attempting to log this client's IP for Azure Package feed telemetry purposes" - try - { - $result = Invoke-WebRequest -Uri "http://co1.msedge.net/fdv2/diagnostics.aspx" -UseBasicParsing - $lines = $result.Content.Split([Environment]::NewLine) - $socketIp = $lines | Select-String -Pattern "^Socket IP:.*" - Write-Host $socketIp - $clientIp = $lines | Select-String -Pattern "^Client IP:.*" - Write-Host $clientIp - } - catch - { - Write-Host "Unable to get this machine's effective IP address for logging: $_" - } -} - # # If $ci flag is set, turn on (and log that we did) special environment variables for improved Nuget client retry logic. # diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 6a4871ef..e555c342 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -188,28 +188,29 @@ function InstallDotNet { GetDotNetInstallScript "$root" local install_script=$_GetDotNetInstallScript - local archArg='' + local installParameters=(--version $version --install-dir "$root") + if [[ -n "${3:-}" ]] && [ "$3" != 'unset' ]; then - archArg="--architecture $3" + installParameters+=(--architecture $3) fi - local runtimeArg='' if [[ -n "${4:-}" ]] && [ "$4" != 'sdk' ]; then - runtimeArg="--runtime $4" + installParameters+=(--runtime $4) fi - local skipNonVersionedFilesArg="" if [[ "$#" -ge "5" ]] && [[ "$5" != 'false' ]]; then - skipNonVersionedFilesArg="--skip-non-versioned-files" + installParameters+=(--skip-non-versioned-files) fi - bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { - local exit_code=$? - echo "Failed to install dotnet SDK from public location (exit code '$exit_code')." - local runtimeSourceFeed='' - if [[ -n "${6:-}" ]]; then - runtimeSourceFeed="--azure-feed $6" - fi + local variations=() # list of variable names with parameter arrays in them + + local public_location=("${installParameters[@]}") + variations+=(public_location) - local runtimeSourceFeedKey='' + local dotnetbuilds=("${installParameters[@]}" --azure-feed "https://dotnetbuilds.azureedge.net/public") + variations+=(dotnetbuilds) + + if [[ -n "${6:-}" ]]; then + variations+=(private_feed) + local private_feed=("${installParameters[@]}" --azure-feed $6) if [[ -n "${7:-}" ]]; then # The 'base64' binary on alpine uses '-d' and doesn't support '--decode' # '-d'. To work around this, do a simple detection and switch the parameter @@ -219,22 +220,27 @@ function InstallDotNet { decodeArg="-d" fi decodedFeedKey=`echo $7 | base64 $decodeArg` - runtimeSourceFeedKey="--feed-credential $decodedFeedKey" + private_feed+=(--feed-credential $decodedFeedKey) fi + fi - if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then - bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || { - local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')." - ExitWithExitCode $exit_code - } - else - if [[ $exit_code != 0 ]]; then - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')." - fi - ExitWithExitCode $exit_code + local installSuccess=0 + for variationName in "${variations[@]}"; do + local name="$variationName[@]" + local variation=("${!name}") + echo "Attempting to install dotnet from $variationName." + bash "$install_script" "${variation[@]}" && installSuccess=1 + if [[ "$installSuccess" -eq 1 ]]; then + break fi - } + + echo "Failed to install dotnet from $variationName." + done + + if [[ "$installSuccess" -eq 0 ]]; then + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from any of the specified locations." + ExitWithExitCode 1 + fi } function with_retries { @@ -399,13 +405,6 @@ function StopProcesses { return 0 } -function TryLogClientIpAddress () { - echo 'Attempting to log this client''s IP for Azure Package feed telemetry purposes' - if command -v curl > /dev/null; then - curl -s 'http://co1.msedge.net/fdv2/diagnostics.aspx' | grep ' IP: ' || true - fi -} - function MSBuild { local args=$@ if [[ "$pipelines_log" == true ]]; then diff --git a/global.json b/global.json index 1b6c589f..6a39d292 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "6.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21559.3" + "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21576.4" } } From 6135f22a3945e86472d04fab03d6b22f0eb715b3 Mon Sep 17 00:00:00 2001 From: tmat Date: Wed, 15 Dec 2021 12:20:37 -0800 Subject: [PATCH 004/282] Workaround nuget bug https://github.com/NuGet/Home/issues/11455 --- .../DotNetSdk/DotNetSdkTestBase.cs | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs b/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs index 8608bf80..4fcb8fa2 100644 --- a/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs +++ b/src/TestUtilities/DotNetSdk/DotNetSdkTestBase.cs @@ -7,6 +7,7 @@ using System.IO; using System.Linq; using System.Reflection; +using System.Text; using System.Text.RegularExpressions; using NuGet.Versioning; using Xunit; @@ -191,7 +192,7 @@ public DotNetSdkTestBase(params string[] packages) { "MSBuildSDKsPath", sdksDir }, { "DOTNET_MSBUILD_SDK_RESOLVER_SDKS_DIR", sdksDir }, { "NUGET_PACKAGES", NuGetCacheDir.Path }, - { "NuGetPackageFolders", NuGetPackageFolders } + { "NuGetPackageFolders", NuGetPackageFolders }, }; ProjectDir = RootDir.CreateDirectory(ProjectName); @@ -241,9 +242,12 @@ protected void VerifyValues( Assert.True(restoreResult.ExitCode == 0, $"Failed with exit code {restoreResult.ExitCode}: {restoreResult.Output}"); Assert.True(File.Exists(Path.Combine(ProjectObjDir.Path, "project.assets.json"))); - Assert.True(File.Exists(Path.Combine(ProjectObjDir.Path, ProjectFileName + ".nuget.g.props"))); + var generatedPropsFilePath = Path.Combine(ProjectObjDir.Path, ProjectFileName + ".nuget.g.props"); + Assert.True(File.Exists(generatedPropsFilePath)); Assert.True(File.Exists(Path.Combine(ProjectObjDir.Path, ProjectFileName + ".nuget.g.targets"))); + FixupGeneratedPropsFilePath(generatedPropsFilePath); + _projectRestored = true; } @@ -301,5 +305,15 @@ static bool diagnosticsEqual(string expected, string actual) } } } + + // Workaround for https://github.com/NuGet/Home/issues/11455 + private void FixupGeneratedPropsFilePath(string generatedPropsFilePath) + { + var content = File.ReadAllText(generatedPropsFilePath, Encoding.UTF8); + int i = 0; + content = Regex.Replace(content, "", + match => (i++ == 0) ? $"" : ""); + File.WriteAllText(generatedPropsFilePath, content, Encoding.UTF8); + } } } From 9792af26bb96a3d866057fe6236c93b887f4650c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Tue, 14 Dec 2021 11:19:50 -0800 Subject: [PATCH 005/282] Use the "latest" VM images --- azure-pipelines.yml | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 98844bf5..28e1a77d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -49,8 +49,7 @@ stages: - job: Windows pool: ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - name: NetCore1ESPool-Public - demands: ImageOverride -equals Build.Windows.10.Amd64.VS2017.Open + vmImage: 'windows-latest' ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: name: NetCore1ESPool-Internal demands: ImageOverride -equals Build.Windows.10.Amd64.VS2017 @@ -108,7 +107,8 @@ stages: - job: MacOS displayName: 'MacOS' - pool: Hosted macOS + pool: + vmImage: 'macOS-latest' strategy: matrix: Debug: @@ -128,12 +128,7 @@ stages: - job: Linux displayName: 'Linux' pool: - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - name: NetCore1ESPool-Public - demands: ImageOverride -equals Build.Ubuntu.1604.Amd64.Open - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - name: NetCore1ESPool-Internal - demands: ImageOverride -equals Build.Ubuntu.1604.Amd64 + vmImage: 'ubuntu-latest' strategy: matrix: Debug: From 5f7474843bfcd3bb1c86df7514eb2480347c3654 Mon Sep 17 00:00:00 2001 From: tmat Date: Wed, 15 Dec 2021 17:32:53 -0800 Subject: [PATCH 006/282] Update libgit to 0.27.0-preview-0119 --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 71744961..46b8d64d 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -19,6 +19,6 @@ 4.5.0 4.7.2 - 0.26.0-preview-0070 + 0.27.0-preview-0119 From 8713c6adfc163fa0caec2d6ae0452bc7ed0cae92 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 16 Dec 2021 16:56:42 +0000 Subject: [PATCH 007/282] [main] Update dependencies from dotnet/arcade (#775) [main] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 +- eng/common/cross/armv6/sources.list.buster | 2 + eng/common/cross/build-rootfs.sh | 17 ++++- eng/common/sdl/packages.config | 2 +- eng/common/templates/job/execute-sdl.yml | 69 +++---------------- eng/common/templates/steps/execute-sdl.yml | 68 ++++++++++++++++++ .../templates/variables/sdl-variables.yml | 7 ++ eng/common/tools.sh | 2 +- global.json | 2 +- 9 files changed, 107 insertions(+), 66 deletions(-) create mode 100644 eng/common/cross/armv6/sources.list.buster create mode 100644 eng/common/templates/steps/execute-sdl.yml create mode 100644 eng/common/templates/variables/sdl-variables.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 4d49a69d..8c1448bd 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 427c05909067bb2e484116ae2239456bb45adb85 + 18adc5b47acce8bb03948baf578fca442d1029d4 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/cross/armv6/sources.list.buster b/eng/common/cross/armv6/sources.list.buster new file mode 100644 index 00000000..f27fc4fb --- /dev/null +++ b/eng/common/cross/armv6/sources.list.buster @@ -0,0 +1,2 @@ +deb http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi +deb-src http://raspbian.raspberrypi.org/raspbian/ buster main contrib non-free rpi diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 6fa2c8aa..5102245b 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -99,6 +99,15 @@ while :; do __AlpineArch=armv7 __QEMUArch=arm ;; + armv6) + __BuildArch=armv6 + __UbuntuArch=armhf + __QEMUArch=arm + __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/" + __CodeName=buster + __LLDB_Package="liblldb-6.0-dev" + __Keyring="/usr/share/keyrings/raspbian-archive-keyring.gpg" + ;; arm64) __BuildArch=arm64 __UbuntuArch=arm64 @@ -236,6 +245,12 @@ while :; do shift done +if [ -e "$__Keyring" ]; then + __Keyring="--keyring=$__Keyring" +else + __Keyring="" +fi + if [ "$__BuildArch" == "armel" ]; then __LLDB_Package="lldb-3.5-dev" fi @@ -337,7 +352,7 @@ elif [[ "$__CodeName" == "illumos" ]]; then wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h elif [[ -n $__CodeName ]]; then - qemu-debootstrap --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo + qemu-debootstrap $__Keyring --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list chroot $__RootfsDir apt-get update chroot $__RootfsDir apt-get -f -y install diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 3bd8b29e..4585cfd6 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 3aafc82e..8cf772b3 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -51,14 +51,9 @@ jobs: value: ${{ parameters.AzDOPipelineId }} - name: AzDOBuildId value: ${{ parameters.AzDOBuildId }} - # The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in - # sync with the packages.config file. - - name: DefaultGuardianVersion - value: 0.53.3 + - template: /eng/common/templates/variables/sdl-variables.yml - name: GuardianVersion value: ${{ coalesce(parameters.overrideGuardianVersion, '$(DefaultGuardianVersion)') }} - - name: GuardianPackagesConfigFile - value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config pool: vmImage: windows-2019 steps: @@ -125,57 +120,11 @@ jobs: displayName: Extract Archive Artifacts continueOnError: ${{ parameters.sdlContinueOnError }} - - ${{ if ne(parameters.overrideGuardianVersion, '') }}: - - powershell: | - $content = Get-Content $(GuardianPackagesConfigFile) - - Write-Host "packages.config content was:`n$content" - - $content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)') - $content | Set-Content $(GuardianPackagesConfigFile) - - Write-Host "packages.config content updated to:`n$content" - displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }} - - - task: NuGetToolInstaller@1 - displayName: 'Install NuGet.exe' - - task: NuGetCommand@2 - displayName: 'Install Guardian' - inputs: - restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config - feedsToUse: config - nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config - externalFeedCredentials: GuardianConnect - restoreDirectory: $(Build.SourcesDirectory)\.packages - - - ${{ if ne(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} - displayName: Execute SDL - continueOnError: ${{ parameters.sdlContinueOnError }} - - ${{ if eq(parameters.overrideParameters, '') }}: - - powershell: ${{ parameters.executeAllSdlToolsScript }} - -GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion) - -NugetPackageDirectory $(Build.SourcesDirectory)\.packages - -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) - ${{ parameters.additionalParameters }} - displayName: Execute SDL - continueOnError: ${{ parameters.sdlContinueOnError }} - - - ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: - # We want to publish the Guardian results and configuration for easy diagnosis. However, the - # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default - # tooling files. Some of these files are large and aren't useful during an investigation, so - # exclude them by simply deleting them before publishing. (As of writing, there is no documented - # way to selectively exclude a dir from the pipeline artifact publish task.) - - task: DeleteFiles@1 - displayName: Delete Guardian dependencies to avoid uploading - inputs: - SourceFolder: $(Agent.BuildDirectory)/.gdn - Contents: | - c - i - condition: succeededOrFailed() - - publish: $(Agent.BuildDirectory)/.gdn - artifact: GuardianConfiguration - displayName: Publish GuardianConfiguration - condition: succeededOrFailed() + - template: /eng/common/templates/steps/execute-sdl.yml + parameters: + overrideGuardianVersion: ${{ parameters.overrideGuardianVersion }} + executeAllSdlToolsScript: ${{ parameters.executeAllSdlToolsScript }} + overrideParameters: ${{ parameters.overrideParameters }} + additionalParameters: ${{ parameters.additionalParameters }} + publishGuardianDirectoryToPipeline: ${{ parameters.publishGuardianDirectoryToPipeline }} + sdlContinueOnError: ${{ parameters.sdlContinueOnError }} diff --git a/eng/common/templates/steps/execute-sdl.yml b/eng/common/templates/steps/execute-sdl.yml new file mode 100644 index 00000000..7b8ee18a --- /dev/null +++ b/eng/common/templates/steps/execute-sdl.yml @@ -0,0 +1,68 @@ +parameters: + overrideGuardianVersion: '' + executeAllSdlToolsScript: '' + overrideParameters: '' + additionalParameters: '' + publishGuardianDirectoryToPipeline: false + sdlContinueOnError: false + condition: '' + +steps: +- ${{ if ne(parameters.overrideGuardianVersion, '') }}: + - powershell: | + $content = Get-Content $(GuardianPackagesConfigFile) + + Write-Host "packages.config content was:`n$content" + + $content = $content.Replace('$(DefaultGuardianVersion)', '$(GuardianVersion)') + $content | Set-Content $(GuardianPackagesConfigFile) + + Write-Host "packages.config content updated to:`n$content" + displayName: Use overridden Guardian version ${{ parameters.overrideGuardianVersion }} + +- task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + +- task: NuGetCommand@2 + displayName: 'Install Guardian' + inputs: + restoreSolution: $(Build.SourcesDirectory)\eng\common\sdl\packages.config + feedsToUse: config + nugetConfigPath: $(Build.SourcesDirectory)\eng\common\sdl\NuGet.config + externalFeedCredentials: GuardianConnect + restoreDirectory: $(Build.SourcesDirectory)\.packages + +- ${{ if ne(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} ${{ parameters.overrideParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if eq(parameters.overrideParameters, '') }}: + - powershell: ${{ parameters.executeAllSdlToolsScript }} + -GuardianPackageName Microsoft.Guardian.Cli.$(GuardianVersion) + -NugetPackageDirectory $(Build.SourcesDirectory)\.packages + -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) + ${{ parameters.additionalParameters }} + displayName: Execute SDL + continueOnError: ${{ parameters.sdlContinueOnError }} + condition: ${{ parameters.condition }} + +- ${{ if ne(parameters.publishGuardianDirectoryToPipeline, 'false') }}: + # We want to publish the Guardian results and configuration for easy diagnosis. However, the + # '.gdn' dir is a mix of configuration, results, extracted dependencies, and Guardian default + # tooling files. Some of these files are large and aren't useful during an investigation, so + # exclude them by simply deleting them before publishing. (As of writing, there is no documented + # way to selectively exclude a dir from the pipeline artifact publish task.) + - task: DeleteFiles@1 + displayName: Delete Guardian dependencies to avoid uploading + inputs: + SourceFolder: $(Agent.BuildDirectory)/.gdn + Contents: | + c + i + condition: succeededOrFailed() + - publish: $(Agent.BuildDirectory)/.gdn + artifact: GuardianConfiguration + displayName: Publish GuardianConfiguration + condition: succeededOrFailed() \ No newline at end of file diff --git a/eng/common/templates/variables/sdl-variables.yml b/eng/common/templates/variables/sdl-variables.yml new file mode 100644 index 00000000..dbdd66d4 --- /dev/null +++ b/eng/common/templates/variables/sdl-variables.yml @@ -0,0 +1,7 @@ +variables: +# The Guardian version specified in 'eng/common/sdl/packages.config'. This value must be kept in +# sync with the packages.config file. +- name: DefaultGuardianVersion + value: 0.109.0 +- name: GuardianPackagesConfigFile + value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config \ No newline at end of file diff --git a/eng/common/tools.sh b/eng/common/tools.sh index e555c342..17f0a365 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -178,7 +178,7 @@ function InstallDotNetSdk { if [[ $# -ge 3 ]]; then architecture=$3 fi - InstallDotNet "$root" "$version" $architecture 'sdk' 'false' $runtime_source_feed $runtime_source_feed_key + InstallDotNet "$root" "$version" $architecture 'sdk' 'true' $runtime_source_feed $runtime_source_feed_key } function InstallDotNet { diff --git a/global.json b/global.json index 6a39d292..22d13d4d 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "6.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21576.4" + "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21610.4" } } From e14f1c4ba657051e40b749d26b61d5680871a716 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Matou=C5=A1ek?= Date: Thu, 16 Dec 2021 08:35:30 -0800 Subject: [PATCH 008/282] Update Versions.props --- eng/Versions.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/Versions.props b/eng/Versions.props index 46b8d64d..df587adf 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -3,7 +3,7 @@ - 1.1.1 + 1.2.0 beta true From fd61d7b341e28abd71cf73083b83dc95eb5694cb Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 17 Dec 2021 01:05:27 +0000 Subject: [PATCH 009/282] [main] Update dependencies from dotnet/source-build-reference-packages (#779) [main] Update dependencies from dotnet/source-build-reference-packages --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8c1448bd..f8eba283 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -7,9 +7,9 @@ https://github.com/dotnet/arcade 18adc5b47acce8bb03948baf578fca442d1029d4 - + https://github.com/dotnet/source-build-reference-packages - def2e2c6dc5064319250e2868a041a3dc07f9579 + 03d849574a22e9a124898f341166082f881a3ad0 From 392126dabae557bf0664f599b5469ccc6d1dfcd8 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Mon, 20 Dec 2021 13:29:04 +0000 Subject: [PATCH 010/282] Update dependencies from https://github.com/dotnet/arcade build 20211217.4 (#781) [main] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 ++-- global.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f8eba283..f4550eb6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 18adc5b47acce8bb03948baf578fca442d1029d4 + 78659a1d4831ce9d62ea817fe13e4e2e70a52961 https://github.com/dotnet/source-build-reference-packages diff --git a/global.json b/global.json index 22d13d4d..e6cc1fc2 100644 --- a/global.json +++ b/global.json @@ -3,6 +3,6 @@ "dotnet": "6.0.100" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21610.4" + "Microsoft.DotNet.Arcade.Sdk": "7.0.0-beta.21617.4" } } From 427936db7c8a37605c4116c4fc785bd0196f34fe Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 29 Dec 2021 13:54:03 +0000 Subject: [PATCH 011/282] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20211228.1 (#784) [main] Update dependencies from dotnet/source-build-reference-packages --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index f4550eb6..97e7035e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -7,9 +7,9 @@ https://github.com/dotnet/arcade 78659a1d4831ce9d62ea817fe13e4e2e70a52961 - + https://github.com/dotnet/source-build-reference-packages - 03d849574a22e9a124898f341166082f881a3ad0 + 2f4e9eb4da9d3eb5d9c4346e0b11449a2995f0f3 From 04c1b6d4da38e8e47593f4f98c3a620715e375ea Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 6 Jan 2022 13:52:22 +0000 Subject: [PATCH 012/282] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20220105.1 (#785) [main] Update dependencies from dotnet/source-build-reference-packages --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 97e7035e..a385d15e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -7,9 +7,9 @@ https://github.com/dotnet/arcade 78659a1d4831ce9d62ea817fe13e4e2e70a52961 - + https://github.com/dotnet/source-build-reference-packages - 2f4e9eb4da9d3eb5d9c4346e0b11449a2995f0f3 + 130938590ba0d472860b8aa7e9885473dee42ac7 From 192fa29996b63bd32b0b112e7757e2ca1c2bfbbb Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 21 Jan 2022 13:35:28 +0000 Subject: [PATCH 013/282] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20220120.2 (#789) [main] Update dependencies from dotnet/source-build-reference-packages --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a385d15e..b06775e0 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -7,9 +7,9 @@ https://github.com/dotnet/arcade 78659a1d4831ce9d62ea817fe13e4e2e70a52961 - + https://github.com/dotnet/source-build-reference-packages - 130938590ba0d472860b8aa7e9885473dee42ac7 + e756a6d20bf6cbcbe3a9cf2f95d3bf5c9fcc09d2 From 87d2071747065ced69cdf96e59b5a6709011648f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Sat, 29 Jan 2022 20:41:14 +0000 Subject: [PATCH 014/282] [main] Update dependencies from dotnet/arcade (#792) [main] Update dependencies from dotnet/arcade --- eng/Version.Details.xml | 4 +- eng/common/cross/arm/tizen-build-rootfs.sh | 35 + eng/common/cross/arm/tizen-fetch.sh | 170 +++++ eng/common/cross/arm/tizen/tizen.patch | 9 + eng/common/cross/build-rootfs.sh | 4 +- eng/common/cross/toolchain.cmake | 54 +- eng/common/dotnet-install.sh | 3 + eng/common/generate-graph-files.ps1 | 86 --- eng/common/internal/NuGet.config | 7 + eng/common/native/CommonLibrary.psm1 | 3 +- eng/common/native/init-compiler.sh | 32 +- eng/common/post-build/publish-using-darc.ps1 | 28 +- eng/common/sdl/configure-sdl-tool.ps1 | 9 +- eng/common/sdl/execute-all-sdl-tools.ps1 | 4 +- eng/common/templates/job/execute-sdl.yml | 13 +- .../templates/job/generate-graph-files.yml | 48 -- eng/common/templates/job/onelocbuild.yml | 3 +- eng/common/templates/jobs/codeql-build.yml | 31 + eng/common/templates/jobs/jobs.yml | 14 +- .../channels/generic-internal-channel.yml | 190 ------ .../channels/generic-public-channel.yml | 192 ------ .../templates/post-build/common-variables.yml | 69 -- .../templates/post-build/post-build.yml | 600 ++++-------------- .../post-build/setup-maestro-vars.yml | 123 ++-- eng/common/templates/steps/execute-codeql.yml | 32 + eng/common/templates/steps/source-build.yml | 2 +- global.json | 2 +- 27 files changed, 569 insertions(+), 1198 deletions(-) create mode 100644 eng/common/cross/arm/tizen-build-rootfs.sh create mode 100644 eng/common/cross/arm/tizen-fetch.sh create mode 100644 eng/common/cross/arm/tizen/tizen.patch delete mode 100644 eng/common/generate-graph-files.ps1 create mode 100644 eng/common/internal/NuGet.config delete mode 100644 eng/common/templates/job/generate-graph-files.yml create mode 100644 eng/common/templates/jobs/codeql-build.yml delete mode 100644 eng/common/templates/post-build/channels/generic-internal-channel.yml delete mode 100644 eng/common/templates/post-build/channels/generic-public-channel.yml create mode 100644 eng/common/templates/steps/execute-codeql.yml diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b06775e0..d711b923 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - 78659a1d4831ce9d62ea817fe13e4e2e70a52961 + e1ea8873d2e0175c1d1e49b3884cd2e9c6ef5007 https://github.com/dotnet/source-build-reference-packages diff --git a/eng/common/cross/arm/tizen-build-rootfs.sh b/eng/common/cross/arm/tizen-build-rootfs.sh new file mode 100644 index 00000000..9fdb32e9 --- /dev/null +++ b/eng/common/cross/arm/tizen-build-rootfs.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -e + +__ARM_HARDFP_CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +__TIZEN_CROSSDIR="$__ARM_HARDFP_CrossDir/tizen" + +if [[ -z "$ROOTFS_DIR" ]]; then + echo "ROOTFS_DIR is not defined." + exit 1; +fi + +TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp +mkdir -p $TIZEN_TMP_DIR + +# Download files +echo ">>Start downloading files" +VERBOSE=1 $__ARM_HARDFP_CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR +echo "<>Start constructing Tizen rootfs" +TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` +cd $ROOTFS_DIR +for f in $TIZEN_RPM_FILES; do + rpm2cpio $f | cpio -idm --quiet +done +echo "<>Start configuring Tizen rootfs" +ln -sfn asm-arm ./usr/include/asm +patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +echo "</dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_URL=http://download.tizen.org/snapshots/tizen +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize arm base" +fetch_tizen_pkgs_init standard base +Inform "fetch common packages" +fetch_tizen_pkgs armv7hl gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils +Inform "fetch coreclr packages" +fetch_tizen_pkgs armv7hl lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7hl libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard unified +Inform "fetch corefx packages" +fetch_tizen_pkgs armv7hl gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/arm/tizen/tizen.patch b/eng/common/cross/arm/tizen/tizen.patch new file mode 100644 index 00000000..fb12ade7 --- /dev/null +++ b/eng/common/cross/arm/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-littlearm) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux-armhf.so.3 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-armhf.so.3 ) ) diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 5102245b..e94d13d6 100755 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -185,8 +185,8 @@ while :; do __LLDB_Package="liblldb-6.0-dev" ;; tizen) - if [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ]; then - echo "Tizen is available only for armel and arm64." + if [ "$__BuildArch" != "arm" ] && [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ]; then + echo "Tizen is available only for arm, armel and arm64." usage; exit 1; fi diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 6501c3a9..fba2afda 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -3,18 +3,26 @@ set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) set(CMAKE_SYSTEM_NAME FreeBSD) + set(FREEBSD 1) elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) set(CMAKE_SYSTEM_NAME SunOS) set(ILLUMOS 1) else() set(CMAKE_SYSTEM_NAME Linux) + set(LINUX 1) endif() set(CMAKE_SYSTEM_VERSION 1) +if(EXISTS ${CROSS_ROOTFS}/etc/tizen-release) + set(TIZEN 1) +elseif(EXISTS ${CROSS_ROOTFS}/android_platform) + set(ANDROID 1) +endif() + if(TARGET_ARCH_NAME STREQUAL "armel") set(CMAKE_SYSTEM_PROCESSOR armv7l) set(TOOLCHAIN "arm-linux-gnueabi") - if("$ENV{__DistroRid}" MATCHES "tizen.*") + if(TIZEN) set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0") endif() elseif(TARGET_ARCH_NAME STREQUAL "arm") @@ -26,6 +34,16 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm") else() set(TOOLCHAIN "arm-linux-gnueabihf") endif() + if(TIZEN) + set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0") + endif() +elseif(TARGET_ARCH_NAME STREQUAL "armv6") + set(CMAKE_SYSTEM_PROCESSOR armv6l) + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf) + set(TOOLCHAIN "armv6-alpine-linux-musleabihf") + else() + set(TOOLCHAIN "arm-linux-gnueabihf") + endif() elseif(TARGET_ARCH_NAME STREQUAL "arm64") set(CMAKE_SYSTEM_PROCESSOR aarch64) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/aarch64-alpine-linux-musl) @@ -33,7 +51,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") else() set(TOOLCHAIN "aarch64-linux-gnu") endif() - if("$ENV{__DistroRid}" MATCHES "tizen.*") + if(TIZEN) set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") endif() elseif(TARGET_ARCH_NAME STREQUAL "s390x") @@ -42,14 +60,14 @@ elseif(TARGET_ARCH_NAME STREQUAL "s390x") elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) set(TOOLCHAIN "i686-linux-gnu") -elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") +elseif (FREEBSD) set(CMAKE_SYSTEM_PROCESSOR "x86_64") set(triple "x86_64-unknown-freebsd12") elseif (ILLUMOS) set(CMAKE_SYSTEM_PROCESSOR "x86_64") set(TOOLCHAIN "x86_64-illumos") else() - message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, arm64, s390x and x86 are supported!") + message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, armv6, arm64, s390x and x86 are supported!") endif() if(DEFINED ENV{TOOLCHAIN}) @@ -57,7 +75,11 @@ if(DEFINED ENV{TOOLCHAIN}) endif() # Specify include paths -if(DEFINED TIZEN_TOOLCHAIN) +if(TIZEN) + if(TARGET_ARCH_NAME STREQUAL "arm") + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7hl-tizen-linux-gnueabihf) + endif() if(TARGET_ARCH_NAME STREQUAL "armel") include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/armv7l-tizen-linux-gnueabi) @@ -68,7 +90,7 @@ if(DEFINED TIZEN_TOOLCHAIN) endif() endif() -if("$ENV{__DistroRid}" MATCHES "android.*") +if(ANDROID) if(TARGET_ARCH_NAME STREQUAL "arm") set(ANDROID_ABI armeabi-v7a) elseif(TARGET_ARCH_NAME STREQUAL "arm64") @@ -76,7 +98,9 @@ if("$ENV{__DistroRid}" MATCHES "android.*") endif() # extract platform number required by the NDK's toolchain - string(REGEX REPLACE ".*\\.([0-9]+)-.*" "\\1" ANDROID_PLATFORM "$ENV{__DistroRid}") + file(READ "${CROSS_ROOTFS}/android_platform" RID_FILE_CONTENTS) + string(REPLACE "RID=" "" ANDROID_RID "${RID_FILE_CONTENTS}") + string(REGEX REPLACE ".*\\.([0-9]+)-.*" "\\1" ANDROID_PLATFORM "${ANDROID_RID}") set(ANDROID_TOOLCHAIN clang) set(FEATURE_EVENT_TRACE 0) # disable event trace as there is no lttng-ust package in termux repository @@ -85,7 +109,7 @@ if("$ENV{__DistroRid}" MATCHES "android.*") # include official NDK toolchain script include(${CROSS_ROOTFS}/../build/cmake/android.toolchain.cmake) -elseif(CMAKE_SYSTEM_NAME STREQUAL "FreeBSD") +elseif(FREEBSD) # we cross-compile by instructing clang set(CMAKE_C_COMPILER_TARGET ${triple}) set(CMAKE_CXX_COMPILER_TARGET ${triple}) @@ -145,20 +169,20 @@ function(add_toolchain_linker_flag Flag) set("CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT" "${CMAKE_SHARED_LINKER_FLAGS${CONFIG_SUFFIX}_INIT} ${Flag}" PARENT_SCOPE) endfunction() -if(CMAKE_SYSTEM_NAME STREQUAL "Linux") +if(LINUX) add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/lib/${TOOLCHAIN}") add_toolchain_linker_flag("-Wl,--rpath-link=${CROSS_ROOTFS}/usr/lib/${TOOLCHAIN}") endif() -if(TARGET_ARCH_NAME STREQUAL "armel") - if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only +if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") + if(TIZEN) add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") endif() elseif(TARGET_ARCH_NAME STREQUAL "arm64") - if(DEFINED TIZEN_TOOLCHAIN) # For Tizen only + if(TIZEN) add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib64") @@ -177,7 +201,7 @@ endif() # Specify compile options -if((TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|s390x)$" AND NOT "$ENV{__DistroRid}" MATCHES "android.*") OR ILLUMOS) +if((TARGET_ARCH_NAME MATCHES "^(arm|armv6|armel|arm64|s390x)$" AND NOT ANDROID) OR ILLUMOS) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) @@ -204,8 +228,8 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") add_compile_options(-Wno-error=unused-command-line-argument) endif() -if(DEFINED TIZEN_TOOLCHAIN) - if(TARGET_ARCH_NAME MATCHES "^(armel|arm64)$") +if(TIZEN) + if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$") add_compile_options(-Wno-deprecated-declarations) # compile-time option add_compile_options(-D__extern_always_inline=inline) # compile-time option endif() diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index fdfeea66..5c94e986 100755 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -55,6 +55,9 @@ case $cpuname in aarch64) buildarch=arm64 ;; + loongarch64) + buildarch=loongarch64 + ;; amd64|x86_64) buildarch=x64 ;; diff --git a/eng/common/generate-graph-files.ps1 b/eng/common/generate-graph-files.ps1 deleted file mode 100644 index 0728b1a8..00000000 --- a/eng/common/generate-graph-files.ps1 +++ /dev/null @@ -1,86 +0,0 @@ -Param( - [Parameter(Mandatory=$true)][string] $barToken, # Token generated at https://maestro-prod.westus2.cloudapp.azure.com/Account/Tokens - [Parameter(Mandatory=$true)][string] $gitHubPat, # GitHub personal access token from https://github.com/settings/tokens (no auth scopes needed) - [Parameter(Mandatory=$true)][string] $azdoPat, # Azure Dev Ops tokens from https://dev.azure.com/dnceng/_details/security/tokens (code read scope needed) - [Parameter(Mandatory=$true)][string] $outputFolder, # Where the graphviz.txt file will be created - [string] $darcVersion, # darc's version - [string] $graphvizVersion = '2.38', # GraphViz version - [switch] $includeToolset # Whether the graph should include toolset dependencies or not. i.e. arcade, optimization. For more about - # toolset dependencies see https://github.com/dotnet/arcade/blob/master/Documentation/Darc.md#toolset-vs-product-dependencies -) - -function CheckExitCode ([string]$stage) -{ - $exitCode = $LASTEXITCODE - if ($exitCode -ne 0) { - Write-PipelineTelemetryError -Category 'Arcade' -Message "Something failed in stage: '$stage'. Check for errors above. Exiting now..." - ExitWithExitCode $exitCode - } -} - -try { - $ErrorActionPreference = 'Stop' - . $PSScriptRoot\tools.ps1 - - Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') - - Push-Location $PSScriptRoot - - Write-Host 'Installing darc...' - . .\darc-init.ps1 -darcVersion $darcVersion - CheckExitCode 'Running darc-init' - - $engCommonBaseDir = Join-Path $PSScriptRoot 'native\' - $graphvizInstallDir = CommonLibrary\Get-NativeInstallDirectory - $nativeToolBaseUri = 'https://netcorenativeassets.blob.core.windows.net/resource-packages/external' - $installBin = Join-Path $graphvizInstallDir 'bin' - - Write-Host 'Installing dot...' - .\native\install-tool.ps1 -ToolName graphviz -InstallPath $installBin -BaseUri $nativeToolBaseUri -CommonLibraryDirectory $engCommonBaseDir -Version $graphvizVersion -Verbose - - $darcExe = "$env:USERPROFILE\.dotnet\tools" - $darcExe = Resolve-Path "$darcExe\darc.exe" - - Create-Directory $outputFolder - - # Generate 3 graph descriptions: - # 1. Flat with coherency information - # 2. Graphviz (dot) file - # 3. Standard dependency graph - $graphVizFilePath = "$outputFolder\graphviz.txt" - $graphVizImageFilePath = "$outputFolder\graph.png" - $normalGraphFilePath = "$outputFolder\graph-full.txt" - $flatGraphFilePath = "$outputFolder\graph-flat.txt" - $baseOptions = @( '--github-pat', "$gitHubPat", '--azdev-pat', "$azdoPat", '--password', "$barToken" ) - - if ($includeToolset) { - Write-Host 'Toolsets will be included in the graph...' - $baseOptions += @( '--include-toolset' ) - } - - Write-Host 'Generating standard dependency graph...' - & "$darcExe" get-dependency-graph @baseOptions --output-file $normalGraphFilePath - CheckExitCode 'Generating normal dependency graph' - - Write-Host 'Generating flat dependency graph and graphviz file...' - & "$darcExe" get-dependency-graph @baseOptions --flat --coherency --graphviz $graphVizFilePath --output-file $flatGraphFilePath - CheckExitCode 'Generating flat and graphviz dependency graph' - - Write-Host "Generating graph image $graphVizFilePath" - $dotFilePath = Join-Path $installBin "graphviz\$graphvizVersion\release\bin\dot.exe" - & "$dotFilePath" -Tpng -o"$graphVizImageFilePath" "$graphVizFilePath" - CheckExitCode 'Generating graphviz image' - - Write-Host "'$graphVizFilePath', '$flatGraphFilePath', '$normalGraphFilePath' and '$graphVizImageFilePath' created!" -} -catch { - if (!$includeToolset) { - Write-Host 'This might be a toolset repo which includes only toolset dependencies. ' -NoNewline -ForegroundColor Yellow - Write-Host 'Since -includeToolset is not set there is no graph to create. Include -includeToolset and try again...' -ForegroundColor Yellow - } - Write-Host $_.ScriptStackTrace - Write-PipelineTelemetryError -Category 'Arcade' -Message $_ - ExitWithExitCode 1 -} finally { - Pop-Location -} \ No newline at end of file diff --git a/eng/common/internal/NuGet.config b/eng/common/internal/NuGet.config new file mode 100644 index 00000000..76965036 --- /dev/null +++ b/eng/common/internal/NuGet.config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/eng/common/native/CommonLibrary.psm1 b/eng/common/native/CommonLibrary.psm1 index adf707c8..ca38268c 100644 --- a/eng/common/native/CommonLibrary.psm1 +++ b/eng/common/native/CommonLibrary.psm1 @@ -276,7 +276,8 @@ function Get-MachineArchitecture { } if (($ProcessorArchitecture -Eq "AMD64") -Or ($ProcessorArchitecture -Eq "IA64") -Or - ($ProcessorArchitecture -Eq "ARM64")) { + ($ProcessorArchitecture -Eq "ARM64") -Or + ($ProcessorArchitecture -Eq "LOONGARCH64")) { return "x64" } return "x86" diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index e361e03f..6d7ba15e 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -2,25 +2,45 @@ # # This file detects the C/C++ compiler and exports it to the CC/CXX environment variables # -# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here! +# NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here! if [[ "$#" -lt 3 ]]; then echo "Usage..." - echo "init-compiler.sh