From 6a60556eb3b24cc1414d62d92dee234c24ef4a62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Thu, 30 Jun 2022 11:25:35 +0200 Subject: [PATCH 1/7] Upgrade to Newtonsoft.Json 13.0.1 (#3815) --- eng/Versions.props | 2 +- .../Microsoft.TestPlatform.Protocol.csproj | 4 +--- scripts/build.ps1 | 5 +++-- scripts/build.sh | 3 ++- scripts/build/TestPlatform.Dependencies.props | 2 +- .../Serialization/TestRunStatisticsConverter.cs | 4 ++-- .../DataCollectorAttachmentProcessorAppDomain.cs | 3 ++- .../DataCollectorAttachmentProcessorWrapper.cs | 2 +- .../Hosting/DotnetTestHostManager.cs | 4 ++-- src/package/ThirdPartyNotices.txt | 2 +- src/testhost.arm64/app.config | 7 ++++++- src/testhost.x86/app.config | 5 +++++ src/testhost/app.config | 7 ++++++- src/vstest.console/app.config | 7 ++++++- .../DotnetArchitectureSwitchTests.Windows.cs | 2 +- .../JsonDataSerializerTests.cs | 2 +- .../Serialization/TestCaseSerializationTests.cs | 2 +- .../Serialization/TestObjectConverterTests.cs | 2 +- .../Serialization/TestResultSerializationTests.cs | 2 +- .../Client/DiscoveryCriteriaTests.cs | 2 +- .../ProtocolV1Tests.cs | 2 +- .../ProtocolV2Tests.cs | 2 +- .../DotnetHostArchitectureVerifierTests.cs | 2 +- .../Hosting/DotnetTestHostManagerTests.cs | 10 +++++----- test/TestAssets/NewtonSoftDependency/App.config | 2 +- .../NewtonSoftDependency/NewtonSoftDependency.csproj | 4 +++- 26 files changed, 57 insertions(+), 34 deletions(-) diff --git a/eng/Versions.props b/eng/Versions.props index b00c4d42bf..4ab3942508 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -38,7 +38,7 @@ 4.16.1 5.3.0.1 2.3.0 - 9.0.1 + 13.0.1 4.6.0-preview4.19202.2 4.4.0 5.3.0 diff --git a/samples/Microsoft.TestPlatform.Protocol/Microsoft.TestPlatform.Protocol.csproj b/samples/Microsoft.TestPlatform.Protocol/Microsoft.TestPlatform.Protocol.csproj index 98f259f429..d1ed57fe43 100644 --- a/samples/Microsoft.TestPlatform.Protocol/Microsoft.TestPlatform.Protocol.csproj +++ b/samples/Microsoft.TestPlatform.Protocol/Microsoft.TestPlatform.Protocol.csproj @@ -7,8 +7,6 @@ - - 9.0.1 - + diff --git a/scripts/build.ps1 b/scripts/build.ps1 index f84e19c59e..2fe9aa0f5b 100644 --- a/scripts/build.ps1 +++ b/scripts/build.ps1 @@ -631,11 +631,12 @@ function Publish-Package { } # Copy dependency of Microsoft.TestPlatform.TestHostRuntimeProvider - $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\9.0.1\lib\net45\Newtonsoft.Json.dll" + $newtonsoftJsonVersion = ([xml](Get-Content $env:TP_ROOT_DIR\eng\Versions.props)).Project.PropertyGroup.NewtonsoftJsonVersion + $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\$newtonsoftJsonVersion\lib\net45\Newtonsoft.Json.dll" Write-Verbose "Copy-Item $newtonsoft $fullCLRPackage451Dir -Force" Copy-Item $newtonsoft $fullCLRPackage451Dir -Force - $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\9.0.1\lib\netstandard1.0\Newtonsoft.Json.dll" + $newtonsoft = Join-Path $env:TP_PACKAGES_DIR "newtonsoft.json\$newtonsoftJsonVersion\lib\netstandard1.0\Newtonsoft.Json.dll" Write-Verbose "Copy-Item $newtonsoft $coreCLR20PackageDir -Force" Copy-Item $newtonsoft $coreCLR20PackageDir -Force diff --git a/scripts/build.sh b/scripts/build.sh index 47e8162eb2..106f33507f 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -150,6 +150,7 @@ TPB_LocalizedBuild=$DISABLE_LOCALIZED_BUILD TPB_Verbose=$VERBOSE TPB_EXTERNALS_VERSION=$(grep TestPlatformExternalsVersion $TP_ROOT_DIR/scripts/build/TestPlatform.Dependencies.props | head -1 | cut -d'>' -f2 | cut -d'<' -f1 || echo $VERSION) TPB_CC_EXTERNALS_VERSION=$(grep MicrosoftInternalCodeCoverageVersion $TP_ROOT_DIR/eng/Versions.props | head -1 | cut -d'>' -f2 | cut -d'<' -f1 || echo $VERSION) +TPB_NEWTONSOFT_JSON_VERSION=$(grep NewtonsoftJsonVersion $TP_ROOT_DIR/eng/Versions.props | head -1 | cut -d'>' -f2 | cut -d'<' -f1 || echo $VERSION) TPB_BRANCH="$(git -C "." rev-parse --abbrev-ref HEAD 2>/dev/null)" || TPB_BRANCH="LOCALBRANCH" # detached HEAD TPB_COMMIT="$(git -C "." rev-parse HEAD 2>/dev/null)" || TPB_COMMIT="LOCALBUILD" # detached HEAD @@ -436,7 +437,7 @@ function publish_package() done #*************************************************************************************************************# - newtonsoft=$TP_PACKAGES_DIR/newtonsoft.json/9.0.1/lib/netstandard1.0/Newtonsoft.Json.dll + newtonsoft=$TP_PACKAGES_DIR/newtonsoft.json/$TPB_NEWTONSOFT_JSON_VERSION/lib/netstandard1.0/Newtonsoft.Json.dll cp $newtonsoft $packageDir done diff --git a/scripts/build/TestPlatform.Dependencies.props b/scripts/build/TestPlatform.Dependencies.props index 1ab12a55e6..5dbe4b63f1 100644 --- a/scripts/build/TestPlatform.Dependencies.props +++ b/scripts/build/TestPlatform.Dependencies.props @@ -57,7 +57,7 @@ 5.11.0 5.0.0 - 9.0.1 + 13.0.1 17.1.0-preview-2-31925-026 diff --git a/src/Microsoft.TestPlatform.CommunicationUtilities/Serialization/TestRunStatisticsConverter.cs b/src/Microsoft.TestPlatform.CommunicationUtilities/Serialization/TestRunStatisticsConverter.cs index 166dcc8c5e..d5b6186b85 100644 --- a/src/Microsoft.TestPlatform.CommunicationUtilities/Serialization/TestRunStatisticsConverter.cs +++ b/src/Microsoft.TestPlatform.CommunicationUtilities/Serialization/TestRunStatisticsConverter.cs @@ -23,13 +23,13 @@ public override bool CanConvert(Type objectType) } /// - public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) + public override object? ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) { return serializer.Deserialize(reader); } /// - public override void WriteJson(JsonWriter writer, object value, JsonSerializer serializer) + public override void WriteJson(JsonWriter writer, object? value, JsonSerializer serializer) { serializer.Serialize(writer, value); } diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentProcessorAppDomain.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentProcessorAppDomain.cs index 940ce8b000..f7f590664a 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentProcessorAppDomain.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentProcessorAppDomain.cs @@ -157,7 +157,8 @@ public async Task> ProcessAttachmentSetsAsync(XmlElem cancellationToken.Register(() => _wrapper.CancelProcessAttachment()); _processAttachmentSetsLogger = logger; _progressReporter = progressReporter; - return JsonDataSerializer.Instance.Deserialize(await Task.Run(() => _wrapper.ProcessAttachment(configurationElement.OuterXml, JsonDataSerializer.Instance.Serialize(attachments.ToArray()))).ConfigureAwait(false)); + var result = await Task.Run(() => _wrapper.ProcessAttachment(configurationElement.OuterXml, JsonDataSerializer.Instance.Serialize(attachments.ToArray()))).ConfigureAwait(false); + return JsonDataSerializer.Instance.Deserialize(result)!; } public void Dispose() diff --git a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentProcessorWrapper.cs b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentProcessorWrapper.cs index df3bda0244..94d7d6bfb9 100644 --- a/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentProcessorWrapper.cs +++ b/src/Microsoft.TestPlatform.CrossPlatEngine/AttachmentsProcessing/DataCollectorAttachmentProcessorWrapper.cs @@ -62,7 +62,7 @@ public DataCollectorAttachmentProcessorRemoteWrapper(string pipeShutdownMessageP { var doc = new XmlDocument(); doc.LoadXml(configurationElement); - AttachmentSet[] attachmentSets = JsonDataSerializer.Instance.Deserialize(attachments); + AttachmentSet[] attachmentSets = JsonDataSerializer.Instance.Deserialize(attachments)!; SynchronousProgress progress = new(Report); _processAttachmentCts = new CancellationTokenSource(); diff --git a/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs b/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs index fc5ff062ca..8e086875d6 100644 --- a/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs +++ b/src/Microsoft.TestPlatform.TestHostProvider/Hosting/DotnetTestHostManager.cs @@ -733,8 +733,8 @@ private string GetTestHostPath(string runtimeConfigDevPath, string depsFilePath, using (JsonTextReader reader = new(file)) { JObject context = (JObject)JToken.ReadFrom(reader); - JObject runtimeOptions = (JObject)context.GetValue("runtimeOptions"); - JToken additionalProbingPaths = runtimeOptions.GetValue("additionalProbingPaths"); + JObject runtimeOptions = (JObject)context.GetValue("runtimeOptions")!; + JToken additionalProbingPaths = runtimeOptions.GetValue("additionalProbingPaths")!; foreach (var x in additionalProbingPaths) { EqtTrace.Verbose("DotnetTestHostmanager: Looking for path {0} in folder {1}", testHostPath, x.ToString()); diff --git a/src/package/ThirdPartyNotices.txt b/src/package/ThirdPartyNotices.txt index 8efcee2fa8..4c85ae8285 100644 --- a/src/package/ThirdPartyNotices.txt +++ b/src/package/ThirdPartyNotices.txt @@ -8,7 +8,7 @@ and the licenses under which Microsoft received such components are set forth be informational purposes only. Microsoft reserves all rights not expressly granted herein, whether by implication, estoppel or otherwise. -1. Newtonsoft version 9.0.1 (https://github.com/JamesNK/Newtonsoft.Json) +1. Newtonsoft version 13.0.1 (https://github.com/JamesNK/Newtonsoft.Json) 2. Mono.Cecil version 0.11.3 (https://github.com/jbevain/cecil) diff --git a/src/testhost.arm64/app.config b/src/testhost.arm64/app.config index 2a67681b5e..e889aff770 100644 --- a/src/testhost.arm64/app.config +++ b/src/testhost.arm64/app.config @@ -46,6 +46,11 @@ + + + + + @@ -61,4 +66,4 @@ - \ No newline at end of file + diff --git a/src/testhost.x86/app.config b/src/testhost.x86/app.config index 254425ccbe..a7be49fb01 100644 --- a/src/testhost.x86/app.config +++ b/src/testhost.x86/app.config @@ -46,6 +46,11 @@ + + + + + diff --git a/src/testhost/app.config b/src/testhost/app.config index 2a67681b5e..e889aff770 100644 --- a/src/testhost/app.config +++ b/src/testhost/app.config @@ -46,6 +46,11 @@ + + + + + @@ -61,4 +66,4 @@ - \ No newline at end of file + diff --git a/src/vstest.console/app.config b/src/vstest.console/app.config index e6f209a541..bbc2c9a12b 100644 --- a/src/vstest.console/app.config +++ b/src/vstest.console/app.config @@ -20,6 +20,11 @@ + + + + + @@ -54,4 +59,4 @@ --> - \ No newline at end of file + diff --git a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs index fdaa3a45b7..0f4a8086b7 100644 --- a/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs +++ b/test/Microsoft.TestPlatform.AcceptanceTests/DotnetArchitectureSwitchTests.Windows.cs @@ -35,7 +35,7 @@ public void Use_EnvironmentVariables(string architectureFrom, string architectur string sdkVersion = GetLatestSdkVersion(dotnetPath); string runtimeConfigFile = Path.Combine(dotnetRunnerPath.FullName, "vstest.console.runtimeconfig.json"); JObject patchRuntimeConfig = JObject.Parse(File.ReadAllText(runtimeConfigFile)); - patchRuntimeConfig["runtimeOptions"]["framework"]["version"] = sdkVersion; + patchRuntimeConfig!["runtimeOptions"]!["framework"]!["version"] = sdkVersion; File.WriteAllText(runtimeConfigFile, patchRuntimeConfig.ToString()); var environmentVariables = new Dictionary diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/JsonDataSerializerTests.cs b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/JsonDataSerializerTests.cs index 0d377f3620..40f9dea8e5 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/JsonDataSerializerTests.cs +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/JsonDataSerializerTests.cs @@ -80,7 +80,7 @@ public void DeserializeShouldDeserializeAnObjectWhichHadSelfReferencingLoopBefor var json = _jsonDataSerializer.SerializePayload("dummy", classWithSelfReferencingLoop); // This line should deserialize properly - var result = _jsonDataSerializer.Deserialize(json, 1); + var result = _jsonDataSerializer.Deserialize(json, 1)!; Assert.AreEqual(typeof(ClassWithSelfReferencingLoop), result.GetType()); Assert.IsNull(result.InfiniteRefernce); diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Serialization/TestCaseSerializationTests.cs b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Serialization/TestCaseSerializationTests.cs index 97fd73d599..23ce72f4d0 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Serialization/TestCaseSerializationTests.cs +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Serialization/TestCaseSerializationTests.cs @@ -260,7 +260,7 @@ private static string Serialize(T data, int version = 1) private static T Deserialize(string json, int version = 1) { - return JsonDataSerializer.Instance.Deserialize(json, version); + return JsonDataSerializer.Instance.Deserialize(json, version)!; } private void VerifyDummyPropertyIsRegistered() diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Serialization/TestObjectConverterTests.cs b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Serialization/TestObjectConverterTests.cs index 77cb64f2fd..04b5089837 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Serialization/TestObjectConverterTests.cs +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Serialization/TestObjectConverterTests.cs @@ -164,6 +164,6 @@ private static string Serialize(T data) private static T Deserialize(string json) { - return JsonDataSerializer.Instance.Deserialize(json); + return JsonDataSerializer.Instance.Deserialize(json)!; } } diff --git a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Serialization/TestResultSerializationTests.cs b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Serialization/TestResultSerializationTests.cs index 21903d3da9..087c01c46e 100644 --- a/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Serialization/TestResultSerializationTests.cs +++ b/test/Microsoft.TestPlatform.CommunicationUtilities.UnitTests/Serialization/TestResultSerializationTests.cs @@ -296,7 +296,7 @@ private static string Serialize(T data, int version) private static T Deserialize(string json, int version) { - return JsonDataSerializer.Instance.Deserialize(json, version); + return JsonDataSerializer.Instance.Deserialize(json, version)!; } private static void VerifyDummyPropertyIsRegistered() diff --git a/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Client/DiscoveryCriteriaTests.cs b/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Client/DiscoveryCriteriaTests.cs index 513fbd21b2..7041273030 100644 --- a/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Client/DiscoveryCriteriaTests.cs +++ b/test/Microsoft.TestPlatform.ObjectModel.UnitTests/Client/DiscoveryCriteriaTests.cs @@ -44,7 +44,7 @@ public void DiscoveryCriteriaShouldBeDeserializable() { var json = "{\"Sources\":[\"sampleTest.dll\"],\"AdapterSourceMap\":{\"_none_\":[\"sampleTest.dll\"]},\"FrequencyOfDiscoveredTestsEvent\":100,\"DiscoveredTestEventTimeout\":\"10675199.02:48:05.4775807\",\"RunSettings\":\"\",\"TestCaseFilter\":\"TestFilter\"}"; - var criteria = JsonConvert.DeserializeObject(json, Settings); + var criteria = JsonConvert.DeserializeObject(json, Settings)!; Assert.AreEqual(TimeSpan.MaxValue, criteria.DiscoveredTestEventTimeout); Assert.AreEqual(100, criteria.FrequencyOfDiscoveredTestsEvent); diff --git a/test/Microsoft.TestPlatform.PerformanceTests/ProtocolV1Tests.cs b/test/Microsoft.TestPlatform.PerformanceTests/ProtocolV1Tests.cs index 58eeca5cc9..fd39dc11f1 100644 --- a/test/Microsoft.TestPlatform.PerformanceTests/ProtocolV1Tests.cs +++ b/test/Microsoft.TestPlatform.PerformanceTests/ProtocolV1Tests.cs @@ -121,6 +121,6 @@ private static string SerializeV1(T data) private static T DeserializeV1(string json) { - return JsonDataSerializer.Instance.Deserialize(json, version: 1); + return JsonDataSerializer.Instance.Deserialize(json, version: 1)!; } } diff --git a/test/Microsoft.TestPlatform.PerformanceTests/ProtocolV2Tests.cs b/test/Microsoft.TestPlatform.PerformanceTests/ProtocolV2Tests.cs index cd5c4a4bf2..bb00c2b1be 100644 --- a/test/Microsoft.TestPlatform.PerformanceTests/ProtocolV2Tests.cs +++ b/test/Microsoft.TestPlatform.PerformanceTests/ProtocolV2Tests.cs @@ -119,6 +119,6 @@ private static string SerializeV2(T data) private static T DeserializeV2(string json) { - return JsonDataSerializer.Instance.Deserialize(json, version: 2); + return JsonDataSerializer.Instance.Deserialize(json, version: 2)!; } } diff --git a/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs b/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs index dd993a721f..456ecf8a7d 100644 --- a/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs +++ b/test/Microsoft.TestPlatform.SmokeTests/DotnetHostArchitectureVerifierTests.cs @@ -34,7 +34,7 @@ public void VerifyHostArchitecture(string architecture) string sdkVersion = GetLatestSdkVersion(dotnetPath); string runtimeConfigFile = Path.Combine(dotnetRunnerPath.FullName, "vstest.console.runtimeconfig.json"); JObject patchRuntimeConfig = JObject.Parse(File.ReadAllText(runtimeConfigFile)); - patchRuntimeConfig["runtimeOptions"]["framework"]["version"] = sdkVersion; + patchRuntimeConfig!["runtimeOptions"]!["framework"]!["version"] = sdkVersion; File.WriteAllText(runtimeConfigFile, patchRuntimeConfig.ToString()); var environmentVariables = new Dictionary diff --git a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs index 289e0799e4..45cd1294d7 100644 --- a/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs +++ b/test/Microsoft.TestPlatform.TestHostProvider.UnitTests/Hosting/DotnetTestHostManagerTests.cs @@ -319,7 +319,7 @@ public void GetTestHostProcessStartInfoShouldUseTestHostExeFromNugetIfNotFoundIn ""microsoft.testplatform.testhost/15.0.0-Dev"": { ""dependencies"": { ""Microsoft.TestPlatform.ObjectModel"": ""15.0.0-Dev"", - ""Newtonsoft.Json"": ""9.0.1"" + ""Newtonsoft.Json"": ""13.0.1"" }, ""runtime"": { ""lib/netstandard1.5/Microsoft.TestPlatform.CommunicationUtilities.dll"": { }, @@ -389,7 +389,7 @@ public void GetTestHostProcessStartInfoShouldUseTestHostX86ExeFromNugetIfNotFoun ""microsoft.testplatform.testhost/15.0.0-Dev"": { ""dependencies"": { ""Microsoft.TestPlatform.ObjectModel"": ""15.0.0-Dev"", - ""Newtonsoft.Json"": ""9.0.1"" + ""Newtonsoft.Json"": ""13.0.1"" }, ""runtime"": { ""lib/netstandard1.5/Microsoft.TestPlatform.CommunicationUtilities.dll"": { }, @@ -709,7 +709,7 @@ public void GetTestHostProcessStartInfoShouldIncludeTestHostPathFromDepsFile() ""microsoft.testplatform.testhost/15.0.0-Dev"": { ""dependencies"": { ""Microsoft.TestPlatform.ObjectModel"": ""15.0.0-Dev"", - ""Newtonsoft.Json"": ""9.0.1"" + ""Newtonsoft.Json"": ""13.0.1"" }, ""runtime"": { ""lib/netstandard1.5/Microsoft.TestPlatform.CommunicationUtilities.dll"": { }, @@ -774,7 +774,7 @@ public void GetTestHostProcessStartInfoShouldIncludeTestHostPathFromSourceDirect ""microsoft.testplatform.testhost/15.0.0-Dev"": { ""dependencies"": { ""Microsoft.TestPlatform.ObjectModel"": ""15.0.0-Dev"", - ""Newtonsoft.Json"": ""9.0.1"" + ""Newtonsoft.Json"": ""13.0.1"" }, ""runtime"": { ""lib/netstandard1.5/Microsoft.TestPlatform.CommunicationUtilities.dll"": { }, @@ -842,7 +842,7 @@ public void GetTestHostProcessStartInfoShouldSkipInvalidAdditionalProbingPaths() ""microsoft.testplatform.testhost/15.0.0-Dev"": { ""dependencies"": { ""Microsoft.TestPlatform.ObjectModel"": ""15.0.0-Dev"", - ""Newtonsoft.Json"": ""9.0.1"" + ""Newtonsoft.Json"": ""13.0.1"" }, ""runtime"": { ""lib/netstandard1.5/Microsoft.TestPlatform.CommunicationUtilities.dll"": { }, diff --git a/test/TestAssets/NewtonSoftDependency/App.config b/test/TestAssets/NewtonSoftDependency/App.config index 776d09c68c..f71f97e6bb 100644 --- a/test/TestAssets/NewtonSoftDependency/App.config +++ b/test/TestAssets/NewtonSoftDependency/App.config @@ -4,7 +4,7 @@ - + diff --git a/test/TestAssets/NewtonSoftDependency/NewtonSoftDependency.csproj b/test/TestAssets/NewtonSoftDependency/NewtonSoftDependency.csproj index 7a30726cdf..802266e3b9 100644 --- a/test/TestAssets/NewtonSoftDependency/NewtonSoftDependency.csproj +++ b/test/TestAssets/NewtonSoftDependency/NewtonSoftDependency.csproj @@ -26,7 +26,9 @@ - + + From f12e93c0d73ae9168e7eec87c2feb118245e708a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakub=20Jare=C5=A1?= Date: Fri, 17 Feb 2023 11:57:18 +0100 Subject: [PATCH 2/7] upgrade arcade --- azure-pipelines.yml | 20 +- .../build-configuration.json | 4 + eng/common/SetupNugetSources.ps1 | 18 +- eng/common/SetupNugetSources.sh | 68 ++-- eng/common/build.ps1 | 5 + eng/common/build.sh | 17 +- eng/common/cross/arm/sources.list.focal | 11 + eng/common/cross/arm/sources.list.jammy | 11 + eng/common/cross/arm64/sources.list.focal | 11 + eng/common/cross/arm64/sources.list.jammy | 11 + eng/common/cross/build-android-rootfs.sh | 4 +- eng/common/cross/build-rootfs.sh | 364 ++++++++++++------ eng/common/cross/riscv64/sources.list.sid | 1 + eng/common/cross/tizen-build-rootfs.sh | 61 +++ eng/common/cross/tizen-fetch.sh | 172 +++++++++ eng/common/cross/toolchain.cmake | 127 ++++-- eng/common/cross/x64/tizen/tizen.patch | 9 + eng/common/cross/x86/sources.list.focal | 11 + eng/common/cross/x86/sources.list.jammy | 11 + eng/common/cross/x86/tizen/tizen.patch | 9 + eng/common/dotnet-install.sh | 4 +- eng/common/generate-locproject.ps1 | 89 ++++- eng/common/generate-sbom-prep.ps1 | 2 + eng/common/generate-sbom-prep.sh | 12 + eng/common/init-tools-native.ps1 | 141 ++++--- eng/common/internal/Tools.csproj | 3 + eng/common/loc/P22DotNetHtmlLocalization.lss | Bin 0 -> 3810 bytes eng/common/native/init-compiler.sh | 83 ++-- eng/common/retain-build.ps1 | 8 +- eng/common/sdk-task.ps1 | 2 +- eng/common/sdl/NuGet.config | 5 + eng/common/sdl/sdl.ps1 | 38 ++ eng/common/templates/job/execute-sdl.yml | 11 +- eng/common/templates/job/job.yml | 63 ++- eng/common/templates/job/onelocbuild.yml | 47 +-- .../templates/job/publish-build-assets.yml | 60 ++- eng/common/templates/job/source-build.yml | 7 +- .../templates/job/source-index-stage1.yml | 26 +- eng/common/templates/jobs/jobs.yml | 20 +- eng/common/templates/jobs/source-build.yml | 2 +- .../templates/post-build/common-variables.yml | 4 - .../templates/post-build/post-build.yml | 104 ++--- eng/common/templates/steps/execute-sdl.yml | 60 ++- eng/common/templates/steps/generate-sbom.yml | 4 + eng/common/templates/steps/send-to-helix.yml | 10 +- eng/common/templates/steps/source-build.yml | 14 +- .../templates/variables/pool-providers.yml | 57 +++ eng/common/tools.ps1 | 25 +- eng/common/tools.sh | 28 +- 49 files changed, 1361 insertions(+), 513 deletions(-) create mode 100644 eng/common/BuildConfiguration/build-configuration.json create mode 100644 eng/common/cross/arm/sources.list.focal create mode 100644 eng/common/cross/arm/sources.list.jammy create mode 100644 eng/common/cross/arm64/sources.list.focal create mode 100644 eng/common/cross/arm64/sources.list.jammy create mode 100644 eng/common/cross/riscv64/sources.list.sid create mode 100644 eng/common/cross/tizen-build-rootfs.sh create mode 100644 eng/common/cross/tizen-fetch.sh create mode 100644 eng/common/cross/x64/tizen/tizen.patch create mode 100644 eng/common/cross/x86/sources.list.focal create mode 100644 eng/common/cross/x86/sources.list.jammy create mode 100644 eng/common/cross/x86/tizen/tizen.patch create mode 100644 eng/common/loc/P22DotNetHtmlLocalization.lss create mode 100644 eng/common/sdl/sdl.ps1 create mode 100644 eng/common/templates/variables/pool-providers.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ed76285bba..295a504337 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,4 +1,10 @@ -# Pipeline YAML for PR +trigger: +- main +- rel/* + +pr: +- main +- rel/* jobs: - job: Windows @@ -15,7 +21,7 @@ jobs: displayName: 'Run script build.cmd' inputs: filename: build.cmd - arguments: '-verbose -configuration $(buildConfiguration) -ci -steps "InstallDotnet, Restore, UpdateLocalization, Build, Publish"' + arguments: '-verbose -configuration $(buildConfiguration) -ci -steps "InstallDotnet, Restore, UpdateLocalization, Build, Publish, Pack, Manifest"' modifyEnvironment: false failOnStandardError: true @@ -165,11 +171,11 @@ jobs: clean: all strategy: matrix: - Ubuntu_18_04: - vmImage: ubuntu-18.04 + Ubuntu_22_04: + vmImage: ubuntu-22.04 pwsh: true - macOS_10_15_Catalina: - vmImage: macOS-10.15 + macOS_11: + vmImage: macOS-11 pwsh: true pool: vmImage: $[ variables['vmImage'] ] @@ -215,5 +221,5 @@ jobs: parameters: platform: name: Managed - container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-3e800f1-20190501005343 + container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7 buildScript: eng/common/build.sh diff --git a/eng/common/BuildConfiguration/build-configuration.json b/eng/common/BuildConfiguration/build-configuration.json new file mode 100644 index 0000000000..3d1cc89894 --- /dev/null +++ b/eng/common/BuildConfiguration/build-configuration.json @@ -0,0 +1,4 @@ +{ + "RetryCountLimit": 1, + "RetryByAnyError": false +} diff --git a/eng/common/SetupNugetSources.ps1 b/eng/common/SetupNugetSources.ps1 index 18823840b1..6e99723945 100644 --- a/eng/common/SetupNugetSources.ps1 +++ b/eng/common/SetupNugetSources.ps1 @@ -146,22 +146,22 @@ $userName = "dn-bot" # Insert credential nodes for Maestro's private feeds InsertMaestroPrivateFeedCredentials -Sources $sources -Creds $creds -Username $userName -Password $Password +# 3.1 uses a different feed url format so it's handled differently here $dotnet31Source = $sources.SelectSingleNode("add[@key='dotnet3.1']") if ($dotnet31Source -ne $null) { AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password AddPackageSource -Sources $sources -SourceName "dotnet3.1-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/_packaging/dotnet3.1-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password } -$dotnet5Source = $sources.SelectSingleNode("add[@key='dotnet5']") -if ($dotnet5Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet5-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password -} +$dotnetVersions = @('5','6','7') -$dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']") -if ($dotnet6Source -ne $null) { - AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password - AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password +foreach ($dotnetVersion in $dotnetVersions) { + $feedPrefix = "dotnet" + $dotnetVersion; + $dotnetSource = $sources.SelectSingleNode("add[@key='$feedPrefix']") + if ($dotnetSource -ne $null) { + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password + AddPackageSource -Sources $sources -SourceName "$feedPrefix-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/$feedPrefix-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password + } } $doc.Save($filename) diff --git a/eng/common/SetupNugetSources.sh b/eng/common/SetupNugetSources.sh index ad3fb74fd2..8af7d899db 100644 --- a/eng/common/SetupNugetSources.sh +++ b/eng/common/SetupNugetSources.sh @@ -105,53 +105,33 @@ if [ "$?" == "0" ]; then PackageSources+=('dotnet3.1-internal-transport') fi -# Ensure dotnet5-internal and dotnet5-internal-transport are in the packageSources if the public dotnet5 feeds are present -grep -i "" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=('dotnet5-internal') - - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding dotnet5-internal-transport to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=('dotnet5-internal-transport') -fi - -# Ensure dotnet6-internal and dotnet6-internal-transport are in the packageSources if the public dotnet6 feeds are present -grep -i "" +DotNetVersions=('5' '6' '7') + +for DotNetVersion in ${DotNetVersions[@]} ; do + FeedPrefix="dotnet${DotNetVersion}"; + grep -i "" + + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=("$FeedPrefix-internal") - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile - fi - PackageSources+=('dotnet6-internal') + grep -i "" $ConfigFile + if [ "$?" != "0" ]; then + echo "Adding $FeedPrefix-internal-transport to the packageSources." + PackageSourcesNodeFooter="" + PackageSourceTemplate="${TB}" - grep -i "" $ConfigFile - if [ "$?" != "0" ]; then - echo "Adding dotnet6-internal-transport to the packageSources." - PackageSourcesNodeFooter="" - PackageSourceTemplate="${TB}" - - sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile + fi + PackageSources+=("$FeedPrefix-internal-transport") fi - PackageSources+=('dotnet6-internal-transport') -fi +done # I want things split line by line PrevIFS=$IFS diff --git a/eng/common/build.ps1 b/eng/common/build.ps1 index 8943da242f..33a6f2d0e2 100644 --- a/eng/common/build.ps1 +++ b/eng/common/build.ps1 @@ -26,6 +26,7 @@ Param( [string] $runtimeSourceFeed = '', [string] $runtimeSourceFeedKey = '', [switch] $excludePrereleaseVS, + [switch] $nativeToolsOnMachine, [switch] $help, [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties ) @@ -67,6 +68,7 @@ function Print-Usage() { Write-Host " -warnAsError Sets warnaserror msbuild parameter ('true' or 'false')" Write-Host " -msbuildEngine Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)." Write-Host " -excludePrereleaseVS Set to exclude build engines in prerelease versions of Visual Studio" + Write-Host " -nativeToolsOnMachine Sets the native tools on machine environment variable (indicating that the script should use native tools on machine)" Write-Host "" Write-Host "Command line arguments not listed above are passed thru to msbuild." @@ -146,6 +148,9 @@ try { $nodeReuse = $false } + if ($nativeToolsOnMachine) { + $env:NativeToolsOnMachine = $true + } if ($restore) { InitializeNativeTools } diff --git a/eng/common/build.sh b/eng/common/build.sh index 55b298f16c..50af40cdd2 100755 --- a/eng/common/build.sh +++ b/eng/common/build.sh @@ -19,6 +19,9 @@ usage() echo "Actions:" echo " --restore Restore dependencies (short: -r)" echo " --build Build solution (short: -b)" + echo " --sourceBuild Source-build the solution (short: -sb)" + echo " Will additionally trigger the following actions: --restore, --build, --pack" + echo " If --configuration is not set explicitly, will also set it to 'Release'" echo " --rebuild Rebuild solution" echo " --test Run all unit tests in the solution (short: -t)" echo " --integrationTest Run all integration tests in the solution" @@ -55,6 +58,7 @@ scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" restore=false build=false +source_build=false rebuild=false test=false integration_test=false @@ -73,7 +77,7 @@ exclude_ci_binary_log=false pipelines_log=false projects='' -configuration='Debug' +configuration='' prepare_machine=false verbosity='minimal' runtime_source_feed='' @@ -119,6 +123,12 @@ while [[ $# > 0 ]]; do -pack) pack=true ;; + -sourcebuild|-sb) + build=true + source_build=true + restore=true + pack=true + ;; -test|-t) test=true ;; @@ -168,6 +178,10 @@ while [[ $# > 0 ]]; do shift done +if [[ -z "$configuration" ]]; then + if [[ "$source_build" = true ]]; then configuration="Release"; else configuration="Debug"; fi +fi + if [[ "$ci" == true ]]; then pipelines_log=true node_reuse=false @@ -205,6 +219,7 @@ function Build { /p:RepoRoot="$repo_root" \ /p:Restore=$restore \ /p:Build=$build \ + /p:ArcadeBuildFromSource=$source_build \ /p:Rebuild=$rebuild \ /p:Test=$test \ /p:Pack=$pack \ diff --git a/eng/common/cross/arm/sources.list.focal b/eng/common/cross/arm/sources.list.focal new file mode 100644 index 0000000000..4de2600c17 --- /dev/null +++ b/eng/common/cross/arm/sources.list.focal @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse diff --git a/eng/common/cross/arm/sources.list.jammy b/eng/common/cross/arm/sources.list.jammy new file mode 100644 index 0000000000..6bb0453029 --- /dev/null +++ b/eng/common/cross/arm/sources.list.jammy @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/sources.list.focal b/eng/common/cross/arm64/sources.list.focal new file mode 100644 index 0000000000..4de2600c17 --- /dev/null +++ b/eng/common/cross/arm64/sources.list.focal @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ focal-security main restricted universe multiverse diff --git a/eng/common/cross/arm64/sources.list.jammy b/eng/common/cross/arm64/sources.list.jammy new file mode 100644 index 0000000000..6bb0453029 --- /dev/null +++ b/eng/common/cross/arm64/sources.list.jammy @@ -0,0 +1,11 @@ +deb http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-updates main restricted universe + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-backports main restricted + +deb http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse +deb-src http://ports.ubuntu.com/ubuntu-ports/ jammy-security main restricted universe multiverse diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index 42516bbeeb..f163fb9dae 100644 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -107,12 +107,12 @@ __AndroidPackages+=" liblzma" __AndroidPackages+=" krb5" __AndroidPackages+=" openssl" -for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\ +for path in $(wget -qO- https://packages.termux.dev/termux-main-21/dists/stable/main/binary-$__AndroidArch/Packages |\ grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do if [[ "$path" != "Filename:" ]]; then echo "Working on: $path" - wget -qO- http://termux.net/$path | dpkg -x - "$__TmpDir" + wget -qO- https://packages.termux.dev/termux-main-21/$path | dpkg -x - "$__TmpDir" fi done diff --git a/eng/common/cross/build-rootfs.sh b/eng/common/cross/build-rootfs.sh index 7e4be9a0cc..1ebf454f3c 100644 --- a/eng/common/cross/build-rootfs.sh +++ b/eng/common/cross/build-rootfs.sh @@ -4,22 +4,27 @@ set -e usage() { - echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [--skipunmount] --rootfsdir ]" - echo "BuildArch can be: arm(default), armel, arm64, x86" + echo "Usage: $0 [BuildArch] [CodeName] [lldbx.y] [llvmx[.y]] [--skipunmount] --rootfsdir ]" + echo "BuildArch can be: arm(default), arm64, armel, armv6, ppc64le, riscv64, s390x, x64, x86" echo "CodeName - optional, Code name for Linux, can be: xenial(default), zesty, bionic, alpine, alpine3.13 or alpine3.14. If BuildArch is armel, LinuxCodeName is jessie(default) or tizen." echo " for FreeBSD can be: freebsd12, freebsd13" - echo " for illumos can be: illumos." + echo " for illumos can be: illumos" + echo " for Haiku can be: haiku." echo "lldbx.y - optional, LLDB version, can be: lldb3.9(default), lldb4.0, lldb5.0, lldb6.0 no-lldb. Ignored for alpine and FreeBSD" + echo "llvmx[.y] - optional, LLVM version for LLVM related packages." echo "--skipunmount - optional, will skip the unmount of rootfs folder." echo "--use-mirror - optional, use mirror URL to fetch resources, when available." + echo "--jobs N - optional, restrict to N jobs." exit 1 } __CodeName=xenial __CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -__InitialDir=$PWD __BuildArch=arm __AlpineArch=armv7 +__FreeBSDArch=arm +__FreeBSDMachineArch=armv7 +__IllumosArch=arm7 __QEMUArch=arm __UbuntuArch=armhf __UbuntuRepo="http://ports.ubuntu.com/" @@ -39,17 +44,20 @@ __AlpinePackages+=" libedit" # symlinks fixer __UbuntuPackages+=" symlinks" -# CoreCLR and CoreFX dependencies +# runtime dependencies __UbuntuPackages+=" libicu-dev" __UbuntuPackages+=" liblttng-ust-dev" __UbuntuPackages+=" libunwind8-dev" +__UbuntuPackages+=" libnuma-dev" __AlpinePackages+=" gettext-dev" __AlpinePackages+=" icu-dev" __AlpinePackages+=" libunwind-dev" __AlpinePackages+=" lttng-ust-dev" +__AlpinePackages+=" compiler-rt-static" +__AlpinePackages+=" numactl-dev" -# CoreFX dependencies +# runtime libraries' dependencies __UbuntuPackages+=" libcurl4-openssl-dev" __UbuntuPackages+=" libkrb5-dev" __UbuntuPackages+=" libssl-dev" @@ -70,26 +78,38 @@ __FreeBSDPackages+=" openssl" __FreeBSDPackages+=" krb5" __FreeBSDPackages+=" terminfo-db" -__IllumosPackages="icu-64.2nb2" -__IllumosPackages+=" mit-krb5-1.16.2nb4" -__IllumosPackages+=" openssl-1.1.1e" -__IllumosPackages+=" zlib-1.2.11" +__IllumosPackages="icu" +__IllumosPackages+=" mit-krb5" +__IllumosPackages+=" openssl" +__IllumosPackages+=" zlib" + +__HaikuPackages="gmp" +__HaikuPackages+=" gmp_devel" +__HaikuPackages+=" krb5" +__HaikuPackages+=" krb5_devel" +__HaikuPackages+=" libiconv" +__HaikuPackages+=" libiconv_devel" +__HaikuPackages+=" llvm12_libunwind" +__HaikuPackages+=" llvm12_libunwind_devel" +__HaikuPackages+=" mpfr" +__HaikuPackages+=" mpfr_devel" # ML.NET dependencies __UbuntuPackages+=" libomp5" __UbuntuPackages+=" libomp-dev" +__Keyring= __UseMirror=0 __UnprocessedBuildArgs= while :; do - if [ $# -le 0 ]; then + if [[ "$#" -le 0 ]]; then break fi - lowerI="$(echo $1 | tr "[:upper:]" "[:lower:]")" + lowerI="$(echo "$1" | tr "[:upper:]" "[:lower:]")" case $lowerI in - -?|-h|--help) + -\?|-h|--help) usage exit 1 ;; @@ -99,20 +119,13 @@ while :; do __AlpineArch=armv7 __QEMUArch=arm ;; - armv6) - __BuildArch=armv6 - __UbuntuArch=armhf - __QEMUArch=arm - __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/" - __CodeName=buster - __LLDB_Package="liblldb-6.0-dev" - __Keyring="/usr/share/keyrings/raspbian-archive-keyring.gpg" - ;; arm64) __BuildArch=arm64 __UbuntuArch=arm64 __AlpineArch=aarch64 __QEMUArch=aarch64 + __FreeBSDArch=arm64 + __FreeBSDMachineArch=aarch64 ;; armel) __BuildArch=armel @@ -120,65 +133,111 @@ while :; do __UbuntuRepo="http://ftp.debian.org/debian/" __CodeName=jessie ;; + armv6) + __BuildArch=armv6 + __UbuntuArch=armhf + __QEMUArch=arm + __UbuntuRepo="http://raspbian.raspberrypi.org/raspbian/" + __CodeName=buster + __LLDB_Package="liblldb-6.0-dev" + + if [[ -e "/usr/share/keyrings/raspbian-archive-keyring.gpg" ]]; then + __Keyring="--keyring /usr/share/keyrings/raspbian-archive-keyring.gpg" + fi + ;; ppc64le) __BuildArch=ppc64le __UbuntuArch=ppc64el __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" - __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//') - __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//') - __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//') + __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" + __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" + __UbuntuPackages="${__UbuntuPackages// libomp5/}" + unset __LLDB_Package + ;; + riscv64) + __BuildArch=riscv64 + __UbuntuArch=riscv64 + __UbuntuRepo="http://deb.debian.org/debian-ports" + __CodeName=sid + __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" unset __LLDB_Package + + if [[ -e "/usr/share/keyrings/debian-ports-archive-keyring.gpg" ]]; then + __Keyring="--keyring /usr/share/keyrings/debian-ports-archive-keyring.gpg --include=debian-ports-archive-keyring" + fi ;; s390x) __BuildArch=s390x __UbuntuArch=s390x __UbuntuRepo="http://ports.ubuntu.com/ubuntu-ports/" - __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libunwind8-dev//') - __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp-dev//') - __UbuntuPackages=$(echo ${__UbuntuPackages} | sed 's/ libomp5//') + __UbuntuPackages="${__UbuntuPackages// libunwind8-dev/}" + __UbuntuPackages="${__UbuntuPackages// libomp-dev/}" + __UbuntuPackages="${__UbuntuPackages// libomp5/}" unset __LLDB_Package ;; + x64) + __BuildArch=x64 + __UbuntuArch=amd64 + __FreeBSDArch=amd64 + __FreeBSDMachineArch=amd64 + __illumosArch=x86_64 + __UbuntuRepo= + ;; x86) __BuildArch=x86 __UbuntuArch=i386 __UbuntuRepo="http://archive.ubuntu.com/ubuntu/" ;; - lldb3.6) - __LLDB_Package="lldb-3.6-dev" - ;; - lldb3.8) - __LLDB_Package="lldb-3.8-dev" - ;; - lldb3.9) - __LLDB_Package="liblldb-3.9-dev" - ;; - lldb4.0) - __LLDB_Package="liblldb-4.0-dev" - ;; - lldb5.0) - __LLDB_Package="liblldb-5.0-dev" - ;; - lldb6.0) - __LLDB_Package="liblldb-6.0-dev" + lldb*) + version="${lowerI/lldb/}" + parts=(${version//./ }) + + # for versions > 6.0, lldb has dropped the minor version + if [[ "${parts[0]}" -gt 6 ]]; then + version="${parts[0]}" + fi + + __LLDB_Package="liblldb-${version}-dev" ;; no-lldb) unset __LLDB_Package ;; + llvm*) + version="${lowerI/llvm/}" + parts=(${version//./ }) + __LLVM_MajorVersion="${parts[0]}" + __LLVM_MinorVersion="${parts[1]}" + + # for versions > 6.0, llvm has dropped the minor version + if [[ -z "$__LLVM_MinorVersion" && "$__LLVM_MajorVersion" -le 6 ]]; then + __LLVM_MinorVersion=0; + fi + ;; xenial) # Ubuntu 16.04 - if [ "$__CodeName" != "jessie" ]; then + if [[ "$__CodeName" != "jessie" ]]; then __CodeName=xenial fi ;; zesty) # Ubuntu 17.04 - if [ "$__CodeName" != "jessie" ]; then + if [[ "$__CodeName" != "jessie" ]]; then __CodeName=zesty fi ;; bionic) # Ubuntu 18.04 - if [ "$__CodeName" != "jessie" ]; then + if [[ "$__CodeName" != "jessie" ]]; then __CodeName=bionic fi ;; + focal) # Ubuntu 20.04 + if [[ "$__CodeName" != "jessie" ]]; then + __CodeName=focal + fi + ;; + jammy) # Ubuntu 22.04 + if [[ "$__CodeName" != "jessie" ]]; then + __CodeName=jammy + fi + ;; jessie) # Debian 8 __CodeName=jessie __UbuntuRepo="http://ftp.debian.org/debian/" @@ -194,11 +253,6 @@ while :; do __LLDB_Package="liblldb-6.0-dev" ;; tizen) - if [ "$__BuildArch" != "arm" ] && [ "$__BuildArch" != "armel" ] && [ "$__BuildArch" != "arm64" ]; then - echo "Tizen is available only for arm, armel and arm64." - usage; - exit 1; - fi __CodeName= __UbuntuRepo= __Tizen=tizen @@ -217,18 +271,20 @@ while :; do ;; freebsd12) __CodeName=freebsd - __BuildArch=x64 __SkipUnmount=1 ;; freebsd13) __CodeName=freebsd __FreeBSDBase="13.0-RELEASE" __FreeBSDABI="13" - __BuildArch=x64 __SkipUnmount=1 ;; illumos) __CodeName=illumos + __SkipUnmount=1 + ;; + haiku) + __CodeName=haiku __BuildArch=x64 __SkipUnmount=1 ;; @@ -237,11 +293,15 @@ while :; do ;; --rootfsdir|-rootfsdir) shift - __RootfsDir=$1 + __RootfsDir="$1" ;; --use-mirror) __UseMirror=1 ;; + --use-jobs) + shift + MAXJOBS=$1 + ;; *) __UnprocessedBuildArgs="$__UnprocessedBuildArgs $1" ;; @@ -250,77 +310,78 @@ while :; do shift done -if [ -e "$__Keyring" ]; then - __Keyring="--keyring=$__Keyring" -else - __Keyring="" -fi - -if [ "$__BuildArch" == "armel" ]; then +if [[ "$__BuildArch" == "armel" ]]; then __LLDB_Package="lldb-3.5-dev" +elif [[ "$__BuildArch" == "arm" && "$__AlpineVersion" == "3.13" ]]; then + __AlpinePackages="${__AlpinePackages//numactl-dev/}" fi + __UbuntuPackages+=" ${__LLDB_Package:-}" -if [ -z "$__RootfsDir" ] && [ ! -z "$ROOTFS_DIR" ]; then - __RootfsDir=$ROOTFS_DIR +if [[ -n "$__LLVM_MajorVersion" ]]; then + __UbuntuPackages+=" libclang-common-${__LLVM_MajorVersion}${__LLVM_MinorVersion:+.$__LLVM_MinorVersion}-dev" fi -if [ -z "$__RootfsDir" ]; then +if [[ -z "$__RootfsDir" && -n "$ROOTFS_DIR" ]]; then + __RootfsDir="$ROOTFS_DIR" +fi + +if [[ -z "$__RootfsDir" ]]; then __RootfsDir="$__CrossDir/../../../.tools/rootfs/$__BuildArch" fi -if [ -d "$__RootfsDir" ]; then - if [ $__SkipUnmount == 0 ]; then - umount $__RootfsDir/* || true +if [[ -d "$__RootfsDir" ]]; then + if [[ "$__SkipUnmount" == "0" ]]; then + umount "$__RootfsDir"/* || true fi - rm -rf $__RootfsDir + rm -rf "$__RootfsDir" fi -mkdir -p $__RootfsDir +mkdir -p "$__RootfsDir" __RootfsDir="$( cd "$__RootfsDir" && pwd )" if [[ "$__CodeName" == "alpine" ]]; then __ApkToolsVersion=2.9.1 - __ApkToolsDir=$(mktemp -d) - wget https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -P $__ApkToolsDir - tar -xf $__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz -C $__ApkToolsDir - mkdir -p $__RootfsDir/usr/bin - cp -v /usr/bin/qemu-$__QEMUArch-static $__RootfsDir/usr/bin - - $__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk \ - -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main \ - -X http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community \ - -U --allow-untrusted --root $__RootfsDir --arch $__AlpineArch --initdb \ + __ApkToolsDir="$(mktemp -d)" + wget "https://github.com/alpinelinux/apk-tools/releases/download/v$__ApkToolsVersion/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz" -P "$__ApkToolsDir" + tar -xf "$__ApkToolsDir/apk-tools-$__ApkToolsVersion-x86_64-linux.tar.gz" -C "$__ApkToolsDir" + mkdir -p "$__RootfsDir"/usr/bin + cp -v "/usr/bin/qemu-$__QEMUArch-static" "$__RootfsDir/usr/bin" + + "$__ApkToolsDir/apk-tools-$__ApkToolsVersion/apk" \ + -X "http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/main" \ + -X "http://dl-cdn.alpinelinux.org/alpine/v$__AlpineVersion/community" \ + -U --allow-untrusted --root "$__RootfsDir" --arch "$__AlpineArch" --initdb \ add $__AlpinePackages - rm -r $__ApkToolsDir + rm -r "$__ApkToolsDir" elif [[ "$__CodeName" == "freebsd" ]]; then - mkdir -p $__RootfsDir/usr/local/etc - JOBS="$(getconf _NPROCESSORS_ONLN)" - wget -O - https://download.freebsd.org/ftp/releases/amd64/${__FreeBSDBase}/base.txz | tar -C $__RootfsDir -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version - echo "ABI = \"FreeBSD:${__FreeBSDABI}:amd64\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > ${__RootfsDir}/usr/local/etc/pkg.conf - echo "FreeBSD: { url: "pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > ${__RootfsDir}/etc/pkg/FreeBSD.conf - mkdir -p $__RootfsDir/tmp + mkdir -p "$__RootfsDir"/usr/local/etc + JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} + wget -O - "https://download.freebsd.org/ftp/releases/${__FreeBSDArch}/${__FreeBSDMachineArch}/${__FreeBSDBase}/base.txz" | tar -C "$__RootfsDir" -Jxf - ./lib ./usr/lib ./usr/libdata ./usr/include ./usr/share/keys ./etc ./bin/freebsd-version + echo "ABI = \"FreeBSD:${__FreeBSDABI}:${__FreeBSDMachineArch}\"; FINGERPRINTS = \"${__RootfsDir}/usr/share/keys\"; REPOS_DIR = [\"${__RootfsDir}/etc/pkg\"]; REPO_AUTOUPDATE = NO; RUN_SCRIPTS = NO;" > "${__RootfsDir}"/usr/local/etc/pkg.conf + echo "FreeBSD: { url: \"pkg+http://pkg.FreeBSD.org/\${ABI}/quarterly\", mirror_type: \"srv\", signature_type: \"fingerprints\", fingerprints: \"${__RootfsDir}/usr/share/keys/pkg\", enabled: yes }" > "${__RootfsDir}"/etc/pkg/FreeBSD.conf + mkdir -p "$__RootfsDir"/tmp # get and build package manager - wget -O - https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz | tar -C $__RootfsDir/tmp -zxf - - cd $__RootfsDir/tmp/pkg-${__FreeBSDPkg} + wget -O - "https://github.com/freebsd/pkg/archive/${__FreeBSDPkg}.tar.gz" | tar -C "$__RootfsDir"/tmp -zxf - + cd "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}" # needed for install to succeed - mkdir -p $__RootfsDir/host/etc - ./autogen.sh && ./configure --prefix=$__RootfsDir/host && make -j "$JOBS" && make install - rm -rf $__RootfsDir/tmp/pkg-${__FreeBSDPkg} + mkdir -p "$__RootfsDir"/host/etc + ./autogen.sh && ./configure --prefix="$__RootfsDir"/host && make -j "$JOBS" && make install + rm -rf "$__RootfsDir/tmp/pkg-${__FreeBSDPkg}" # install packages we need. - INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf update - INSTALL_AS_USER=$(whoami) $__RootfsDir/host/sbin/pkg -r $__RootfsDir -C $__RootfsDir/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages + INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf update + INSTALL_AS_USER=$(whoami) "$__RootfsDir"/host/sbin/pkg -r "$__RootfsDir" -C "$__RootfsDir"/usr/local/etc/pkg.conf install --yes $__FreeBSDPackages elif [[ "$__CodeName" == "illumos" ]]; then mkdir "$__RootfsDir/tmp" pushd "$__RootfsDir/tmp" - JOBS="$(getconf _NPROCESSORS_ONLN)" + JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} echo "Downloading sysroot." wget -O - https://github.com/illumos/sysroot/releases/download/20181213-de6af22ae73b-v1/illumos-sysroot-i386-20181213-de6af22ae73b-v1.tar.gz | tar -C "$__RootfsDir" -xzf - echo "Building binutils. Please wait.." wget -O - https://ftp.gnu.org/gnu/binutils/binutils-2.33.1.tar.bz2 | tar -xjf - mkdir build-binutils && cd build-binutils - ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" + ../binutils-2.33.1/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" make -j "$JOBS" && make install && cd .. echo "Building gcc. Please wait.." wget -O - https://ftp.gnu.org/gnu/gcc/gcc-8.4.0/gcc-8.4.0.tar.xz | tar -xJf - @@ -330,22 +391,27 @@ elif [[ "$__CodeName" == "illumos" ]]; then CFLAGS_FOR_TARGET="-fPIC" export CFLAGS CXXFLAGS CXXFLAGS_FOR_TARGET CFLAGS_FOR_TARGET mkdir build-gcc && cd build-gcc - ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="x86_64-sun-solaris2.10" --program-prefix="x86_64-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \ + ../gcc-8.4.0/configure --prefix="$__RootfsDir" --target="${__illumosArch}-sun-solaris2.10" --program-prefix="${__illumosArch}-illumos-" --with-sysroot="$__RootfsDir" --with-gnu-as \ --with-gnu-ld --disable-nls --disable-libgomp --disable-libquadmath --disable-libssp --disable-libvtv --disable-libcilkrts --disable-libada --disable-libsanitizer \ --disable-libquadmath-support --disable-shared --enable-tls make -j "$JOBS" && make install && cd .. - BaseUrl=https://pkgsrc.joyent.com + BaseUrl=https://pkgsrc.smartos.org if [[ "$__UseMirror" == 1 ]]; then - BaseUrl=http://pkgsrc.smartos.skylime.net + BaseUrl=https://pkgsrc.smartos.skylime.net fi - BaseUrl="$BaseUrl"/packages/SmartOS/2020Q1/x86_64/All + BaseUrl="$BaseUrl/packages/SmartOS/trunk/${__illumosArch}/All" + echo "Downloading manifest" + wget "$BaseUrl" echo "Downloading dependencies." read -ra array <<<"$__IllumosPackages" for package in "${array[@]}"; do - echo "Installing $package..." + echo "Installing '$package'" + # find last occurrence of package in listing and extract its name + package="$(sed -En '/.*href="('"$package"'-[0-9].*).tgz".*/h;$!d;g;s//\1/p' All)" + echo "Resolved name '$package'" wget "$BaseUrl"/"$package".tgz ar -x "$package".tgz - tar --skip-old-files -xzf "$package".tmp.tgz -C "$__RootfsDir" 2>/dev/null + tar --skip-old-files -xzf "$package".tmp.tg* -C "$__RootfsDir" 2>/dev/null done echo "Cleaning up temporary files." popd @@ -356,26 +422,90 @@ elif [[ "$__CodeName" == "illumos" ]]; then wget -P "$__RootfsDir"/usr/include/net https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/io/bpf/net/dlt.h wget -P "$__RootfsDir"/usr/include/netpacket https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/inet/sockmods/netpacket/packet.h wget -P "$__RootfsDir"/usr/include/sys https://raw.githubusercontent.com/illumos/illumos-gate/master/usr/src/uts/common/sys/sdt.h -elif [[ -n $__CodeName ]]; then - qemu-debootstrap $__Keyring --arch $__UbuntuArch $__CodeName $__RootfsDir $__UbuntuRepo - cp $__CrossDir/$__BuildArch/sources.list.$__CodeName $__RootfsDir/etc/apt/sources.list - chroot $__RootfsDir apt-get update - chroot $__RootfsDir apt-get -f -y install - chroot $__RootfsDir apt-get -y install $__UbuntuPackages - chroot $__RootfsDir symlinks -cr /usr - chroot $__RootfsDir apt-get clean - - if [ $__SkipUnmount == 0 ]; then - umount $__RootfsDir/* || true +elif [[ "$__CodeName" == "haiku" ]]; then + JOBS=${MAXJOBS:="$(getconf _NPROCESSORS_ONLN)"} + + echo "Building Haiku sysroot for x86_64" + mkdir -p "$__RootfsDir/tmp" + cd "$__RootfsDir/tmp" + git clone -b hrev56235 https://review.haiku-os.org/haiku + git clone -b btrev43195 https://review.haiku-os.org/buildtools + cd "$__RootfsDir/tmp/buildtools" && git checkout 7487388f5110021d400b9f3b88e1a7f310dc066d + + # Fetch some unmerged patches + cd "$__RootfsDir/tmp/haiku" + ## Add development build profile (slimmer than nightly) + git fetch origin refs/changes/64/4164/1 && git -c commit.gpgsign=false cherry-pick FETCH_HEAD + + # Build jam + cd "$__RootfsDir/tmp/buildtools/jam" + make + + # Configure cross tools + echo "Building cross-compiler" + mkdir -p "$__RootfsDir/generated" + cd "$__RootfsDir/generated" + "$__RootfsDir/tmp/haiku/configure" -j"$JOBS" --sysroot "$__RootfsDir" --cross-tools-source "$__RootfsDir/tmp/buildtools" --build-cross-tools x86_64 + + # Build Haiku packages + echo "Building Haiku" + echo 'HAIKU_BUILD_PROFILE = "development-raw" ;' > UserProfileConfig + "$__RootfsDir/tmp/buildtools/jam/jam0" -j"$JOBS" -q 'package' 'Haiku' + + BaseUrl="https://depot.haiku-os.org/__api/v2/pkg/get-pkg" + + # Download additional packages + echo "Downloading additional required packages" + read -ra array <<<"$__HaikuPackages" + for package in "${array[@]}"; do + echo "Downloading $package..." + # API documented here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L60 + # The schema here: https://github.com/haiku/haikudepotserver/blob/master/haikudepotserver-api2/src/main/resources/api2/pkg.yaml#L598 + hpkgDownloadUrl="$(wget -qO- --post-data='{"name":"'"$package"'","repositorySourceCode":"haikuports_x86_64","versionType":"LATEST","naturalLanguageCode":"en"}' \ + --header='Content-Type:application/json' "$BaseUrl" | jq -r '.result.versions[].hpkgDownloadURL')" + wget -P "$__RootfsDir/generated/download" "$hpkgDownloadUrl" + done + + # Setup the sysroot + echo "Setting up sysroot and extracting needed packages" + mkdir -p "$__RootfsDir/boot/system" + for file in "$__RootfsDir/generated/objects/haiku/x86_64/packaging/packages/"*.hpkg; do + "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file" + done + for file in "$__RootfsDir/generated/download/"*.hpkg; do + "$__RootfsDir/generated/objects/linux/x86_64/release/tools/package/package" extract -C "$__RootfsDir/boot/system" "$file" + done + + # Cleaning up temporary files + echo "Cleaning up temporary files" + rm -rf "$__RootfsDir/tmp" + for name in "$__RootfsDir/generated/"*; do + if [[ "$name" =~ "cross-tools-" ]]; then + : # Keep the cross-compiler + else + rm -rf "$name" + fi + done +elif [[ -n "$__CodeName" ]]; then + qemu-debootstrap $__Keyring --arch "$__UbuntuArch" "$__CodeName" "$__RootfsDir" "$__UbuntuRepo" + cp "$__CrossDir/$__BuildArch/sources.list.$__CodeName" "$__RootfsDir/etc/apt/sources.list" + chroot "$__RootfsDir" apt-get update + chroot "$__RootfsDir" apt-get -f -y install + chroot "$__RootfsDir" apt-get -y install $__UbuntuPackages + chroot "$__RootfsDir" symlinks -cr /usr + chroot "$__RootfsDir" apt-get clean + + if [[ "$__SkipUnmount" == "0" ]]; then + umount "$__RootfsDir"/* || true fi if [[ "$__BuildArch" == "armel" && "$__CodeName" == "jessie" ]]; then - pushd $__RootfsDir - patch -p1 < $__CrossDir/$__BuildArch/armel.jessie.patch + pushd "$__RootfsDir" + patch -p1 < "$__CrossDir/$__BuildArch/armel.jessie.patch" popd fi elif [[ "$__Tizen" == "tizen" ]]; then - ROOTFS_DIR=$__RootfsDir $__CrossDir/$__BuildArch/tizen-build-rootfs.sh + ROOTFS_DIR="$__RootfsDir" "$__CrossDir/tizen-build-rootfs.sh" "$__BuildArch" else echo "Unsupported target platform." usage; diff --git a/eng/common/cross/riscv64/sources.list.sid b/eng/common/cross/riscv64/sources.list.sid new file mode 100644 index 0000000000..65f730d224 --- /dev/null +++ b/eng/common/cross/riscv64/sources.list.sid @@ -0,0 +1 @@ +deb http://deb.debian.org/debian-ports sid main diff --git a/eng/common/cross/tizen-build-rootfs.sh b/eng/common/cross/tizen-build-rootfs.sh new file mode 100644 index 0000000000..ac84173d44 --- /dev/null +++ b/eng/common/cross/tizen-build-rootfs.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -e + +ARCH=$1 +LINK_ARCH=$ARCH + +case "$ARCH" in + arm) + TIZEN_ARCH="armv7hl" + ;; + armel) + TIZEN_ARCH="armv7l" + LINK_ARCH="arm" + ;; + arm64) + TIZEN_ARCH="aarch64" + ;; + x86) + TIZEN_ARCH="i686" + ;; + x64) + TIZEN_ARCH="x86_64" + LINK_ARCH="x86" + ;; + *) + echo "Unsupported architecture for tizen: $ARCH" + exit 1 +esac + +__CrossDir=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +__TIZEN_CROSSDIR="$__CrossDir/${ARCH}/tizen" + +if [[ -z "$ROOTFS_DIR" ]]; then + echo "ROOTFS_DIR is not defined." + exit 1; +fi + +TIZEN_TMP_DIR=$ROOTFS_DIR/tizen_tmp +mkdir -p $TIZEN_TMP_DIR + +# Download files +echo ">>Start downloading files" +VERBOSE=1 $__CrossDir/tizen-fetch.sh $TIZEN_TMP_DIR $TIZEN_ARCH +echo "<>Start constructing Tizen rootfs" +TIZEN_RPM_FILES=`ls $TIZEN_TMP_DIR/*.rpm` +cd $ROOTFS_DIR +for f in $TIZEN_RPM_FILES; do + rpm2cpio $f | cpio -idm --quiet +done +echo "<>Start configuring Tizen rootfs" +ln -sfn asm-${LINK_ARCH} ./usr/include/asm +patch -p1 < $__TIZEN_CROSSDIR/tizen.patch +echo "</dev/null; then + VERBOSE=0 +fi + +Log() +{ + if [ $VERBOSE -ge $1 ]; then + echo ${@:2} + fi +} + +Inform() +{ + Log 1 -e "\x1B[0;34m$@\x1B[m" +} + +Debug() +{ + Log 2 -e "\x1B[0;32m$@\x1B[m" +} + +Error() +{ + >&2 Log 0 -e "\x1B[0;31m$@\x1B[m" +} + +Fetch() +{ + URL=$1 + FILE=$2 + PROGRESS=$3 + if [ $VERBOSE -ge 1 ] && [ $PROGRESS ]; then + CURL_OPT="--progress-bar" + else + CURL_OPT="--silent" + fi + curl $CURL_OPT $URL > $FILE +} + +hash curl 2> /dev/null || { Error "Require 'curl' Aborting."; exit 1; } +hash xmllint 2> /dev/null || { Error "Require 'xmllint' Aborting."; exit 1; } +hash sha256sum 2> /dev/null || { Error "Require 'sha256sum' Aborting."; exit 1; } + +TMPDIR=$1 +if [ ! -d $TMPDIR ]; then + TMPDIR=./tizen_tmp + Debug "Create temporary directory : $TMPDIR" + mkdir -p $TMPDIR +fi + +TIZEN_ARCH=$2 + +TIZEN_URL=http://download.tizen.org/snapshots/TIZEN/Tizen +BUILD_XML=build.xml +REPOMD_XML=repomd.xml +PRIMARY_XML=primary.xml +TARGET_URL="http://__not_initialized" + +Xpath_get() +{ + XPATH_RESULT='' + XPATH=$1 + XML_FILE=$2 + RESULT=$(xmllint --xpath $XPATH $XML_FILE) + if [[ -z ${RESULT// } ]]; then + Error "Can not find target from $XML_FILE" + Debug "Xpath = $XPATH" + exit 1 + fi + XPATH_RESULT=$RESULT +} + +fetch_tizen_pkgs_init() +{ + TARGET=$1 + PROFILE=$2 + Debug "Initialize TARGET=$TARGET, PROFILE=$PROFILE" + + TMP_PKG_DIR=$TMPDIR/tizen_${PROFILE}_pkgs + if [ -d $TMP_PKG_DIR ]; then rm -rf $TMP_PKG_DIR; fi + mkdir -p $TMP_PKG_DIR + + PKG_URL=$TIZEN_URL/$PROFILE/latest + + BUILD_XML_URL=$PKG_URL/$BUILD_XML + TMP_BUILD=$TMP_PKG_DIR/$BUILD_XML + TMP_REPOMD=$TMP_PKG_DIR/$REPOMD_XML + TMP_PRIMARY=$TMP_PKG_DIR/$PRIMARY_XML + TMP_PRIMARYGZ=${TMP_PRIMARY}.gz + + Fetch $BUILD_XML_URL $TMP_BUILD + + Debug "fetch $BUILD_XML_URL to $TMP_BUILD" + + TARGET_XPATH="//build/buildtargets/buildtarget[@name=\"$TARGET\"]/repo[@type=\"binary\"]/text()" + Xpath_get $TARGET_XPATH $TMP_BUILD + TARGET_PATH=$XPATH_RESULT + TARGET_URL=$PKG_URL/$TARGET_PATH + + REPOMD_URL=$TARGET_URL/repodata/repomd.xml + PRIMARY_XPATH='string(//*[local-name()="data"][@type="primary"]/*[local-name()="location"]/@href)' + + Fetch $REPOMD_URL $TMP_REPOMD + + Debug "fetch $REPOMD_URL to $TMP_REPOMD" + + Xpath_get $PRIMARY_XPATH $TMP_REPOMD + PRIMARY_XML_PATH=$XPATH_RESULT + PRIMARY_URL=$TARGET_URL/$PRIMARY_XML_PATH + + Fetch $PRIMARY_URL $TMP_PRIMARYGZ + + Debug "fetch $PRIMARY_URL to $TMP_PRIMARYGZ" + + gunzip $TMP_PRIMARYGZ + + Debug "unzip $TMP_PRIMARYGZ to $TMP_PRIMARY" +} + +fetch_tizen_pkgs() +{ + ARCH=$1 + PACKAGE_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="location"]/@href)' + + PACKAGE_CHECKSUM_XPATH_TPL='string(//*[local-name()="metadata"]/*[local-name()="package"][*[local-name()="name"][text()="_PKG_"]][*[local-name()="arch"][text()="_ARCH_"]]/*[local-name()="checksum"]/text())' + + for pkg in ${@:2} + do + Inform "Fetching... $pkg" + XPATH=${PACKAGE_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + PKG_PATH=$XPATH_RESULT + + XPATH=${PACKAGE_CHECKSUM_XPATH_TPL/_PKG_/$pkg} + XPATH=${XPATH/_ARCH_/$ARCH} + Xpath_get $XPATH $TMP_PRIMARY + CHECKSUM=$XPATH_RESULT + + PKG_URL=$TARGET_URL/$PKG_PATH + PKG_FILE=$(basename $PKG_PATH) + PKG_PATH=$TMPDIR/$PKG_FILE + + Debug "Download $PKG_URL to $PKG_PATH" + Fetch $PKG_URL $PKG_PATH true + + echo "$CHECKSUM $PKG_PATH" | sha256sum -c - > /dev/null + if [ $? -ne 0 ]; then + Error "Fail to fetch $PKG_URL to $PKG_PATH" + Debug "Checksum = $CHECKSUM" + exit 1 + fi + done +} + +Inform "Initialize ${TIZEN_ARCH} base" +fetch_tizen_pkgs_init standard Tizen-Base +Inform "fetch common packages" +fetch_tizen_pkgs ${TIZEN_ARCH} gcc gcc-devel-static glibc glibc-devel libicu libicu-devel libatomic linux-glibc-devel keyutils keyutils-devel libkeyutils +Inform "fetch coreclr packages" +fetch_tizen_pkgs ${TIZEN_ARCH} lldb lldb-devel libgcc libstdc++ libstdc++-devel libunwind libunwind-devel lttng-ust-devel lttng-ust userspace-rcu-devel userspace-rcu +Inform "fetch corefx packages" +fetch_tizen_pkgs ${TIZEN_ARCH} libcom_err libcom_err-devel zlib zlib-devel libopenssl11 libopenssl1.1-devel krb5 krb5-devel + +Inform "Initialize standard unified" +fetch_tizen_pkgs_init standard Tizen-Unified +Inform "fetch corefx packages" +fetch_tizen_pkgs ${TIZEN_ARCH} gssdp gssdp-devel tizen-release + diff --git a/eng/common/cross/toolchain.cmake b/eng/common/cross/toolchain.cmake index 9fd345bde6..ccfb9951a5 100644 --- a/eng/common/cross/toolchain.cmake +++ b/eng/common/cross/toolchain.cmake @@ -1,5 +1,12 @@ set(CROSS_ROOTFS $ENV{ROOTFS_DIR}) +# reset platform variables (e.g. cmake 3.25 sets LINUX=1) +unset(LINUX) +unset(FREEBSD) +unset(ILLUMOS) +unset(ANDROID) +unset(TIZEN) + set(TARGET_ARCH_NAME $ENV{TARGET_BUILD_ARCH}) if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) set(CMAKE_SYSTEM_NAME FreeBSD) @@ -7,6 +14,8 @@ if(EXISTS ${CROSS_ROOTFS}/bin/freebsd-version) elseif(EXISTS ${CROSS_ROOTFS}/usr/platform/i86pc) set(CMAKE_SYSTEM_NAME SunOS) set(ILLUMOS 1) +elseif(EXISTS ${CROSS_ROOTFS}/boot/system/develop/headers/config/HaikuConfig.h) + set(CMAKE_SYSTEM_NAME Haiku) else() set(CMAKE_SYSTEM_NAME Linux) set(LINUX 1) @@ -19,13 +28,7 @@ elseif(EXISTS ${CROSS_ROOTFS}/android_platform) set(ANDROID 1) endif() -if(TARGET_ARCH_NAME STREQUAL "armel") - set(CMAKE_SYSTEM_PROCESSOR armv7l) - set(TOOLCHAIN "arm-linux-gnueabi") - if(TIZEN) - set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0") - endif() -elseif(TARGET_ARCH_NAME STREQUAL "arm") +if(TARGET_ARCH_NAME STREQUAL "arm") set(CMAKE_SYSTEM_PROCESSOR armv7l) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv7-alpine-linux-musleabihf) set(TOOLCHAIN "armv7-alpine-linux-musleabihf") @@ -37,40 +40,62 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm") if(TIZEN) set(TIZEN_TOOLCHAIN "armv7hl-tizen-linux-gnueabihf/9.2.0") endif() -elseif(TARGET_ARCH_NAME STREQUAL "armv6") - set(CMAKE_SYSTEM_PROCESSOR armv6l) - if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf) - set(TOOLCHAIN "armv6-alpine-linux-musleabihf") - else() - set(TOOLCHAIN "arm-linux-gnueabihf") - endif() elseif(TARGET_ARCH_NAME STREQUAL "arm64") set(CMAKE_SYSTEM_PROCESSOR aarch64) if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/aarch64-alpine-linux-musl) set(TOOLCHAIN "aarch64-alpine-linux-musl") - else() + elseif(LINUX) set(TOOLCHAIN "aarch64-linux-gnu") + if(TIZEN) + set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") + endif() + elseif(FREEBSD) + set(triple "aarch64-unknown-freebsd12") endif() +elseif(TARGET_ARCH_NAME STREQUAL "armel") + set(CMAKE_SYSTEM_PROCESSOR armv7l) + set(TOOLCHAIN "arm-linux-gnueabi") if(TIZEN) - set(TIZEN_TOOLCHAIN "aarch64-tizen-linux-gnu/9.2.0") + set(TIZEN_TOOLCHAIN "armv7l-tizen-linux-gnueabi/9.2.0") + endif() +elseif(TARGET_ARCH_NAME STREQUAL "armv6") + set(CMAKE_SYSTEM_PROCESSOR armv6l) + if(EXISTS ${CROSS_ROOTFS}/usr/lib/gcc/armv6-alpine-linux-musleabihf) + set(TOOLCHAIN "armv6-alpine-linux-musleabihf") + else() + set(TOOLCHAIN "arm-linux-gnueabihf") endif() elseif(TARGET_ARCH_NAME STREQUAL "ppc64le") set(CMAKE_SYSTEM_PROCESSOR ppc64le) set(TOOLCHAIN "powerpc64le-linux-gnu") +elseif(TARGET_ARCH_NAME STREQUAL "riscv64") + set(CMAKE_SYSTEM_PROCESSOR riscv64) + set(TOOLCHAIN "riscv64-linux-gnu") elseif(TARGET_ARCH_NAME STREQUAL "s390x") set(CMAKE_SYSTEM_PROCESSOR s390x) set(TOOLCHAIN "s390x-linux-gnu") +elseif(TARGET_ARCH_NAME STREQUAL "x64") + set(CMAKE_SYSTEM_PROCESSOR x86_64) + if(LINUX) + set(TOOLCHAIN "x86_64-linux-gnu") + if(TIZEN) + set(TIZEN_TOOLCHAIN "x86_64-tizen-linux-gnu/9.2.0") + endif() + elseif(FREEBSD) + set(triple "x86_64-unknown-freebsd12") + elseif(ILLUMOS) + set(TOOLCHAIN "x86_64-illumos") + elseif(HAIKU) + set(TOOLCHAIN "x64_64-unknown-haiku") + endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") set(CMAKE_SYSTEM_PROCESSOR i686) set(TOOLCHAIN "i686-linux-gnu") -elseif (FREEBSD) - set(CMAKE_SYSTEM_PROCESSOR "x86_64") - set(triple "x86_64-unknown-freebsd12") -elseif (ILLUMOS) - set(CMAKE_SYSTEM_PROCESSOR "x86_64") - set(TOOLCHAIN "x86_64-illumos") + if(TIZEN) + set(TIZEN_TOOLCHAIN "i586-tizen-linux-gnu/9.2.0") + endif() else() - message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only armel, arm, armv6, arm64, ppc64le, s390x and x86 are supported!") + message(FATAL_ERROR "Arch is ${TARGET_ARCH_NAME}. Only arm, arm64, armel, armv6, ppc64le, riscv64, s390x, x64 and x86 are supported!") endif() if(DEFINED ENV{TOOLCHAIN}) @@ -91,6 +116,14 @@ if(TIZEN) include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/) include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/aarch64-tizen-linux-gnu) endif() + if(TARGET_ARCH_NAME STREQUAL "x86") + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}/include/c++/i586-tizen-linux-gnu) + endif() + if(TARGET_ARCH_NAME STREQUAL "x64") + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/) + include_directories(SYSTEM ${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}/include/c++/x86_64-tizen-linux-gnu) + endif() endif() if(ANDROID) @@ -152,6 +185,41 @@ elseif(ILLUMOS) set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") +elseif(HAIKU) + set(CMAKE_SYSROOT "${CROSS_ROOTFS}") + + set(TOOLSET_PREFIX ${TOOLCHAIN}-) + function(locate_toolchain_exec exec var) + string(TOUPPER ${exec} EXEC_UPPERCASE) + if(NOT "$ENV{CLR_${EXEC_UPPERCASE}}" STREQUAL "") + set(${var} "$ENV{CLR_${EXEC_UPPERCASE}}" PARENT_SCOPE) + return() + endif() + + set(SEARCH_PATH "${CROSS_ROOTFS}/generated/cross-tools-x86_64/bin") + + find_program(EXEC_LOCATION_${exec} + PATHS ${SEARCH_PATH} + NAMES + "${TOOLSET_PREFIX}${exec}${CLR_CMAKE_COMPILER_FILE_NAME_VERSION}" + "${TOOLSET_PREFIX}${exec}") + + if (EXEC_LOCATION_${exec} STREQUAL "EXEC_LOCATION_${exec}-NOTFOUND") + message(FATAL_ERROR "Unable to find toolchain executable. Name: ${exec}, Prefix: ${TOOLSET_PREFIX}.") + endif() + set(${var} ${EXEC_LOCATION_${exec}} PARENT_SCOPE) + endfunction() + + set(CMAKE_SYSTEM_PREFIX_PATH "${CROSS_ROOTFS}") + + locate_toolchain_exec(gcc CMAKE_C_COMPILER) + locate_toolchain_exec(g++ CMAKE_CXX_COMPILER) + + set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lssp") + set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lssp") + + # let CMake set up the correct search paths + include(Platform/Haiku) else() set(CMAKE_SYSROOT "${CROSS_ROOTFS}") @@ -184,7 +252,7 @@ if(TARGET_ARCH_NAME MATCHES "^(arm|armel)$") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") endif() -elseif(TARGET_ARCH_NAME STREQUAL "arm64") +elseif(TARGET_ARCH_NAME MATCHES "^(arm64|x64)$") if(TIZEN) add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib64/gcc/${TIZEN_TOOLCHAIN}") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib64") @@ -197,6 +265,13 @@ elseif(TARGET_ARCH_NAME STREQUAL "arm64") endif() elseif(TARGET_ARCH_NAME STREQUAL "x86") add_toolchain_linker_flag(-m32) + + if(TIZEN) + add_toolchain_linker_flag("-B${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib") + add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/lib/gcc/${TIZEN_TOOLCHAIN}") + endif() elseif(ILLUMOS) add_toolchain_linker_flag("-L${CROSS_ROOTFS}/lib/amd64") add_toolchain_linker_flag("-L${CROSS_ROOTFS}/usr/amd64/lib") @@ -204,7 +279,7 @@ endif() # Specify compile options -if((TARGET_ARCH_NAME MATCHES "^(arm|armv6|armel|arm64|ppc64le|s390x)$" AND NOT ANDROID) OR ILLUMOS) +if((TARGET_ARCH_NAME MATCHES "^(arm|arm64|armel|armv6|ppc64le|riscv64|s390x)$" AND NOT ANDROID AND NOT FREEBSD) OR ILLUMOS OR HAIKU) set(CMAKE_C_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_CXX_COMPILER_TARGET ${TOOLCHAIN}) set(CMAKE_ASM_COMPILER_TARGET ${TOOLCHAIN}) @@ -232,7 +307,7 @@ elseif(TARGET_ARCH_NAME STREQUAL "x86") endif() if(TIZEN) - if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64)$") + if(TARGET_ARCH_NAME MATCHES "^(arm|armel|arm64|x86)$") add_compile_options(-Wno-deprecated-declarations) # compile-time option add_compile_options(-D__extern_always_inline=inline) # compile-time option endif() diff --git a/eng/common/cross/x64/tizen/tizen.patch b/eng/common/cross/x64/tizen/tizen.patch new file mode 100644 index 0000000000..56fbc88109 --- /dev/null +++ b/eng/common/cross/x64/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib64/libc.so b/usr/lib64/libc.so +--- a/usr/lib64/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib64/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf64-x86-64) +-GROUP ( /lib64/libc.so.6 /usr/lib64/libc_nonshared.a AS_NEEDED ( /lib64/ld-linux-x86-64.so.2 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux-x86-64.so.2 ) ) diff --git a/eng/common/cross/x86/sources.list.focal b/eng/common/cross/x86/sources.list.focal new file mode 100644 index 0000000000..99d5731330 --- /dev/null +++ b/eng/common/cross/x86/sources.list.focal @@ -0,0 +1,11 @@ +deb http://archive.ubuntu.com/ubuntu/ focal main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ focal main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ focal-updates main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ focal-updates main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ focal-backports main restricted +deb-src http://archive.ubuntu.com/ubuntu/ focal-backports main restricted + +deb http://archive.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu/ focal-security main restricted universe multiverse diff --git a/eng/common/cross/x86/sources.list.jammy b/eng/common/cross/x86/sources.list.jammy new file mode 100644 index 0000000000..af1c1feaea --- /dev/null +++ b/eng/common/cross/x86/sources.list.jammy @@ -0,0 +1,11 @@ +deb http://archive.ubuntu.com/ubuntu/ jammy main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ jammy main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted universe +deb-src http://archive.ubuntu.com/ubuntu/ jammy-updates main restricted universe + +deb http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted +deb-src http://archive.ubuntu.com/ubuntu/ jammy-backports main restricted + +deb http://archive.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse +deb-src http://archive.ubuntu.com/ubuntu/ jammy-security main restricted universe multiverse diff --git a/eng/common/cross/x86/tizen/tizen.patch b/eng/common/cross/x86/tizen/tizen.patch new file mode 100644 index 0000000000..f4fe8838ad --- /dev/null +++ b/eng/common/cross/x86/tizen/tizen.patch @@ -0,0 +1,9 @@ +diff -u -r a/usr/lib/libc.so b/usr/lib/libc.so +--- a/usr/lib/libc.so 2016-12-30 23:00:08.284951863 +0900 ++++ b/usr/lib/libc.so 2016-12-30 23:00:32.140951815 +0900 +@@ -2,4 +2,4 @@ + Use the shared library, but some functions are only in + the static library, so try that secondarily. */ + OUTPUT_FORMAT(elf32-i386) +-GROUP ( /lib/libc.so.6 /usr/lib/libc_nonshared.a AS_NEEDED ( /lib/ld-linux.so.2 ) ) ++GROUP ( libc.so.6 libc_nonshared.a AS_NEEDED ( ld-linux.so.2 ) ) diff --git a/eng/common/dotnet-install.sh b/eng/common/dotnet-install.sh index 5c94e98632..abd045a324 100644 --- a/eng/common/dotnet-install.sh +++ b/eng/common/dotnet-install.sh @@ -52,7 +52,7 @@ done # Use uname to determine what the CPU is, see https://en.wikipedia.org/wiki/Uname#Examples cpuname=$(uname -m) case $cpuname in - aarch64) + arm64|aarch64) buildarch=arm64 ;; loongarch64) @@ -64,7 +64,7 @@ case $cpuname in armv*l) buildarch=arm ;; - i686) + i[3-6]86) buildarch=x86 ;; *) diff --git a/eng/common/generate-locproject.ps1 b/eng/common/generate-locproject.ps1 index 25e97ac007..bcb579e37a 100644 --- a/eng/common/generate-locproject.ps1 +++ b/eng/common/generate-locproject.ps1 @@ -10,9 +10,7 @@ Param( Set-StrictMode -Version 2.0 $ErrorActionPreference = "Stop" -. $PSScriptRoot\tools.ps1 - -Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1') +. $PSScriptRoot\pipeline-logging-functions.ps1 $exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json" $exclusions = @{ Exclusions = @() } @@ -28,13 +26,34 @@ $jsonFiles = @() $jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern $jsonTemplateFiles | ForEach-Object { $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json - + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json" $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru } $jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern +$wxlFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\.+\.wxl" -And -Not( $_.Directory.Name -Match "\d{4}" ) } # localized files live in four digit lang ID directories; this excludes them +if (-not $wxlFiles) { + $wxlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\\1033\\.+\.wxl" } # pick up en files (1033 = en) specifically so we can copy them to use as the neutral xlf files + if ($wxlEnFiles) { + $wxlFiles = @() + $wxlEnFiles | ForEach-Object { + $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" + $wxlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru + } + } +} + +$macosHtmlEnFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\.lproj\\.+\.html" } # add installer HTML files +$macosHtmlFiles = @() +if ($macosHtmlEnFiles) { + $macosHtmlEnFiles | ForEach-Object { + $destinationFile = "$($_.Directory.Parent.FullName)\$($_.Name)" + $macosHtmlFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru + } +} + $xlfFiles = @() $allXlfFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory\*\*.xlf" @@ -46,7 +65,7 @@ if ($allXlfFiles) { } $langXlfFiles | ForEach-Object { $null = $_.Name -Match "(.+)\.[\w-]+\.xlf" # matches '[filename].[langcode].xlf - + $destinationFile = "$($_.Directory.FullName)\$($Matches.1).xlf" $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru } @@ -59,10 +78,10 @@ $locJson = @{ LanguageSet = $LanguageSet LocItems = @( $locFiles | ForEach-Object { - $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" + $outputPath = "$(($_.DirectoryName | Resolve-Path -Relative) + "\")" $continue = $true foreach ($exclusion in $exclusions.Exclusions) { - if ($outputPath.Contains($exclusion)) + if ($_.FullName.Contains($exclusion)) { $continue = $false } @@ -79,8 +98,7 @@ $locJson = @{ CopyOption = "LangIDOnPath" OutputPath = "$($_.Directory.Parent.FullName | Resolve-Path -Relative)\" } - } - else { + } else { return @{ SourceFile = $sourceFile CopyOption = "LangIDOnName" @@ -90,6 +108,55 @@ $locJson = @{ } } ) + }, + @{ + LanguageSet = $LanguageSet + CloneLanguageSet = "WiX_CloneLanguages" + LssFiles = @( "wxl_loc.lss" ) + LocItems = @( + $wxlFiles | ForEach-Object { + $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if ($continue) + { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = $outputPath + } + } + } + ) + }, + @{ + LanguageSet = $LanguageSet + CloneLanguageSet = "VS_macOS_CloneLanguages" + LssFiles = @( ".\eng\common\loc\P22DotNetHtmlLocalization.lss" ) + LocItems = @( + $macosHtmlFiles | ForEach-Object { + $outputPath = "$($_.Directory.FullName | Resolve-Path -Relative)\" + $continue = $true + foreach ($exclusion in $exclusions.Exclusions) { + if ($_.FullName.Contains($exclusion)) { + $continue = $false + } + } + $sourceFile = ($_.FullName | Resolve-Path -Relative) + if ($continue) { + return @{ + SourceFile = $sourceFile + CopyOption = "LangIDOnPath" + OutputPath = $outputPath + } + } + } + ) } ) } @@ -108,10 +175,10 @@ else { if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) { Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them." - + exit 1 } else { Write-Host "Generated LocProject.json and current LocProject.json are identical." } -} \ No newline at end of file +} diff --git a/eng/common/generate-sbom-prep.ps1 b/eng/common/generate-sbom-prep.ps1 index a733a88858..3e5c1c74a1 100644 --- a/eng/common/generate-sbom-prep.ps1 +++ b/eng/common/generate-sbom-prep.ps1 @@ -2,6 +2,8 @@ Param( [Parameter(Mandatory=$true)][string] $ManifestDirPath # Manifest directory where sbom will be placed ) +. $PSScriptRoot\pipeline-logging-functions.ps1 + Write-Host "Creating dir $ManifestDirPath" # create directory for sbom manifest to be placed if (!(Test-Path -path $ManifestDirPath)) diff --git a/eng/common/generate-sbom-prep.sh b/eng/common/generate-sbom-prep.sh index f6c7745314..d5c76dc827 100644 --- a/eng/common/generate-sbom-prep.sh +++ b/eng/common/generate-sbom-prep.sh @@ -2,6 +2,18 @@ source="${BASH_SOURCE[0]}" +# resolve $SOURCE until the file is no longer a symlink +while [[ -h $source ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" +. $scriptroot/pipeline-logging-functions.sh + manifest_dir=$1 if [ ! -d "$manifest_dir" ] ; then diff --git a/eng/common/init-tools-native.ps1 b/eng/common/init-tools-native.ps1 index db830c00a6..27ccdb9ecc 100644 --- a/eng/common/init-tools-native.ps1 +++ b/eng/common/init-tools-native.ps1 @@ -31,6 +31,10 @@ Wait time between retry attempts in seconds .PARAMETER GlobalJsonFile File path to global.json file +.PARAMETER PathPromotion +Optional switch to enable either promote native tools specified in the global.json to the path (in Azure Pipelines) +or break the build if a native tool is not found on the path (on a local dev machine) + .NOTES #> [CmdletBinding(PositionalBinding=$false)] @@ -41,7 +45,8 @@ Param ( [switch] $Force = $False, [int] $DownloadRetries = 5, [int] $RetryWaitTimeInSeconds = 30, - [string] $GlobalJsonFile + [string] $GlobalJsonFile, + [switch] $PathPromotion ) if (!$GlobalJsonFile) { @@ -77,53 +82,99 @@ try { ConvertFrom-Json | Select-Object -Expand 'native-tools' -ErrorAction SilentlyContinue if ($NativeTools) { - $NativeTools.PSObject.Properties | ForEach-Object { - $ToolName = $_.Name - $ToolVersion = $_.Value - $LocalInstallerArguments = @{ ToolName = "$ToolName" } - $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } - $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } - $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } - $LocalInstallerArguments += @{ Version = "$ToolVersion" } - - if ($Verbose) { - $LocalInstallerArguments += @{ Verbose = $True } - } - if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { - if($Force) { - $LocalInstallerArguments += @{ Force = $True } - } - } - if ($Clean) { - $LocalInstallerArguments += @{ Clean = $True } - } - - Write-Verbose "Installing $ToolName version $ToolVersion" - Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" - & $InstallerPath @LocalInstallerArguments - if ($LASTEXITCODE -Ne "0") { - $errMsg = "$ToolName installation failed" - if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { - $showNativeToolsWarning = $true - if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { - $showNativeToolsWarning = $false + if ($PathPromotion -eq $True) { + $ArcadeToolsDirectory = "$env:SYSTEMDRIVE\arcade-tools" + if (Test-Path $ArcadeToolsDirectory) { # if this directory exists, we should use native tools on machine + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $InstalledTools = @{} + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + if ($ToolVersion -eq "latest") { + $ToolVersion = "" + } + $ToolDirectories = (Get-ChildItem -Path "$ArcadeToolsDirectory" -Filter "$ToolName-$ToolVersion*" | Sort-Object -Descending) + if ($ToolDirectories -eq $null) { + Write-Error "Unable to find directory for $ToolName $ToolVersion; please make sure the tool is installed on this image." + exit 1 } - if ($showNativeToolsWarning) { - Write-Warning $errMsg + $ToolDirectory = $ToolDirectories[0] + $BinPathFile = "$($ToolDirectory.FullName)\binpath.txt" + if (-not (Test-Path -Path "$BinPathFile")) { + Write-Error "Unable to find binpath.txt in '$($ToolDirectory.FullName)' ($ToolName $ToolVersion); artifact is either installed incorrectly or is not a bootstrappable tool." + exit 1 } - $toolInstallationFailure = $true - } else { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host $errMsg - exit 1 + $BinPath = Get-Content "$BinPathFile" + $ToolPath = Convert-Path -Path $BinPath + Write-Host "Adding $ToolName to the path ($ToolPath)..." + Write-Host "##vso[task.prependpath]$ToolPath" + $env:PATH = "$ToolPath;$env:PATH" + $InstalledTools += @{ $ToolName = $ToolDirectory.FullName } + } } + return $InstalledTools + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + + if ((Get-Command "$ToolName" -ErrorAction SilentlyContinue) -eq $null) { + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "$ToolName not found on path. Please install $ToolName $ToolVersion before proceeding." + Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message "If this is running on a build machine, the arcade-tools directory was not found, which means there's an error with the image." + } + } + exit 0 + } + } else { + $NativeTools.PSObject.Properties | ForEach-Object { + $ToolName = $_.Name + $ToolVersion = $_.Value + $LocalInstallerArguments = @{ ToolName = "$ToolName" } + $LocalInstallerArguments += @{ InstallPath = "$InstallBin" } + $LocalInstallerArguments += @{ BaseUri = "$BaseUri" } + $LocalInstallerArguments += @{ CommonLibraryDirectory = "$EngCommonBaseDir" } + $LocalInstallerArguments += @{ Version = "$ToolVersion" } + + if ($Verbose) { + $LocalInstallerArguments += @{ Verbose = $True } + } + if (Get-Variable 'Force' -ErrorAction 'SilentlyContinue') { + if($Force) { + $LocalInstallerArguments += @{ Force = $True } + } + } + if ($Clean) { + $LocalInstallerArguments += @{ Clean = $True } + } + + Write-Verbose "Installing $ToolName version $ToolVersion" + Write-Verbose "Executing '$InstallerPath $($LocalInstallerArguments.Keys.ForEach({"-$_ '$($LocalInstallerArguments.$_)'"}) -join ' ')'" + & $InstallerPath @LocalInstallerArguments + if ($LASTEXITCODE -Ne "0") { + $errMsg = "$ToolName installation failed" + if ((Get-Variable 'DoNotAbortNativeToolsInstallationOnFailure' -ErrorAction 'SilentlyContinue') -and $DoNotAbortNativeToolsInstallationOnFailure) { + $showNativeToolsWarning = $true + if ((Get-Variable 'DoNotDisplayNativeToolsInstallationWarnings' -ErrorAction 'SilentlyContinue') -and $DoNotDisplayNativeToolsInstallationWarnings) { + $showNativeToolsWarning = $false + } + if ($showNativeToolsWarning) { + Write-Warning $errMsg + } + $toolInstallationFailure = $true + } else { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host $errMsg + exit 1 + } + } + } + + if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { + # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 + Write-Host 'Native tools bootstrap failed' + exit 1 } - } - - if ((Get-Variable 'toolInstallationFailure' -ErrorAction 'SilentlyContinue') -and $toolInstallationFailure) { - # We cannot change this to Write-PipelineTelemetryError because of https://github.com/dotnet/arcade/issues/4482 - Write-Host 'Native tools bootstrap failed' - exit 1 } } else { @@ -139,7 +190,7 @@ try { Write-Host "##vso[task.prependpath]$(Convert-Path -Path $InstallBin)" return $InstallBin } - else { + elseif (-not ($PathPromotion)) { Write-PipelineTelemetryError -Category 'NativeToolsBootstrap' -Message 'Native tools install directory does not exist, installation failed' exit 1 } diff --git a/eng/common/internal/Tools.csproj b/eng/common/internal/Tools.csproj index beb9c4648e..7f5ce6d608 100644 --- a/eng/common/internal/Tools.csproj +++ b/eng/common/internal/Tools.csproj @@ -8,6 +8,9 @@ + + + diff --git a/eng/common/loc/P22DotNetHtmlLocalization.lss b/eng/common/loc/P22DotNetHtmlLocalization.lss new file mode 100644 index 0000000000000000000000000000000000000000..6661fed566e49b0c206665bc21f135e06c9b89c4 GIT binary patch literal 3810 zcmd^CT~8BH5S?ce|HG9Bo&v?0;P_m6l= znU-wb=}VLT7UH{>{5H;0M4iLmRE8QeBRr|>&k=uV*L;heKY+dq>B$0^=0I}|x`)s{ zht4t9zaiEh5Fe>E-;zVT;c4G?v;9N0G=rWwo~*(Cs`OS6ZL`HL;vsL0J@I%$+0YvE zx{9ukK|FtFx1PlPD5M;6ZM>f;1BhCf?`8y6QH*?RT9T>XwF z#~m_N+i^UKE^j{e;KdNW`kH9Rbj{G8tDY}mafCgG+m3H`I@_PhrDmcIzxD&IX@s083kV|lLUE^0(h6wWRPN0QN1n^PU5eX8r6OZ*s^g)tt77#SZCB}znxye#U$Dtinr6lnVu z!LzA{A}0~no7p$thFGJAnI}oSW||9H=Bz}I7kD#2MLg7WfrlE5o9sQjePc>qmv+6iQCmdp(y}(Vr literal 0 HcmV?d00001 diff --git a/eng/common/native/init-compiler.sh b/eng/common/native/init-compiler.sh index 6d7ba15e5f..7aee4213e1 100644 --- a/eng/common/native/init-compiler.sh +++ b/eng/common/native/init-compiler.sh @@ -1,30 +1,25 @@ -#!/usr/bin/env bash +#!/bin/sh # # This file detects the C/C++ compiler and exports it to the CC/CXX environment variables # # NOTE: some scripts source this file and rely on stdout being empty, make sure to not output anything here! -if [[ "$#" -lt 3 ]]; then +if [ -z "$build_arch" ] || [ -z "$compiler" ]; then echo "Usage..." - echo "init-compiler.sh