public static BuildTargetResult FinalizeBuild(BuildTargetContext c) { if (CheckIfAllBuildsHavePublished()) { string targetContainer = $"{Channel}/Binaries/Latest/"; string targetVersionFile = $"{targetContainer}{CliNuGetVersion}"; string semaphoreBlob = $"{Channel}/Binaries/publishSemaphore"; AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob); string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob); // Prevent race conditions by dropping a version hint of what version this is. If we see this file // and it is the same as our version then we know that a race happened where two+ builds finished // at the same time and someone already took care of publishing and we have no work to do. if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile)) { AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); return(c.Success()); } else { Regex versionFileRegex = new Regex(@"(?<version>\d\.\d\.\d)-(?<release>.*)?"); // Delete old version files AzurePublisherTool.ListBlobs($"{targetContainer}") .Select(s => s.Replace("/dotnet/", "")) .Where(s => versionFileRegex.IsMatch(s)) .ToList() .ForEach(f => AzurePublisherTool.TryDeleteBlob(f)); // Drop the version file signaling such for any race-condition builds (see above comment). AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile); } try { // Copy the latest CLI bits CopyBlobs($"{Channel}/Binaries/{CliNuGetVersion}/", targetContainer); // Copy the latest installer files CopyBlobs($"{Channel}/Installers/{CliNuGetVersion}/", $"{Channel}/Installers/Latest/"); // Generate the SDK Version text files List <string> versionFiles = new List <string>() { "win.x86.version", "win.x64.version", "ubuntu.x64.version", "rhel.x64.version", "osx.x64.version", "debian.x64.version", "centos.x64.version" }; string cliVersion = Utils.GetCliVersionFileContent(c); foreach (string version in versionFiles) { AzurePublisherTool.PublishStringToBlob($"{Channel}/dnvm/latest.{version}", cliVersion); } } finally { AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); } } return(c.Success()); }
private static bool ShouldDownloadAndPush(string hostBlob, string buildVersion) { // Set of runtime ids to look for to act as the signal that the build // as finished each of these legs of the build. Dictionary <string, bool> runtimes = new Dictionary <string, bool>() { { "win7-x64", false }, { "win7-x86", false }, { "osx.10.10-x64", false }, { "rhel.7-x64", false }, { "ubuntu.14.04-x64", false }, { "debian.8-x64", false }, }; var buildFiles = AzurePublisherTool.ListBlobs(hostBlob + buildVersion); foreach (var bf in buildFiles) { string buildFile = Path.GetFileName(bf); if (buildFile == PackagePushedSemaphoreFileName) { Console.WriteLine($"Found '{PackagePushedSemaphoreFileName}' for build version {buildVersion} so skipping this drop."); // Nothing to do because the latest build is uploaded. return(false); } foreach (var runtime in runtimes.Keys) { if (buildFile.StartsWith($"runtime.{runtime}")) { runtimes[runtime] = true; break; } } } bool missingRuntime = false; foreach (var runtime in runtimes) { if (!runtime.Value) { missingRuntime = true; Console.WriteLine($"Version {buildVersion} missing packages for runtime {runtime.Key}"); } } if (missingRuntime) { Console.WriteLine($"Build version {buildVersion} is missing some runtime packages so not pushing this drop."); } return(!missingRuntime); }
private static void CopyBlobsToLatest(string destinationFolder) { foreach (string blob in AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, CliNuGetVersion)) { string targetName = Path.GetFileName(blob) .Replace(CliNuGetVersion, "latest"); string target = $"{destinationFolder}/{targetName}"; AzurePublisherTool.CopyBlob(blob, target); } }
private static void CopyBlobs(string sourceFolder, string destinationFolder) { foreach (string blob in AzurePublisherTool.ListBlobs(sourceFolder)) { string source = blob.Replace("/dotnet/", ""); string targetName = Path.GetFileName(blob) .Replace(SharedFrameworkNugetVersion, "latest") .Replace(SharedHostNugetVersion, "latest"); string target = $"{destinationFolder}{targetName}"; AzurePublisherTool.CopyBlob(source, target); } }
private static bool CheckIfAllBuildsHavePublished() { Dictionary <string, bool> badges = new Dictionary <string, bool>() { { "sharedfx_Windows_x86", false }, { "sharedfx_Windows_x64", false }, { "sharedfx_Ubuntu_x64", false }, { "sharedfx_Ubuntu_16_04_x64", false }, { "sharedfx_Ubuntu_16_10_x64", false }, { "sharedfx_RHEL_x64", false }, { "sharedfx_OSX_x64", false }, { "sharedfx_Debian_x64", false }, { "sharedfx_CentOS_x64", false }, { "sharedfx_Fedora_23_x64", false }, { "sharedfx_openSUSE_13_2_x64", false }, { "sharedfx_openSUSE_42_1_x64", false } }; List <string> blobs = new List <string>(AzurePublisherTool.ListBlobs($"{Channel}/Binaries/{SharedFrameworkNugetVersion}/")); var versionBadgeName = $"sharedfx_{Monikers.GetBadgeMoniker()}"; if (badges.ContainsKey(versionBadgeName) == false) { throw new ArgumentException($"A new OS build ({versionBadgeName}) was added without adding the moniker to the {nameof(badges)} lookup"); } foreach (string file in blobs) { string name = Path.GetFileName(file); string key = string.Empty; foreach (string img in badges.Keys) { if ((name.StartsWith($"{img}")) && (name.EndsWith(".svg"))) { key = img; break; } } if (string.IsNullOrEmpty(key) == false) { badges[key] = true; } } return(badges.Keys.All(key => badges[key])); }
public static BuildTargetResult FinalizeBuild(BuildTargetContext c) { if (CheckIfAllBuildsHavePublished()) { string targetContainer = $"{AzurePublisher.Product.Sdk}/{Channel}"; string targetVersionFile = $"{targetContainer}/{CommitHash}"; string semaphoreBlob = $"{targetContainer}/publishSemaphore"; AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob); string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob); // Prevent race conditions by dropping a version hint of what version this is. If we see this file // and it is the same as our version then we know that a race happened where two+ builds finished // at the same time and someone already took care of publishing and we have no work to do. if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile)) { AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); return(c.Success()); } else { Regex versionFileRegex = new Regex(@"(?<CommitHash>[\w\d]{40})"); // Delete old version files AzurePublisherTool.ListBlobs(targetContainer) .Where(s => versionFileRegex.IsMatch(s)) .ToList() .ForEach(f => AzurePublisherTool.TryDeleteBlob(f)); // Drop the version file signaling such for any race-condition builds (see above comment). AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile); } try { CopyBlobsToLatest(targetContainer); string cliVersion = Utils.GetCliVersionFileContent(c); AzurePublisherTool.PublishStringToBlob($"{targetContainer}/latest.version", cliVersion); UpdateVersionsRepo(c); } finally { AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); } } return(c.Success()); }
public static BuildTargetResult PullNupkgFilesFromBlob(BuildTargetContext c) { Directory.CreateDirectory(Dirs.PackagesNoRID); var hostBlob = $"{Channel}/Binaries/"; string forcePushBuild = Environment.GetEnvironmentVariable("FORCE_PUBLISH_BLOB_BUILD_VERSION"); if (!string.IsNullOrEmpty(forcePushBuild)) { Console.WriteLine($"Forcing all nupkg packages for build version {forcePushBuild}."); DownloadPackagesForPush(hostBlob + forcePushBuild); return(c.Success()); } List <string> buildVersions = new List <string>(); Regex buildVersionRegex = new Regex(@"Binaries/(?<version>\d+\.\d+\.\d+(?<prerelease>-[^-]+-\d{6})?)/$"); foreach (string file in AzurePublisherTool.ListBlobs(hostBlob)) { var match = buildVersionRegex.Match(file); if (match.Success) { buildVersions.Add(match.Groups["version"].Value); } } // Sort decending buildVersions.Sort(); buildVersions.Reverse(); // Try to publish the last 10 builds foreach (var bv in buildVersions.Take(10)) { Console.WriteLine($"Checking drop version: {bv}"); if (ShouldDownloadAndPush(hostBlob, bv)) { DownloadPackagesForPush(hostBlob + bv); } } return(c.Success()); }
private static bool CheckIfAllBuildsHavePublished() { Dictionary <string, bool> badges = new Dictionary <string, bool>() { { "Windows_x86", false }, { "Windows_x64", false }, { "Ubuntu_x64", false }, { "RHEL_x64", false }, { "OSX_x64", false }, { "Debian_x64", false }, { "CentOS_x64", false } }; List <string> blobs = new List <string>(AzurePublisherTool.ListBlobs($"{Channel}/Binaries/{CliNuGetVersion}/")); var config = Environment.GetEnvironmentVariable("CONFIGURATION"); var versionBadgeName = $"{CurrentPlatform.Current}_{CurrentArchitecture.Current}"; if (badges.ContainsKey(versionBadgeName) == false) { throw new ArgumentException("A new OS build was added without adding the moniker to the {nameof(badges)} lookup"); } foreach (string file in blobs) { string name = Path.GetFileName(file); string key = string.Empty; foreach (string img in badges.Keys) { if ((name.StartsWith($"{img}")) && (name.EndsWith(".svg"))) { key = img; break; } } if (string.IsNullOrEmpty(key) == false) { badges[key] = true; } } return(badges.Keys.All(key => badges[key])); }
private static bool CheckIfAllBuildsHavePublished() { Dictionary <string, bool> badges = new Dictionary <string, bool>() { { "Windows_x86", false }, { "Windows_x64", false }, { "Ubuntu_x64", false }, { "Ubuntu_16_04_x64", false }, { "RHEL_x64", false }, { "OSX_x64", false }, { "Debian_x64", false }, { "CentOS_x64", false }, { "Fedora_23_x64", false }, { "openSUSE_13_2_x64", false } }; var versionBadgeName = $"{Monikers.GetBadgeMoniker()}"; if (!badges.ContainsKey(versionBadgeName)) { throw new ArgumentException($"A new OS build '{versionBadgeName}' was added without adding the moniker to the {nameof(badges)} lookup"); } IEnumerable <string> blobs = AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, CliNuGetVersion); foreach (string file in blobs) { string name = Path.GetFileName(file); foreach (string img in badges.Keys) { if ((name.StartsWith($"{img}")) && (name.EndsWith(".svg"))) { badges[img] = true; break; } } } return(badges.Values.All(v => v)); }
public static BuildTargetResult FinalizeBuild(BuildTargetContext c) { if (CheckIfAllBuildsHavePublished()) { string targetContainer = $"{Channel}/Binaries/Latest/"; string targetVersionFile = $"{targetContainer}{CommitHash}"; string semaphoreBlob = $"{Channel}/Binaries/sharedFxPublishSemaphore"; AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob); string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob); // Prevent race conditions by dropping a version hint of what version this is. If we see this file // and it is the same as our version then we know that a race happened where two+ builds finished // at the same time and someone already took care of publishing and we have no work to do. if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile)) { AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); return(c.Success()); } else { Regex versionFileRegex = new Regex(@"(?<CommitHash>[\w\d]{40})"); // Delete old version files AzurePublisherTool.ListBlobs($"{targetContainer}") .Select(s => s.Replace("/dotnet/", "")) .Where(s => versionFileRegex.IsMatch(s)) .ToList() .ForEach(f => AzurePublisherTool.TryDeleteBlob(f)); // Drop the version file signaling such for any race-condition builds (see above comment). AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile); } try { // Copy the shared framework + host Archives CopyBlobs($"{Channel}/Binaries/{SharedFrameworkNugetVersion}/", targetContainer); // Copy the shared framework installers CopyBlobs($"{Channel}/Installers/{SharedFrameworkNugetVersion}/", $"{Channel}/Installers/Latest/"); // Copy the shared host installers CopyBlobs($"{Channel}/Installers/{SharedHostNugetVersion}/", $"{Channel}/Installers/Latest/"); // Generate the Sharedfx Version text files List <string> versionFiles = new List <string>() { "win.x86.version", "win.x64.version", "win.arm.version", "win.arm64.version", "linux.x64.version", "ubuntu.x64.version", "ubuntu.14.04.arm.version", "ubuntu.16.04.x64.version", "ubuntu.16.04.arm.version", "ubuntu.16.10.x64.version", "rhel.x64.version", "osx.x64.version", "debian.x64.version", "centos.x64.version", "fedora.23.x64.version", "fedora.24.x64.version", "opensuse.13.2.x64.version", "opensuse.42.1.x64.version" }; PublishCoreHostPackagesToFeed(); string sfxVersion = Utils.GetSharedFrameworkVersionFileContent(c); foreach (string version in versionFiles) { AzurePublisherTool.PublishStringToBlob($"{Channel}/dnvm/latest.sharedfx.{version}", sfxVersion); } } finally { AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId); } } return(c.Success()); }
private static bool CheckIfAllBuildsHavePublished() { Dictionary <string, bool> badges = new Dictionary <string, bool>() { { "sharedfx_Windows_x86", false }, { "sharedfx_Windows_x64", false }, { "sharedfx_Windows_arm", false }, { "sharedfx_Windows_arm64", false }, { "sharedfx_Linux_x64", false }, { "sharedfx_Ubuntu_x64", false }, // { "sharedfx_Ubuntu_14_04_arm", false }, { "sharedfx_Ubuntu_16_04_x64", false }, // { "sharedfx_Ubuntu_16_04_arm", false }, { "sharedfx_Ubuntu_16_10_x64", false }, { "sharedfx_RHEL_x64", false }, { "sharedfx_OSX_x64", false }, { "sharedfx_Debian_x64", false }, { "sharedfx_CentOS_x64", false }, { "sharedfx_Fedora_23_x64", false }, { "sharedfx_Fedora_24_x64", false }, { "sharedfx_openSUSE_13_2_x64", false }, { "sharedfx_openSUSE_42_1_x64", false } }; List <string> blobs = new List <string>(AzurePublisherTool.ListBlobs($"{Channel}/Binaries/{SharedFrameworkNugetVersion}/")); var versionBadgeName = $"sharedfx_{Monikers.GetBadgeMoniker()}"; if (badges.ContainsKey(versionBadgeName) == false) { throw new ArgumentException($"A new OS build ({versionBadgeName}) was added without adding the moniker to the {nameof(badges)} lookup"); } foreach (string file in blobs) { string name = Path.GetFileName(file); if (!name.EndsWith(".svg")) { continue; } // Include _ delimiter when matching to prevent finding both arm and arm64 badges // when checking the arm badge file. string[] matchingBadgeKeys = badges.Keys .Where(badgeName => name.StartsWith($"{badgeName}_")) .ToArray(); if (matchingBadgeKeys.Length == 1) { badges[matchingBadgeKeys[0]] = true; } else if (matchingBadgeKeys.Length > 1) { throw new BuildFailureException( $"Expected 0 or 1 badges matching file '{name}', " + $"found {matchingBadgeKeys.Length}: " + string.Join(", ", matchingBadgeKeys)); } } foreach (string unfinishedBadge in badges.Where(pair => !pair.Value).Select(pair => pair.Key)) { Console.WriteLine($"Not all builds complete, badge not found: {unfinishedBadge}"); } return(badges.Keys.All(key => badges[key])); }