public static BuildTargetResult FinalizeBuild(BuildTargetContext c)
{
if (CheckIfAllBuildsHavePublished())
{
string targetContainer = $"{Channel}/Binaries/Latest/";
string targetVersionFile = $"{targetContainer}{CliNuGetVersion}";
string semaphoreBlob = $"{Channel}/Binaries/publishSemaphore";
AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob);
string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob);
// Prevent race conditions by dropping a version hint of what version this is. If we see this file
// and it is the same as our version then we know that a race happened where two+ builds finished
// at the same time and someone already took care of publishing and we have no work to do.
if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile))
{
AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
return(c.Success());
}
else
{
Regex versionFileRegex = new Regex(@"(?<version>\d\.\d\.\d)-(?<release>.*)?");
// Delete old version files
AzurePublisherTool.ListBlobs($"{targetContainer}")
.Select(s => s.Replace("/dotnet/", ""))
.Where(s => versionFileRegex.IsMatch(s))
.ToList()
.ForEach(f => AzurePublisherTool.TryDeleteBlob(f));
// Drop the version file signaling such for any race-condition builds (see above comment).
AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile);
}
try
{
// Copy the latest CLI bits
CopyBlobs($"{Channel}/Binaries/{CliNuGetVersion}/", targetContainer);
// Copy the latest installer files
CopyBlobs($"{Channel}/Installers/{CliNuGetVersion}/", $"{Channel}/Installers/Latest/");
// Generate the SDK Version text files
List <string> versionFiles = new List <string>()
{
"win.x86.version", "win.x64.version", "ubuntu.x64.version", "rhel.x64.version", "osx.x64.version", "debian.x64.version", "centos.x64.version"
};
string cliVersion = Utils.GetCliVersionFileContent(c);
foreach (string version in versionFiles)
{
AzurePublisherTool.PublishStringToBlob($"{Channel}/dnvm/latest.{version}", cliVersion);
}
}
finally
{
AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
}
}
return(c.Success());
}
private static bool ShouldDownloadAndPush(string hostBlob, string buildVersion)
{
// Set of runtime ids to look for to act as the signal that the build
// as finished each of these legs of the build.
Dictionary <string, bool> runtimes = new Dictionary <string, bool>()
{
{ "win7-x64", false },
{ "win7-x86", false },
{ "osx.10.10-x64", false },
{ "rhel.7-x64", false },
{ "ubuntu.14.04-x64", false },
{ "debian.8-x64", false },
};
var buildFiles = AzurePublisherTool.ListBlobs(hostBlob + buildVersion);
foreach (var bf in buildFiles)
{
string buildFile = Path.GetFileName(bf);
if (buildFile == PackagePushedSemaphoreFileName)
{
Console.WriteLine($"Found '{PackagePushedSemaphoreFileName}' for build version {buildVersion} so skipping this drop.");
// Nothing to do because the latest build is uploaded.
return(false);
}
foreach (var runtime in runtimes.Keys)
{
if (buildFile.StartsWith($"runtime.{runtime}"))
{
runtimes[runtime] = true;
break;
}
}
}
bool missingRuntime = false;
foreach (var runtime in runtimes)
{
if (!runtime.Value)
{
missingRuntime = true;
Console.WriteLine($"Version {buildVersion} missing packages for runtime {runtime.Key}");
}
}
if (missingRuntime)
{
Console.WriteLine($"Build version {buildVersion} is missing some runtime packages so not pushing this drop.");
}
return(!missingRuntime);
}
private static void CopyBlobsToLatest(string destinationFolder)
{
foreach (string blob in AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, CliNuGetVersion))
{
string targetName = Path.GetFileName(blob)
.Replace(CliNuGetVersion, "latest");
string target = $"{destinationFolder}/{targetName}";
AzurePublisherTool.CopyBlob(blob, target);
}
}
private static void CopyBlobs(string sourceFolder, string destinationFolder)
{
foreach (string blob in AzurePublisherTool.ListBlobs(sourceFolder))
{
string source = blob.Replace("/dotnet/", "");
string targetName = Path.GetFileName(blob)
.Replace(SharedFrameworkNugetVersion, "latest")
.Replace(SharedHostNugetVersion, "latest");
string target = $"{destinationFolder}{targetName}";
AzurePublisherTool.CopyBlob(source, target);
}
}
private static bool CheckIfAllBuildsHavePublished()
{
Dictionary <string, bool> badges = new Dictionary <string, bool>()
{
{ "sharedfx_Windows_x86", false },
{ "sharedfx_Windows_x64", false },
{ "sharedfx_Ubuntu_x64", false },
{ "sharedfx_Ubuntu_16_04_x64", false },
{ "sharedfx_Ubuntu_16_10_x64", false },
{ "sharedfx_RHEL_x64", false },
{ "sharedfx_OSX_x64", false },
{ "sharedfx_Debian_x64", false },
{ "sharedfx_CentOS_x64", false },
{ "sharedfx_Fedora_23_x64", false },
{ "sharedfx_openSUSE_13_2_x64", false },
{ "sharedfx_openSUSE_42_1_x64", false }
};
List <string> blobs = new List <string>(AzurePublisherTool.ListBlobs($"{Channel}/Binaries/{SharedFrameworkNugetVersion}/"));
var versionBadgeName = $"sharedfx_{Monikers.GetBadgeMoniker()}";
if (badges.ContainsKey(versionBadgeName) == false)
{
throw new ArgumentException($"A new OS build ({versionBadgeName}) was added without adding the moniker to the {nameof(badges)} lookup");
}
foreach (string file in blobs)
{
string name = Path.GetFileName(file);
string key = string.Empty;
foreach (string img in badges.Keys)
{
if ((name.StartsWith($"{img}")) && (name.EndsWith(".svg")))
{
key = img;
break;
}
}
if (string.IsNullOrEmpty(key) == false)
{
badges[key] = true;
}
}
return(badges.Keys.All(key => badges[key]));
}
public static BuildTargetResult FinalizeBuild(BuildTargetContext c)
{
if (CheckIfAllBuildsHavePublished())
{
string targetContainer = $"{AzurePublisher.Product.Sdk}/{Channel}";
string targetVersionFile = $"{targetContainer}/{CommitHash}";
string semaphoreBlob = $"{targetContainer}/publishSemaphore";
AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob);
string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob);
// Prevent race conditions by dropping a version hint of what version this is. If we see this file
// and it is the same as our version then we know that a race happened where two+ builds finished
// at the same time and someone already took care of publishing and we have no work to do.
if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile))
{
AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
return(c.Success());
}
else
{
Regex versionFileRegex = new Regex(@"(?<CommitHash>[\w\d]{40})");
// Delete old version files
AzurePublisherTool.ListBlobs(targetContainer)
.Where(s => versionFileRegex.IsMatch(s))
.ToList()
.ForEach(f => AzurePublisherTool.TryDeleteBlob(f));
// Drop the version file signaling such for any race-condition builds (see above comment).
AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile);
}
try
{
CopyBlobsToLatest(targetContainer);
string cliVersion = Utils.GetCliVersionFileContent(c);
AzurePublisherTool.PublishStringToBlob($"{targetContainer}/latest.version", cliVersion);
UpdateVersionsRepo(c);
}
finally
{
AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
}
}
return(c.Success());
}
public static BuildTargetResult PullNupkgFilesFromBlob(BuildTargetContext c)
{
Directory.CreateDirectory(Dirs.PackagesNoRID);
var hostBlob = $"{Channel}/Binaries/";
string forcePushBuild = Environment.GetEnvironmentVariable("FORCE_PUBLISH_BLOB_BUILD_VERSION");
if (!string.IsNullOrEmpty(forcePushBuild))
{
Console.WriteLine($"Forcing all nupkg packages for build version {forcePushBuild}.");
DownloadPackagesForPush(hostBlob + forcePushBuild);
return(c.Success());
}
List <string> buildVersions = new List <string>();
Regex buildVersionRegex = new Regex(@"Binaries/(?<version>\d+\.\d+\.\d+(?<prerelease>-[^-]+-\d{6})?)/$");
foreach (string file in AzurePublisherTool.ListBlobs(hostBlob))
{
var match = buildVersionRegex.Match(file);
if (match.Success)
{
buildVersions.Add(match.Groups["version"].Value);
}
}
// Sort decending
buildVersions.Sort();
buildVersions.Reverse();
// Try to publish the last 10 builds
foreach (var bv in buildVersions.Take(10))
{
Console.WriteLine($"Checking drop version: {bv}");
if (ShouldDownloadAndPush(hostBlob, bv))
{
DownloadPackagesForPush(hostBlob + bv);
}
}
return(c.Success());
}
private static bool CheckIfAllBuildsHavePublished()
{
Dictionary <string, bool> badges = new Dictionary <string, bool>()
{
{ "Windows_x86", false },
{ "Windows_x64", false },
{ "Ubuntu_x64", false },
{ "RHEL_x64", false },
{ "OSX_x64", false },
{ "Debian_x64", false },
{ "CentOS_x64", false }
};
List <string> blobs = new List <string>(AzurePublisherTool.ListBlobs($"{Channel}/Binaries/{CliNuGetVersion}/"));
var config = Environment.GetEnvironmentVariable("CONFIGURATION");
var versionBadgeName = $"{CurrentPlatform.Current}_{CurrentArchitecture.Current}";
if (badges.ContainsKey(versionBadgeName) == false)
{
throw new ArgumentException("A new OS build was added without adding the moniker to the {nameof(badges)} lookup");
}
foreach (string file in blobs)
{
string name = Path.GetFileName(file);
string key = string.Empty;
foreach (string img in badges.Keys)
{
if ((name.StartsWith($"{img}")) && (name.EndsWith(".svg")))
{
key = img;
break;
}
}
if (string.IsNullOrEmpty(key) == false)
{
badges[key] = true;
}
}
return(badges.Keys.All(key => badges[key]));
}
private static bool CheckIfAllBuildsHavePublished()
{
Dictionary <string, bool> badges = new Dictionary <string, bool>()
{
{ "Windows_x86", false },
{ "Windows_x64", false },
{ "Ubuntu_x64", false },
{ "Ubuntu_16_04_x64", false },
{ "RHEL_x64", false },
{ "OSX_x64", false },
{ "Debian_x64", false },
{ "CentOS_x64", false },
{ "Fedora_23_x64", false },
{ "openSUSE_13_2_x64", false }
};
var versionBadgeName = $"{Monikers.GetBadgeMoniker()}";
if (!badges.ContainsKey(versionBadgeName))
{
throw new ArgumentException($"A new OS build '{versionBadgeName}' was added without adding the moniker to the {nameof(badges)} lookup");
}
IEnumerable <string> blobs = AzurePublisherTool.ListBlobs(AzurePublisher.Product.Sdk, CliNuGetVersion);
foreach (string file in blobs)
{
string name = Path.GetFileName(file);
foreach (string img in badges.Keys)
{
if ((name.StartsWith($"{img}")) && (name.EndsWith(".svg")))
{
badges[img] = true;
break;
}
}
}
return(badges.Values.All(v => v));
}
public static BuildTargetResult FinalizeBuild(BuildTargetContext c)
{
if (CheckIfAllBuildsHavePublished())
{
string targetContainer = $"{Channel}/Binaries/Latest/";
string targetVersionFile = $"{targetContainer}{CommitHash}";
string semaphoreBlob = $"{Channel}/Binaries/sharedFxPublishSemaphore";
AzurePublisherTool.CreateBlobIfNotExists(semaphoreBlob);
string leaseId = AzurePublisherTool.AcquireLeaseOnBlob(semaphoreBlob);
// Prevent race conditions by dropping a version hint of what version this is. If we see this file
// and it is the same as our version then we know that a race happened where two+ builds finished
// at the same time and someone already took care of publishing and we have no work to do.
if (AzurePublisherTool.IsLatestSpecifiedVersion(targetVersionFile))
{
AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
return(c.Success());
}
else
{
Regex versionFileRegex = new Regex(@"(?<CommitHash>[\w\d]{40})");
// Delete old version files
AzurePublisherTool.ListBlobs($"{targetContainer}")
.Select(s => s.Replace("/dotnet/", ""))
.Where(s => versionFileRegex.IsMatch(s))
.ToList()
.ForEach(f => AzurePublisherTool.TryDeleteBlob(f));
// Drop the version file signaling such for any race-condition builds (see above comment).
AzurePublisherTool.DropLatestSpecifiedVersion(targetVersionFile);
}
try
{
// Copy the shared framework + host Archives
CopyBlobs($"{Channel}/Binaries/{SharedFrameworkNugetVersion}/", targetContainer);
// Copy the shared framework installers
CopyBlobs($"{Channel}/Installers/{SharedFrameworkNugetVersion}/", $"{Channel}/Installers/Latest/");
// Copy the shared host installers
CopyBlobs($"{Channel}/Installers/{SharedHostNugetVersion}/", $"{Channel}/Installers/Latest/");
// Generate the Sharedfx Version text files
List <string> versionFiles = new List <string>()
{
"win.x86.version",
"win.x64.version",
"win.arm.version",
"win.arm64.version",
"linux.x64.version",
"ubuntu.x64.version",
"ubuntu.14.04.arm.version",
"ubuntu.16.04.x64.version",
"ubuntu.16.04.arm.version",
"ubuntu.16.10.x64.version",
"rhel.x64.version",
"osx.x64.version",
"debian.x64.version",
"centos.x64.version",
"fedora.23.x64.version",
"fedora.24.x64.version",
"opensuse.13.2.x64.version",
"opensuse.42.1.x64.version"
};
PublishCoreHostPackagesToFeed();
string sfxVersion = Utils.GetSharedFrameworkVersionFileContent(c);
foreach (string version in versionFiles)
{
AzurePublisherTool.PublishStringToBlob($"{Channel}/dnvm/latest.sharedfx.{version}", sfxVersion);
}
}
finally
{
AzurePublisherTool.ReleaseLeaseOnBlob(semaphoreBlob, leaseId);
}
}
return(c.Success());
}
private static bool CheckIfAllBuildsHavePublished()
{
Dictionary <string, bool> badges = new Dictionary <string, bool>()
{
{ "sharedfx_Windows_x86", false },
{ "sharedfx_Windows_x64", false },
{ "sharedfx_Windows_arm", false },
{ "sharedfx_Windows_arm64", false },
{ "sharedfx_Linux_x64", false },
{ "sharedfx_Ubuntu_x64", false },
// { "sharedfx_Ubuntu_14_04_arm", false },
{ "sharedfx_Ubuntu_16_04_x64", false },
// { "sharedfx_Ubuntu_16_04_arm", false },
{ "sharedfx_Ubuntu_16_10_x64", false },
{ "sharedfx_RHEL_x64", false },
{ "sharedfx_OSX_x64", false },
{ "sharedfx_Debian_x64", false },
{ "sharedfx_CentOS_x64", false },
{ "sharedfx_Fedora_23_x64", false },
{ "sharedfx_Fedora_24_x64", false },
{ "sharedfx_openSUSE_13_2_x64", false },
{ "sharedfx_openSUSE_42_1_x64", false }
};
List <string> blobs = new List <string>(AzurePublisherTool.ListBlobs($"{Channel}/Binaries/{SharedFrameworkNugetVersion}/"));
var versionBadgeName = $"sharedfx_{Monikers.GetBadgeMoniker()}";
if (badges.ContainsKey(versionBadgeName) == false)
{
throw new ArgumentException($"A new OS build ({versionBadgeName}) was added without adding the moniker to the {nameof(badges)} lookup");
}
foreach (string file in blobs)
{
string name = Path.GetFileName(file);
if (!name.EndsWith(".svg"))
{
continue;
}
// Include _ delimiter when matching to prevent finding both arm and arm64 badges
// when checking the arm badge file.
string[] matchingBadgeKeys = badges.Keys
.Where(badgeName => name.StartsWith($"{badgeName}_"))
.ToArray();
if (matchingBadgeKeys.Length == 1)
{
badges[matchingBadgeKeys[0]] = true;
}
else if (matchingBadgeKeys.Length > 1)
{
throw new BuildFailureException(
$"Expected 0 or 1 badges matching file '{name}', " +
$"found {matchingBadgeKeys.Length}: " +
string.Join(", ", matchingBadgeKeys));
}
}
foreach (string unfinishedBadge in badges.Where(pair => !pair.Value).Select(pair => pair.Key))
{
Console.WriteLine($"Not all builds complete, badge not found: {unfinishedBadge}");
}
return(badges.Keys.All(key => badges[key]));
}