private void SaveTagInfoToImageInfoFile(IEnumerable <ImageInfo> imageInfos, DateTime createdDate)
{
this.loggerService.WriteSubheading("SETTING TAG INFO");
ImageArtifactDetails imageArtifactDetails = ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest);
// Find the images from the image info file that correspond to the images that were published
IEnumerable <ImageData> imageDataList = imageArtifactDetails.Repos
.SelectMany(repo => repo.Images)
.Where(image => imageInfos.Contains(image.ManifestImage));
if (imageDataList.Count() != imageInfos.Count())
{
this.loggerService.WriteError(
$"There is a mismatch between the number of images being published and the number of images contained in the image info file ({imageInfos.Count()} vs {imageDataList.Count()}, respectively).");
this.environmentService.Exit(1);
}
Parallel.ForEach(imageDataList, image =>
{
image.Manifest.Created = createdDate;
TagInfo sharedTag = image.ManifestImage.SharedTags.First();
JArray tagManifest = this.manifestToolService.Inspect(sharedTag.FullyQualifiedName, Options.IsDryRun);
string digest = tagManifest?
.OfType <JObject>()
.First(manifestType => manifestType["MediaType"].Value <string>() == ManifestListMediaType)
["Digest"].Value <string>();
image.Manifest.Digest = digest;
});
string imageInfoString = JsonHelper.SerializeObject(imageArtifactDetails);
File.WriteAllText(Options.ImageInfoPath, imageInfoString);
}
private async Task <GitObject> GetUpdatedImageInfoGitObjectAsync()
{
ImageArtifactDetails srcImageArtifactDetails = ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest);
string repoPath = await GitHelper.DownloadAndExtractGitRepoArchiveAsync(httpClient, Options.GitOptions);
try
{
string repoImageInfoPath = Path.Combine(repoPath, Options.GitOptions.Path);
string originalTargetImageInfoContents = File.ReadAllText(repoImageInfoPath);
ImageArtifactDetails newImageArtifactDetails;
if (originalTargetImageInfoContents != null)
{
ImageArtifactDetails targetImageArtifactDetails = ImageInfoHelper.LoadFromContent(
originalTargetImageInfoContents, Manifest, skipManifestValidation: true);
RemoveOutOfDateContent(targetImageArtifactDetails);
ImageInfoMergeOptions options = new ImageInfoMergeOptions
{
ReplaceTags = true
};
ImageInfoHelper.MergeImageArtifactDetails(srcImageArtifactDetails, targetImageArtifactDetails, options);
newImageArtifactDetails = targetImageArtifactDetails;
}
else
{
// If there is no existing file to update, there's nothing to merge with so the source data
// becomes the target data.
newImageArtifactDetails = srcImageArtifactDetails;
}
string newTargetImageInfoContents =
JsonHelper.SerializeObject(newImageArtifactDetails) + Environment.NewLine;
if (originalTargetImageInfoContents != newTargetImageInfoContents)
{
return(new GitObject
{
Path = Options.GitOptions.Path,
Type = GitObject.TypeBlob,
Mode = GitObject.ModeFile,
Content = newTargetImageInfoContents
});
}
else
{
return(null);
}
}
finally
{
Directory.Delete(repoPath, recursive: true);
}
}
private (string imageInfo, string layerInfo) GetImageInfoCsv()
{
StringBuilder imageInfo = new();
StringBuilder layerInfo = new();
foreach (RepoData repo in ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest).Repos)
{
foreach (ImageData image in repo.Images)
{
foreach (PlatformData platform in image.Platforms)
{
string timestamp = platform.Created.ToUniversalTime().ToString("yyyy'-'MM'-'dd' 'HH':'mm':'ss");
string sha = DockerHelper.GetDigestSha(platform.Digest);
imageInfo.AppendLine(FormatImageCsv(sha, platform, image, repo.Repo, timestamp));
IEnumerable <TagInfo> tagInfos = (platform.PlatformInfo?.Tags ?? Enumerable.Empty <TagInfo>())
.Where(tagInfo => platform.SimpleTags.Contains(tagInfo.Name))
.ToList();
IEnumerable <string> syndicatedRepos = tagInfos
.Select(tag => tag.SyndicatedRepo)
.Where(repo => repo != null)
.Distinct();
foreach (string syndicatedRepo in syndicatedRepos)
{
imageInfo.AppendLine(
FormatImageCsv(sha, platform, image, syndicatedRepo, timestamp));
}
foreach (TagInfo tag in tagInfos)
{
imageInfo.AppendLine(FormatImageCsv(tag.Name, platform, image, repo.Repo, timestamp));
if (tag.SyndicatedRepo != null)
{
foreach (string destinationTag in tag.SyndicatedDestinationTags)
{
imageInfo.AppendLine(
FormatImageCsv(destinationTag, platform, image, tag.SyndicatedRepo, timestamp));
}
}
}
for (int i = 0; i < platform.Layers.Count; i++)
{
// TODO: Track layer size (currently set to 0) https://github.com/dotnet/docker-tools/issues/745
layerInfo.AppendLine(FormatLayerCsv(
platform.Layers[i], 0, platform.Layers.Count - i, sha, platform, image, repo.Repo, timestamp));
}
}
}
}
// Kusto ingest API does not handle an empty line, therefore the last line must be trimmed.
return(imageInfo.ToString().TrimEnd(Environment.NewLine), layerInfo.ToString().TrimEnd(Environment.NewLine));
}
private async Task <IEnumerable <ImageResultInfo> > WaitForImageIngestionAsync(IMcrStatusClient statusClient)
{
ImageArtifactDetails imageArtifactDetails = ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest);
List <Task <ImageResultInfo> > tasks = GetImageDigestInfos(imageArtifactDetails)
.Select(digestInfo => ReportImageStatusWithContinuationAsync(statusClient, digestInfo))
.ToList();
return(await TaskHelper.WhenAll(tasks, Options.WaitTimeout));
}
public CopyAcrImagesCommand(IAzureManagementFactory azureManagementFactory, IEnvironmentService environmentService) : base()
{
this.azureManagementFactory = azureManagementFactory ?? throw new ArgumentNullException(nameof(azureManagementFactory));
this.environmentService = environmentService ?? throw new ArgumentNullException(nameof(environmentService));
this.imageArtifactDetails = new Lazy <ImageArtifactDetails>(() =>
{
if (!String.IsNullOrEmpty(Options.ImageInfoPath))
{
return(ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest));
}
return(null);
});
}
public CopyAcrImagesCommand(
IAzureManagementFactory azureManagementFactory, ILoggerService loggerService)
: base(azureManagementFactory, loggerService)
{
_imageArtifactDetails = new Lazy <ImageArtifactDetails>(() =>
{
if (!string.IsNullOrEmpty(Options.ImageInfoPath))
{
return(ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest));
}
return(null);
});
}
private string GetImageInfoCsv()
{
StringBuilder builder = new StringBuilder();
foreach (RepoData repo in ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest).Repos)
{
foreach (ImageData image in repo.Images)
{
foreach (PlatformData platform in image.Platforms)
{
string timestamp = platform.Created.ToUniversalTime().ToString("yyyy'-'MM'-'dd' 'HH':'mm':'ss");
string sha = DockerHelper.GetDigestSha(platform.Digest);
builder.AppendLine(FormatCsv(sha, platform, image, repo.Repo, timestamp));
IEnumerable <TagInfo> tagInfos = platform.PlatformInfo.Tags
.Where(tagInfo => platform.SimpleTags.Contains(tagInfo.Name))
.ToList();
IEnumerable <string> syndicatedRepos = tagInfos
.Select(tag => tag.SyndicatedRepo)
.Where(repo => repo != null)
.Distinct();
foreach (string syndicatedRepo in syndicatedRepos)
{
builder.AppendLine(
FormatCsv(sha, platform, image, syndicatedRepo, timestamp));
}
foreach (TagInfo tag in tagInfos)
{
builder.AppendLine(FormatCsv(tag.Name, platform, image, repo.Repo, timestamp));
if (tag.SyndicatedRepo != null)
{
foreach (string destinationTag in tag.SyndicatedDestinationTags)
{
builder.AppendLine(
FormatCsv(destinationTag, platform, image, tag.SyndicatedRepo, timestamp));
}
}
}
}
}
}
// Kusto ingest API does not handle an empty line, therefore the last line must be trimmed.
return(builder.ToString().TrimEnd(Environment.NewLine));
}
public override Task ExecuteAsync()
{
IEnumerable <(string Tag, string Platform)> platformTags;
if (string.IsNullOrEmpty(Options.ImageInfoPath))
{
platformTags = Manifest.GetFilteredPlatforms()
.Where(platform => platform.Tags.Any())
.Select(platform => (platform.Tags.First().FullyQualifiedName, platform.PlatformLabel));
}
else
{
// We want to apply manifest filtering to the loading of the image info file. This allows, for example,
// only images of a specific architecture to be pulled.
ImageArtifactDetails imageArtifactDetails = ImageInfoHelper.LoadFromFile(
Options.ImageInfoPath, Manifest, skipManifestValidation: true, useFilteredManifest: true);
platformTags = imageArtifactDetails.Repos
.SelectMany(repo => repo.Images)
.SelectMany(image => image.Platforms)
// If the platform doesn't have an associated manifest instance, it means the manifest filter
// options had filtered out the platform. In that case, it doesn't apply and shouldn't be pulled.
.Where(platform => platform.PlatformInfo is not null && platform.SimpleTags.Any())
.Select(platform => (
TagInfo.GetFullyQualifiedName(platform.PlatformInfo !.FullRepoModelName, platform.SimpleTags.First()),
platform.PlatformInfo !.PlatformLabel));
}
platformTags = platformTags
.Distinct()
.ToList();
_loggerService.WriteHeading("PULLING IMAGES");
foreach ((string tag, string platform) in platformTags)
{
_dockerService.PullImage(tag, platform, Options.IsDryRun);
}
if (Options.OutputVariableName is not null)
{
_loggerService.WriteMessage(
PipelineHelper.FormatOutputVariable(
Options.OutputVariableName,
string.Join(',', platformTags.Select(platformTag => platformTag.Tag))));
}
return(Task.CompletedTask);
}
private string?GetUpdatedImageInfo(string repoPath)
{
ImageArtifactDetails srcImageArtifactDetails = ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest);
string repoImageInfoPath = Path.Combine(repoPath, Options.GitOptions.Path);
string originalTargetImageInfoContents = File.ReadAllText(repoImageInfoPath);
ImageArtifactDetails newImageArtifactDetails;
if (originalTargetImageInfoContents != null)
{
ImageArtifactDetails targetImageArtifactDetails = ImageInfoHelper.LoadFromContent(
originalTargetImageInfoContents, Manifest, skipManifestValidation: true);
RemoveOutOfDateContent(targetImageArtifactDetails);
ImageInfoMergeOptions options = new ImageInfoMergeOptions
{
IsPublish = true
};
ImageInfoHelper.MergeImageArtifactDetails(srcImageArtifactDetails, targetImageArtifactDetails, options);
newImageArtifactDetails = targetImageArtifactDetails;
}
else
{
// If there is no existing file to update, there's nothing to merge with so the source data
// becomes the target data.
newImageArtifactDetails = srcImageArtifactDetails;
}
string newTargetImageInfoContents =
JsonHelper.SerializeObject(newImageArtifactDetails) + Environment.NewLine;
if (originalTargetImageInfoContents != newTargetImageInfoContents)
{
return(newTargetImageInfoContents);
}
else
{
return(null);
}
}
private void WriteImagesMarkdown(StringBuilder notificationMarkdown)
{
if (!File.Exists(Options.ImageInfoPath))
{
return;
}
ImageArtifactDetails imageArtifactDetails = ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest);
IEnumerable <ImageData> images = imageArtifactDetails.Repos.SelectMany(repo => repo.Images);
List <(string digestSha, IEnumerable <string> tags)> publishedImages = new();
foreach (ImageData image in images)
{
if (image.Manifest is not null)
{
string digestSha = DockerHelper.GetDigestSha(image.Manifest.Digest);
IEnumerable <string> tags = GetTags(image.ManifestImage.SharedTags);
publishedImages.Add((digestSha, tags));
}
publishedImages.AddRange(
image.Platforms
.Where(platform => platform.PlatformInfo.Tags.Any())
.Select(platform =>
{
string digestSha = DockerHelper.GetDigestSha(platform.Digest);
IEnumerable <string> tags = GetTags(platform.PlatformInfo.Tags);
return(digestSha, tags);
}));
}
notificationMarkdown.AppendLine("## Images");
notificationMarkdown.AppendLine();
foreach ((string digestSha, IEnumerable <string> tags) in publishedImages.OrderBy(digestTags => digestTags.digestSha))
{
notificationMarkdown.AppendLine($"* {digestSha}");
foreach (string tag in tags.OrderBy(tag => tag))
{
notificationMarkdown.AppendLine($" * {tag}");
}
}
}
public override Task ExecuteAsync()
{
_loggerService.WriteHeading("GENERATING MANIFESTS");
ImageArtifactDetails imageArtifactDetails = ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest);
ExecuteWithUser(() =>
{
IEnumerable <string> manifests = Manifest.FilteredRepos
.SelectMany(repo =>
repo.FilteredImages
.Where(image => image.SharedTags.Any())
.Select(image => (repo, image)))
.SelectMany(repoImage => GenerateManifests(repoImage.repo, repoImage.image))
.ToList();
DateTime createdDate = DateTime.Now.ToUniversalTime();
Parallel.ForEach(manifests, manifest =>
{
string manifestFilename = $"manifest.{Guid.NewGuid()}.yml";
_loggerService.WriteSubheading($"PUBLISHING MANIFEST: '{manifestFilename}'{Environment.NewLine}{manifest}");
File.WriteAllText(manifestFilename, manifest);
try
{
_manifestToolService.PushFromSpec(manifestFilename, Options.IsDryRun);
}
finally
{
File.Delete(manifestFilename);
}
});
WriteManifestSummary();
SaveTagInfoToImageInfoFile(createdDate, imageArtifactDetails);
});
return(Task.CompletedTask);
}
private string GetImageInfoCsv()
{
StringBuilder builder = new StringBuilder();
foreach (RepoData repo in ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest).Repos)
{
foreach (ImageData image in repo.Images)
{
foreach (PlatformData platform in image.Platforms)
{
string timestamp = platform.Created.ToUniversalTime().ToString("yyyy'-'MM'-'dd' 'HH':'mm':'ss");
builder.AppendLine(FormatCsv(platform.Digest, platform, image, repo, timestamp));
foreach (string tag in platform.SimpleTags)
{
builder.AppendLine(FormatCsv(tag, platform, image, repo, timestamp));
}
}
}
}
// Kusto ingest API does not handle an empty line, therefore the last line must be trimmed.
return(builder.ToString().TrimEnd(Environment.NewLine));
}
private void BuildImages()
{
_loggerService.WriteHeading("BUILDING IMAGES");
ImageArtifactDetails?srcImageArtifactDetails = null;
if (Options.ImageInfoSourcePath != null)
{
srcImageArtifactDetails = ImageInfoHelper.LoadFromFile(Options.ImageInfoSourcePath, Manifest, skipManifestValidation: true);
}
foreach (RepoInfo repoInfo in Manifest.FilteredRepos)
{
RepoData?repoData = CreateRepoData(repoInfo);
RepoData?srcRepoData = srcImageArtifactDetails?.Repos.FirstOrDefault(srcRepo => srcRepo.Repo == repoInfo.Name);
foreach (ImageInfo image in repoInfo.FilteredImages)
{
ImageData?imageData = CreateImageData(image);
repoData?.Images.Add(imageData);
ImageData?srcImageData = srcRepoData?.Images.FirstOrDefault(srcImage => srcImage.ManifestImage == image);
foreach (PlatformInfo platform in image.FilteredPlatforms)
{
// Tag the built images with the shared tags as well as the platform tags.
// Some tests and image FROM instructions depend on these tags.
IEnumerable <TagInfo> allTagInfos = platform.Tags
.Concat(image.SharedTags)
.ToList();
_builtTags.AddRange(allTagInfos);
IEnumerable <string> allTags = allTagInfos
.Select(tag => tag.FullyQualifiedName)
.ToList();
PlatformData?platformData = CreatePlatformData(image, platform);
imageData?.Platforms.Add(platformData);
bool isCachedImage = !Options.NoCache && CheckForCachedImage(srcImageData, repoInfo, platform, allTags, platformData);
if (!isCachedImage)
{
BuildImage(platform, allTags);
if (platformData != null)
{
platformData.BaseImageDigest =
_imageDigestCache.GetImageDigest(platform.FinalStageFromImage, Options.IsDryRun);
}
}
}
}
if (repoData?.Images.Any() == true)
{
_imageArtifactDetails?.Repos.Add(repoData);
}
}
}
public override Task ExecuteAsync()
{
var imageInfoFiles = Directory.EnumerateFiles(
Options.SourceImageInfoFolderPath,
"*.json",
SearchOption.AllDirectories);
List <ImageArtifactDetails> srcImageArtifactDetailsList = imageInfoFiles
.OrderBy(file => file) // Ensure the files are ordered for testing consistency between OS's.
.Select(imageDataPath => ImageInfoHelper.LoadFromFile(imageDataPath, Manifest))
.ToList();
if (!srcImageArtifactDetailsList.Any())
{
throw new InvalidOperationException(
$"No JSON files found in source folder '{Options.SourceImageInfoFolderPath}'");
}
ImageArtifactDetails targetImageArtifactDetails = new ImageArtifactDetails();
foreach (ImageArtifactDetails srcImageArtifactDetails in srcImageArtifactDetailsList)
{
ImageInfoHelper.MergeImageArtifactDetails(srcImageArtifactDetails, targetImageArtifactDetails);
}
string destinationContents = JsonHelper.SerializeObject(targetImageArtifactDetails) + Environment.NewLine;
File.WriteAllText(Options.DestinationImageInfoPath, destinationContents);
return(Task.CompletedTask);
}