private void OnCacheHit(RepoInfo repo, IEnumerable <string> allTags, bool pullImage, string sourceDigest) { _loggerService.WriteMessage(); _loggerService.WriteMessage("CACHE HIT"); _loggerService.WriteMessage(); // Pull the image instead of building it if (pullImage) { _dockerService.PullImage(sourceDigest, Options.IsDryRun); } // Tag the image as if it were locally built so that subsequent built images can reference it Parallel.ForEach(allTags, tag => { _dockerService.CreateTag(sourceDigest, tag, Options.IsDryRun); // Rewrite the digest to match the repo of the tags being associated with it. This is necessary // in order to handle scenarios where shared Dockerfiles are being used across different repositories. // In that scenario, the digest that is retrieved will be based on the repo of the first repository // encountered. For subsequent cache hits on different repositories, we need to prepopulate the digest // cache with a digest value that would correspond to that repository, not the original repository. string newDigest = DockerHelper.GetImageName( Manifest.Model.Registry, repo.Model.Name, digest: DockerHelper.GetDigestSha(sourceDigest)); // Populate the digest cache with the known digest value for the tags assigned to the image. // This is needed in order to prevent a call to the manifest tool to get the digest for these tags // because they haven't yet been pushed to staging by that time. _imageDigestCache.AddDigest(tag, newDigest); }); }
private (string imageInfo, string layerInfo) GetImageInfoCsv() { StringBuilder imageInfo = new(); StringBuilder layerInfo = new(); foreach (RepoData repo in ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest).Repos) { foreach (ImageData image in repo.Images) { foreach (PlatformData platform in image.Platforms) { string timestamp = platform.Created.ToUniversalTime().ToString("yyyy'-'MM'-'dd' 'HH':'mm':'ss"); string sha = DockerHelper.GetDigestSha(platform.Digest); imageInfo.AppendLine(FormatImageCsv(sha, platform, image, repo.Repo, timestamp)); IEnumerable <TagInfo> tagInfos = (platform.PlatformInfo?.Tags ?? Enumerable.Empty <TagInfo>()) .Where(tagInfo => platform.SimpleTags.Contains(tagInfo.Name)) .ToList(); IEnumerable <string> syndicatedRepos = tagInfos .Select(tag => tag.SyndicatedRepo) .Where(repo => repo != null) .Distinct(); foreach (string syndicatedRepo in syndicatedRepos) { imageInfo.AppendLine( FormatImageCsv(sha, platform, image, syndicatedRepo, timestamp)); } foreach (TagInfo tag in tagInfos) { imageInfo.AppendLine(FormatImageCsv(tag.Name, platform, image, repo.Repo, timestamp)); if (tag.SyndicatedRepo != null) { foreach (string destinationTag in tag.SyndicatedDestinationTags) { imageInfo.AppendLine( FormatImageCsv(destinationTag, platform, image, tag.SyndicatedRepo, timestamp)); } } } for (int i = 0; i < platform.Layers.Count; i++) { // TODO: Track layer size (currently set to 0) https://github.com/dotnet/docker-tools/issues/745 layerInfo.AppendLine(FormatLayerCsv( platform.Layers[i], 0, platform.Layers.Count - i, sha, platform, image, repo.Repo, timestamp)); } } } } // Kusto ingest API does not handle an empty line, therefore the last line must be trimmed. return(imageInfo.ToString().TrimEnd(Environment.NewLine), layerInfo.ToString().TrimEnd(Environment.NewLine)); }
private bool IsBaseImageDigestUpToDate(PlatformInfo platform, PlatformData srcPlatformData) { string currentBaseImageDigest = _imageDigestCache.GetImageDigest(platform.FinalStageFromImage, Options.IsDryRun); bool baseImageDigestMatches = DockerHelper.GetDigestSha(srcPlatformData.BaseImageDigest)?.Equals( DockerHelper.GetDigestSha(currentBaseImageDigest), StringComparison.OrdinalIgnoreCase) == true; _loggerService.WriteMessage(); _loggerService.WriteMessage($"Image info's base image digest: {srcPlatformData.BaseImageDigest}"); _loggerService.WriteMessage($"Latest base image digest: {currentBaseImageDigest}"); _loggerService.WriteMessage($"Base image digests match: {baseImageDigestMatches}"); return(baseImageDigestMatches); }
private IEnumerable <DigestInfo> GetImageDigestInfos(ImageData image, RepoData repo) { if (image.Manifest?.Digest != null) { string digestSha = DockerHelper.GetDigestSha(image.Manifest.Digest); yield return(new DigestInfo(digestSha, repo.Repo, image.Manifest.SharedTags)); // Find all syndicated shared tags grouped by their syndicated repo IEnumerable <IGrouping <string, TagInfo> > syndicatedTagGroups = image.ManifestImage.SharedTags .Where(tag => image.Manifest.SharedTags.Contains(tag.Name) && tag.SyndicatedRepo != null) .GroupBy(tag => tag.SyndicatedRepo); foreach (IGrouping <string, TagInfo> syndicatedTags in syndicatedTagGroups) { string syndicatedRepo = Options.RepoPrefix + syndicatedTags.Key; string fullyQualifiedRepo = DockerHelper.GetImageName(Manifest.Model.Registry, syndicatedRepo); string?syndicatedDigest = image.Manifest.SyndicatedDigests .FirstOrDefault(digest => digest.StartsWith($"{fullyQualifiedRepo}@")); if (syndicatedDigest is null) { throw new InvalidOperationException($"Unable to find syndicated digest for '{fullyQualifiedRepo}'"); } yield return(new DigestInfo( DockerHelper.GetDigestSha(syndicatedDigest), syndicatedRepo, syndicatedTags.SelectMany(tag => tag.SyndicatedDestinationTags))); } } foreach (PlatformData platform in image.Platforms) { string sha = DockerHelper.GetDigestSha(platform.Digest); yield return(new DigestInfo(sha, repo.Repo, platform.SimpleTags)); // Find all syndicated simple tags grouped by their syndicated repo IEnumerable <IGrouping <string, TagInfo> > syndicatedTagGroups = (platform.PlatformInfo?.Tags ?? Enumerable.Empty <TagInfo>()) .Where(tagInfo => platform.SimpleTags.Contains(tagInfo.Name) && tagInfo.SyndicatedRepo != null) .GroupBy(tagInfo => tagInfo.SyndicatedRepo); foreach (IGrouping <string, TagInfo> syndicatedTags in syndicatedTagGroups) { string syndicatedRepo = syndicatedTags.Key; yield return(new DigestInfo( sha, Options.RepoPrefix + syndicatedRepo, syndicatedTags.SelectMany(tag => tag.SyndicatedDestinationTags))); } } }
private string GetImageInfoCsv() { StringBuilder builder = new StringBuilder(); foreach (RepoData repo in ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest).Repos) { foreach (ImageData image in repo.Images) { foreach (PlatformData platform in image.Platforms) { string timestamp = platform.Created.ToUniversalTime().ToString("yyyy'-'MM'-'dd' 'HH':'mm':'ss"); string sha = DockerHelper.GetDigestSha(platform.Digest); builder.AppendLine(FormatCsv(sha, platform, image, repo.Repo, timestamp)); IEnumerable <TagInfo> tagInfos = platform.PlatformInfo.Tags .Where(tagInfo => platform.SimpleTags.Contains(tagInfo.Name)) .ToList(); IEnumerable <string> syndicatedRepos = tagInfos .Select(tag => tag.SyndicatedRepo) .Where(repo => repo != null) .Distinct(); foreach (string syndicatedRepo in syndicatedRepos) { builder.AppendLine( FormatCsv(sha, platform, image, syndicatedRepo, timestamp)); } foreach (TagInfo tag in tagInfos) { builder.AppendLine(FormatCsv(tag.Name, platform, image, repo.Repo, timestamp)); if (tag.SyndicatedRepo != null) { foreach (string destinationTag in tag.SyndicatedDestinationTags) { builder.AppendLine( FormatCsv(destinationTag, platform, image, tag.SyndicatedRepo, timestamp)); } } } } } } // Kusto ingest API does not handle an empty line, therefore the last line must be trimmed. return(builder.ToString().TrimEnd(Environment.NewLine)); }
private IEnumerable <DigestInfo> GetImageDigestInfos(ImageData image, RepoData repo) { if (image.Manifest?.Digest != null) { string digestSha = DockerHelper.GetDigestSha(image.Manifest.Digest); yield return(new DigestInfo(digestSha, repo.Repo, image.Manifest.SharedTags)); // Find all syndicated shared tags grouped by their syndicated repo IEnumerable <IGrouping <string, TagInfo> > syndicatedTagGroups = image.ManifestImage.SharedTags .Where(tag => image.Manifest.SharedTags.Contains(tag.Name) && tag.SyndicatedRepo != null) .GroupBy(tag => tag.SyndicatedRepo); foreach (IGrouping <string, TagInfo> syndicatedTags in syndicatedTagGroups) { string syndicatedRepo = Options.RepoPrefix + syndicatedTags.Key; string tag = syndicatedTags.First().SyndicatedDestinationTags.First(); tag = DockerHelper.GetImageName(Manifest.Registry, syndicatedRepo, tag); yield return(new DigestInfo( digestSha, syndicatedRepo, syndicatedTags.SelectMany(tag => tag.SyndicatedDestinationTags))); } } foreach (PlatformData platform in image.Platforms) { string sha = DockerHelper.GetDigestSha(platform.Digest); yield return(new DigestInfo(sha, repo.Repo, platform.SimpleTags)); // Find all syndicated simple tags grouped by their syndicated repo IEnumerable <IGrouping <string, TagInfo> > syndicatedTagGroups = platform.PlatformInfo.Tags .Where(tagInfo => platform.SimpleTags.Contains(tagInfo.Name) && tagInfo.SyndicatedRepo != null) .GroupBy(tagInfo => tagInfo.SyndicatedRepo); foreach (IGrouping <string, TagInfo> syndicatedTags in syndicatedTagGroups) { string syndicatedRepo = syndicatedTags.Key; yield return(new DigestInfo( sha, Options.RepoPrefix + syndicatedRepo, syndicatedTags.SelectMany(tag => tag.SyndicatedDestinationTags))); } } }
private void WriteImagesMarkdown(StringBuilder notificationMarkdown) { if (!File.Exists(Options.ImageInfoPath)) { return; } ImageArtifactDetails imageArtifactDetails = ImageInfoHelper.LoadFromFile(Options.ImageInfoPath, Manifest); IEnumerable <ImageData> images = imageArtifactDetails.Repos.SelectMany(repo => repo.Images); List <(string digestSha, IEnumerable <string> tags)> publishedImages = new(); foreach (ImageData image in images) { if (image.Manifest is not null) { string digestSha = DockerHelper.GetDigestSha(image.Manifest.Digest); IEnumerable <string> tags = GetTags(image.ManifestImage.SharedTags); publishedImages.Add((digestSha, tags)); } publishedImages.AddRange( image.Platforms .Where(platform => platform.PlatformInfo.Tags.Any()) .Select(platform => { string digestSha = DockerHelper.GetDigestSha(platform.Digest); IEnumerable <string> tags = GetTags(platform.PlatformInfo.Tags); return(digestSha, tags); })); } notificationMarkdown.AppendLine("## Images"); notificationMarkdown.AppendLine(); foreach ((string digestSha, IEnumerable <string> tags) in publishedImages.OrderBy(digestTags => digestTags.digestSha)) { notificationMarkdown.AppendLine($"* {digestSha}"); foreach (string tag in tags.OrderBy(tag => tag)) { notificationMarkdown.AppendLine($" * {tag}"); } } }
private void SetPlatformDataDigest(PlatformData platform, string tag) { // The digest of an image that is pushed to ACR is guaranteed to be the same when transferred to MCR. string digest = _imageDigestCache.GetImageDigest(tag, Options.IsDryRun); if (digest != null) { digest = DockerHelper.GetDigestString(platform.PlatformInfo.FullRepoModelName, DockerHelper.GetDigestSha(digest)); } if (platform.Digest != null && platform.Digest != digest) { // Pushing the same image with different tags should result in the same digest being output throw new InvalidOperationException( $"Tag '{tag}' was pushed with a resulting digest value that differs from the corresponding image's digest value." + Environment.NewLine + $"\tDigest value from image info: {platform.Digest}{Environment.NewLine}" + $"\tDigest value retrieved from query: {digest}"); } platform.Digest = digest; }