public async Task PublishImageInfoCommand_ReplaceTags()
{
using (TempFolderContext tempFolderContext = TestHelper.UseTempFolder())
{
string repo1Image1DockerfilePath = DockerfileHelper.CreateDockerfile("1.0/runtime/os", tempFolderContext);
string repo2Image2DockerfilePath = DockerfileHelper.CreateDockerfile("2.0/runtime/os", tempFolderContext);
Manifest manifest = CreateManifest(
CreateRepo("repo1",
CreateImage(
CreatePlatform(repo1Image1DockerfilePath, new string[0]))),
CreateRepo("repo2",
CreateImage(
CreatePlatform(repo2Image2DockerfilePath, new string[0])))
);
RepoData repo2;
ImageArtifactDetails srcImageArtifactDetails = new ImageArtifactDetails
{
Repos =
{
new RepoData
{
Repo = "repo1",
Images =
{
new ImageData
{
Platforms =
{
Helpers.ImageInfoHelper.CreatePlatform(repo1Image1DockerfilePath,
simpleTags: new List <string>
{
"newtag"
})
}
}
}
},
{
repo2 = new RepoData
{
Repo = "repo2",
Images =
{
new ImageData
{
Platforms =
{
Helpers.ImageInfoHelper.CreatePlatform(repo2Image2DockerfilePath,
simpleTags: new List <string>
{
"tag1"
})
}
}
}
}
}
}
};
string file = Path.Combine(tempFolderContext.Path, "image-info.json");
File.WriteAllText(file, JsonHelper.SerializeObject(srcImageArtifactDetails));
ImageArtifactDetails targetImageArtifactDetails = new ImageArtifactDetails
{
Repos =
{
new RepoData
{
Repo = "repo1",
Images =
{
new ImageData
{
Platforms =
{
Helpers.ImageInfoHelper.CreatePlatform(repo1Image1DockerfilePath,
simpleTags: new List <string>
{
"oldtag"
})
}
}
}
}
}
};
Mock <IGitHubClient> gitHubClientMock = GetGitHubClientMock();
Mock <IGitHubClientFactory> gitHubClientFactoryMock = new Mock <IGitHubClientFactory>();
gitHubClientFactoryMock
.Setup(o => o.GetClient(It.IsAny <GitHubAuth>(), false))
.Returns(gitHubClientMock.Object);
GitOptions gitOptions = new GitOptions
{
AuthToken = "token",
Repo = "testRepo",
Branch = "testBranch",
Path = "imageinfo.json"
};
PublishImageInfoCommand command = new PublishImageInfoCommand(
gitHubClientFactoryMock.Object, Mock.Of <ILoggerService>(),
CreateHttpClientFactory(gitOptions, targetImageArtifactDetails));
command.Options.ImageInfoPath = file;
command.Options.GitOptions = gitOptions;
command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json");
File.WriteAllText(Path.Combine(tempFolderContext.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest));
command.LoadManifest();
await command.ExecuteAsync();
ImageArtifactDetails expectedImageArtifactDetails = new ImageArtifactDetails
{
Repos =
{
new RepoData
{
Repo = "repo1",
Images =
{
new ImageData
{
Platforms =
{
Helpers.ImageInfoHelper.CreatePlatform(repo1Image1DockerfilePath,
simpleTags: new List <string>
{
"newtag"
})
}
}
}
},
repo2
}
};
Func <GitObject[], bool> verifyGitObjects = (gitObjects) =>
{
if (gitObjects.Length != 1)
{
return(false);
}
return(gitObjects[0].Content.Trim() == JsonHelper.SerializeObject(expectedImageArtifactDetails).Trim());
};
gitHubClientMock.Verify(
o => o.PostTreeAsync(It.IsAny <GitHubProject>(), It.IsAny <string>(), It.Is <GitObject[]>(gitObjects => verifyGitObjects(gitObjects))));
}
}
public void ImageInfoHelper_MergeRepos_RemoveTag()
{
ImageData srcImage1;
ImageData targetImage2;
RepoData[] repoDataSet = new RepoData[]
{
new RepoData
{
Repo = "repo1",
Images = new SortedDictionary <string, ImageData>
{
{
"image1",
srcImage1 = new ImageData
{
SimpleTags =
{
"tag1",
"tag3"
}
}
}
}
}
};
List <RepoData> targetRepos = new List <RepoData>
{
new RepoData
{
Repo = "repo1",
Images = new SortedDictionary <string, ImageData>
{
{
"image1",
new ImageData
{
SimpleTags =
{
"tag1",
"tag2",
"tag4"
}
}
},
{
"image2",
targetImage2 = new ImageData
{
SimpleTags =
{
"a"
}
}
}
}
}
};
ImageInfoMergeOptions options = new ImageInfoMergeOptions
{
ReplaceTags = true
};
ImageInfoHelper.MergeRepos(repoDataSet, targetRepos, options);
List <RepoData> expected = new List <RepoData>
{
new RepoData
{
Repo = "repo1",
Images = new SortedDictionary <string, ImageData>
{
{
"image1",
srcImage1
},
{
"image2",
targetImage2
}
}
}
};
CompareRepos(expected, targetRepos);
}
public async Task PublishImageInfoCommand_ReplaceTags()
{
using (TempFolderContext tempFolderContext = TestHelper.UseTempFolder())
{
RepoData repo2;
RepoData[] sourceRepos = new RepoData[]
{
new RepoData
{
Repo = "repo1",
Images = new SortedDictionary <string, ImageData>
{
{
"image1",
new ImageData
{
SimpleTags =
{
"newtag"
}
}
}
}
},
repo2 = new RepoData
{
Repo = "repo2",
Images = new SortedDictionary <string, ImageData>
{
{
"image2",
new ImageData
{
SimpleTags =
{
"tag1"
}
}
}
}
}
};
string file = Path.Combine(tempFolderContext.Path, "image-info.json");
File.WriteAllText(file, JsonHelper.SerializeObject(sourceRepos));
RepoData[] targetRepos = new RepoData[]
{
new RepoData
{
Repo = "repo1",
Images = new SortedDictionary <string, ImageData>
{
{
"image1",
new ImageData
{
SimpleTags =
{
"oldtag"
}
}
}
}
}
};
Mock <IGitHubClient> gitHubClientMock = GetGitHubClientMock(targetRepos);
Mock <IGitHubClientFactory> gitHubClientFactoryMock = new Mock <IGitHubClientFactory>();
gitHubClientFactoryMock
.Setup(o => o.GetClient(It.IsAny <GitHubAuth>(), false))
.Returns(gitHubClientMock.Object);
PublishImageInfoCommand command = new PublishImageInfoCommand(gitHubClientFactoryMock.Object);
command.Options.ImageInfoPath = file;
command.Options.GitOptions.AuthToken = "token";
await command.ExecuteAsync();
RepoData[] expectedRepos = new RepoData[]
{
new RepoData
{
Repo = "repo1",
Images = new SortedDictionary <string, ImageData>
{
{
"image1",
new ImageData
{
SimpleTags =
{
"newtag"
}
}
}
}
},
repo2
};
Func <GitObject[], bool> verifyGitObjects = (gitObjects) =>
{
if (gitObjects.Length != 1)
{
return(false);
}
return(gitObjects[0].Content.Trim() == JsonHelper.SerializeObject(expectedRepos).Trim());
};
gitHubClientMock.Verify(
o => o.PostTreeAsync(It.IsAny <GitHubProject>(), It.IsAny <string>(), It.Is <GitObject[]>(gitObjects => verifyGitObjects(gitObjects))));
}
}
private async Task <List <string> > GetPathsToRebuildAsync(
IEnumerable <PlatformInfo> allPlatforms, PlatformInfo platform, RepoData repoData)
{
bool foundImageInfo = false;
List <string> pathsToRebuild = new List <string>();
void processPlatformWithMissingImageInfo(PlatformInfo platform)
{
this.loggerService.WriteMessage(
$"WARNING: Image info not found for '{platform.DockerfilePath}'. Adding path to build to be queued anyway.");
IEnumerable <PlatformInfo> dependentPlatforms = platform.GetDependencyGraph(allPlatforms);
pathsToRebuild.AddRange(dependentPlatforms.Select(p => p.Model.Dockerfile));
}
if (repoData == null || repoData.Images == null)
{
processPlatformWithMissingImageInfo(platform);
return(pathsToRebuild);
}
foreach (ImageData imageData in repoData.Images)
{
PlatformData platformData = imageData.Platforms
.FirstOrDefault(platformData => platformData.Equals(platform));
if (platformData != null)
{
foundImageInfo = true;
string fromImage = platform.FinalStageFromImage;
string currentDigest;
await this.imageDigestsSemaphore.WaitAsync();
try
{
if (!this.imageDigests.TryGetValue(fromImage, out currentDigest))
{
this.dockerService.PullImage(fromImage, Options.IsDryRun);
currentDigest = this.dockerService.GetImageDigest(fromImage, Options.IsDryRun);
this.imageDigests.Add(fromImage, currentDigest);
}
}
finally
{
this.imageDigestsSemaphore.Release();
}
bool rebuildImage = platformData.BaseImageDigest != currentDigest;
this.loggerService.WriteMessage(
$"Checking base image '{fromImage}' from '{platform.DockerfilePath}'{Environment.NewLine}"
+ $"\tLast build digest: {platformData.BaseImageDigest}{Environment.NewLine}"
+ $"\tCurrent digest: {currentDigest}{Environment.NewLine}"
+ $"\tImage is up-to-date: {!rebuildImage}{Environment.NewLine}");
if (rebuildImage)
{
IEnumerable <PlatformInfo> dependentPlatforms = platform.GetDependencyGraph(allPlatforms);
pathsToRebuild.AddRange(dependentPlatforms.Select(p => p.Model.Dockerfile));
}
break;
}
}
if (!foundImageInfo)
{
processPlatformWithMissingImageInfo(platform);
}
return(pathsToRebuild);
}
private async Task <IEnumerable <string> > GetPathsToRebuildAsync(Subscription subscription)
{
// If the command is filtered with an OS type that does not match the OsType filter of the subscription,
// then there are no images that need to be inspected.
string osTypeRegexPattern = ManifestFilter.GetFilterRegexPattern(Options.FilterOptions.OsType);
if (!String.IsNullOrEmpty(subscription.OsType) &&
!Regex.IsMatch(subscription.OsType, osTypeRegexPattern, RegexOptions.IgnoreCase))
{
return(Enumerable.Empty <string>());
}
RepoData[] repos = await GetImageInfoForSubscriptionAsync(subscription);
string repoPath = await GetGitRepoPath(subscription);
TempManifestOptions manifestOptions = new TempManifestOptions(Options.FilterOptions)
{
Manifest = Path.Combine(repoPath, subscription.ManifestPath)
};
ManifestInfo manifest = ManifestInfo.Load(manifestOptions);
List <string> pathsToRebuild = new List <string>();
IEnumerable <PlatformInfo> allPlatforms = manifest.GetAllPlatforms().ToList();
foreach (RepoInfo repo in manifest.FilteredRepos)
{
IEnumerable <PlatformInfo> platforms = repo.FilteredImages
.SelectMany(image => image.FilteredPlatforms);
RepoData repoData = repos
.FirstOrDefault(s => s.Repo == repo.Model.Name);
foreach (var platform in platforms)
{
if (repoData != null &&
repoData.Images != null &&
repoData.Images.TryGetValue(platform.DockerfilePathRelativeToManifest, out ImageData imageData))
{
bool hasDigestChanged = false;
foreach (string fromImage in platform.ExternalFromImages)
{
string currentDigest;
await this.imageDigestsSemaphore.WaitAsync();
try
{
if (!this.imageDigests.TryGetValue(fromImage, out currentDigest))
{
this.dockerService.PullImage(fromImage, Options.IsDryRun);
currentDigest = this.dockerService.GetImageDigest(fromImage, Options.IsDryRun);
this.imageDigests.Add(fromImage, currentDigest);
}
}
finally
{
this.imageDigestsSemaphore.Release();
}
string lastDigest = null;
imageData.BaseImages?.TryGetValue(fromImage, out lastDigest);
if (lastDigest != currentDigest)
{
hasDigestChanged = true;
break;
}
}
if (hasDigestChanged)
{
IEnumerable <PlatformInfo> dependentPlatforms = platform.GetDependencyGraph(allPlatforms);
pathsToRebuild.AddRange(dependentPlatforms.Select(p => p.Model.Dockerfile));
}
}
else
{
this.loggerService.WriteMessage(
$"WARNING: Image info not found for '{platform.DockerfilePath}'. Adding path to build to be queued anyway.");
pathsToRebuild.Add(platform.Model.Dockerfile);
}
}
}
return(pathsToRebuild.Distinct().ToList());
}
private void BuildImages()
{
this.loggerService.WriteHeading("BUILDING IMAGES");
foreach (RepoInfo repoInfo in Manifest.FilteredRepos)
{
RepoData repoData = new RepoData
{
Repo = repoInfo.Name
};
imageArtifactDetails.Repos.Add(repoData);
foreach (ImageInfo image in repoInfo.FilteredImages)
{
ImageData imageData = new ImageData
{
ProductVersion = image.ProductVersion
};
if (image.SharedTags.Any())
{
imageData.Manifest = new ManifestData
{
SharedTags = image.SharedTags
.Select(tag => tag.Name)
.ToList()
};
}
repoData.Images.Add(imageData);
foreach (PlatformInfo platform in image.FilteredPlatforms)
{
PlatformData platformData = PlatformData.FromPlatformInfo(platform);
imageData.Platforms.Add(platformData);
bool createdPrivateDockerfile = UpdateDockerfileFromCommands(platform, out string dockerfilePath);
IEnumerable <string> allTags;
try
{
InvokeBuildHook("pre-build", platform.BuildContextPath);
// Tag the built images with the shared tags as well as the platform tags.
// Some tests and image FROM instructions depend on these tags.
allTags = platform.Tags
.Concat(image.SharedTags)
.Select(tag => tag.FullyQualifiedName)
.ToList();
this.dockerService.BuildImage(
dockerfilePath,
platform.BuildContextPath,
allTags,
platform.BuildArgs,
Options.IsRetryEnabled,
Options.IsDryRun);
if (!Options.IsDryRun)
{
EnsureArchitectureMatches(platform, allTags);
}
InvokeBuildHook("post-build", platform.BuildContextPath);
}
finally
{
if (createdPrivateDockerfile)
{
File.Delete(dockerfilePath);
}
}
platformData.BaseImageDigest = this.dockerService.GetImageDigest(platform.FinalStageFromImage, Options.IsDryRun);
platformData.SimpleTags = GetPushTags(platform.Tags)
.Select(tag => tag.Name)
.OrderBy(name => name)
.ToList();
platformData.FullyQualifiedSimpleTags = platformData.SimpleTags
.Select(tag => TagInfo.GetFullyQualifiedName(repoInfo.QualifiedName, tag))
.ToList();
platformData.AllTags = allTags;
}
}
}
}
public async Task SyndicatedTags()
{
const string subscriptionId = "my subscription";
using TempFolderContext tempFolderContext = TestHelper.UseTempFolder();
Mock <IRegistriesOperations> registriesOperationsMock = AzureHelper.CreateRegistriesOperationsMock();
IAzure azure = AzureHelper.CreateAzureMock(registriesOperationsMock);
Mock <IAzureManagementFactory> azureManagementFactoryMock =
AzureHelper.CreateAzureManagementFactoryMock(subscriptionId, azure);
Mock <IEnvironmentService> environmentServiceMock = new Mock <IEnvironmentService>();
CopyAcrImagesCommand command = new CopyAcrImagesCommand(azureManagementFactoryMock.Object, Mock.Of <ILoggerService>());
command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json");
command.Options.Subscription = subscriptionId;
command.Options.ResourceGroup = "my resource group";
command.Options.SourceRepoPrefix = command.Options.RepoPrefix = "test/";
command.Options.ImageInfoPath = "image-info.json";
const string runtimeRelativeDir = "1.0/runtime/os";
Directory.CreateDirectory(Path.Combine(tempFolderContext.Path, runtimeRelativeDir));
string dockerfileRelativePath = Path.Combine(runtimeRelativeDir, "Dockerfile");
File.WriteAllText(Path.Combine(tempFolderContext.Path, dockerfileRelativePath), "FROM repo:tag");
Manifest manifest = ManifestHelper.CreateManifest(
ManifestHelper.CreateRepo("runtime",
ManifestHelper.CreateImage(
ManifestHelper.CreatePlatform(dockerfileRelativePath, new string[] { "tag1", "tag2", "tag3" })))
);
manifest.Registry = "mcr.microsoft.com";
const string syndicatedRepo2 = "runtime2";
const string syndicatedRepo3 = "runtime3";
Platform platform = manifest.Repos.First().Images.First().Platforms.First();
platform.Tags["tag2"].Syndication = new TagSyndication
{
Repo = syndicatedRepo2,
};
platform.Tags["tag3"].Syndication = new TagSyndication
{
Repo = syndicatedRepo3,
DestinationTags = new string[]
{
"tag3a",
"tag3b"
}
};
File.WriteAllText(Path.Combine(tempFolderContext.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest));
RepoData runtimeRepo;
ImageArtifactDetails imageArtifactDetails = new ImageArtifactDetails
{
Repos =
{
{
runtimeRepo = new RepoData
{
Repo = "runtime",
Images =
{
new ImageData
{
Platforms =
{
CreatePlatform(
PathHelper.NormalizePath(dockerfileRelativePath),
simpleTags: new List <string>
{
"tag1",
"tag2",
"tag3"
})
}
}
}
}
}
}
};
File.WriteAllText(command.Options.ImageInfoPath, JsonConvert.SerializeObject(imageArtifactDetails));
command.LoadManifest();
await command.ExecuteAsync();
List <string> expectedTags = new List <string>
{
$"{command.Options.RepoPrefix}{runtimeRepo.Repo}:tag1",
$"{command.Options.RepoPrefix}{runtimeRepo.Repo}:tag2",
$"{command.Options.RepoPrefix}{runtimeRepo.Repo}:tag3",
$"{command.Options.RepoPrefix}{syndicatedRepo2}:tag2",
$"{command.Options.RepoPrefix}{syndicatedRepo3}:tag3a",
$"{command.Options.RepoPrefix}{syndicatedRepo3}:tag3b"
};
foreach (string expectedTag in expectedTags)
{
registriesOperationsMock
.Verify(o => o.ImportImageWithHttpMessagesAsync(
command.Options.ResourceGroup,
manifest.Registry,
It.Is <ImportImageParametersInner>(parameters =>
VerifyImportImageParameters(parameters, new List <string> {
expectedTag
})),
It.IsAny <Dictionary <string, List <string> > >(),
It.IsAny <CancellationToken>()));
}
}
private void OnGUI()
{
Color c = Handles.color;
Handles.color = new Color(0.3f, 0.3f, 0.3f, 0.6f);
GUILayout.BeginHorizontal();
GUILayout.BeginVertical(GUILayout.Width(128));
for (int i = 0; i < _categoryNames.Count; ++i)
{
GUI.enabled = !(i == _currentIdx);
if (GUILayout.Button(_categoryNames[i]))
{
_currentIdx = i;
}
}
GUI.enabled = true;
GUILayout.EndVertical();
GUILayout.BeginHorizontal();
_scrollValue = GUILayout.BeginScrollView(_scrollValue);
for (int i = 0; i < _categories[_categoryNames[_currentIdx]].Count; ++i)
{
int idx = _categories[_categoryNames[_currentIdx]][i];
RepoData val = _repoData[idx];
GUILayout.BeginHorizontal();
Rect iconeRect = GUILayoutUtility.GetRect(64, 64);
if (val.icone != null)
{
GUI.DrawTexture(iconeRect, val.icone, ScaleMode.ScaleToFit);
}
GUILayout.BeginVertical();
GUILayout.BeginHorizontal();
if (val.currentDownLoadRequest == null)
{
if (GUILayout.Button("Import", GUILayout.Width(64)))
{
ImportRepo(idx);
}
}
else
{
GUI.enabled = false;
GUILayout.Button(Mathf.FloorToInt(val.currentDownLoadRequest.request.downloadProgress * 100) + "%", GUILayout.Width(64));
GUI.enabled = true;
Repaint();
}
EditorGUILayout.LabelField(val.name, EditorStyles.boldLabel);
GUILayout.FlexibleSpace();
GUILayout.EndHorizontal();
EditorGUILayout.LabelField(val.description);
GUILayout.EndVertical();
GUILayout.EndHorizontal();
GUILayout.Space(8);
Rect r = GUILayoutUtility.GetLastRect();
Handles.DrawLine(new Vector3(0, r.yMax), new Vector3(position.width, r.yMax));
GUILayout.Space(8);
}
GUILayout.EndScrollView();
GUILayout.EndVertical();
GUILayout.EndHorizontal();
Handles.color = c;
}
public async Task PublishImageInfoCommand_RemoveOutOfDateContent()
{
using (TempFolderContext tempFolderContext = TestHelper.UseTempFolder())
{
string repo1Image1DockerfilePath = DockerfileHelper.CreateDockerfile("1.0/runtime/os", tempFolderContext);
string repo2Image2DockerfilePath = DockerfileHelper.CreateDockerfile("2.0/runtime/os", tempFolderContext);
Manifest manifest = CreateManifest(
CreateRepo("repo1",
CreateImage(
new Platform[]
{
CreatePlatform(repo1Image1DockerfilePath, new string[0])
},
productVersion: "1.0")),
CreateRepo("repo2",
CreateImage(
new Platform[]
{
CreatePlatform(repo2Image2DockerfilePath, new string[0])
},
productVersion: "2.0"))
);
manifest.Registry = "mcr.microsoft.com";
RepoData repo2;
ImageArtifactDetails srcImageArtifactDetails = new ImageArtifactDetails
{
Repos =
{
new RepoData
{
Repo = "repo1",
Images =
{
new ImageData
{
Platforms =
{
Helpers.ImageInfoHelper.CreatePlatform(repo1Image1DockerfilePath)
},
ProductVersion = "1.0"
}
}
},
{
repo2 = new RepoData
{
Repo = "repo2",
Images =
{
new ImageData
{
Platforms =
{
Helpers.ImageInfoHelper.CreatePlatform(repo2Image2DockerfilePath)
},
ProductVersion = "2.0"
}
}
}
}
}
};
string file = Path.Combine(tempFolderContext.Path, "image-info.json");
File.WriteAllText(file, JsonHelper.SerializeObject(srcImageArtifactDetails));
ImageArtifactDetails targetImageArtifactDetails = new ImageArtifactDetails
{
Repos =
{
new RepoData
{
Repo = "repo1",
Images =
{
new ImageData
{
Platforms =
{
Helpers.ImageInfoHelper.CreatePlatform(repo1Image1DockerfilePath)
},
ProductVersion = "1.0"
},
new ImageData
{
Platforms =
{
Helpers.ImageInfoHelper.CreatePlatform(
DockerfileHelper.CreateDockerfile("1.0/runtime2/os", tempFolderContext))
},
ProductVersion = "1.0"
}
}
},
new RepoData
{
Repo = "repo4"
}
}
};
GitOptions gitOptions = new GitOptions
{
AuthToken = "token",
Repo = "repo",
Owner = "owner",
Path = "imageinfo.json",
Branch = "branch",
Email = "*****@*****.**",
Username = "******"
};
AzdoOptions azdoOptions = new AzdoOptions
{
AccessToken = "azdo-token",
Branch = "testBranch",
Repo = "testRepo",
Organization = "azdo-org",
Project = "azdo-project",
Path = "imageinfo.json"
};
Mock <IRepository> repositoryMock = GetRepositoryMock();
Mock <IGitService> gitServiceMock = GetGitServiceMock(repositoryMock.Object, gitOptions.Path, targetImageArtifactDetails);
PublishImageInfoCommand command = new PublishImageInfoCommand(gitServiceMock.Object, Mock.Of <ILoggerService>());
command.Options.ImageInfoPath = file;
command.Options.GitOptions = gitOptions;
command.Options.AzdoOptions = azdoOptions;
command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json");
File.WriteAllText(Path.Combine(tempFolderContext.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest));
command.LoadManifest();
await command.ExecuteAsync();
ImageArtifactDetails expectedImageArtifactDetails = new ImageArtifactDetails
{
Repos =
{
new RepoData
{
Repo = "repo1",
Images =
{
new ImageData
{
Platforms =
{
Helpers.ImageInfoHelper.CreatePlatform(repo1Image1DockerfilePath)
},
ProductVersion = "1.0"
}
}
},
repo2
}
};
VerifyMocks(repositoryMock);
}
}
public async Task CopyAcrImagesCommand_RuntimeDepsSharing()
{
const string subscriptionId = "my subscription";
using TempFolderContext tempFolderContext = TestHelper.UseTempFolder();
Mock <IRegistriesOperations> registriesOperationsMock = AzureHelper.CreateRegistriesOperationsMock();
IAzure azure = AzureHelper.CreateAzureMock(registriesOperationsMock);
Mock <IAzureManagementFactory> azureManagementFactoryMock =
AzureHelper.CreateAzureManagementFactoryMock(subscriptionId, azure);
Mock <IEnvironmentService> environmentServiceMock = new Mock <IEnvironmentService>();
CopyAcrImagesCommand command = new CopyAcrImagesCommand(azureManagementFactoryMock.Object, Mock.Of <ILoggerService>());
command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json");
command.Options.Subscription = subscriptionId;
command.Options.ResourceGroup = "my resource group";
command.Options.SourceRepoPrefix = command.Options.RepoPrefix = "test/";
command.Options.ImageInfoPath = "image-info.json";
string dockerfileRelativePath = DockerfileHelper.CreateDockerfile("3.1/runtime-deps/os", tempFolderContext);
Manifest manifest = CreateManifest(
CreateRepo("runtime-deps",
CreateImage(
new Platform[]
{
CreatePlatform(dockerfileRelativePath, new string[] { "3.1" }, osVersion: "focal")
},
productVersion: "3.1"),
CreateImage(
new Platform[]
{
CreatePlatform(dockerfileRelativePath, new string[] { "5.0" }, osVersion: "focal")
},
productVersion: "5.0"))
);
manifest.Registry = "mcr.microsoft.com";
File.WriteAllText(Path.Combine(tempFolderContext.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest));
RepoData runtimeRepo;
ImageArtifactDetails imageArtifactDetails = new ImageArtifactDetails
{
Repos =
{
{
runtimeRepo = new RepoData
{
Repo = "runtime-deps",
Images =
{
new ImageData
{
Platforms =
{
CreatePlatform(
PathHelper.NormalizePath(dockerfileRelativePath),
simpleTags: new List <string>
{
"3.1"
},
osVersion: "focal")
},
ProductVersion = "3.1"
},
new ImageData
{
Platforms =
{
CreatePlatform(
PathHelper.NormalizePath(dockerfileRelativePath),
simpleTags: new List <string>
{
"5.0"
},
osVersion: "focal")
},
ProductVersion = "5.0"
}
}
}
}
}
};
File.WriteAllText(command.Options.ImageInfoPath, JsonConvert.SerializeObject(imageArtifactDetails));
command.LoadManifest();
await command.ExecuteAsync();
List <string> expectedTags = new List <string>
{
$"{command.Options.RepoPrefix}{runtimeRepo.Repo}:3.1",
$"{command.Options.RepoPrefix}{runtimeRepo.Repo}:5.0"
};
foreach (string expectedTag in expectedTags)
{
registriesOperationsMock
.Verify(o => o.ImportImageWithHttpMessagesAsync(
command.Options.ResourceGroup,
manifest.Registry,
It.Is <ImportImageParametersInner>(parameters =>
VerifyImportImageParameters(parameters, new List <string> {
expectedTag
})),
It.IsAny <Dictionary <string, List <string> > >(),
It.IsAny <CancellationToken>()));
}
}
public DigestInfo(string digest, RepoData repo, IEnumerable <string> tags)
{
Digest = digest;
Repo = repo;
RemainingTags = tags.OrderBy(tag => tag).ToList();
}
private void BuildImages()
{
Logger.WriteHeading("BUILDING IMAGES");
List <RepoData> reposList = new List <RepoData>();
foreach (RepoInfo repoInfo in Manifest.FilteredRepos)
{
RepoData repoData = new RepoData
{
Repo = repoInfo.Model.Name
};
reposList.Add(repoData);
SortedDictionary <string, ImageData> images = new SortedDictionary <string, ImageData>();
foreach (ImageInfo image in repoInfo.FilteredImages)
{
foreach (PlatformInfo platform in image.FilteredPlatforms)
{
ImageData imageData = new ImageData();
images.Add(platform.DockerfilePathRelativeToManifest, imageData);
bool createdPrivateDockerfile = UpdateDockerfileFromCommands(platform, out string dockerfilePath);
try
{
InvokeBuildHook("pre-build", platform.BuildContextPath);
// Tag the built images with the shared tags as well as the platform tags.
// Some tests and image FROM instructions depend on these tags.
IEnumerable <string> allTags = platform.Tags
.Concat(image.SharedTags)
.Select(tag => tag.FullyQualifiedName);
this.dockerService.BuildImage(
dockerfilePath,
platform.BuildContextPath,
allTags,
platform.BuildArgs,
Options.IsRetryEnabled,
Options.IsDryRun);
if (!Options.IsDryRun)
{
EnsureArchitectureMatches(platform, allTags);
}
InvokeBuildHook("post-build", platform.BuildContextPath);
BuiltTags = BuiltTags.Concat(platform.Tags);
}
finally
{
if (createdPrivateDockerfile)
{
File.Delete(dockerfilePath);
}
}
SortedDictionary <string, string> baseImageDigests = GetBaseImageDigests(platform);
if (baseImageDigests.Any())
{
imageData.BaseImages = baseImageDigests;
}
imageData.SimpleTags = GetPushTags(platform.Tags)
.Select(tag => tag.Name)
.OrderBy(name => name)
.ToList();
}
}
if (images.Any())
{
repoData.Images = images;
}
}
BuiltTags = BuiltTags.ToArray();
if (!String.IsNullOrEmpty(Options.ImageInfoOutputPath))
{
string digestsString = JsonHelper.SerializeObject(reposList.OrderBy(r => r.Repo).ToArray());
File.WriteAllText(Options.ImageInfoOutputPath, digestsString);
}
}
public async void GetRepoUsers(string projectDir)
{
if (projectDir == null || projectDir.Equals(""))
{
return;
}
IDictionary <string, string> resourceInfo = this.GetResourceInfo(projectDir);
if (resourceInfo != null && resourceInfo.ContainsKey("identifier"))
{
string identifier = "";
resourceInfo.TryGetValue("identifier", out identifier);
if (identifier != null && !identifier.Equals(""))
{
string tag = "";
resourceInfo.TryGetValue("tag", out tag);
string branch = "";
resourceInfo.TryGetValue("branch", out branch);
string gitLogData = SoftwareCoUtil.RunCommand("git log --pretty=%an,%ae | sort", projectDir);
IDictionary <string, string> memberMap = new Dictionary <string, string>();
List <RepoMember> repoMembers = new List <RepoMember>();
if (gitLogData != null && !gitLogData.Equals(""))
{
string[] lines = gitLogData.Split(
new string[] { "\r\n", "\r", "\n" }, StringSplitOptions.RemoveEmptyEntries);
if (lines != null && lines.Length > 0)
{
for (int i = 0; i < lines.Length; i++)
{
string line = lines[i];
string[] memberInfos = line.Split(',');
if (memberInfos != null && memberInfos.Length > 1)
{
string name = memberInfos[0].Trim();
string email = memberInfos[1].Trim();
if (!memberMap.ContainsKey(email))
{
memberMap.Add(email, name);
repoMembers.Add(new RepoMember(name, email));
}
}
}
}
}
if (memberMap.Count > 0)
{
RepoData repoData = new RepoData(identifier, tag, branch, repoMembers);
string jsonContent = SimpleJson.SerializeObject(repoData);
// send the members
HttpResponseMessage response = await SoftwareHttpManager.SendRequestAsync(
HttpMethod.Post, "/repo/members", jsonContent);
if (!SoftwareHttpManager.IsOk(response))
{
Logger.Error(response.ToString());
}
}
}
}
}
public void ImageInfoHelper_MergeRepos_ExistingTarget()
{
ImageData repo2Image1;
ImageData repo2Image2;
ImageData repo2Image3;
ImageData repo3Image1;
RepoData[] repoDataSet = new RepoData[]
{
new RepoData
{
Repo = "repo2",
Images = new SortedDictionary <string, ImageData>
{
{
"image1",
repo2Image1 = new ImageData
{
BaseImages = new SortedDictionary <string, string>
{
{ "base1", "base1digest-NEW" }
}
}
}
,
{
"image3",
repo2Image3 = new ImageData()
}
}
},
new RepoData
{
Repo = "repo3",
Images = new SortedDictionary <string, ImageData>
{
{
"image1",
repo3Image1 = new ImageData()
}
}
},
new RepoData
{
Repo = "repo4",
}
};
List <RepoData> targetRepos = new List <RepoData>
{
new RepoData
{
Repo = "repo1"
},
new RepoData
{
Repo = "repo2",
Images = new SortedDictionary <string, ImageData>
{
{
"image1",
new ImageData
{
BaseImages = new SortedDictionary <string, string>
{
{ "base1", "base1digest" }
}
}
},
{
"image2",
repo2Image2 = new ImageData
{
BaseImages = new SortedDictionary <string, string>
{
{ "base2", "base2digest" }
}
}
}
}
},
new RepoData
{
Repo = "repo3"
}
};
ImageInfoHelper.MergeRepos(repoDataSet, targetRepos);
List <RepoData> expected = new List <RepoData>
{
new RepoData
{
Repo = "repo1"
},
new RepoData
{
Repo = "repo2",
Images = new SortedDictionary <string, ImageData>
{
{
"image1",
repo2Image1
},
{
"image2",
repo2Image2
},
{
"image3",
repo2Image3
}
}
},
new RepoData
{
Repo = "repo3",
Images = new SortedDictionary <string, ImageData>
{
{
"image1",
repo3Image1
}
}
},
new RepoData
{
Repo = "repo4",
}
};
CompareRepos(expected, targetRepos);
}
public async Task MergeImageInfoFilesCommand_HappyPath()
{
using (TempFolderContext context = TestHelper.UseTempFolder())
{
List <RepoData[]> repoDataSets = new List <RepoData[]>
{
new RepoData[]
{
new RepoData
{
Repo = "repo1"
},
new RepoData
{
Repo = "repo2",
Images = new SortedDictionary <string, ImageData>
{
{
"image1",
new ImageData()
}
}
},
new RepoData
{
Repo = "repo4",
Images = new SortedDictionary <string, ImageData>
{
{
"image2",
new ImageData
{
SimpleTags =
{
"tag1"
}
}
}
}
},
},
new RepoData[]
{
new RepoData
{
Repo = "repo2",
Images = new SortedDictionary <string, ImageData>
{
{
"image1",
new ImageData
{
BaseImages = new SortedDictionary <string, string>
{
{ "base1", "base1hash" }
},
SimpleTags =
{
"tag1"
}
}
}
}
},
new RepoData
{
Repo = "repo3",
},
new RepoData
{
Repo = "repo4",
Images = new SortedDictionary <string, ImageData>
{
{
"image2",
new ImageData
{
SimpleTags =
{
"tag2"
}
}
}
}
},
}
};
for (int i = 0; i < repoDataSets.Count; i++)
{
string file = Path.Combine(context.Path, $"{i}.json");
File.WriteAllText(file, JsonHelper.SerializeObject(repoDataSets[i]));
}
MergeImageInfoCommand command = new MergeImageInfoCommand();
command.Options.SourceImageInfoFolderPath = context.Path;
command.Options.DestinationImageInfoPath = Path.Combine(context.Path, "output.json");
await command.ExecuteAsync();
string resultsContent = File.ReadAllText(command.Options.DestinationImageInfoPath);
RepoData[] actual = JsonConvert.DeserializeObject <RepoData[]>(resultsContent);
RepoData[] expected = new RepoData[]
{
new RepoData
{
Repo = "repo1"
},
new RepoData
{
Repo = "repo2",
Images = new SortedDictionary <string, ImageData>
{
{
"image1",
new ImageData {
BaseImages = new SortedDictionary <string, string>
{
{ "base1", "base1hash" }
},
SimpleTags =
{
"tag1"
}
}
}
}
},
new RepoData
{
Repo = "repo3",
},
new RepoData
{
Repo = "repo4",
Images = new SortedDictionary <string, ImageData>
{
{
"image2",
new ImageData {
SimpleTags =
{
"tag1",
"tag2"
}
}
}
}
},
};
ImageInfoHelperTests.CompareRepos(expected, actual);
}
}
private string FormatCsv(string imageId, PlatformData platform, ImageData image, RepoData repo, string timestamp) =>
$"\"{imageId}\",\"{platform.Architecture}\",\"{platform.OsType}\",\"{platform.OsVersion}\","
+ $"\"{image.ProductVersion}\",\"{platform.Dockerfile}\",\"{repo.Repo}\",\"{timestamp}\"";
public async Task PublishImageInfoCommand_ReplaceContent()
{
using (TempFolderContext tempFolderContext = TestHelper.UseTempFolder())
{
string repo1Image1DockerfilePath = DockerfileHelper.CreateDockerfile("1.0/runtime/os", tempFolderContext);
string repo2Image2DockerfilePath = DockerfileHelper.CreateDockerfile("2.0/runtime/os", tempFolderContext);
Manifest manifest = CreateManifest(
CreateRepo("repo1",
CreateImage(
new Platform[]
{
CreatePlatform(repo1Image1DockerfilePath, new string[] { "tag1" })
},
productVersion: "1.0")),
CreateRepo("repo2",
CreateImage(
new Platform[]
{
CreatePlatform(repo2Image2DockerfilePath, new string[] { "tag1" })
},
productVersion: "2.0"))
);
RepoData repo2;
ImageArtifactDetails srcImageArtifactDetails = new ImageArtifactDetails
{
Repos =
{
new RepoData
{
Repo = "repo1",
Images =
{
new ImageData
{
Platforms =
{
Helpers.ImageInfoHelper.CreatePlatform(repo1Image1DockerfilePath,
simpleTags: new List <string>
{
"newtag"
})
},
ProductVersion = "1.0",
Manifest = new ManifestData
{
SyndicatedDigests = new List <string>
{
"newdigest1",
"newdigest2"
}
}
}
}
},
{
repo2 = new RepoData
{
Repo = "repo2",
Images =
{
new ImageData
{
Platforms =
{
Helpers.ImageInfoHelper.CreatePlatform(repo2Image2DockerfilePath,
simpleTags: new List <string>
{
"tag1"
})
},
ProductVersion = "2.0"
}
}
}
}
}
};
string file = Path.Combine(tempFolderContext.Path, "image-info.json");
File.WriteAllText(file, JsonHelper.SerializeObject(srcImageArtifactDetails));
ImageArtifactDetails targetImageArtifactDetails = new ImageArtifactDetails
{
Repos =
{
new RepoData
{
Repo = "repo1",
Images =
{
new ImageData
{
Platforms =
{
Helpers.ImageInfoHelper.CreatePlatform(repo1Image1DockerfilePath,
simpleTags: new List <string>
{
"oldtag"
})
},
ProductVersion = "1.0",
Manifest = new ManifestData
{
SyndicatedDigests = new List <string>
{
"olddigest1",
"olddigest2"
}
}
}
}
}
}
};
GitOptions gitOptions = new GitOptions
{
AuthToken = "token",
Repo = "PublishImageInfoCommand_ReplaceContent",
Branch = "testBranch",
Path = "imageinfo.json",
Email = "*****@*****.**",
Username = "******"
};
AzdoOptions azdoOptions = new AzdoOptions
{
AccessToken = "azdo-token",
AzdoBranch = "testBranch",
AzdoRepo = "testRepo",
Organization = "azdo-org",
Project = "azdo-project",
AzdoPath = "imageinfo.json"
};
Mock <IRepository> repositoryMock = GetRepositoryMock();
Mock <IGitService> gitServiceMock = GetGitServiceMock(repositoryMock.Object, gitOptions.Path, targetImageArtifactDetails);
string actualImageArtifactDetailsContents = null;
gitServiceMock
.Setup(o => o.Stage(It.IsAny <IRepository>(), It.IsAny <string>()))
.Callback((IRepository repo, string path) =>
{
actualImageArtifactDetailsContents = File.ReadAllText(path);
});
PublishImageInfoCommand command = new PublishImageInfoCommand(gitServiceMock.Object, Mock.Of <ILoggerService>());
command.Options.ImageInfoPath = file;
command.Options.GitOptions = gitOptions;
command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json");
File.WriteAllText(Path.Combine(tempFolderContext.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest));
command.LoadManifest();
await command.ExecuteAsync();
ImageArtifactDetails expectedImageArtifactDetails = new ImageArtifactDetails
{
Repos =
{
new RepoData
{
Repo = "repo1",
Images =
{
new ImageData
{
Platforms =
{
Helpers.ImageInfoHelper.CreatePlatform(repo1Image1DockerfilePath,
simpleTags: new List <string>
{
"newtag"
})
},
ProductVersion = "1.0",
Manifest = new ManifestData
{
SyndicatedDigests = new List <string>
{
"newdigest1",
"newdigest2"
}
}
}
}
},
repo2
}
};
Assert.Equal(JsonHelper.SerializeObject(expectedImageArtifactDetails), actualImageArtifactDetailsContents.Trim());
VerifyMocks(repositoryMock);
}
}
private List <string> GetPathsToRebuild(
IEnumerable <PlatformInfo> allPlatforms, PlatformInfo platform, RepoData repoData)
{
bool foundImageInfo = false;
List <string> pathsToRebuild = new List <string>();
void processPlatformWithMissingImageInfo(PlatformInfo platform)
{
_loggerService.WriteMessage(
$"WARNING: Image info not found for '{platform.DockerfilePath}'. Adding path to build to be queued anyway.");
IEnumerable <PlatformInfo> dependentPlatforms = platform.GetDependencyGraph(allPlatforms);
pathsToRebuild.AddRange(dependentPlatforms.Select(p => p.Model.Dockerfile));
}
if (repoData == null || repoData.Images == null)
{
processPlatformWithMissingImageInfo(platform);
return(pathsToRebuild);
}
foreach (ImageData imageData in repoData.Images)
{
PlatformData platformData = imageData.Platforms
.FirstOrDefault(platformData => platformData.PlatformInfo == platform);
if (platformData != null)
{
foundImageInfo = true;
string fromImage = platform.FinalStageFromImage;
string currentDigest;
currentDigest = LockHelper.DoubleCheckedLockLookup(_imageDigestsLock, _imageDigests, fromImage,
() =>
{
string digest = _manifestToolService.GetManifestDigestSha(ManifestMediaType.Any, fromImage, Options.IsDryRun);
return(DockerHelper.GetDigestString(DockerHelper.GetRepo(fromImage), digest));
});
bool rebuildImage = platformData.BaseImageDigest != currentDigest;
_loggerService.WriteMessage(
$"Checking base image '{fromImage}' from '{platform.DockerfilePath}'{Environment.NewLine}"
+ $"\tLast build digest: {platformData.BaseImageDigest}{Environment.NewLine}"
+ $"\tCurrent digest: {currentDigest}{Environment.NewLine}"
+ $"\tImage is up-to-date: {!rebuildImage}{Environment.NewLine}");
if (rebuildImage)
{
IEnumerable <PlatformInfo> dependentPlatforms = platform.GetDependencyGraph(allPlatforms);
pathsToRebuild.AddRange(dependentPlatforms.Select(p => p.Model.Dockerfile));
}
break;
}
}
if (!foundImageInfo)
{
processPlatformWithMissingImageInfo(platform);
}
return(pathsToRebuild);
}
public async Task CopyAcrImagesCommand_CustomDockerfileName()
{
const string subscriptionId = "my subscription";
using (TempFolderContext tempFolderContext = TestHelper.UseTempFolder())
{
Mock <IRegistriesOperations> registriesOperationsMock = CreateRegistriesOperationsMock();
IAzure azure = CreateAzureMock(registriesOperationsMock);
Mock <IAzureManagementFactory> azureManagementFactoryMock = CreateAzureManagementFactoryMock(subscriptionId, azure);
Mock <IEnvironmentService> environmentServiceMock = new Mock <IEnvironmentService>();
CopyAcrImagesCommand command = new CopyAcrImagesCommand(
azureManagementFactoryMock.Object, environmentServiceMock.Object);
command.Options.Manifest = Path.Combine(tempFolderContext.Path, "manifest.json");
command.Options.Subscription = subscriptionId;
command.Options.ResourceGroup = "my resource group";
command.Options.SourceRepoPrefix = command.Options.RepoPrefix = "test/";
command.Options.ImageInfoPath = "image-info.json";
const string runtimeRelativeDir = "1.0/runtime/os";
Directory.CreateDirectory(Path.Combine(tempFolderContext.Path, runtimeRelativeDir));
string dockerfileRelativePath = Path.Combine(runtimeRelativeDir, "Dockerfile.custom");
File.WriteAllText(Path.Combine(tempFolderContext.Path, dockerfileRelativePath), "FROM repo:tag");
Manifest manifest = ManifestHelper.CreateManifest(
ManifestHelper.CreateRepo("runtime",
ManifestHelper.CreateImage(
ManifestHelper.CreatePlatform(dockerfileRelativePath, new string[] { "runtime" })))
);
manifest.Registry = "mcr.microsoft.com";
File.WriteAllText(Path.Combine(tempFolderContext.Path, command.Options.Manifest), JsonConvert.SerializeObject(manifest));
RepoData runtimeRepo;
RepoData[] repos = new RepoData[]
{
runtimeRepo = new RepoData
{
Repo = "runtime",
Images = new SortedDictionary <string, ImageData>
{
{
PathHelper.NormalizePath(dockerfileRelativePath),
new ImageData
{
SimpleTags =
{
"tag1",
"tag2"
}
}
}
}
}
};
File.WriteAllText(command.Options.ImageInfoPath, JsonConvert.SerializeObject(repos));
command.LoadManifest();
await command.ExecuteAsync();
IList <string> expectedTags = runtimeRepo.Images.First().Value.SimpleTags
.Select(tag => $"{command.Options.RepoPrefix}{runtimeRepo.Repo}:{tag}")
.ToList();
foreach (string expectedTag in expectedTags)
{
registriesOperationsMock
.Verify(o => o.ImportImageWithHttpMessagesAsync(
command.Options.ResourceGroup,
manifest.Registry,
It.Is <ImportImageParametersInner>(parameters =>
VerifyImportImageParameters(parameters, new List <string> {
expectedTag
})),
It.IsAny <Dictionary <string, List <string> > >(),
It.IsAny <CancellationToken>()));
}
environmentServiceMock.Verify(o => o.Exit(It.IsAny <int>()), Times.Never);
}
}