private static List <ContainerImagePayload> ImagePayloadWhenNotBuildingImages(string acrRepoBase, string testRepoBase, string prodRepoBase, List <DirectoryInfo> directoryInfos)
{
List <ContainerImagePayload> payloads = new List <ContainerImagePayload>();
foreach (var di in directoryInfos)
{
//we aren't building the images, so read the version file for both parent and child
var childVersionTag = TryReadVersionFileForDirectory(di);
if (childVersionTag == null)
{
PublishContainerImages.WriteError($"No version found in target directory {di.FullName}");
throw new FileNotFoundException($"No version file found in target directory {di.FullName}");
}
if (di.Parent != null)
{
var parentVersion = TryReadVersionFileForDirectory(di.Parent);
//we aren't building the parent, so use the parent tag from the version file in the directory above
var payload = PayloadForDirectory(acrRepoBase, testRepoBase, prodRepoBase, di, ExtractBuildVersionFromFullTag(childVersionTag), parentVersion);
payloads.Add(payload);
}
}
return(payloads);
}
private static FileInfo _fileInfoForContainerImageDefinition(DirectoryInfo root)
{
FileInfo fileInfo = null;
try
{
fileInfo = root.GetFiles(PublishContainerImages.ContainerImagesDefinitionFilename).Single();
}
catch (UnauthorizedAccessException e)
{
PublishContainerImages.WriteError(e.Message);
}
catch (DirectoryNotFoundException e)
{
PublishContainerImages.WriteError(e.Message);
}
catch (ArgumentNullException e)
{
PublishContainerImages.WriteError(e.Message);
}
catch (InvalidOperationException e)
{
PublishContainerImages.WriteError(e.Message);
}
return(fileInfo);
}
private static ContainerImageDefinition ReadContainerImageDefinition(DirectoryInfo directory)
{
var containerDefinitionFileInfo = _fileInfoForContainerImageDefinition(directory);
if (containerDefinitionFileInfo != null)
{
ContainerImageDefinition containerImageDefinition;
dynamic json = Json.ReadJsonFileToDynamic(containerDefinitionFileInfo.FullName);
try
{
containerImageDefinition = json.ToObject <ContainerImageDefinition>();
containerImageDefinition.PathToDockerFile = System.IO.Path.GetRelativePath(
PublishContainerImages.DockerInstallScriptsRootDir, directory.FullName);
}
catch (JsonSerializationException ex)
{
PublishContainerImages.WriteError(
$"Invalid Json read in file {containerDefinitionFileInfo}, Json was: {json}. Exception: {ex}");
throw;
}
return(containerImageDefinition);
}
return(null);
}
private static Tuple <TestConfigurationDefinition, TestParametersDefinition> TryReadTestConfigAndParams(DirectoryInfo directory)
{
var testConfigFiles = directory.GetFiles(PublishContainerImages.TestConfigurationFilename);
var testParamsFiles = directory.GetFiles(PublishContainerImages.TestParametersFilename);
if (!testConfigFiles.Any())
{
PublishContainerImages.WriteLog($"No Test will be run, testConfiguration.json not found in directory {directory}");
return(new Tuple <TestConfigurationDefinition, TestParametersDefinition>(null, null));
}
if (!testParamsFiles.Any())
{
PublishContainerImages.WriteLog($"No Test will be run, testParameters.json not found in directory {directory}");
return(new Tuple <TestConfigurationDefinition, TestParametersDefinition>(null, null));
}
if (testConfigFiles.Length > 1)
{
PublishContainerImages.WriteLog($"No test will be run, more than one TestConfiguration.json file found in {directory}");
return(new Tuple <TestConfigurationDefinition, TestParametersDefinition>(null, null));
}
if (testParamsFiles.Length > 1)
{
PublishContainerImages.WriteLog($"No test will be run, more than one TestParameters.json file found in {directory}");
return(new Tuple <TestConfigurationDefinition, TestParametersDefinition>(null, null));
}
var testConfigJson = _readJsonFileToDynamic(testConfigFiles.Single().FullName);
TestConfigurationDefinition testConfig;
TestParametersDefinition testParams;
try
{
testConfig = testConfigJson.ToObject <TestConfigurationDefinition>();
}
catch (JsonSerializationException ex)
{
PublishContainerImages.WriteError($"No test will be run, invalid Json read in file {testConfigFiles.Single().FullName}, Json was: {testConfigJson}. Exception: {ex}");
return(new Tuple <TestConfigurationDefinition, TestParametersDefinition>(null, null));
}
var testParamsJson = _readJsonFileToDynamic(testParamsFiles.Single().FullName);
try
{
testParams = testParamsJson.ToObject <TestParametersDefinition>();
}
catch (JsonSerializationException ex)
{
PublishContainerImages.WriteError($"No test will be run, invalid Json read in file {testParamsJson.Single().FullName}, Json was: {testParamsJson}. Exception: {ex}");
return(new Tuple <TestConfigurationDefinition, TestParametersDefinition>(null, null));
}
return(new Tuple <TestConfigurationDefinition, TestParametersDefinition>(testConfig, testParams));
}
public static void _outputTestImagesFile(List <string> testImages)
{
PublishContainerImages.WriteLog($"\nWriting the following entries to {PublishContainerImages.TestImagesFilename}:");
testImages.ForEach(PublishContainerImages.WriteLog);
var builtImagesFilePath = Path.Combine(OutputImagesTxtFilePath(), PublishContainerImages.TestImagesFilename);
Directory.CreateDirectory(new FileInfo(builtImagesFilePath).DirectoryName);
File.WriteAllLines(builtImagesFilePath, testImages);
}
public static void MergeJson(string masterFile, string newImagesFile, string outputFile, string exclusivelyForApp)
{
var masterContainerImages = ReadRenderingContainerImagesJsonFile(masterFile);
var newContainerImages = ReadRenderingContainerImagesJsonFile(newImagesFile);
var outputContainerImages = masterContainerImages.ContainerImages.ToList();
foreach (var newImage in newContainerImages.ContainerImages)
{
var preExistingContainerImage = outputContainerImages.Find(masterImage => ImageMetadataMatches(masterImage, newImage));
if (preExistingContainerImage == null)
{
outputContainerImages.Add(newImage);
}
else
{
preExistingContainerImage.ContainerImage = newImage.ContainerImage;
}
}
if (!string.IsNullOrEmpty(exclusivelyForApp))
{
outputContainerImages.RemoveAll(image => !image.App.Equals(exclusivelyForApp, StringComparison.OrdinalIgnoreCase));
}
var output = new RenderingContainerImages()
{
ContainerImages = outputContainerImages.ToArray(),
ImageReferences = masterContainerImages.ImageReferences
};
string outputJson = Json.JsonSerializeObject(output);
File.WriteAllText(outputFile, outputJson);
PublishContainerImages.WriteLog($"\nWrote metadata file at: {outputFile}, file contents:");
PublishContainerImages.WriteLog(outputJson);
dynamic metaDataOutput = new ExpandoObject();
//metaDataOutput.containerImages = new dynamic[imagesWithMetadata.Count];
//int index = 0;
//foreach (var image in imagesWithMetadata)
//{
// dynamic imageEntry = new ExpandoObject();
// imageEntry.containerImage = $"{image.ProdRepoName}:{image.VersionTag}";
// imageEntry.os = image.Metadata.Os;
// imageEntry.app = image.Metadata.App;
// imageEntry.appVersion = image.Metadata.AppVersion;
// imageEntry.renderer = image.Metadata.Renderer;
// imageEntry.rendererVersion = image.Metadata.RendererVersion;
// imageEntry.imageReferenceId = image.Metadata.ImageReferenceId;
// metaDataOutput.containerImages[index++] = imageEntry;
//}
}
public static string BuildVersionTag(string containerImage, string parentVersionTag, string currentVersion)
{
var imageAndTag = containerImage
.Replace($"$VER", currentVersion)
.Replace($"$PARENT_TAG", parentVersionTag)
.Split(':');
PublishContainerImages.WriteLog($"Split image and Tag into: {string.Join(", ", imageAndTag)}");
return(imageAndTag.Last());
}
public static void _outputTestFiles(List <ContainerImagePayload> payloads, List <string> builtImages)
{
PublishContainerImages.WriteLog($"\nWriting test files for the following images:\n { string.Join("\n", builtImages)}");
var payloadsWithTests = _removeInvalidPayloads(payloads);
var payloadsWithTestsAndImagesTagged = _updateTestConfigAndParametersWithTaggedImage(payloadsWithTests, builtImages);
payloadsWithTestsAndImagesTagged.ForEach(payload =>
{
var parametersPath = Path.Combine(OutputTestPath(), payload.TestConfigurationDefinition.Parameters);
var parametersJson = JsonSerializeObject(payload.TestParametersDefinition);
FileInfo paramsFile = new FileInfo(parametersPath);
Directory.CreateDirectory(paramsFile.DirectoryName);
File.WriteAllText(parametersPath, parametersJson);
PublishContainerImages.WriteLog($"\nWrote parameters file at: {paramsFile.FullName}, file contents:");
PublishContainerImages.WriteLog(parametersJson);
});
var testsConfiguration = new TestsDefinition
{
Tests = payloadsWithTestsAndImagesTagged.Select(payload =>
{
var config = payload.TestConfigurationDefinition;
config.Parameters = Path.Combine(OutputTestPath(), config.Parameters).Replace("\\", "/");
return(config);
}).ToArray(),
Images = new[]
{
new TestMarketplaceImageDefinition
{
Offer = "microsoft-azure-batch",
OsType = "linux",
Publisher = "centos-container",
Sku = "7-5",
Version = "latest",
}
}
};
var testsConfigurationFilepath = Path.Combine(OutputTestPath(), PublishContainerImages.TestConfigurationFilename);
var testsConfigurationJson = JsonSerializeObject(testsConfiguration);
FileInfo configFile = new FileInfo(testsConfigurationFilepath);
Directory.CreateDirectory(configFile.DirectoryName);
File.WriteAllText(testsConfigurationFilepath, testsConfigurationJson);
PublishContainerImages.WriteLog($"\nWrote configuration file at: {configFile.FullName}, file contents:");
PublishContainerImages.WriteLog(testsConfigurationJson);
}
private static RenderingContainerImages ReadRenderingContainerImagesJsonFile(string masterFilePath)
{
RenderingContainerImages renderingContainerImages;
dynamic json = Json.ReadJsonFileToDynamic(masterFilePath);
try
{
renderingContainerImages = json.ToObject <RenderingContainerImages>();
}
catch (JsonSerializationException ex)
{
PublishContainerImages.WriteError(
$"Invalid Json read in file {masterFilePath}, Json was: {json}. Exception: {ex}");
throw;
}
return(renderingContainerImages);
}
public static void _outputContainerImageMetadataFile(List <OutputContainerImage> builtImages, bool outputFullMetadataFile)
{
PublishContainerImages.WriteLog($"\nWriting the following entries to {PublishContainerImages.BuiltImageMetadataFilename}:");
builtImages.Select(x =>
$"{x.ProdRepoName}:{x.VersionTag}")
.ToList().ForEach(PublishContainerImages.WriteLog);
var builtImagesFilePath = Path.Combine(OutputImagesTxtFilePath(), PublishContainerImages.BuiltImageMetadataFilename);
Directory.CreateDirectory(new FileInfo(builtImagesFilePath).DirectoryName);
var imagesWithMetadata = builtImages.Where(im => im.Metadata != null).ToList();
dynamic metaDataOutput = new ExpandoObject();
if (outputFullMetadataFile)
{
metaDataOutput.imageReferences = ImageReference.GetAllImageReferences();
}
metaDataOutput.containerImages = new dynamic[imagesWithMetadata.Count];
int index = 0;
foreach (var image in imagesWithMetadata)
{
dynamic imageEntry = new ExpandoObject();
imageEntry.containerImage = $"{image.ProdRepoName}:{image.VersionTag}";
imageEntry.os = image.Metadata.Os;
imageEntry.app = image.Metadata.App;
imageEntry.appVersion = image.Metadata.AppVersion;
imageEntry.renderer = image.Metadata.Renderer;
imageEntry.rendererVersion = image.Metadata.RendererVersion;
imageEntry.imageReferenceId = image.Metadata.ImageReferenceId;
metaDataOutput.containerImages[index++] = imageEntry;
}
string metadataJson = Json.JsonSerializeObject(metaDataOutput);
File.WriteAllText(builtImagesFilePath, metadataJson);
PublishContainerImages.WriteLog($"\nWrote metadata file at: {builtImagesFilePath}, file contents:");
PublishContainerImages.WriteLog(metadataJson);
}
private static ContainerImageDefinition ReadContainerImageDefinition(DirectoryInfo directory)
{
var containerDefinitionFileInfo = _fileInfoForContainerImageDefinition(directory);
if (containerDefinitionFileInfo != null)
{
ContainerImageDefinition containerImageDefinition;
dynamic json = _readJsonFileToDynamic(containerDefinitionFileInfo.FullName);
try
{
containerImageDefinition = json.ToObject <ContainerImageDefinition>();
}
catch (JsonSerializationException ex)
{
PublishContainerImages.WriteError(
$"Invalid Json read in file {containerDefinitionFileInfo}, Json was: {json}. Exception: {ex}");
throw;
}
return(containerImageDefinition);
}
return(null);
}
public static void _outputTestFiles(string testRepoUsername, string testRepoPassword, string testRepoRegistry, string batchExtensionTemplatesRootDir, List <OutputContainerImage> imageDefinitions, List <string> imagesWithTagsToTest)
{
PublishContainerImages.WriteLog($"\nWriting test files for the following images:\n { string.Join("\n", imageDefinitions.Select(image => image.TestContainerImage))}");
var imagesWithTests = _removeInvalidPayloads(imageDefinitions);
var updatedImageDefinitions = _updateTestConfigAndParametersWithTaggedImage(imagesWithTests);
updatedImageDefinitions.ForEach(payload =>
{
var parametersPath = Path.Combine(OutputTestPath(), payload.Payload.TestConfigurationDefinition.GetParametersFileName());
var parametersJson = Json.JsonSerializeObject(payload.Payload.TestParametersDefinition);
FileInfo paramsFile = new FileInfo(parametersPath);
Directory.CreateDirectory(paramsFile.DirectoryName);
File.WriteAllText(parametersPath, parametersJson);
PublishContainerImages.WriteLog($"\nWrote parameters file at: {paramsFile.FullName}, file contents:");
PublishContainerImages.WriteLog(parametersJson);
});
updatedImageDefinitions.ForEach(payload =>
{
var runnerDirectory = Path.Combine(ExecutableDirectory(), batchExtensionTemplatesRootDir, "runner");
var poolTemplateOriginalFullPath = Path.Combine(runnerDirectory, payload.Payload.TestConfigurationDefinition.PoolTemplate);
var poolTemplateDynamic = Json.ReadJsonFileToDynamic(poolTemplateOriginalFullPath);
if (!string.IsNullOrEmpty(testRepoUsername))
{
var containerRegistries = JArray.FromObject(new[]
{
new
{
username = testRepoUsername, password = testRepoPassword, registryServer = testRepoRegistry
}
});
poolTemplateDynamic.variables.osType.containerConfiguration.containerRegistries = containerRegistries;
}
var poolTemplateJson = Json.JsonSerializeObject(poolTemplateDynamic);
var relativeFinalPathForPoolTemplate = payload.Payload.TestConfigurationDefinition.PoolTemplate.Replace("../", "");
var poolTemplateFinalPath = Path.Combine(OutputTestPath(), relativeFinalPathForPoolTemplate);
payload.Payload.TestConfigurationDefinition.PoolTemplate = poolTemplateFinalPath.Replace("\\", "/");
FileInfo poolTemplateFinal = new FileInfo(poolTemplateFinalPath);
Directory.CreateDirectory(poolTemplateFinal.DirectoryName);
File.WriteAllText(poolTemplateFinalPath, poolTemplateJson);
PublishContainerImages.WriteLog($"\nWrote pool template file at: {poolTemplateFinalPath}, file contents:");
PublishContainerImages.WriteLog(poolTemplateJson);
});
var testsConfiguration = new TestsDefinition
{
Tests = updatedImageDefinitions.Select(payload =>
{
var config = payload.Payload.TestConfigurationDefinition;
config.Parameters = Path.Combine(OutputTestPath(), config.GetParametersFileName()).Replace("\\", "/");
return(config);
}).ToArray(),
Images = new[]
{
new TestMarketplaceImageDefinition
{
Offer = "microsoft-azure-batch",
OsType = "linux",
Publisher = "centos-container",
Sku = "7-5",
Version = "latest",
},
new TestMarketplaceImageDefinition
{
Offer = "microsoft-azure-batch",
OsType = "linux",
Publisher = "ubuntu-server-container",
Sku = "16-04-lts",
Version = "latest",
}
}
};
var testsConfigurationFilepath = Path.Combine(OutputTestPath(), PublishContainerImages.TestConfigurationFilename);
var testsConfigurationJson = Json.JsonSerializeObject(testsConfiguration);
FileInfo configFile = new FileInfo(testsConfigurationFilepath);
Directory.CreateDirectory(configFile.DirectoryName);
File.WriteAllText(testsConfigurationFilepath, testsConfigurationJson);
PublishContainerImages.WriteLog($"\nWrote configuration file at: {configFile.FullName}, file contents:");
PublishContainerImages.WriteLog(testsConfigurationJson);
}
private static string[] _runCmdProcess(string commandLine, int timeoutInMs)
{
ProcessStartInfo processStartInfo;
if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux))
{
processStartInfo = new ProcessStartInfo
{
FileName = "docker",
Arguments = commandLine,
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardOutput = true,
RedirectStandardError = true,
};
}
else
{
processStartInfo = new ProcessStartInfo
{
FileName = "cmd.exe",
Arguments = $"/c docker " + commandLine,
UseShellExecute = false,
CreateNoWindow = true,
RedirectStandardOutput = true,
RedirectStandardError = true,
};
}
PublishContainerImages.WriteLog($"Running commandLine: {processStartInfo.FileName} {processStartInfo.Arguments}");
using (var process = new Process
{
StartInfo = processStartInfo
})
{
var output = new List <string>();
process.OutputDataReceived += delegate(object sender, DataReceivedEventArgs e)
{
if (!String.IsNullOrEmpty(e.Data))
{
PublishContainerImages.WriteLog(e.Data);
output.Add(e.Data);
}
};
process.ErrorDataReceived += delegate(object sender, DataReceivedEventArgs e)
{
if (!String.IsNullOrEmpty(e.Data))
{
PublishContainerImages.WriteError(e.Data);
output.Add("ERROR: " + e.Data);
throw new Exception($"Error in external process: {commandLine}, {e.Data}");
}
};
process.Start();
process.BeginErrorReadLine();
process.BeginOutputReadLine();
process.WaitForExit(timeoutInMs);
process.WaitForExit();
return(output.ToArray());
}
}