private static List <ContainerImagePayload> ImagePayloadWhenNotBuildingImages(string acrRepoBase, string testRepoBase, string prodRepoBase, List <DirectoryInfo> directoryInfos) { List <ContainerImagePayload> payloads = new List <ContainerImagePayload>(); foreach (var di in directoryInfos) { //we aren't building the images, so read the version file for both parent and child var childVersionTag = TryReadVersionFileForDirectory(di); if (childVersionTag == null) { PublishContainerImages.WriteError($"No version found in target directory {di.FullName}"); throw new FileNotFoundException($"No version file found in target directory {di.FullName}"); } if (di.Parent != null) { var parentVersion = TryReadVersionFileForDirectory(di.Parent); //we aren't building the parent, so use the parent tag from the version file in the directory above var payload = PayloadForDirectory(acrRepoBase, testRepoBase, prodRepoBase, di, ExtractBuildVersionFromFullTag(childVersionTag), parentVersion); payloads.Add(payload); } } return(payloads); }
private static FileInfo _fileInfoForContainerImageDefinition(DirectoryInfo root) { FileInfo fileInfo = null; try { fileInfo = root.GetFiles(PublishContainerImages.ContainerImagesDefinitionFilename).Single(); } catch (UnauthorizedAccessException e) { PublishContainerImages.WriteError(e.Message); } catch (DirectoryNotFoundException e) { PublishContainerImages.WriteError(e.Message); } catch (ArgumentNullException e) { PublishContainerImages.WriteError(e.Message); } catch (InvalidOperationException e) { PublishContainerImages.WriteError(e.Message); } return(fileInfo); }
private static ContainerImageDefinition ReadContainerImageDefinition(DirectoryInfo directory) { var containerDefinitionFileInfo = _fileInfoForContainerImageDefinition(directory); if (containerDefinitionFileInfo != null) { ContainerImageDefinition containerImageDefinition; dynamic json = Json.ReadJsonFileToDynamic(containerDefinitionFileInfo.FullName); try { containerImageDefinition = json.ToObject <ContainerImageDefinition>(); containerImageDefinition.PathToDockerFile = System.IO.Path.GetRelativePath( PublishContainerImages.DockerInstallScriptsRootDir, directory.FullName); } catch (JsonSerializationException ex) { PublishContainerImages.WriteError( $"Invalid Json read in file {containerDefinitionFileInfo}, Json was: {json}. Exception: {ex}"); throw; } return(containerImageDefinition); } return(null); }
private static Tuple <TestConfigurationDefinition, TestParametersDefinition> TryReadTestConfigAndParams(DirectoryInfo directory) { var testConfigFiles = directory.GetFiles(PublishContainerImages.TestConfigurationFilename); var testParamsFiles = directory.GetFiles(PublishContainerImages.TestParametersFilename); if (!testConfigFiles.Any()) { PublishContainerImages.WriteLog($"No Test will be run, testConfiguration.json not found in directory {directory}"); return(new Tuple <TestConfigurationDefinition, TestParametersDefinition>(null, null)); } if (!testParamsFiles.Any()) { PublishContainerImages.WriteLog($"No Test will be run, testParameters.json not found in directory {directory}"); return(new Tuple <TestConfigurationDefinition, TestParametersDefinition>(null, null)); } if (testConfigFiles.Length > 1) { PublishContainerImages.WriteLog($"No test will be run, more than one TestConfiguration.json file found in {directory}"); return(new Tuple <TestConfigurationDefinition, TestParametersDefinition>(null, null)); } if (testParamsFiles.Length > 1) { PublishContainerImages.WriteLog($"No test will be run, more than one TestParameters.json file found in {directory}"); return(new Tuple <TestConfigurationDefinition, TestParametersDefinition>(null, null)); } var testConfigJson = _readJsonFileToDynamic(testConfigFiles.Single().FullName); TestConfigurationDefinition testConfig; TestParametersDefinition testParams; try { testConfig = testConfigJson.ToObject <TestConfigurationDefinition>(); } catch (JsonSerializationException ex) { PublishContainerImages.WriteError($"No test will be run, invalid Json read in file {testConfigFiles.Single().FullName}, Json was: {testConfigJson}. Exception: {ex}"); return(new Tuple <TestConfigurationDefinition, TestParametersDefinition>(null, null)); } var testParamsJson = _readJsonFileToDynamic(testParamsFiles.Single().FullName); try { testParams = testParamsJson.ToObject <TestParametersDefinition>(); } catch (JsonSerializationException ex) { PublishContainerImages.WriteError($"No test will be run, invalid Json read in file {testParamsJson.Single().FullName}, Json was: {testParamsJson}. Exception: {ex}"); return(new Tuple <TestConfigurationDefinition, TestParametersDefinition>(null, null)); } return(new Tuple <TestConfigurationDefinition, TestParametersDefinition>(testConfig, testParams)); }
public static void _outputTestImagesFile(List <string> testImages) { PublishContainerImages.WriteLog($"\nWriting the following entries to {PublishContainerImages.TestImagesFilename}:"); testImages.ForEach(PublishContainerImages.WriteLog); var builtImagesFilePath = Path.Combine(OutputImagesTxtFilePath(), PublishContainerImages.TestImagesFilename); Directory.CreateDirectory(new FileInfo(builtImagesFilePath).DirectoryName); File.WriteAllLines(builtImagesFilePath, testImages); }
public static void MergeJson(string masterFile, string newImagesFile, string outputFile, string exclusivelyForApp) { var masterContainerImages = ReadRenderingContainerImagesJsonFile(masterFile); var newContainerImages = ReadRenderingContainerImagesJsonFile(newImagesFile); var outputContainerImages = masterContainerImages.ContainerImages.ToList(); foreach (var newImage in newContainerImages.ContainerImages) { var preExistingContainerImage = outputContainerImages.Find(masterImage => ImageMetadataMatches(masterImage, newImage)); if (preExistingContainerImage == null) { outputContainerImages.Add(newImage); } else { preExistingContainerImage.ContainerImage = newImage.ContainerImage; } } if (!string.IsNullOrEmpty(exclusivelyForApp)) { outputContainerImages.RemoveAll(image => !image.App.Equals(exclusivelyForApp, StringComparison.OrdinalIgnoreCase)); } var output = new RenderingContainerImages() { ContainerImages = outputContainerImages.ToArray(), ImageReferences = masterContainerImages.ImageReferences }; string outputJson = Json.JsonSerializeObject(output); File.WriteAllText(outputFile, outputJson); PublishContainerImages.WriteLog($"\nWrote metadata file at: {outputFile}, file contents:"); PublishContainerImages.WriteLog(outputJson); dynamic metaDataOutput = new ExpandoObject(); //metaDataOutput.containerImages = new dynamic[imagesWithMetadata.Count]; //int index = 0; //foreach (var image in imagesWithMetadata) //{ // dynamic imageEntry = new ExpandoObject(); // imageEntry.containerImage = $"{image.ProdRepoName}:{image.VersionTag}"; // imageEntry.os = image.Metadata.Os; // imageEntry.app = image.Metadata.App; // imageEntry.appVersion = image.Metadata.AppVersion; // imageEntry.renderer = image.Metadata.Renderer; // imageEntry.rendererVersion = image.Metadata.RendererVersion; // imageEntry.imageReferenceId = image.Metadata.ImageReferenceId; // metaDataOutput.containerImages[index++] = imageEntry; //} }
public static string BuildVersionTag(string containerImage, string parentVersionTag, string currentVersion) { var imageAndTag = containerImage .Replace($"$VER", currentVersion) .Replace($"$PARENT_TAG", parentVersionTag) .Split(':'); PublishContainerImages.WriteLog($"Split image and Tag into: {string.Join(", ", imageAndTag)}"); return(imageAndTag.Last()); }
public static void _outputTestFiles(List <ContainerImagePayload> payloads, List <string> builtImages) { PublishContainerImages.WriteLog($"\nWriting test files for the following images:\n { string.Join("\n", builtImages)}"); var payloadsWithTests = _removeInvalidPayloads(payloads); var payloadsWithTestsAndImagesTagged = _updateTestConfigAndParametersWithTaggedImage(payloadsWithTests, builtImages); payloadsWithTestsAndImagesTagged.ForEach(payload => { var parametersPath = Path.Combine(OutputTestPath(), payload.TestConfigurationDefinition.Parameters); var parametersJson = JsonSerializeObject(payload.TestParametersDefinition); FileInfo paramsFile = new FileInfo(parametersPath); Directory.CreateDirectory(paramsFile.DirectoryName); File.WriteAllText(parametersPath, parametersJson); PublishContainerImages.WriteLog($"\nWrote parameters file at: {paramsFile.FullName}, file contents:"); PublishContainerImages.WriteLog(parametersJson); }); var testsConfiguration = new TestsDefinition { Tests = payloadsWithTestsAndImagesTagged.Select(payload => { var config = payload.TestConfigurationDefinition; config.Parameters = Path.Combine(OutputTestPath(), config.Parameters).Replace("\\", "/"); return(config); }).ToArray(), Images = new[] { new TestMarketplaceImageDefinition { Offer = "microsoft-azure-batch", OsType = "linux", Publisher = "centos-container", Sku = "7-5", Version = "latest", } } }; var testsConfigurationFilepath = Path.Combine(OutputTestPath(), PublishContainerImages.TestConfigurationFilename); var testsConfigurationJson = JsonSerializeObject(testsConfiguration); FileInfo configFile = new FileInfo(testsConfigurationFilepath); Directory.CreateDirectory(configFile.DirectoryName); File.WriteAllText(testsConfigurationFilepath, testsConfigurationJson); PublishContainerImages.WriteLog($"\nWrote configuration file at: {configFile.FullName}, file contents:"); PublishContainerImages.WriteLog(testsConfigurationJson); }
private static RenderingContainerImages ReadRenderingContainerImagesJsonFile(string masterFilePath) { RenderingContainerImages renderingContainerImages; dynamic json = Json.ReadJsonFileToDynamic(masterFilePath); try { renderingContainerImages = json.ToObject <RenderingContainerImages>(); } catch (JsonSerializationException ex) { PublishContainerImages.WriteError( $"Invalid Json read in file {masterFilePath}, Json was: {json}. Exception: {ex}"); throw; } return(renderingContainerImages); }
public static void _outputContainerImageMetadataFile(List <OutputContainerImage> builtImages, bool outputFullMetadataFile) { PublishContainerImages.WriteLog($"\nWriting the following entries to {PublishContainerImages.BuiltImageMetadataFilename}:"); builtImages.Select(x => $"{x.ProdRepoName}:{x.VersionTag}") .ToList().ForEach(PublishContainerImages.WriteLog); var builtImagesFilePath = Path.Combine(OutputImagesTxtFilePath(), PublishContainerImages.BuiltImageMetadataFilename); Directory.CreateDirectory(new FileInfo(builtImagesFilePath).DirectoryName); var imagesWithMetadata = builtImages.Where(im => im.Metadata != null).ToList(); dynamic metaDataOutput = new ExpandoObject(); if (outputFullMetadataFile) { metaDataOutput.imageReferences = ImageReference.GetAllImageReferences(); } metaDataOutput.containerImages = new dynamic[imagesWithMetadata.Count]; int index = 0; foreach (var image in imagesWithMetadata) { dynamic imageEntry = new ExpandoObject(); imageEntry.containerImage = $"{image.ProdRepoName}:{image.VersionTag}"; imageEntry.os = image.Metadata.Os; imageEntry.app = image.Metadata.App; imageEntry.appVersion = image.Metadata.AppVersion; imageEntry.renderer = image.Metadata.Renderer; imageEntry.rendererVersion = image.Metadata.RendererVersion; imageEntry.imageReferenceId = image.Metadata.ImageReferenceId; metaDataOutput.containerImages[index++] = imageEntry; } string metadataJson = Json.JsonSerializeObject(metaDataOutput); File.WriteAllText(builtImagesFilePath, metadataJson); PublishContainerImages.WriteLog($"\nWrote metadata file at: {builtImagesFilePath}, file contents:"); PublishContainerImages.WriteLog(metadataJson); }
private static ContainerImageDefinition ReadContainerImageDefinition(DirectoryInfo directory) { var containerDefinitionFileInfo = _fileInfoForContainerImageDefinition(directory); if (containerDefinitionFileInfo != null) { ContainerImageDefinition containerImageDefinition; dynamic json = _readJsonFileToDynamic(containerDefinitionFileInfo.FullName); try { containerImageDefinition = json.ToObject <ContainerImageDefinition>(); } catch (JsonSerializationException ex) { PublishContainerImages.WriteError( $"Invalid Json read in file {containerDefinitionFileInfo}, Json was: {json}. Exception: {ex}"); throw; } return(containerImageDefinition); } return(null); }
public static void _outputTestFiles(string testRepoUsername, string testRepoPassword, string testRepoRegistry, string batchExtensionTemplatesRootDir, List <OutputContainerImage> imageDefinitions, List <string> imagesWithTagsToTest) { PublishContainerImages.WriteLog($"\nWriting test files for the following images:\n { string.Join("\n", imageDefinitions.Select(image => image.TestContainerImage))}"); var imagesWithTests = _removeInvalidPayloads(imageDefinitions); var updatedImageDefinitions = _updateTestConfigAndParametersWithTaggedImage(imagesWithTests); updatedImageDefinitions.ForEach(payload => { var parametersPath = Path.Combine(OutputTestPath(), payload.Payload.TestConfigurationDefinition.GetParametersFileName()); var parametersJson = Json.JsonSerializeObject(payload.Payload.TestParametersDefinition); FileInfo paramsFile = new FileInfo(parametersPath); Directory.CreateDirectory(paramsFile.DirectoryName); File.WriteAllText(parametersPath, parametersJson); PublishContainerImages.WriteLog($"\nWrote parameters file at: {paramsFile.FullName}, file contents:"); PublishContainerImages.WriteLog(parametersJson); }); updatedImageDefinitions.ForEach(payload => { var runnerDirectory = Path.Combine(ExecutableDirectory(), batchExtensionTemplatesRootDir, "runner"); var poolTemplateOriginalFullPath = Path.Combine(runnerDirectory, payload.Payload.TestConfigurationDefinition.PoolTemplate); var poolTemplateDynamic = Json.ReadJsonFileToDynamic(poolTemplateOriginalFullPath); if (!string.IsNullOrEmpty(testRepoUsername)) { var containerRegistries = JArray.FromObject(new[] { new { username = testRepoUsername, password = testRepoPassword, registryServer = testRepoRegistry } }); poolTemplateDynamic.variables.osType.containerConfiguration.containerRegistries = containerRegistries; } var poolTemplateJson = Json.JsonSerializeObject(poolTemplateDynamic); var relativeFinalPathForPoolTemplate = payload.Payload.TestConfigurationDefinition.PoolTemplate.Replace("../", ""); var poolTemplateFinalPath = Path.Combine(OutputTestPath(), relativeFinalPathForPoolTemplate); payload.Payload.TestConfigurationDefinition.PoolTemplate = poolTemplateFinalPath.Replace("\\", "/"); FileInfo poolTemplateFinal = new FileInfo(poolTemplateFinalPath); Directory.CreateDirectory(poolTemplateFinal.DirectoryName); File.WriteAllText(poolTemplateFinalPath, poolTemplateJson); PublishContainerImages.WriteLog($"\nWrote pool template file at: {poolTemplateFinalPath}, file contents:"); PublishContainerImages.WriteLog(poolTemplateJson); }); var testsConfiguration = new TestsDefinition { Tests = updatedImageDefinitions.Select(payload => { var config = payload.Payload.TestConfigurationDefinition; config.Parameters = Path.Combine(OutputTestPath(), config.GetParametersFileName()).Replace("\\", "/"); return(config); }).ToArray(), Images = new[] { new TestMarketplaceImageDefinition { Offer = "microsoft-azure-batch", OsType = "linux", Publisher = "centos-container", Sku = "7-5", Version = "latest", }, new TestMarketplaceImageDefinition { Offer = "microsoft-azure-batch", OsType = "linux", Publisher = "ubuntu-server-container", Sku = "16-04-lts", Version = "latest", } } }; var testsConfigurationFilepath = Path.Combine(OutputTestPath(), PublishContainerImages.TestConfigurationFilename); var testsConfigurationJson = Json.JsonSerializeObject(testsConfiguration); FileInfo configFile = new FileInfo(testsConfigurationFilepath); Directory.CreateDirectory(configFile.DirectoryName); File.WriteAllText(testsConfigurationFilepath, testsConfigurationJson); PublishContainerImages.WriteLog($"\nWrote configuration file at: {configFile.FullName}, file contents:"); PublishContainerImages.WriteLog(testsConfigurationJson); }
private static string[] _runCmdProcess(string commandLine, int timeoutInMs) { ProcessStartInfo processStartInfo; if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) { processStartInfo = new ProcessStartInfo { FileName = "docker", Arguments = commandLine, UseShellExecute = false, CreateNoWindow = true, RedirectStandardOutput = true, RedirectStandardError = true, }; } else { processStartInfo = new ProcessStartInfo { FileName = "cmd.exe", Arguments = $"/c docker " + commandLine, UseShellExecute = false, CreateNoWindow = true, RedirectStandardOutput = true, RedirectStandardError = true, }; } PublishContainerImages.WriteLog($"Running commandLine: {processStartInfo.FileName} {processStartInfo.Arguments}"); using (var process = new Process { StartInfo = processStartInfo }) { var output = new List <string>(); process.OutputDataReceived += delegate(object sender, DataReceivedEventArgs e) { if (!String.IsNullOrEmpty(e.Data)) { PublishContainerImages.WriteLog(e.Data); output.Add(e.Data); } }; process.ErrorDataReceived += delegate(object sender, DataReceivedEventArgs e) { if (!String.IsNullOrEmpty(e.Data)) { PublishContainerImages.WriteError(e.Data); output.Add("ERROR: " + e.Data); throw new Exception($"Error in external process: {commandLine}, {e.Data}"); } }; process.Start(); process.BeginErrorReadLine(); process.BeginOutputReadLine(); process.WaitForExit(timeoutInMs); process.WaitForExit(); return(output.ToArray()); } }