public IEnumerable <string> GenerateScript(IGraph <IArtifact, Dependency> graph, INode <IArtifact> node, Func <IArtifact, bool> artifactSelector)
{
if (graph == null)
{
throw new ArgumentNullException(nameof(graph));
}
if (node == null)
{
throw new ArgumentNullException(nameof(node));
}
if (artifactSelector == null)
{
throw new ArgumentNullException(nameof(artifactSelector));
}
var artifacts = _buildPathProvider.GetPath(graph, node).Select(i => i.Value).ToList();
var images = artifacts.OfType <Image>().ToList();
if (images.Any())
{
foreach (var reference in artifacts.OfType <Reference>())
{
if (artifactSelector(reference))
{
yield return($"docker pull {reference.RepoTag}");
}
}
// ReSharper disable once IdentifierTypo
var dockerignore = Path.Combine(_options.ContextPath, ".dockerignore").Replace("\\", "/");
var ignores = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase);
var isFirst = true;
foreach (var image in images)
{
if (artifactSelector(image))
{
if (ignores.Except(image.File.Ignores).Any())
{
yield return($"echo 2> {dockerignore}");
ignores.Clear();
isFirst = false;
}
foreach (var ignore in image.File.Ignores.Except(ignores))
{
var redirection = isFirst ? ">" : ">>";
isFirst = false;
yield return($"echo {ignore} {redirection} {dockerignore}");
ignores.Add(ignore);
}
var dockerFilePath = _pathService.Normalize(Path.Combine(_options.TargetPath, image.File.Path, "Dockerfile"));
yield return($"docker build -f \"{dockerFilePath}\" -t {image.File.ImageId}:{image.File.Tags.First()} \"{_options.ContextPath}\"");
}
}
}
}
private FileArtifact AddFile(string fileName, IEnumerable <string> lines)
{
var curLines = new List <string>
{
"package generated",
string.Empty,
"import jetbrains.buildServer.configs.kotlin.v2019_2.*",
"import jetbrains.buildServer.configs.kotlin.v2019_2.ui.*",
"import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script",
"import jetbrains.buildServer.configs.kotlin.v2019_2.vcs.GitVcsRoot",
"import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.dockerSupport",
"import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.freeDiskSpace",
"import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.swabra",
"import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.dockerCommand",
"import common.TeamCityDockerImagesRepo.TeamCityDockerImagesRepo",
string.Empty
};
// ReSharper disable once StringLiteralTypo
curLines.AddRange(lines);
return(new FileArtifact(_pathService.Normalize(Path.Combine(_options.TeamCityDslPath, NormalizeFileName(fileName) + ".kts")), curLines));
}
private void AddScriptNode(IGraph <IArtifact, Dependency> graph, INode <IArtifact> node, string newLine, string header, string extension)
{
var image = (Image)node.Value;
var lines = new List <string>();
var scriptFileName = $"{image.File.ImageId}-{image.File.Tags.First()}{extension}";
var root = string.Join("/", Enumerable.Repeat("..", _options.TargetPath.Split('\\', '/').Length));
if (!string.IsNullOrWhiteSpace(header))
{
lines.Add(header);
}
lines.Add($"cd {root}");
lines.AddRange(_scriptGenerator.GenerateScript(graph, node, _ => true));
var scriptFilePath = _pathService.Normalize(Path.Combine(_options.TargetPath, scriptFileName));
graph.TryAddNode(new FileArtifact(scriptFilePath, new[] { string.Join(newLine, lines) }), out var _);
}
public Task <Result> Run()
{
var templates = _configurationExplorer.Explore(_options.SourcePath, _options.ConfigurationFiles);
if (templates.State == Result.Error)
{
return(Task.FromResult(Result.Error));
}
var graph = _buildGraphFactory.Create(templates.Value);
if (graph.State == Result.Error)
{
return(Task.FromResult(Result.Error));
}
using (_logger.CreateBlock("Generate"))
{
foreach (var generator in _generators)
{
generator.Generate(graph.Value);
}
var dockerFiles = graph.Value.Nodes.Select(i => i.Value).OfType <GeneratedDockerfile>();
foreach (var dockerfile in dockerFiles)
{
var path = _pathService.Normalize(Path.Combine(_options.TargetPath, dockerfile.Path));
_logger.Log(path);
_fileSystem.WriteLines(path, dockerfile.Lines.Select(i => i.Text));
}
var artifacts = graph.Value.Nodes.Select(i => i.Value).OfType <FileArtifact>();
foreach (var artifact in artifacts)
{
_logger.Log(artifact.Path);
_fileSystem.WriteLines(artifact.Path, artifact.Lines);
}
}
return(Task.FromResult(graph.State));
}
public async Task <Result <Stream> > Create(string dockerFilesRootPath, IEnumerable <Dockerfile> dockerFiles)
{
if (dockerFilesRootPath == null)
{
throw new ArgumentNullException(nameof(dockerFilesRootPath));
}
if (dockerFiles == null)
{
throw new ArgumentNullException(nameof(dockerFiles));
}
using (_logger.CreateBlock("Context"))
{
var context = new MemoryStream();
using (var archive = new TarOutputStream(context)
{
IsStreamOwner = false
})
{
var number = 0;
if (!string.IsNullOrWhiteSpace(_options.ContextPath))
{
var path = Path.GetFullPath(_options.ContextPath);
if (!_fileSystem.IsDirectoryExist(path))
{
throw new InvalidOperationException($"The docker build context directory \"{path}\" does not exist.");
}
foreach (var file in _fileSystem.EnumerateFileSystemEntries(path))
{
if (!_fileSystem.IsFileExist(file))
{
continue;
}
number++;
using (var fileStream = _fileSystem.OpenRead(file))
{
var filePathInArchive = _pathService.Normalize(Path.GetRelativePath(path, file));
var result = await AddEntry(archive, filePathInArchive, fileStream);
_logger.Details($"{number:0000} \"{filePathInArchive}\" was added.");
if (result == Result.Error)
{
return(new Result <Stream>(new MemoryStream(), Result.Error));
}
}
}
_logger.Log($"{number} files were added to docker build context from the directory \"{_options.ContextPath}\" (\"{path}\").");
}
else
{
_logger.Log("The path for docker build context was not defined.", Result.Warning);
}
number = 0;
using (_logger.CreateBlock("Docker files"))
{
foreach (var dockerfile in dockerFiles)
{
number++;
var dockerFilePathInArchive = _pathService.Normalize(Path.Combine(dockerFilesRootPath, dockerfile.Path));
using (var dockerFileStream = new MemoryStream())
{
using (var writer = new StreamWriter(dockerFileStream, Encoding.UTF8, 0xff, true))
{
foreach (var line in dockerfile.Lines)
{
writer.WriteLine(line.Text);
}
}
dockerFileStream.Position = 0;
var result = await AddEntry(archive, dockerFilePathInArchive, dockerFileStream);
if (result == Result.Error)
{
return(new Result <Stream>(new MemoryStream(), Result.Error));
}
_logger.Log($"{number:0000} \"{dockerFilePathInArchive}\" was added ({dockerFileStream.Length} bytes).");
}
}
}
archive.Close();
return(new Result <Stream>(context));
}
}
}
public async Task <Result> Run()
{
var templates = _configurationExplorer.Explore(_options.SourcePath, _options.ConfigurationFiles);
if (templates.State == Result.Error)
{
return(Result.Error);
}
var buildGraphResult = _buildGraphFactory.Create(templates.Value);
if (buildGraphResult.State == Result.Error)
{
return(Result.Error);
}
var buildGraphsResult = _buildGraphsFactory.Create(buildGraphResult.Value);
if (buildGraphsResult.State == Result.Error)
{
return(Result.Error);
}
var buildGraphs = buildGraphsResult.Value.ToList();
if (!buildGraphs.Any())
{
_logger.Log("Nothing to build.", Result.Error);
return(Result.Error);
}
var dockerFilesRootPath = _fileSystem.UniqueName;
var contextStreamResult = await _contextFactory.Create(dockerFilesRootPath, buildGraphResult.Value.Nodes.Select(i => i.Value).OfType <Image>().Select(i => i.File));
if (contextStreamResult.State == Result.Error)
{
return(Result.Error);
}
var contextStream = contextStreamResult.Value;
using (contextStream)
using (_logger.CreateBlock("Build"))
{
var labels = new Dictionary <string, string>();
foreach (var buildGraph in buildGraphs)
{
var name = _graphNameFactory.Create(buildGraph).Value ?? "Unnamed graph";
if (!string.IsNullOrWhiteSpace(_options.FilterRegex) && !new Regex(_options.FilterRegex).IsMatch(name))
{
_logger.Log($"\"{name}\" was skipped according to filter \"{_options.FilterRegex}\".", Result.Warning);
continue;
}
using (_logger.CreateBlock(name))
{
var buildPath = _buildPathProvider.GetPath(buildGraph).ToList();
foreach (var buildNode in buildPath)
{
switch (buildNode.Value)
{
case Image image:
var dockerFile = image.File;
using (_logger.CreateBlock(dockerFile.ToString()))
{
contextStream.Position = 0;
var dockerFilePathInContext = _pathService.Normalize(Path.Combine(dockerFilesRootPath, dockerFile.Path));
var buildParameters = new ImageBuildParameters
{
Dockerfile = dockerFilePathInContext,
Tags = dockerFile.Tags.Distinct().ToList(),
Labels = labels
};
using (var buildEventStream = await _dockerClient.Images.BuildImageFromDockerfileAsync(
contextStream,
buildParameters,
_cancellationTokenSource.Token))
{
_streamService.ProcessLines(buildEventStream, line => { _messageLogger.Log(line); });
}
}
break;
}
}
}
}
}
return(Result.Success);
}
public void Generate([NotNull] IGraph <IArtifact, Dependency> graph)
{
if (graph == null)
{
throw new ArgumentNullException(nameof(graph));
}
var groups = (
from node in graph.Nodes
let image = node.Value as Image
where image != null
group node by image.File.ImageId
into groupsByImageId
from groupByImageId in
from groupByImageId in groupsByImageId
let image = groupByImageId.Value as Image
where image != null
orderby image.File
group groupByImageId by image.File
group groupByImageId by groupsByImageId.Key)
.ToList();
foreach (var groupByImageId in groups)
{
var imageId = groupByImageId.Key;
var lines = new List <string>();
graph.TryAddNode(new FileArtifact(GetReadmeFile(imageId), lines), out var readmeNode);
var groupByImage = groupByImageId.ToList();
lines.Add($"## {imageId} tags");
lines.Add(string.Empty);
var otherImages = groups.Where(i => i.Key != imageId).ToList();
if (otherImages.Any())
{
lines.Add("Other tags");
lines.Add(string.Empty);
foreach (var image in otherImages)
{
lines.Add($"- [{image.Key}]({GetReadmeFilePath(image.Key)})");
}
}
lines.Add(string.Empty);
// ReSharper disable once IdentifierTypo
var mutliArch = (
from grp in (
from image in groupByImage
where image.Key.Repositories.Any()
from tag in image.Key.Tags.Skip(1)
where tag.Length > 0 && char.IsLetterOrDigit(tag[0])
orderby tag descending
group image by tag)
orderby $"{grp.Count()} {grp.Key}" descending
select grp)
.ToList();
if (mutliArch.Any())
{
lines.Add("#### multi-architecture");
lines.Add(string.Empty);
lines.Add("When running an image with multi-architecture support, docker will automatically select an image variant which matches your OS and architecture.");
lines.Add(string.Empty);
foreach (var dockerfileByMultiArchTag in mutliArch)
{
lines.Add($"- [{dockerfileByMultiArchTag.Key}](#{Normalize(dockerfileByMultiArchTag.Key)})");
}
lines.Add(string.Empty);
}
var dockerfileGroups =
from image in groupByImage
orderby image.Key.Platform
group image by image.Key.Platform
into grp1
from grp2 in (
from image in grp1
orderby image.Key.Description descending
group image by image.Key.Description)
group grp2 by grp1.Key;
foreach (var dockerfileByPlatform in dockerfileGroups)
{
lines.Add($"#### {dockerfileByPlatform.Key}");
lines.Add(string.Empty);
foreach (var dockerfileByDescription in dockerfileByPlatform)
{
lines.Add($"- {dockerfileByDescription.Key}");
foreach (var dockerfile in dockerfileByDescription)
{
lines.Add($" - [{GetReadmeTagName(dockerfile.Key)}](#{GetTagLink(dockerfile.Key)})");
}
}
lines.Add(string.Empty);
}
lines.Add(string.Empty);
foreach (var dockerfileByMultiArchTag in mutliArch)
{
lines.Add($"### {dockerfileByMultiArchTag.Key}");
lines.Add(string.Empty);
var platforms = string.Join(", ", dockerfileByMultiArchTag.Select(i => $"{i.Key.Platform} {i.Key.Description}").Distinct().OrderBy(i => i));
lines.Add($"Supported platforms: {platforms}");
lines.Add(string.Empty);
lines.Add("#### Content");
lines.Add(string.Empty);
foreach (var dockerfile in dockerfileByMultiArchTag)
{
lines.Add($"- [{GetReadmeTagName(dockerfile.Key)}](#{GetTagLink(dockerfile.Key)})");
}
lines.Add(string.Empty);
}
lines.Add(string.Empty);
foreach (var groupByFile in groupByImage)
{
var dockerFile = groupByFile.Key;
lines.Add($"### {GetReadmeTagName(dockerFile)}");
lines.Add(string.Empty);
lines.Add($"[Dockerfile]({_pathService.Normalize(Path.Combine(dockerFile.Path, "Dockerfile"))})");
if (dockerFile.Comments.Any())
{
lines.Add(string.Empty);
foreach (var comment in dockerFile.Comments)
{
lines.Add(comment);
}
}
if (dockerFile.Repositories.Any(i => !string.IsNullOrWhiteSpace(i)))
{
lines.Add(string.Empty);
lines.Add("The docker image is available on:");
lines.Add(string.Empty);
foreach (var repo in dockerFile.Repositories.Where(i => !string.IsNullOrWhiteSpace(i)))
{
lines.Add($"- [{repo}{dockerFile.ImageId}]({repo}{dockerFile.ImageId})");
}
}
else
{
lines.Add("The docker image is not available and may be created manually.");
}
if (dockerFile.Components.Any())
{
lines.Add(string.Empty);
lines.Add("Installed components:");
lines.Add(string.Empty);
foreach (var component in dockerFile.Components)
{
lines.Add($"- {component}");
}
}
lines.Add(string.Empty);
lines.Add($"Container platform: {dockerFile.Platform}");
var publishRepo = dockerFile
.Repositories
.Select(i =>
{
try
{
return(new Uri(i));
}
catch
{
return(null);
}
})
.FirstOrDefault(i => i != null);
foreach (var node in groupByFile)
{
var artifacts = _buildPathProvider.GetPath(graph, node).Select(i => i.Value).ToList();
var images = artifacts.OfType <Image>().ToList();
var weight = 0;
if (images.Any())
{
lines.Add(string.Empty);
lines.Add("Docker build commands:");
lines.Add(string.Empty);
lines.Add("```");
foreach (var reference in artifacts.OfType <Reference>())
{
lines.Add(GeneratePullCommand(reference.RepoTag));
weight += reference.Weight.Value;
}
var dockerignore = Path.Combine(_options.ContextPath, ".dockerignore").Replace("\\", "/");
var ignores = new HashSet <string>(StringComparer.InvariantCultureIgnoreCase);
var isFirst = true;
foreach (var image in images)
{
if (ignores.Except(image.File.Ignores).Any())
{
lines.Add($"echo 2> {dockerignore}");
ignores.Clear();
isFirst = false;
}
foreach (var ignore in image.File.Ignores.Except(ignores))
{
var redirection = isFirst ? ">" : ">>";
isFirst = false;
lines.Add($"echo {ignore} {redirection} {dockerignore}");
ignores.Add(ignore);
}
lines.Add(GenerateBuildCommand(image));
weight += image.Weight.Value;
}
lines.Add("```");
}
if (weight > 0)
{
lines.Add(string.Empty);
lines.Add($"_The required free space to generate image(s) is about **{weight} GB**._");
}
lines.Add(string.Empty);
}
foreach (var node in groupByFile)
{
graph.TryAddLink(node, GenerateDependency, readmeNode, out _);
}
}
}
}
public Result <IGraph <IArtifact, Dependency> > Create(IEnumerable <Template> templates)
{
var graph = new Graph <IArtifact, Dependency>();
var nodeDict = new Dictionary <string, INode <IArtifact> >();
foreach (var template in templates)
{
foreach (var variant in template.Variants)
{
var lines = _contentParser.Parse(template.Lines, variant.Variables).ToImmutableList();
var imageId = "unknown";
var tags = new List <string>();
var platform = string.Empty;
var references = new List <Reference>();
var components = new List <string>();
var repositories = new List <string>();
var comments = new List <string>();
var dockerfileLines = new List <Line>();
var weight = 0;
foreach (var line in lines)
{
var isMetadata = false;
if (line.Type == LineType.Comment)
{
isMetadata =
TrySetByPrefix(line.Text, CommentPrefix, value => comments.Add(value.Trim())) ||
TrySetByPrefix(line.Text, IdPrefix, value => imageId = value) ||
TrySetByPrefix(line.Text, TagPrefix, value => tags.Add(value)) ||
TrySetByPrefix(line.Text, PlatformPrefix, value => platform = value) ||
TrySetByPrefix(line.Text, BasedOnPrefix, value =>
{
var match = ReferenceRegex.Match(value);
if (match.Success)
{
var weightValue = 0;
if (int.TryParse(match.Groups["weight"].Value, out var refWeightValue))
{
weightValue = refWeightValue;
}
references.Add(new Reference(match.Groups["reference"].Value, new Weight(weightValue)));
}
}) ||
TrySetByPrefix(line.Text, ComponentsPrefix, value => components.Add(value)) ||
TrySetByPrefix(line.Text, RepoPrefix, value => repositories.Add(value)) ||
TrySetByPrefix(line.Text, WeightPrefix, value =>
{
if (int.TryParse(value, out var weightValue))
{
weight = weightValue;
}
});
}
if (!isMetadata)
{
dockerfileLines.Add(line);
}
}
var dockerfile = new Dockerfile(_pathService.Normalize(variant.BuildPath), imageId, platform, tags, components, repositories, comments, references, new Weight(weight), dockerfileLines);
if (graph.TryAddNode(new Image(dockerfile), out var dockerImageNode))
{
foreach (var tag in tags)
{
nodeDict[$"{imageId}:{tag}"] = dockerImageNode;
}
if (graph.TryAddNode(new GeneratedDockerfile(_pathService.Normalize(Path.Combine(dockerfile.Path, "Dockerfile")), dockerfile.Lines), out var dockerfileNode))
{
graph.TryAddLink(dockerImageNode, GenerateDependency, dockerfileNode, out _);
}
}
}
}
var imageNodes =
from node in graph.Nodes
let image = node.Value as Image
where image != null
select new { node, image };
// Add references
foreach (var from in imageNodes.ToList())
{
foreach (var reference in from.image.File.References)
{
if (nodeDict.TryGetValue(reference.RepoTag, out var toNode))
{
graph.TryAddLink(from.node, new Dependency(DependencyType.Build), toNode, out _);
}
else
{
if (graph.TryAddNode(reference, out var referenceNode))
{
nodeDict[reference.RepoTag] = referenceNode;
}
graph.TryAddLink(from.node, new Dependency(DependencyType.Pull), referenceNode, out _);
}
}
}
return(new Result <IGraph <IArtifact, Dependency> >(graph));
}
public void Generate([NotNull] IGraph <IArtifact, Dependency> graph)
{
if (graph == null)
{
throw new ArgumentNullException(nameof(graph));
}
var groups =
from node in graph.Nodes
let image = node.Value as Image
where image != null
group node by image.File.ImageId
into groupsByImageId
from groupByImageId in
from groupByImageId in groupsByImageId
let image = groupByImageId.Value as Image
where image != null
let repoCount = image.File.Repositories.Count(i => !string.IsNullOrWhiteSpace(i))
orderby repoCount descending
group groupByImageId by image.File
group groupByImageId by groupsByImageId.Key;
foreach (var groupByImageId in groups)
{
var imageId = groupByImageId.Key;
var lines = new List <string>();
graph.TryAddNode(new FileArtifact(_pathService.Normalize(Path.Combine(_options.TargetPath, GetReadmeFilePath(imageId))), lines), out var readmeNode);
var groupByImage = groupByImageId.ToList();
lines.Add("### Tags");
lines.Add(string.Empty);
foreach (var groupByFile in groupByImage)
{
var dockerFile = groupByFile.Key;
lines.Add($"- [{GetReadmeTagName(dockerFile)}](#whale-{GetTagLink(dockerFile)})");
}
lines.Add(string.Empty);
foreach (var groupByFile in groupByImage)
{
var dockerFile = groupByFile.Key;
lines.Add($"### :whale: {GetReadmeTagName(dockerFile)}");
lines.Add(string.Empty);
lines.Add($"[Dockerfile]({_pathService.Normalize(Path.Combine(dockerFile.Path, "Dockerfile"))})");
if (dockerFile.Comments.Any())
{
lines.Add(string.Empty);
foreach (var comment in dockerFile.Comments)
{
lines.Add(comment);
}
}
if (dockerFile.Repositories.Any(i => !string.IsNullOrWhiteSpace(i)))
{
lines.Add(string.Empty);
lines.Add("The docker image is available on:");
lines.Add(string.Empty);
foreach (var repo in dockerFile.Repositories.Where(i => !string.IsNullOrWhiteSpace(i)))
{
lines.Add($"- [{repo}{dockerFile.ImageId}]({repo}{dockerFile.ImageId})");
}
}
else
{
lines.Add("The docker image is not available and may be created manually.");
}
if (dockerFile.Components.Any())
{
lines.Add(string.Empty);
lines.Add("Installed components:");
lines.Add(string.Empty);
foreach (var component in dockerFile.Components)
{
lines.Add($"- {component}");
}
}
lines.Add(string.Empty);
lines.Add($"Container Platform: {dockerFile.Platform}");
foreach (var node in groupByFile)
{
var artifacts = _buildPathProvider.GetPath(graph, node).Select(i => i.Value).ToList();
var images = artifacts.OfType <Image>().ToList();
var weight = 0;
if (images.Any())
{
lines.Add(string.Empty);
lines.Add("Build commands:");
lines.Add(string.Empty);
lines.Add("```");
foreach (var image in images)
{
lines.Add(GenerateCommand(image));
weight += image.Weight.Value;
}
lines.Add("```");
}
var references = artifacts.OfType <Reference>();
if (references.Any())
{
lines.Add(string.Empty);
lines.Add("Base images:");
lines.Add(string.Empty);
lines.Add("```");
foreach (var reference in artifacts.OfType <Reference>())
{
lines.Add(GeneratePullCommand(reference));
weight += reference.Weight.Value;
}
lines.Add("```");
}
if (weight > 0)
{
lines.Add(string.Empty);
lines.Add($"_The required free space to generate image(s) is about **{weight} GB**._");
}
}
foreach (var node in groupByFile)
{
graph.TryAddLink(node, GenerateDependency, readmeNode, out _);
}
}
}
}
public void Generate([NotNull] IGraph <IArtifact, Dependency> graph)
{
if (graph == null)
{
throw new ArgumentNullException(nameof(graph));
}
if (string.IsNullOrWhiteSpace(_options.TeamCityDslPath))
{
return;
}
if (string.IsNullOrWhiteSpace(_options.TeamCityBuildConfigurationId))
{
return;
}
// ReSharper disable once UseObjectOrCollectionInitializer
var lines = new List <string>();
lines.Add("import jetbrains.buildServer.configs.kotlin.v2019_2.*");
lines.Add("import jetbrains.buildServer.configs.kotlin.v2019_2.ui.*");
lines.Add("import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.script");
lines.Add("import jetbrains.buildServer.configs.kotlin.v2019_2.vcs.GitVcsRoot");
lines.Add("import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.dockerSupport");
lines.Add("import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.freeDiskSpace");
// ReSharper disable once StringLiteralTypo
lines.Add("import jetbrains.buildServer.configs.kotlin.v2019_2.buildFeatures.swabra");
lines.Add("import jetbrains.buildServer.configs.kotlin.v2019_2.buildSteps.dockerCommand");
lines.Add("version = \"2019.2\"");
lines.Add(string.Empty);
var buildGraphResult = _buildGraphsFactory.Create(graph);
if (buildGraphResult.State == Result.Error)
{
return;
}
var names = new Dictionary <string, int>();
var buildGraphs = (
from buildGraph in buildGraphResult.Value
let hasRepoToPush = buildGraph.Nodes.Select(i => i.Value).OfType <Image>().Any(i => i.File.Repositories.Any())
where hasRepoToPush
let weight = buildGraph.Nodes.Select(i => i.Value.Weight.Value).Sum()
let nodesDescription = _nodesDescriptionsFactory.Create(buildGraph.Nodes)
orderby nodesDescription.State != Result.Error ? nodesDescription.Value.Name : string.Empty
select new { graph = buildGraph, weight })
.ToList();
var localBuildTypes = new List <string>();
var hubBuildTypes = new List <string>();
var allImages = buildGraphs
.SelectMany(i => i.graph.Nodes.Select(j => j.Value).OfType <Image>())
.ToList();
// Build and push on local registry
var buildAndPushLocalBuildTypes = new List <string>();
foreach (var buildGraph in buildGraphs)
{
var path = _buildPathProvider.GetPath(buildGraph.graph).ToList();
var description = _nodesDescriptionsFactory.Create(path);
var name = description.State != Result.Error
? description.Value.Name
: string.Join("", path.Select(i => i.Value).OfType <Image>().Select(i => i.File.Platform).Distinct().OrderBy(i => i));
if (names.TryGetValue(name, out var counter))
{
name = $"{name} {++counter}";
}
else
{
names[name] = 1;
}
var buildTypeId = $"push_local_{NormalizeName(name)}";
buildAndPushLocalBuildTypes.Add(buildTypeId);
lines.AddRange(GenerateBuildAndPushType(buildTypeId, name, path, buildGraph.weight));
}
localBuildTypes.AddRange(buildAndPushLocalBuildTypes);
// Publish on local registry
var localPublishGroups =
from image in allImages
from tag in image.File.Tags.Skip(1)
group image by tag;
var publishLocalId = "publish_local";
localBuildTypes.Add(publishLocalId);
lines.AddRange(CreateManifestBuildConfiguration(publishLocalId, BuildRepositoryName, "Publish", localPublishGroups, true, buildAndPushLocalBuildTypes.ToArray()));
// Push on docker hub
var pushOnHubBuildTypes = new List <string>();
var platforms = allImages.Select(i => i.File.Platform).Distinct();
foreach (var platform in platforms)
{
var buildTypeId = $"push_hub_{NormalizeName(platform)}";
pushOnHubBuildTypes.Add(buildTypeId);
lines.AddRange(CreatePushBuildConfiguration(buildTypeId, platform, allImages, publishLocalId));
}
hubBuildTypes.AddRange(pushOnHubBuildTypes);
// Publish on docker hub
var publishOnHubBuildTypes = new List <string>();
var publishOnHubGroups =
from grp in
from image in allImages
from tag in image.File.Tags.Skip(1)
group image by tag
group grp by grp.Key.ToLowerInvariant() == "latest" ? "latest" : "version";
foreach (var group in publishOnHubGroups)
{
var buildTypeId = $"publish_hub_{NormalizeName(group.Key)}";
publishOnHubBuildTypes.Add(buildTypeId);
lines.AddRange(CreateManifestBuildConfiguration(buildTypeId, DeployRepositoryName, $"Publish as {group.Key}", group, false, pushOnHubBuildTypes.ToArray()));
}
hubBuildTypes.AddRange(publishOnHubBuildTypes);
// Local project
lines.Add("object LocalProject : Project({");
lines.Add("name = \"Local registry\"");
foreach (var buildType in localBuildTypes.Distinct())
{
lines.Add($"buildType({buildType})");
}
lines.Add("})");
// Hub project
lines.Add("object HubProject : Project({");
lines.Add("name = \"Docker hub\"");
foreach (var buildType in hubBuildTypes.Distinct())
{
lines.Add($"buildType({buildType})");
}
lines.Add("})");
// root project
lines.Add("project {");
lines.Add("vcsRoot(RemoteTeamcityImages)");
lines.Add("subProject(LocalProject)");
lines.Add("subProject(HubProject)");
lines.Add("params {");
lines.Add($"param(\"{BuildNumberParam}\", \"%dep.{_options.TeamCityBuildConfigurationId}.build.number%\")");
lines.Add("}");
lines.Add("}");
lines.Add(string.Empty);
// vcs
lines.Add("object RemoteTeamcityImages : GitVcsRoot({");
lines.Add("name = \"remote teamcity images\"");
lines.Add("url = \"https://github.com/JetBrains/teamcity-docker-images.git\"");
lines.Add("})");
// vcs
graph.TryAddNode(new FileArtifact(_pathService.Normalize(Path.Combine(_options.TeamCityDslPath, "settings.kts")), lines), out _);
}
public async Task <Result> Run()
{
var templates = _configurationExplorer.Explore(_options.SourcePath, _options.ConfigurationFiles);
if (templates.State == Result.Error)
{
return(Result.Error);
}
var buildGraphResult = _buildGraphFactory.Create(templates.Value);
if (buildGraphResult.State == Result.Error)
{
return(Result.Error);
}
var buildGraphsResult = _buildGraphsFactory.Create(buildGraphResult.Value);
if (buildGraphsResult.State == Result.Error)
{
return(Result.Error);
}
var buildGraphs = buildGraphsResult.Value.ToList();
if (!buildGraphs.Any())
{
_logger.Log("Nothing to build.", Result.Error);
return(Result.Error);
}
var dockerFilesRootPath = _fileSystem.UniqueName;
var contextStreamResult = await _contextFactory.Create(dockerFilesRootPath, buildGraphResult.Value.Nodes.Select(i => i.Value).OfType <Image>().Select(i => i.File));
if (contextStreamResult.State == Result.Error)
{
return(Result.Error);
}
var contextStream = contextStreamResult.Value;
using (contextStream)
using (_logger.CreateBlock("Build"))
{
foreach (var buildGraph in buildGraphs)
{
var nameResult = _nodesDescriptionFactory.Create(buildGraph.Nodes);
var name = nameResult.State != Result.Error ? nameResult.Value.Name : "Unnamed graph";
if (!string.IsNullOrWhiteSpace(_options.FilterRegex) && !new Regex(_options.FilterRegex).IsMatch(name))
{
_logger.Log($"\"{name}\" was skipped according to filter \"{_options.FilterRegex}\".", Result.Warning);
continue;
}
using (_logger.CreateBlock(name))
{
var buildPath = _buildPathProvider.GetPath(buildGraph).ToList();
foreach (var buildNode in buildPath)
{
switch (buildNode.Value)
{
case Image image:
var dockerFile = image.File;
using (_logger.CreateBlock(dockerFile.ToString()))
{
var id = Guid.NewGuid().ToString();
var labels = new Dictionary <string, string> {
{ "InternalImageId", id }
};
var tags = (
from tag in dockerFile.Tags
select $"{dockerFile.ImageId}:{tag}")
.Distinct()
.ToList();
contextStream.Position = 0;
var dockerFilePathInContext = _pathService.Normalize(Path.Combine(dockerFilesRootPath, dockerFile.Path));
var buildParameters = new ImageBuildParameters
{
Dockerfile = dockerFilePathInContext,
Tags = tags,
PullParent = true,
Labels = labels
};
using (var buildEventStream = await _dockerClient.Images.BuildImageFromDockerfileAsync(
contextStream,
buildParameters,
_cancellationTokenSource.Token))
{
_streamService.ProcessLines(buildEventStream, line => { _messageLogger.Log(line); });
}
var filter = new Dictionary <string, IDictionary <string, bool> >
{
{ "label", labels.ToDictionary(i => $"{i.Key}={i.Value}", _ => true) }
};
var images = await _dockerClient.Images.ListImagesAsync(new ImagesListParameters { Filters = filter });
if (images.Count == 0)
{
_logger.Log($"Error while building the image {dockerFile}", Result.Error);
return(Result.Error);
}
}
break;
}
}
}
}
}
return(Result.Success);
}