private static void PatchDeps(string currentModuleFullPath, List <string> modulesToInsert, List <string> cementConfigs)
{
foreach (var config in cementConfigs)
{
foreach (var module in modulesToInsert)
{
var dep = new Dep(module);
var workspace = Directory.GetParent(currentModuleFullPath).FullName;
var moduleName = Path.GetFileName(currentModuleFullPath);
var kvp = new KeyValuePair <string, string>(Path.Combine(workspace, moduleName, config), module);
if (patchedDeps.Contains(kvp))
{
continue;
}
patchedDeps.Add(kvp);
try
{
new DepsPatcher(workspace, moduleName, dep).Patch(config);
}
catch (Exception exception)
{
ConsoleWriter.WriteError($"Fail adding {dep} to {moduleName}/{config} deps:\n\t{exception.Message}");
Log.LogError($"Fail adding {dep} to deps: {exception.Message}");
}
}
}
}
private static DepsContent GetCurrentModuleDeps(Dep dep)
{
Log.Info($"{"[" + dep.Name + "]",-30}Getting deps for configuration {dep.Configuration ?? "full-build"}");
var deps = new DepsParser(Path.Combine(Helper.CurrentWorkspace, dep.Name)).Get(dep.Configuration);
return(deps);
}
private void AddDepLine(string configuration, Dep dep, bool isOn)
{
var path = Path.Combine(workspace, patchModule, Helper.YamlSpecFile);
var moduleYamlFile = new ModuleYamlFile(new FileInfo(path));
var lines = moduleYamlFile.Lines;
var configIndex = lines.FindIndex(l => l.StartsWith(configuration + ":") ||
l.StartsWith(configuration + " "));
var depsIndex = -1;
for (int index = configIndex + 1; index < lines.Count && (lines[index].Length == 0 || lines[index].StartsWith(" ")); index++)
{
if (lines[index].EndsWith("deps:"))
{
depsIndex = index;
break;
}
}
if (depsIndex == -1)
{
lines.Insert(configIndex + 1, GetSpacesStart(lines, configIndex) + "deps:");
depsIndex = configIndex + 1;
}
var prefix = GetSpacesStart(lines, depsIndex) + (isOn ? "- " : "- -");
lines.Insert(depsIndex + 1, prefix + GetDepLine(dep));
moduleYamlFile.Save(path, lines);
}
private void RemoveDepLine(string configuration, Dep dep, bool isOn)
{
var path = Path.Combine(workspace, patchModule, Helper.YamlSpecFile);
var moduleYamlFile = new ModuleYamlFile(new FileInfo(path));
var lines = moduleYamlFile.Lines;
var configIndex = lines.FindIndex(l => l.StartsWith(configuration + ":") ||
l.StartsWith(configuration + " "));
var depsIndex = -1;
for (int index = configIndex + 1; index < lines.Count && lines[index].StartsWith(" "); index++)
{
if (lines[index].EndsWith("deps:"))
{
depsIndex = index;
break;
}
}
if (depsIndex == -1)
{
return;
}
for (int index = depsIndex + 1; index < lines.Count && lines[index].StartsWith(GetSpacesStart(lines, depsIndex)); index++)
{
var str = (isOn ? " - " : " - -") + dep.Name;
if (lines[index].Contains(str + "/") || lines[index].Contains(str + "@") || lines[index].EndsWith(str))
{
lines.RemoveAt(index);
break;
}
}
moduleYamlFile.Save(path, lines);
}
private Dep FindLca(Dep dep1, Dep dep2)
{
if (dep1.Name != dep2.Name)
{
throw new BadArgumentException();
}
var result = new Dep(dep1.Name, dep1.Treeish ?? dep2.Treeish);
var configParser = new ConfigurationYamlParser(new FileInfo(Path.Combine(workspace, dep1.Name)));
var configs = configParser.GetConfigurations();
var commonAncestorsList = new List <string>();
foreach (var parent in configs)
{
var cm = new ConfigurationManager(new[] { new Dep(null, null, parent) }, configParser);
if (cm.ProcessedParent(dep1) && cm.ProcessedParent(dep2))
{
commonAncestorsList.Add(parent);
}
}
var lowestAncestor = commonAncestorsList.Where(c =>
!new ConfigurationManager(commonAncestorsList.Select(cc => new Dep(null, null, cc)), configParser).ProcessedChildrenConfigurations(new Dep(null, null, c)).Any()).ToList();
if (!commonAncestorsList.Any() || !lowestAncestor.Any())
{
throw new CementException("failed get common ancestor for configurations '" + dep1 + "' and '" + dep2 + "'");
}
result.Configuration = lowestAncestor.First();
return(result);
}
private void FixChild(Dep currentParrentDep, string childConfiguration)
{
RemoveDepLine(childConfiguration, currentParrentDep, false);
var parser = new DepsYamlParser(new FileInfo(Directory.GetParent(yamlPath).FullName));
var hadInSameSection = parser.GetDepsFromConfig(childConfiguration).Deps.Where(d => d.Name == patchDep.Name).ToList();
if (!hadInSameSection.Any())
{
return;
}
if (hadInSameSection.Count > 1)
{
ThrowDuplicate();
}
var had = hadInSameSection.First();
RemoveDepLine(childConfiguration, had, true);
if (GetDepLine(had) == GetDepLine(currentParrentDep))
{
return;
}
if (FindLca(had, currentParrentDep).Configuration == currentParrentDep.Configuration)
{
return;
}
AddDepLine(childConfiguration, had, true);
AddDepLine(childConfiguration, currentParrentDep, false);
}
public static string GetBuildScriptName(Dep dep)
{
var configToBuild = dep.Configuration == "full-build" || dep.Configuration == null ? "" : "." + dep.Configuration;
return(Path.Combine(Helper.CurrentWorkspace, dep.Name,
"build" + configToBuild + ".cmd"));
}
public bool TreeishAlreadyProcessed(Dep dep, IList <Dep> processed)
{
return
(processed.Select(d => d.Treeish)
.Any(
dtreeish =>
(dtreeish == null && dep.Treeish == null) ||
(dtreeish != null && (dtreeish.Equals(dep.Treeish) || dep.Treeish == null))));
}
private bool HaveToForce(Dep dep, string force, GitRepository repo)
{
if (!localBranchForce && repo.HasLocalBranch(force) && !repo.HasRemoteBranch(force))
{
ConsoleWriter.WriteWarning(
$"Module '{repo.ModuleName}' has local-only branch '{force}' which will not be forced.\nUse --allow-local-branch-force key to force it");
return(false);
}
return(dep.Treeish == null && force != null && repo.HasRemoteBranch(force));
}
public bool Build(Dep dep)
{
log.Debug($"{dep.ToBuildString()}");
if (BuildSingleModule(dep))
{
return(true);
}
log.Debug($"{dep.ToBuildString(),-40} *build failed");
return(false);
}
private bool IsModuleUpdate(Dictionary <string, string> currentCommitHashes, Dep module)
{
return
(!modulesWithDeps.ContainsKey(module) ||
modulesWithDeps[module].Any(
dep => !currentCommitHashes.ContainsKey(dep.Dep.Name) ||
currentCommitHashes[dep.Dep.Name] != dep.CommitHash ||
!buildHelper.HasAllOutput(dep.Dep.Name, dep.Dep.Configuration, false)) ||
!buildHelper.HasAllOutput(module.Name, module.Configuration, false));
}
public static List <Dep> GetTopologicallySortedGraph(Dictionary <Dep, List <Dep> > graph, string root, string config, bool printCycle = true)
{
var visitedConfigurations = new HashSet <Dep>();
var processingConfigs = new List <Dep>();
var result = new List <Dep>();
var rootDep = new Dep(root, null, config);
TopSort(rootDep, graph, visitedConfigurations, processingConfigs, result, printCycle);
return(result);
}
private string GetDepLine(Dep dep)
{
var parser = new ConfigurationYamlParser(new FileInfo(Path.Combine(workspace, dep.Name)));
if (parser.GetDefaultConfigurationName() == dep.Configuration)
{
dep.Configuration = null;
}
return(dep.ToYamlString());
}
public static Dictionary <string, object> ParseGet(string[] args)
{
var parsedArguments = new Dictionary <string, object>
{
{ "module", null },
{ "treeish", null },
{ "reset", 0 },
{ "force", 0 },
{ "pullAnyway", 0 },
{ "configuration", null },
{ "merged", null },
{ "verbose", false },
{ "gitDepth", null }
};
var parser = new OptionSet
{
{ "r|reset", r => parsedArguments["reset"] = 1 },
{ "p|pull-anyway", p => parsedArguments["pullAnyway"] = 1 },
{ "c|configuration=", conf => parsedArguments["configuration"] = conf },
{ "f|force", f => parsedArguments["force"] = 1 },
{ "m|merged:", m => parsedArguments["merged"] = m ?? "master" },
{ "v|verbose", v => parsedArguments["verbose"] = true },
{ "git-depth=", d => parsedArguments["gitDepth"] = int.Parse(d) }
};
var extraArgs = parser.Parse(args.Skip(1));
if (extraArgs.Count > 0)
{
var module = new Dep(extraArgs[0]);
if (module.Configuration != null)
{
parsedArguments["configuration"] = module.Configuration;
}
if (module.Treeish != null)
{
parsedArguments["treeish"] = module.Treeish;
}
parsedArguments["module"] = module.Name;
if (extraArgs.Count > 1)
{
parsedArguments["treeish"] = extraArgs[1];
}
ThrowIfHasExtraArgs(extraArgs.Skip(2).ToList());
}
if ((int)parsedArguments["force"] + (int)parsedArguments["reset"] + (int)parsedArguments["pullAnyway"] > 1)
{
throw new BadArgumentException();
}
return(parsedArguments);
}
public ModuleGetter(List <Module> modules, Dep rootModule,
LocalChangesPolicy userLocalChangesPolicy,
string mergedBranch, bool verbose = false, bool localBranchForce = false)
{
this.modules = modules;
this.rootModule = rootModule;
this.userLocalChangesPolicy = userLocalChangesPolicy;
this.localBranchForce = localBranchForce;
this.verbose = verbose;
this.mergedBranch = mergedBranch;
}
private static void CloneInEmptyFolder(Dep dep, Module module, GitRepository repo)
{
if (GitRepository.HasRemoteBranch(module.Url, dep.Treeish))
{
repo.Clone(module.Url, dep.Treeish);
}
else
{
repo.Clone(module.Url);
}
}
private void ReplaceDepLine(Dep was, Dep shouldBe, int removeIndex, List <string> lines, int depsIndex)
{
if (removeIndex == -1)
{
ConsoleWriter.WriteWarning("Fail to replace " + was + ", not found in module.yaml.");
return;
}
var suffix = lines[removeIndex].EndsWith(":") ? ":" : "";
lines[removeIndex] = GetSpacesStart(lines, depsIndex) + "- " + GetDepLine(shouldBe) + suffix;
}
public DepsPatcher(string workspace, string patchModule, Dep patchDep)
{
this.workspace = workspace;
this.patchModule = patchModule;
this.patchDep = patchDep;
yamlPath = Path.Combine(workspace, patchModule, Helper.YamlSpecFile);
if (!File.Exists(yamlPath))
{
throw new CementException("module.yaml not found in " + yamlPath);
}
ModuleYamlFile.ReplaceTabs(yamlPath);
}
private static string MakeScript(Dep dep, BuildData buildSection)
{
switch (buildSection.Tool.Name)
{
case "msbuild":
case "dotnet":
return(BuildMsbuildScript(buildSection, dep.Name));
default:
return(BuildShellScript(buildSection));
}
}
public ModuleGetter(List <Module> modules, Dep rootModule,
LocalChangesPolicy userLocalChangesPolicy,
string mergedBranch, bool verbose = false, bool localBranchForce = false, int?gitDepth = null)
{
this.modules = modules;
this.rootModule = rootModule;
this.userLocalChangesPolicy = userLocalChangesPolicy;
this.localBranchForce = localBranchForce;
this.gitDepth = gitDepth;
this.verbose = verbose;
this.mergedBranch = mergedBranch;
cycleDetector = new CycleDetector();
}
private bool BuildSingleModule(Dep dep)
{
var moduleYaml = Path.Combine(Helper.CurrentWorkspace, dep.Name, Helper.YamlSpecFile);
var cmdFile = Path.Combine(Helper.CurrentWorkspace, ModuleBuilderHelper.GetBuildScriptName(dep));
if (!Build(dep, moduleYaml, cmdFile))
{
return(false);
}
CheckHasInstall(dep);
return(true);
}
private void PrintBuildResult(Dep dep, string buildName, int warnCount, string elapsedTime, List <string> obsoleteUsages)
{
ConsoleWriter.WriteOk(
$"{dep.ToBuildString() + " " + buildName,-40}{(warnCount == 0 || buildSettings.ShowWarningsSummary ? "" : "warnings: " + warnCount),-15}{elapsedTime,10}");
var obsoleteCount = obsoleteUsages.Count;
if (buildSettings.ShowWarningsSummary && warnCount > 0)
{
ConsoleWriter.WriteBuildWarning(
$" warnings: {warnCount}{(obsoleteCount == 0 ? "" : ", obsolete usages: " + obsoleteCount)} (Use -w key to print warnings or -W to print obsolete usages. You can also use ReSharper to find them.)");
}
}
private string ToJsonValueString(Dep dep)
{
var str = dep.Name;
if (dep.Configuration != null)
{
str += "/" + EscapeBadChars(dep.Configuration);
}
if (dep.Treeish != null)
{
str += "@" + EscapeBadChars(dep.Treeish);
}
return(str);
}
public static void PatchDepsForSolution(string currentModuleFullPath, Dep dep, string solutionFile)
{
var installData = InstallParser.Get(dep.Name, dep.Configuration);
Log.Info("Adding deps to module.yaml");
Log.Info("Getting cement configurations insert to");
var usedConfigs = GetUsedCementConfigsForSolution(currentModuleFullPath, solutionFile);
var toPatch = GetSmallerCementConfigs(currentModuleFullPath, usedConfigs);
PatchDeps(
currentModuleFullPath,
installData.ExternalModules.Concat(new[] { dep.ToYamlString() }).ToList(),
toPatch);
}
private InstallData ProceedExternalModule(string moduleNameWithConfiguration, HashSet <string> proceededModules)
{
var dep = new Dep(moduleNameWithConfiguration);
var externalModulePath = Path.Combine(path, "..", dep.Name);
var externalInstallData = new InstallYamlParser(new FileInfo(externalModulePath)).Get(dep.Configuration);
return(new InstallData(
externalInstallData.BuildFiles
.Select(f => Path.Combine(dep.Name, f))
.ToList(),
externalInstallData.ExternalModules
.Where(m => !proceededModules.Contains(m))
.ToList()));
}
public List <string> ChildrenConfigurations(Dep dep)
{
configHierarchy = ReverseHierarchy(parser.GetConfigurationsHierarchy());
var config = dep.Configuration ?? parser.GetDefaultConfigurationName();
if (config == null || !configHierarchy.ContainsKey(config))
{
return(new List <string>());
}
var children = new HashSet <string>();
GetChildrenDfs(dep.Configuration, children);
return(children.ToList());
}
private void PrintProcessedModuleInfo(Dep dep, GetInfo getInfo, string treeish)
{
var longestModuleName = modules.Select(m => m.Name.Length).Max() + 5;
dep.UpdateConfigurationIfNull();
var name = dep.Name + (dep.Configuration == null || dep.Configuration.Equals("full-build") ? "" : Helper.ConfigurationDelimiter + dep.Configuration);
var outputTreeish = getInfo.Forced && !treeish.Equals("master") ? treeish + " *forced" : dep.Treeish ?? "master";
if (getInfo.HookUpdated)
{
outputTreeish += " (hook updated) ";
}
if (getInfo.Pulled)
{
outputTreeish += " *pulled";
ConsoleWriter.Shared.WriteUpdate(GetPrintString(longestModuleName, name, outputTreeish, getInfo.CommitInfo));
Log.LogDebug($"{"[" + dep.Name + "]",-30}{outputTreeish,-18} {getInfo.CommitInfo}");
return;
}
if (getInfo.ForcedLocal)
{
outputTreeish += " *forced local";
ConsoleWriter.Shared.WriteWarning(GetPrintString(longestModuleName, name, outputTreeish, getInfo.CommitInfo));
Log.LogDebug($"{"[" + dep.Name + "]",-30}{outputTreeish,-18} {getInfo.CommitInfo}");
return;
}
if (getInfo.Reset)
{
outputTreeish += " *reset";
ConsoleWriter.Shared.WriteWarning(GetPrintString(longestModuleName, name, outputTreeish, getInfo.CommitInfo));
Log.LogDebug($"{"[" + dep.Name + "]",-30}{outputTreeish,-18} {getInfo.CommitInfo}");
return;
}
if (getInfo.Changed || getInfo.Cloned)
{
Log.LogDebug($"{"[" + dep.Name + "]",-30}{outputTreeish + (getInfo.Cloned ? " *cloned" : " *changed"),-18} {getInfo.CommitInfo}");
ConsoleWriter.Shared.WriteUpdate(GetPrintString(longestModuleName, name, outputTreeish, getInfo.CommitInfo));
}
else
{
Log.LogDebug($"{"[" + dep.Name + "]",-30}{outputTreeish + " *skipped",-18} {getInfo.CommitInfo}");
ConsoleWriter.Shared.WriteSkip(GetPrintString(longestModuleName, name, outputTreeish, getInfo.CommitInfo));
}
}
private static void PrintBuildFailResult(Dep dep, string buildName, ShellRunner runner)
{
ConsoleWriter.WriteBuildError(
$"Failed to build {dep.Name}{(dep.Configuration == null ? "" : "/" + dep.Configuration)} {buildName}");
foreach (var line in runner.Output.Split('\n'))
{
ModuleBuilderHelper.WriteLine(line);
}
ConsoleWriter.WriteLine();
ConsoleWriter.WriteInfo("Errors summary:");
foreach (var line in runner.Output.Split('\n'))
{
ModuleBuilderHelper.WriteIfErrorToStandartStream(line);
}
}
private void CloneInEmptyFolder(Dep dep, Module module, GitRepository repo)
{
if (GitRepository.HasRemoteBranch(module.Url, dep.Treeish))
{
repo.Clone(module.Url, dep.Treeish, gitDepth);
}
else
{
repo.Clone(module.Url, depth: gitDepth);
}
if (module.Pushurl != null)
{
repo.SetPushUrl(module.Pushurl);
}
}
public void AddBuiltModule(Dep module, Dictionary <string, string> currentCommitHashes)
{
var configs = new ConfigurationParser(new FileInfo(Path.Combine(Helper.CurrentWorkspace, module.Name))).GetConfigurations();
var childConfigs = new ConfigurationManager(module.Name, configs).ProcessedChildrenConfigurations(module);
childConfigs.Add(module.Configuration);
foreach (var childConfig in childConfigs)
{
var deps = BuildPreparer.Shared.BuildConfigsGraph(module.Name, childConfig).Keys.ToList();
var depsWithCommit = deps
.Where(dep => currentCommitHashes.ContainsKey(dep.Name) && currentCommitHashes[dep.Name] != null)
.Select(dep => new DepWithCommitHash(dep, currentCommitHashes[dep.Name]))
.ToList();
modulesWithDeps[new Dep(module.Name, null, childConfig)] = depsWithCommit;
}
}