private static string DescribeProcess(Process p, BuildGraph graph)
{
StringBuilder desc = new StringBuilder();
desc.AppendLine("Consumes:");
foreach (var file in p.Consumes)
{
File f;
if (graph.Files.TryGetValue(file, out f))
{
desc.Append(" ");
desc.AppendLine(f.Location);
}
}
desc.AppendLine("Produces:");
foreach (var file in p.Produces)
{
File f;
if (graph.Files.TryGetValue(file, out f))
{
desc.Append(" ");
desc.AppendLine(f.Location);
}
}
return(desc.ToString());
}
public BuildWriter(string outputDirectory, bool writeInputFiles, double inputScaleFactor, BuildGraph buildGraph, bool ignoreResponseFiles, Language language)
{
m_outputDirectory = outputDirectory;
m_writeInputFiles = writeInputFiles;
m_inputScaleFactor = inputScaleFactor;
m_buildGraph = buildGraph;
m_specFileToPipsLookup = buildGraph.Pips.Values.ToLookup(pip => pip.Spec, StringComparer.Ordinal);
m_ignoreResponseFiles = ignoreResponseFiles;
m_language = language;
}
private static ProjectStats GetStats(IEnumerable <Pip> pips, BuildGraph graph)
{
ProjectStats stats = default(ProjectStats);
HashSet <string> inputFiles = new HashSet <string>();
HashSet <string> outputFiles = new HashSet <string>();
foreach (Pip pip in pips)
{
Process process = pip as Process;
if (process != null)
{
stats.ProcessCount++;
foreach (int consumes in process.Consumes)
{
File f;
if (graph.Files.TryGetValue(consumes, out f))
{
inputFiles.Add(f.Location);
}
}
foreach (int produces in process.Produces)
{
File f;
if (graph.Files.TryGetValue(produces, out f))
{
outputFiles.Add(f.Location);
}
}
continue;
}
CopyFile copyFile = pip as CopyFile;
if (copyFile != null)
{
stats.CopyFileCount++;
continue;
}
WriteFile writeFile = pip as WriteFile;
if (writeFile != null)
{
stats.WriteFileCount++;
continue;
}
}
stats.CppFileCount = inputFiles.Where(f => f.EndsWith(".cpp", StringComparison.OrdinalIgnoreCase)).Count();
stats.CsFileCount = inputFiles.Where(f => f.EndsWith(".cs", StringComparison.OrdinalIgnoreCase)).Count();
stats.OutputFileCount = outputFiles.Count;
return(stats);
}
public static void Main(string[] arguments)
{
string debug = Environment.GetEnvironmentVariable("MimicGeneratorDebugOnStart");
if (debug != null && debug != "0")
{
Debugger.Launch();
}
Stopwatch sw = Stopwatch.StartNew();
Args args = new Args(arguments);
if (args.HelpDisplayed)
{
return;
}
try
{
GraphReader reader = new GraphReader(args.JsonGraph, args.ObservedInputs);
BuildGraph graph = reader.ReadGraph();
if (args.DuplicateGraph > 1)
{
GraphMultiplier.DuplicateAsParallelGraphs(graph, args.DuplicateGraph, args.MaxPipsPerSpec);
}
BuildWriter writer = new BuildWriter(args.Dest, args.WriteInputs, args.InputScaleFactor, graph, args.IgnoreResponseFiles, args.Language);
if (!writer.WriteBuildFiles())
{
ExitError(sw);
}
Console.WriteLine("MimicGenerator completed successfully in {0} seconds.", sw.Elapsed.TotalSeconds);
Environment.Exit(0);
}
catch (Exception ex)
{
ExitError(sw, ex);
}
}
private void Duplicate(int multiplicationFactor, int maxPipsPerSpec)
{
Console.WriteLine("Duplicating Graph {0} times", multiplicationFactor);
// Take a snapshot of the pips currently in the graph before any multiplication is applied.
Process[] originalProcesses = m_graph.Pips.Values.OfType <Process>().ToArray();
WriteFile[] originalWriteFiles = m_graph.Pips.Values.OfType <WriteFile>().ToArray();
// The basic strategy here is to clone every pip in a parallelizeable way. Each pip gets n copies with same
// source inputs, but different outputs. Dependencies within the original set are translated to the new set
for (int i = 0; i < multiplicationFactor; i++)
{
Console.WriteLine("Duplicating Graph iteration {0}", i);
string newRoot = i.ToString(CultureInfo.InvariantCulture);
for (int j = 0; j < originalProcesses.Length; j++)
{
Process p = originalProcesses[j];
List <int> clonedConsumes = new List <int>(p.Consumes.Count);
List <int> clonedProduces = new List <int>(p.Produces.Count);
foreach (int consume in p.Consumes)
{
int producingPip;
if (m_graph.OutputArtifactToProducingPip.TryGetValue(consume, out producingPip))
{
Process process = m_graph.Pips[producingPip] as Process;
WriteFile writeFile = m_graph.Pips[producingPip] as WriteFile;
if (process != null || writeFile != null)
{
// This is an output file created by a pip that will also be cloned. We need to translate it to the new path
File f;
if (!m_graph.Files.TryGetValue(consume, out f))
{
throw new MimicGeneratorException("Failed to find referenced file with id: {0}", consume);
}
clonedConsumes.Add(m_graph.DuplicateFile(f, newRoot));
continue;
}
}
// If the path isn't translated based on cloning, just consume it directly.
clonedConsumes.Add(consume);
}
foreach (int produce in p.Produces)
{
File f;
if (!m_graph.Files.TryGetValue(produce, out f))
{
throw new MimicGeneratorException("Failed to find referenced file with id: {0}", produce);
}
clonedProduces.Add(m_graph.DuplicateFile(f, newRoot));
}
Process cloned = new Process(0, string.Empty, BuildGraph.DuplicatePath(GetSpecName(p.Spec, j, maxPipsPerSpec), newRoot), clonedProduces, clonedConsumes, p.Semaphores);
int newPipId = m_graph.AddWithNewPipId(cloned, p.PipId);
// Need to register all of the outputs of the cloned pip so consumers reference it as an output
// rather than a source file when specs are generated
foreach (var file in clonedProduces)
{
m_graph.OutputArtifactToProducingPip.TryAdd(file, newPipId);
}
}
for (int j = 0; j < originalWriteFiles.Length; j++)
{
WriteFile wf = originalWriteFiles[j];
File f;
if (!m_graph.Files.TryGetValue(wf.Destination, out f))
{
throw new MimicGeneratorException("Failed to find referenced file with id: {0}", wf.Destination);
}
int clonedDestination = m_graph.DuplicateFile(f, newRoot);
WriteFile cloned = new WriteFile(0, string.Empty, BuildGraph.DuplicatePath(GetSpecName(wf.Spec, j, maxPipsPerSpec), newRoot), clonedDestination);
int newPipId = m_graph.AddWithNewPipId(cloned, wf.PipId);
m_graph.OutputArtifactToProducingPip.TryAdd(clonedDestination, newPipId);
}
}
// TODO: Mirror CopyFile pips
}
/// <summary>
/// Mutates the graph by duplicating the graph into n additional parallelizable graphs.
/// </summary>
public static void DuplicateAsParallelGraphs(BuildGraph graph, int duplicationFactor, int maxPipsPerSpec)
{
GraphMultiplier multiplier = new GraphMultiplier(graph);
multiplier.Duplicate(duplicationFactor, maxPipsPerSpec);
}
/// <summary>
/// Constructor
/// </summary>
private GraphMultiplier(BuildGraph sourceGraph)
{
m_graph = sourceGraph;
}
public static void CompareWrapperAndConvPhoneBuilds()
{
// TODO:= Set these paths if you intend on using this
var wrapperTask = ReadGraph(@"D:\mimic\graphs\\DBuild.M.x86fre.json");
var convTask = ReadGraph(@"D:\mimic\graphs\PhonePartialNativeSpecX86fre.json");
BuildGraph wrapperGraph = wrapperTask.Result;
BuildGraph convGraph = convTask.Result;
Dictionary <string, Tuple <List <Pip>, List <Pip> > > pipsBySpec = new Dictionary <string, Tuple <List <Pip>, List <Pip> > >();
HashSet <string> convertedPaths = new HashSet <string>();
foreach (var pip in wrapperGraph.Pips.Values)
{
string relativeSpec = PrepareCollection(pip, pipsBySpec);
if (relativeSpec == null)
{
continue;
}
pipsBySpec[relativeSpec].Item1.Add(pip);
}
foreach (var pip in convGraph.Pips.Values)
{
string relativeSpec = PrepareCollection(pip, pipsBySpec);
if (pip.Spec.EndsWith(ConvertedSpecName, StringComparison.OrdinalIgnoreCase))
{
convertedPaths.Add(relativeSpec);
}
if (relativeSpec == null)
{
continue;
}
pipsBySpec[relativeSpec].Item2.Add(pip);
}
Console.WriteLine("WrapperProcesses,WrapperCopyFile,WrapperWriteFile,WrapperOutputCount,WasConverted,ConvProcesses,ConvCopyFile,ConvWriteFile,CppCount,CsCount,ConvPrediction,SpecDir");
foreach (var item in pipsBySpec)
{
ProjectStats wrapperStats = GetStats(item.Value.Item1, wrapperGraph);
ProjectStats convStats = GetStats(item.Value.Item2, convGraph);
int convPrediction = 0;
if (wrapperStats.CppFileCount > 0)
{
convPrediction = wrapperStats.CppFileCount + 2;
}
else
{
convPrediction = wrapperStats.ProcessCount;
}
bool wasConverted = convertedPaths.Contains(item.Key);
Console.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}", wrapperStats.ProcessCount, wrapperStats.CopyFileCount, wrapperStats.WriteFileCount, wrapperStats.OutputFileCount,
wasConverted,
convStats.ProcessCount, convStats.CopyFileCount, convStats.WriteFileCount,
wrapperStats.CppFileCount, wrapperStats.CsFileCount,
convPrediction,
item.Key);
}
}
