private static string DescribeProcess(Process p, BuildGraph graph) { StringBuilder desc = new StringBuilder(); desc.AppendLine("Consumes:"); foreach (var file in p.Consumes) { File f; if (graph.Files.TryGetValue(file, out f)) { desc.Append(" "); desc.AppendLine(f.Location); } } desc.AppendLine("Produces:"); foreach (var file in p.Produces) { File f; if (graph.Files.TryGetValue(file, out f)) { desc.Append(" "); desc.AppendLine(f.Location); } } return(desc.ToString()); }
public BuildWriter(string outputDirectory, bool writeInputFiles, double inputScaleFactor, BuildGraph buildGraph, bool ignoreResponseFiles, Language language) { m_outputDirectory = outputDirectory; m_writeInputFiles = writeInputFiles; m_inputScaleFactor = inputScaleFactor; m_buildGraph = buildGraph; m_specFileToPipsLookup = buildGraph.Pips.Values.ToLookup(pip => pip.Spec, StringComparer.Ordinal); m_ignoreResponseFiles = ignoreResponseFiles; m_language = language; }
private static ProjectStats GetStats(IEnumerable <Pip> pips, BuildGraph graph) { ProjectStats stats = default(ProjectStats); HashSet <string> inputFiles = new HashSet <string>(); HashSet <string> outputFiles = new HashSet <string>(); foreach (Pip pip in pips) { Process process = pip as Process; if (process != null) { stats.ProcessCount++; foreach (int consumes in process.Consumes) { File f; if (graph.Files.TryGetValue(consumes, out f)) { inputFiles.Add(f.Location); } } foreach (int produces in process.Produces) { File f; if (graph.Files.TryGetValue(produces, out f)) { outputFiles.Add(f.Location); } } continue; } CopyFile copyFile = pip as CopyFile; if (copyFile != null) { stats.CopyFileCount++; continue; } WriteFile writeFile = pip as WriteFile; if (writeFile != null) { stats.WriteFileCount++; continue; } } stats.CppFileCount = inputFiles.Where(f => f.EndsWith(".cpp", StringComparison.OrdinalIgnoreCase)).Count(); stats.CsFileCount = inputFiles.Where(f => f.EndsWith(".cs", StringComparison.OrdinalIgnoreCase)).Count(); stats.OutputFileCount = outputFiles.Count; return(stats); }
public static void Main(string[] arguments) { string debug = Environment.GetEnvironmentVariable("MimicGeneratorDebugOnStart"); if (debug != null && debug != "0") { Debugger.Launch(); } Stopwatch sw = Stopwatch.StartNew(); Args args = new Args(arguments); if (args.HelpDisplayed) { return; } try { GraphReader reader = new GraphReader(args.JsonGraph, args.ObservedInputs); BuildGraph graph = reader.ReadGraph(); if (args.DuplicateGraph > 1) { GraphMultiplier.DuplicateAsParallelGraphs(graph, args.DuplicateGraph, args.MaxPipsPerSpec); } BuildWriter writer = new BuildWriter(args.Dest, args.WriteInputs, args.InputScaleFactor, graph, args.IgnoreResponseFiles, args.Language); if (!writer.WriteBuildFiles()) { ExitError(sw); } Console.WriteLine("MimicGenerator completed successfully in {0} seconds.", sw.Elapsed.TotalSeconds); Environment.Exit(0); } catch (Exception ex) { ExitError(sw, ex); } }
private void Duplicate(int multiplicationFactor, int maxPipsPerSpec) { Console.WriteLine("Duplicating Graph {0} times", multiplicationFactor); // Take a snapshot of the pips currently in the graph before any multiplication is applied. Process[] originalProcesses = m_graph.Pips.Values.OfType <Process>().ToArray(); WriteFile[] originalWriteFiles = m_graph.Pips.Values.OfType <WriteFile>().ToArray(); // The basic strategy here is to clone every pip in a parallelizeable way. Each pip gets n copies with same // source inputs, but different outputs. Dependencies within the original set are translated to the new set for (int i = 0; i < multiplicationFactor; i++) { Console.WriteLine("Duplicating Graph iteration {0}", i); string newRoot = i.ToString(CultureInfo.InvariantCulture); for (int j = 0; j < originalProcesses.Length; j++) { Process p = originalProcesses[j]; List <int> clonedConsumes = new List <int>(p.Consumes.Count); List <int> clonedProduces = new List <int>(p.Produces.Count); foreach (int consume in p.Consumes) { int producingPip; if (m_graph.OutputArtifactToProducingPip.TryGetValue(consume, out producingPip)) { Process process = m_graph.Pips[producingPip] as Process; WriteFile writeFile = m_graph.Pips[producingPip] as WriteFile; if (process != null || writeFile != null) { // This is an output file created by a pip that will also be cloned. We need to translate it to the new path File f; if (!m_graph.Files.TryGetValue(consume, out f)) { throw new MimicGeneratorException("Failed to find referenced file with id: {0}", consume); } clonedConsumes.Add(m_graph.DuplicateFile(f, newRoot)); continue; } } // If the path isn't translated based on cloning, just consume it directly. clonedConsumes.Add(consume); } foreach (int produce in p.Produces) { File f; if (!m_graph.Files.TryGetValue(produce, out f)) { throw new MimicGeneratorException("Failed to find referenced file with id: {0}", produce); } clonedProduces.Add(m_graph.DuplicateFile(f, newRoot)); } Process cloned = new Process(0, string.Empty, BuildGraph.DuplicatePath(GetSpecName(p.Spec, j, maxPipsPerSpec), newRoot), clonedProduces, clonedConsumes, p.Semaphores); int newPipId = m_graph.AddWithNewPipId(cloned, p.PipId); // Need to register all of the outputs of the cloned pip so consumers reference it as an output // rather than a source file when specs are generated foreach (var file in clonedProduces) { m_graph.OutputArtifactToProducingPip.TryAdd(file, newPipId); } } for (int j = 0; j < originalWriteFiles.Length; j++) { WriteFile wf = originalWriteFiles[j]; File f; if (!m_graph.Files.TryGetValue(wf.Destination, out f)) { throw new MimicGeneratorException("Failed to find referenced file with id: {0}", wf.Destination); } int clonedDestination = m_graph.DuplicateFile(f, newRoot); WriteFile cloned = new WriteFile(0, string.Empty, BuildGraph.DuplicatePath(GetSpecName(wf.Spec, j, maxPipsPerSpec), newRoot), clonedDestination); int newPipId = m_graph.AddWithNewPipId(cloned, wf.PipId); m_graph.OutputArtifactToProducingPip.TryAdd(clonedDestination, newPipId); } } // TODO: Mirror CopyFile pips }
/// <summary> /// Mutates the graph by duplicating the graph into n additional parallelizable graphs. /// </summary> public static void DuplicateAsParallelGraphs(BuildGraph graph, int duplicationFactor, int maxPipsPerSpec) { GraphMultiplier multiplier = new GraphMultiplier(graph); multiplier.Duplicate(duplicationFactor, maxPipsPerSpec); }
/// <summary> /// Constructor /// </summary> private GraphMultiplier(BuildGraph sourceGraph) { m_graph = sourceGraph; }
public static void CompareWrapperAndConvPhoneBuilds() { // TODO:= Set these paths if you intend on using this var wrapperTask = ReadGraph(@"D:\mimic\graphs\\DBuild.M.x86fre.json"); var convTask = ReadGraph(@"D:\mimic\graphs\PhonePartialNativeSpecX86fre.json"); BuildGraph wrapperGraph = wrapperTask.Result; BuildGraph convGraph = convTask.Result; Dictionary <string, Tuple <List <Pip>, List <Pip> > > pipsBySpec = new Dictionary <string, Tuple <List <Pip>, List <Pip> > >(); HashSet <string> convertedPaths = new HashSet <string>(); foreach (var pip in wrapperGraph.Pips.Values) { string relativeSpec = PrepareCollection(pip, pipsBySpec); if (relativeSpec == null) { continue; } pipsBySpec[relativeSpec].Item1.Add(pip); } foreach (var pip in convGraph.Pips.Values) { string relativeSpec = PrepareCollection(pip, pipsBySpec); if (pip.Spec.EndsWith(ConvertedSpecName, StringComparison.OrdinalIgnoreCase)) { convertedPaths.Add(relativeSpec); } if (relativeSpec == null) { continue; } pipsBySpec[relativeSpec].Item2.Add(pip); } Console.WriteLine("WrapperProcesses,WrapperCopyFile,WrapperWriteFile,WrapperOutputCount,WasConverted,ConvProcesses,ConvCopyFile,ConvWriteFile,CppCount,CsCount,ConvPrediction,SpecDir"); foreach (var item in pipsBySpec) { ProjectStats wrapperStats = GetStats(item.Value.Item1, wrapperGraph); ProjectStats convStats = GetStats(item.Value.Item2, convGraph); int convPrediction = 0; if (wrapperStats.CppFileCount > 0) { convPrediction = wrapperStats.CppFileCount + 2; } else { convPrediction = wrapperStats.ProcessCount; } bool wasConverted = convertedPaths.Contains(item.Key); Console.WriteLine("{0},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}", wrapperStats.ProcessCount, wrapperStats.CopyFileCount, wrapperStats.WriteFileCount, wrapperStats.OutputFileCount, wasConverted, convStats.ProcessCount, convStats.CopyFileCount, convStats.WriteFileCount, wrapperStats.CppFileCount, wrapperStats.CsFileCount, convPrediction, item.Key); } }