private static ProjectStats GetStats(IEnumerable <Pip> pips, BuildGraph graph)
{
ProjectStats stats = default(ProjectStats);
HashSet <string> inputFiles = new HashSet <string>();
HashSet <string> outputFiles = new HashSet <string>();
foreach (Pip pip in pips)
{
Process process = pip as Process;
if (process != null)
{
stats.ProcessCount++;
foreach (int consumes in process.Consumes)
{
File f;
if (graph.Files.TryGetValue(consumes, out f))
{
inputFiles.Add(f.Location);
}
}
foreach (int produces in process.Produces)
{
File f;
if (graph.Files.TryGetValue(produces, out f))
{
outputFiles.Add(f.Location);
}
}
continue;
}
CopyFile copyFile = pip as CopyFile;
if (copyFile != null)
{
stats.CopyFileCount++;
continue;
}
WriteFile writeFile = pip as WriteFile;
if (writeFile != null)
{
stats.WriteFileCount++;
continue;
}
}
stats.CppFileCount = inputFiles.Where(f => f.EndsWith(".cpp", StringComparison.OrdinalIgnoreCase)).Count();
stats.CsFileCount = inputFiles.Where(f => f.EndsWith(".cs", StringComparison.OrdinalIgnoreCase)).Count();
stats.OutputFileCount = outputFiles.Count;
return(stats);
}
private void Duplicate(int multiplicationFactor, int maxPipsPerSpec)
{
Console.WriteLine("Duplicating Graph {0} times", multiplicationFactor);
// Take a snapshot of the pips currently in the graph before any multiplication is applied.
Process[] originalProcesses = m_graph.Pips.Values.OfType <Process>().ToArray();
WriteFile[] originalWriteFiles = m_graph.Pips.Values.OfType <WriteFile>().ToArray();
// The basic strategy here is to clone every pip in a parallelizeable way. Each pip gets n copies with same
// source inputs, but different outputs. Dependencies within the original set are translated to the new set
for (int i = 0; i < multiplicationFactor; i++)
{
Console.WriteLine("Duplicating Graph iteration {0}", i);
string newRoot = i.ToString(CultureInfo.InvariantCulture);
for (int j = 0; j < originalProcesses.Length; j++)
{
Process p = originalProcesses[j];
List <int> clonedConsumes = new List <int>(p.Consumes.Count);
List <int> clonedProduces = new List <int>(p.Produces.Count);
foreach (int consume in p.Consumes)
{
int producingPip;
if (m_graph.OutputArtifactToProducingPip.TryGetValue(consume, out producingPip))
{
Process process = m_graph.Pips[producingPip] as Process;
WriteFile writeFile = m_graph.Pips[producingPip] as WriteFile;
if (process != null || writeFile != null)
{
// This is an output file created by a pip that will also be cloned. We need to translate it to the new path
File f;
if (!m_graph.Files.TryGetValue(consume, out f))
{
throw new MimicGeneratorException("Failed to find referenced file with id: {0}", consume);
}
clonedConsumes.Add(m_graph.DuplicateFile(f, newRoot));
continue;
}
}
// If the path isn't translated based on cloning, just consume it directly.
clonedConsumes.Add(consume);
}
foreach (int produce in p.Produces)
{
File f;
if (!m_graph.Files.TryGetValue(produce, out f))
{
throw new MimicGeneratorException("Failed to find referenced file with id: {0}", produce);
}
clonedProduces.Add(m_graph.DuplicateFile(f, newRoot));
}
Process cloned = new Process(0, string.Empty, BuildGraph.DuplicatePath(GetSpecName(p.Spec, j, maxPipsPerSpec), newRoot), clonedProduces, clonedConsumes, p.Semaphores);
int newPipId = m_graph.AddWithNewPipId(cloned, p.PipId);
// Need to register all of the outputs of the cloned pip so consumers reference it as an output
// rather than a source file when specs are generated
foreach (var file in clonedProduces)
{
m_graph.OutputArtifactToProducingPip.TryAdd(file, newPipId);
}
}
for (int j = 0; j < originalWriteFiles.Length; j++)
{
WriteFile wf = originalWriteFiles[j];
File f;
if (!m_graph.Files.TryGetValue(wf.Destination, out f))
{
throw new MimicGeneratorException("Failed to find referenced file with id: {0}", wf.Destination);
}
int clonedDestination = m_graph.DuplicateFile(f, newRoot);
WriteFile cloned = new WriteFile(0, string.Empty, BuildGraph.DuplicatePath(GetSpecName(wf.Spec, j, maxPipsPerSpec), newRoot), clonedDestination);
int newPipId = m_graph.AddWithNewPipId(cloned, wf.PipId);
m_graph.OutputArtifactToProducingPip.TryAdd(clonedDestination, newPipId);
}
}
// TODO: Mirror CopyFile pips
}
/// <summary>
/// Writes out build files
/// </summary>
public bool WriteBuildFiles()
{
bool success = true;
Console.WriteLine("Writing build files and inputs");
Directory.CreateDirectory(m_outputDirectory);
using (var textWriter = new StreamWriter(Path.Combine(m_outputDirectory, "stats.txt")))
{
m_buildGraph.OutputFileStats.Write(textWriter);
m_buildGraph.SourceFileStats.Write(textWriter);
m_buildGraph.PipDurationStats.Write(textWriter);
m_buildGraph.BuildInterval.Write(textWriter);
m_buildGraph.ProcessPipStats.Write(textWriter);
}
// Write out the cache config file
System.IO.File.WriteAllText(
Path.Combine(m_outputDirectory, "cacheConfig.json"),
GetEmbeddedResourceFile("Tool.MimicGenerator.Content.CacheConfig.json"));
WriteBuildScript();
var provider = new LanguageProvider(m_language);
using (var configWriter = provider.CreateConfigWriter(Path.Combine(m_outputDirectory, "mimic")))
{
using (var moduleWriter = provider.CreateModuleWriter(m_outputDirectory, "MimicModule", new string[] { "{EngineLayout.DefaultMounts.DeploymentRootPath.Path.Combine('BuildXL.Transformers.Runners.dll')}" }))
{
configWriter.AddModule(moduleWriter);
// Write each pip into its spec file
using (Timer updateTimer = new Timer(ReportProgress, null, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(30)))
{
Parallel.ForEach(m_specFileToPipsLookup, specPips =>
{
using (var writer = provider.CreateSpecWriter(RemapPath(specPips.Key)))
{
List <Process> allProcesses = new List <Process>();
foreach (var p in specPips)
{
Interlocked.Increment(ref m_processesEncountered);
lock (moduleWriter)
{
int lengthToStrip = m_outputDirectory.Length;
if (!(m_outputDirectory.EndsWith(@"/", StringComparison.Ordinal) || m_outputDirectory.EndsWith(@"\", StringComparison.Ordinal)))
{
lengthToStrip++;
}
string specPath = writer.AbsolutePath.Remove(0, lengthToStrip);
moduleWriter.AddSpec(specPath, writer);
}
Process process = p as Process;
if (process != null)
{
allProcesses.Add(process);
}
CopyFile copyFile = p as CopyFile;
if (copyFile != null)
{
bool isDirectory;
bool isResponseFile;
writer.AddCopyFile(
GetProcessOutputName(p.PipId),
GetInputExpression(copyFile.Source, writer, configWriter, out isDirectory, out isResponseFile),
GetOutputExpression(copyFile.Destination, configWriter));
if (isDirectory)
{
throw new MimicGeneratorException("CopyFile shouldn't produce a directory");
}
}
WriteFile writeFile = p as WriteFile;
if (writeFile != null)
{
bool ignore = false;
if (m_ignoreResponseFiles)
{
File f;
if (m_buildGraph.Files.TryGetValue(writeFile.Destination, out f))
{
if (f.Location.EndsWith(ResponseFileName, StringComparison.OrdinalIgnoreCase))
{
ignore = true;
}
}
}
if (!ignore)
{
writer.AddWriteFile(GetProcessOutputName(p.PipId), GetOutputExpression(writeFile.Destination, configWriter));
}
}
SealDirectory sealDirectory = p as SealDirectory;
if (sealDirectory != null)
{
HandleSealDirectory(sealDirectory, writer, configWriter);
}
}
HandleProcesses(writer, configWriter, allProcesses);
}
});
success &= WriteQueuedInputFiles();
}
}
}
Console.WriteLine("Write {0} input files", m_inputsWritten);
Console.WriteLine("{0} Input files were using the default size", m_inputsWithDefaultSize);
return(success);
}
