private DirectoryEntry CreateEntry(DirectoryArtifact dir)
{
var sealId = PipGraph.GetSealedDirectoryNode(dir).ToPipId();
var sealKind = PipTable.GetSealDirectoryKind(sealId);
var pipId = PipGraph.TryGetProducer(dir);
var producer = GetEntry(pipId);
var pip = (SealDirectory)GetPip(sealId);
var kind = pip.Kind.ToString();
if (pip.IsComposite)
{
kind = "Composite" + kind;
}
var entry = new DirectoryEntry()
{
Directory = dir,
Producer = GetEntry(pipId),
FileCount = GetContents(dir).Length,
Kind = sealKind,
SemistableHash = producer?.Identifier ?? string.Empty,
Identifier = $"{pip.FormattedSemiStableHash} [{kind}]",
Id = dir.PartialSealId.ToString(),
};
if (pip.Provenance.Token.Path.IsValid)
{
entry.SpecFileName = pip.Provenance.Token.Path.GetName(PathTable).ToString(StringTable);
}
return(entry);
}
private void GetNestedSealDirectories()
{
Dictionary <AbsolutePath, DirectoryArtifact> directories = new Dictionary <AbsolutePath, DirectoryArtifact>();
foreach (var directory in PipGraph.AllSealDirectories)
{
directories[directory.Path] = directory;
}
foreach (var directory in PipGraph.AllSealDirectories)
{
var path = directory.Path;
var current = path;
while (current.IsValid)
{
DirectoryArtifact containerArtifact;
if (current != path && directories.TryGetValue(current, out containerArtifact))
{
var innerNode = PipGraph.GetSealedDirectoryNode(directory);
var containerNode = PipGraph.GetSealedDirectoryNode(containerArtifact);
m_writer.WriteLine($"Path: {path.ToString(PathTable)}");
m_writer.WriteLine($"Child: {GetSealDescription(GetPip(innerNode))}");
m_writer.WriteLine($"Container Path: {containerArtifact.Path.ToString(PathTable)}");
m_writer.WriteLine($"Container: {GetSealDescription(GetPip(containerNode))}");
m_writer.WriteLine();
}
current = current.GetParent(PathTable);
}
}
}
private static ContentFingerprint CreateFingerprintForSharedOpaque(DirectoryArtifact sharedOpaqueDirectory, PipGraph graph)
{
var sealDirectoryPipId = graph.GetSealedDirectoryNode(sharedOpaqueDirectory).ToPipId();
XAssert.IsTrue(graph.TryGetPipFingerprint(sealDirectoryPipId, out ContentFingerprint fingerprint));
return(fingerprint);
}
public override void Prepare()
{
Directory.CreateDirectory(OutputFilePath);
m_writer = new StreamWriter(Path.Combine(OutputFilePath, "results.txt"));
Console.WriteLine("Creating nodes");
foreach (var node in DataflowGraph.Nodes)
{
m_mutableGraph.CreateNode();
}
Console.WriteLine("Created nodes");
foreach (var entry in PipGraph.AllOutputDirectoriesAndProducers)
{
var sealId = PipGraph.GetSealedDirectoryNode(entry.Key).ToPipId();
m_dynamicDirectoryProducers[sealId] = entry.Value;
}
foreach (CopyFile copyFile in PipGraph.RetrievePipsOfType(PipType.CopyFile))
{
m_copiedFilesByTarget[copyFile.Destination] = copyFile.Source;
}
foreach (var directory in PipGraph.AllSealDirectories)
{
var sealId = PipGraph.GetSealedDirectoryNode(directory).ToPipId();
var sealKind = PipTable.GetSealDirectoryKind(sealId);
// Populate map of whether this is a source only seal
IsSourceOnlySeal(sealId);
if (sealKind == SealDirectoryKind.Full || sealKind == SealDirectoryKind.Partial)
{
PipId?singleProducer = null;
foreach (var file in PipGraph.ListSealedDirectoryContents(directory))
{
if (file.IsOutputFile)
{
var producer = PipGraph.TryGetProducer(file);
if (singleProducer == null)
{
singleProducer = producer;
}
else if (singleProducer != producer)
{
singleProducer = PipId.Invalid;
}
}
}
if (singleProducer.HasValue && singleProducer.Value.IsValid)
{
m_dynamicDirectoryProducers[sealId] = singleProducer.Value;
}
}
}
}
private static List <string> GetDirectoryDependencies(ReadOnlyArray <DirectoryArtifact> dependencies, PathTable pathTable, PipGraph graph)
{
return(GetJsonFriendlyList(
dependencies
.Where(value => value.Path.IsValid)
.Select(value => {
var pipHash = graph.GetFormattedSemiStableHash(graph.GetSealedDirectoryNode(value).ToPipId());
return $"{value.Path.ToString(pathTable)} (SealDirectorySemiStableHash: {pipHash}, PartialSealId: {value.PartialSealId}, IsSharedOpaque: {(value.IsSharedOpaque ? 1 : 0)})";
})
));
}
private void GetPathProducers(AbsolutePath path)
{
Dictionary <AbsolutePath, DirectoryArtifact> directories = new Dictionary <AbsolutePath, DirectoryArtifact>();
foreach (var directory in PipGraph.AllSealDirectories)
{
directories[directory.Path] = directory;
}
var latest = PipGraph.TryGetLatestFileArtifactForPath(path);
if (!latest.IsValid)
{
m_writer.WriteLine("No declared file producers");
}
else
{
for (int i = 0; i <= latest.RewriteCount; i++)
{
var producerId = PipGraph.TryGetProducer(new FileArtifact(path, i));
if (producerId.IsValid)
{
m_writer.WriteLine("File Producer: ({0})", i);
m_producers.Add(producerId);
m_writer.WriteLine(GetDescription(GetPip(producerId)));
m_writer.WriteLine();
}
}
}
while (path.IsValid)
{
DirectoryArtifact containingDirectory;
if (directories.TryGetValue(path, out containingDirectory))
{
var directoryNode = PipGraph.GetSealedDirectoryNode(containingDirectory);
var sealDirectory = (SealDirectory)GetPip(directoryNode);
if (sealDirectory.Kind == SealDirectoryKind.Opaque)
{
m_writer.WriteLine("Directory Producer:");
m_producers.Add(PipGraph.GetProducer(containingDirectory));
m_writer.WriteLine(GetDescription(GetPip(PipGraph.GetProducer(containingDirectory))));
m_writer.WriteLine("Directory: " + GetDescription(sealDirectory));
m_writer.WriteLine();
}
}
path = path.GetParent(PathTable);
}
}
private ReadOnlyArray <FileArtifact> GetContents(DirectoryArtifact directoryArtifact)
{
var sealId = PipGraph.GetSealedDirectoryNode(directoryArtifact).ToPipId();
var sealKind = PipTable.GetSealDirectoryKind(sealId);
if (sealKind == SealDirectoryKind.Full || sealKind == SealDirectoryKind.Partial)
{
return(PipGraph.ListSealedDirectoryContents(directoryArtifact));
}
else if (sealKind.IsOpaqueOutput())
{
return(m_dynamicContents[directoryArtifact]);
}
return(ReadOnlyArray <FileArtifact> .Empty);
}
public override int Analyze()
{
var rootExpander = new RootExpander(PathTable);
HashSet <ContentHash> hashes = new HashSet <ContentHash>();
hashes.Add(ContentHashingUtilities.ZeroHash);
HashSet <FileArtifact> files = new HashSet <FileArtifact>();
foreach (var root in Roots)
{
rootExpander.Add(AbsolutePath.Create(PathTable, root.Key), root.Value);
}
Func <AbsolutePath, string> expandRoot = absPath => absPath.ToString(PathTable, rootExpander);
var orderedPips = CachedGraph.PipGraph.RetrievePipReferencesOfType(PipType.Process)
.Where(lazyPip => TargetSemiStableHash == null || TargetSemiStableHash == lazyPip.SemiStableHash)
.Select(lazyPip => (Process)lazyPip.HydratePip())
.ToLookup(process => process.Provenance.Token.Path)
.OrderBy(grouping => grouping.Key.ToString(PathTable, rootExpander))
.ToList();
using (var fingerprintStream = File.Create(DumpFilePath, bufferSize: 64 << 10 /* 64 KB */))
using (var hashWriter = new StreamWriter(DumpFilePath + ".hashes.txt"))
{
using (
var fingerprintArchive = CompressFile
? new ZipArchive(fingerprintStream, ZipArchiveMode.Create)
: null)
{
using (
var writer =
XmlWriter.Create(
CompressFile
? fingerprintArchive.CreateEntry("dump.xml", CompressionLevel.Fastest).Open()
: fingerprintStream, new XmlWriterSettings()
{
Indent = true
}))
{
int doneProcesses = 0;
var t = new Timer(
o =>
{
var done = doneProcesses;
Console.WriteLine("Processes Done: {0} of {1}", done, orderedPips.Count);
},
null,
5000,
5000);
try
{
writer.WriteStartElement("ProcessDump");
writer.WriteAttributeString("Count", orderedPips.Count.ToString(CultureInfo.InvariantCulture));
foreach (var specPipGroup in orderedPips)
{
writer.WriteStartElement("SpecFile");
writer.WriteAttributeString("Path", specPipGroup.Key.ToString(PathTable, rootExpander));
foreach (var pip in specPipGroup)
{
doneProcesses++;
writer.WriteStartElement("Process");
writer.WriteAttributeString("Name", pip.Executable.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("CMD", RenderProcessArguments(pip));
writer.WriteElementString("Description", pip.GetDescription(PipGraph.Context));
writer.WriteStartElement("EnvironmentVariables");
foreach (var environmentVariable in pip.EnvironmentVariables)
{
writer.WriteStartElement("Environment");
writer.WriteAttributeString("Name", environmentVariable.Name.ToString(PathTable.StringTable));
if (environmentVariable.Value.IsValid)
{
writer.WriteAttributeString("Value", environmentVariable.Value.ToString(expandRoot, PathTable.StringTable, PipData.MaxMonikerRenderer));
}
else
{
writer.WriteAttributeString("Value", "Unset");
}
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("Dependencies");
foreach (var input in pip.Dependencies)
{
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", input.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(input, ContentHashingUtilities.ZeroHash).Item.Value.ToString());
writer.WriteAttributeString("RewriteCount", input.RewriteCount.ToString());
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("DirectoryDependencies");
foreach (var input in pip.DirectoryDependencies)
{
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", input.Path.ToString(PathTable, rootExpander));
var kind = PipTable.GetSealDirectoryKind(PipGraph.GetSealedDirectoryNode(input).ToPipId());
writer.WriteAttributeString("Kind", kind.ToString());
// Print directory dependency file details when dumping a specific process
if (TargetSemiStableHash != null && (kind == SealDirectoryKind.Full || kind == SealDirectoryKind.Partial))
{
foreach (var file in PipGraph.ListSealedDirectoryContents(input))
{
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", file.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(file, ContentHashingUtilities.ZeroHash).Item.Value.ToString());
writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString());
writer.WriteEndElement();
}
}
else if (m_contents.TryGetValue(input, out var contents))
{
m_observedInputs.TryGetValue(pip.PipId.Value, out var observedInputs);
foreach (var file in contents)
{
// skip the files that were not accessed
if (observedInputs != null && !observedInputs.Contains(file.Path))
{
continue;
}
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", file.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(file, ContentHashingUtilities.ZeroHash).Item.Value.ToString());
writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString());
writer.WriteEndElement();
}
}
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("Outputs");
foreach (var input in pip.FileOutputs)
{
writer.WriteStartElement("Item");
if (input.RewriteCount > 1)
{
writer.WriteAttributeString("RewriteCount", input.RewriteCount.ToString());
}
writer.WriteString(input.Path.ToString(PathTable, rootExpander));
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("DirectoryOutputs");
foreach (var output in pip.DirectoryOutputs)
{
writer.WriteStartElement("Directory");
{
writer.WriteAttributeString("Path", output.Path.ToString(PathTable, rootExpander));
if (m_contents.TryGetValue(output, out var contents))
{
writer.WriteStartElement("Contents");
{
foreach (var file in contents)
{
writer.WriteStartElement("Item");
if (file.RewriteCount > 1)
{
writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString());
}
writer.WriteString(file.Path.ToString(PathTable, rootExpander));
writer.WriteEndElement();
}
}
writer.WriteEndElement();
}
}
writer.WriteEndElement();
}
writer.WriteEndElement();
if (pip.TempDirectory.IsValid)
{
writer.WriteElementString("TempDirectory", pip.TempDirectory.ToString(PathTable, rootExpander));
}
writer.WriteStartElement("AdditionalTempDirectories");
foreach (var item in pip.AdditionalTempDirectories)
{
writer.WriteElementString("Item", item.ToString(PathTable, rootExpander));
}
writer.WriteEndElement();
writer.WriteEndElement(); // Process
}
writer.WriteEndElement(); // SpecFile
}
writer.WriteEndElement(); // ProcessDump
}
finally
{
// kill and wait for the status timer to die...
using (var e = new AutoResetEvent(false))
{
t.Dispose(e);
e.WaitOne();
}
}
}
}
}
return(0);
}