private DirectoryEntry CreateEntry(DirectoryArtifact dir) { var sealId = PipGraph.GetSealedDirectoryNode(dir).ToPipId(); var sealKind = PipTable.GetSealDirectoryKind(sealId); var pipId = PipGraph.TryGetProducer(dir); var producer = GetEntry(pipId); var pip = (SealDirectory)GetPip(sealId); var kind = pip.Kind.ToString(); if (pip.IsComposite) { kind = "Composite" + kind; } var entry = new DirectoryEntry() { Directory = dir, Producer = GetEntry(pipId), FileCount = GetContents(dir).Length, Kind = sealKind, SemistableHash = producer?.Identifier ?? string.Empty, Identifier = $"{pip.FormattedSemiStableHash} [{kind}]", Id = dir.PartialSealId.ToString(), }; if (pip.Provenance.Token.Path.IsValid) { entry.SpecFileName = pip.Provenance.Token.Path.GetName(PathTable).ToString(StringTable); } return(entry); }
private void GetNestedSealDirectories() { Dictionary <AbsolutePath, DirectoryArtifact> directories = new Dictionary <AbsolutePath, DirectoryArtifact>(); foreach (var directory in PipGraph.AllSealDirectories) { directories[directory.Path] = directory; } foreach (var directory in PipGraph.AllSealDirectories) { var path = directory.Path; var current = path; while (current.IsValid) { DirectoryArtifact containerArtifact; if (current != path && directories.TryGetValue(current, out containerArtifact)) { var innerNode = PipGraph.GetSealedDirectoryNode(directory); var containerNode = PipGraph.GetSealedDirectoryNode(containerArtifact); m_writer.WriteLine($"Path: {path.ToString(PathTable)}"); m_writer.WriteLine($"Child: {GetSealDescription(GetPip(innerNode))}"); m_writer.WriteLine($"Container Path: {containerArtifact.Path.ToString(PathTable)}"); m_writer.WriteLine($"Container: {GetSealDescription(GetPip(containerNode))}"); m_writer.WriteLine(); } current = current.GetParent(PathTable); } } }
private static ContentFingerprint CreateFingerprintForSharedOpaque(DirectoryArtifact sharedOpaqueDirectory, PipGraph graph) { var sealDirectoryPipId = graph.GetSealedDirectoryNode(sharedOpaqueDirectory).ToPipId(); XAssert.IsTrue(graph.TryGetPipFingerprint(sealDirectoryPipId, out ContentFingerprint fingerprint)); return(fingerprint); }
public override void Prepare() { Directory.CreateDirectory(OutputFilePath); m_writer = new StreamWriter(Path.Combine(OutputFilePath, "results.txt")); Console.WriteLine("Creating nodes"); foreach (var node in DataflowGraph.Nodes) { m_mutableGraph.CreateNode(); } Console.WriteLine("Created nodes"); foreach (var entry in PipGraph.AllOutputDirectoriesAndProducers) { var sealId = PipGraph.GetSealedDirectoryNode(entry.Key).ToPipId(); m_dynamicDirectoryProducers[sealId] = entry.Value; } foreach (CopyFile copyFile in PipGraph.RetrievePipsOfType(PipType.CopyFile)) { m_copiedFilesByTarget[copyFile.Destination] = copyFile.Source; } foreach (var directory in PipGraph.AllSealDirectories) { var sealId = PipGraph.GetSealedDirectoryNode(directory).ToPipId(); var sealKind = PipTable.GetSealDirectoryKind(sealId); // Populate map of whether this is a source only seal IsSourceOnlySeal(sealId); if (sealKind == SealDirectoryKind.Full || sealKind == SealDirectoryKind.Partial) { PipId?singleProducer = null; foreach (var file in PipGraph.ListSealedDirectoryContents(directory)) { if (file.IsOutputFile) { var producer = PipGraph.TryGetProducer(file); if (singleProducer == null) { singleProducer = producer; } else if (singleProducer != producer) { singleProducer = PipId.Invalid; } } } if (singleProducer.HasValue && singleProducer.Value.IsValid) { m_dynamicDirectoryProducers[sealId] = singleProducer.Value; } } } }
private static List <string> GetDirectoryDependencies(ReadOnlyArray <DirectoryArtifact> dependencies, PathTable pathTable, PipGraph graph) { return(GetJsonFriendlyList( dependencies .Where(value => value.Path.IsValid) .Select(value => { var pipHash = graph.GetFormattedSemiStableHash(graph.GetSealedDirectoryNode(value).ToPipId()); return $"{value.Path.ToString(pathTable)} (SealDirectorySemiStableHash: {pipHash}, PartialSealId: {value.PartialSealId}, IsSharedOpaque: {(value.IsSharedOpaque ? 1 : 0)})"; }) )); }
private void GetPathProducers(AbsolutePath path) { Dictionary <AbsolutePath, DirectoryArtifact> directories = new Dictionary <AbsolutePath, DirectoryArtifact>(); foreach (var directory in PipGraph.AllSealDirectories) { directories[directory.Path] = directory; } var latest = PipGraph.TryGetLatestFileArtifactForPath(path); if (!latest.IsValid) { m_writer.WriteLine("No declared file producers"); } else { for (int i = 0; i <= latest.RewriteCount; i++) { var producerId = PipGraph.TryGetProducer(new FileArtifact(path, i)); if (producerId.IsValid) { m_writer.WriteLine("File Producer: ({0})", i); m_producers.Add(producerId); m_writer.WriteLine(GetDescription(GetPip(producerId))); m_writer.WriteLine(); } } } while (path.IsValid) { DirectoryArtifact containingDirectory; if (directories.TryGetValue(path, out containingDirectory)) { var directoryNode = PipGraph.GetSealedDirectoryNode(containingDirectory); var sealDirectory = (SealDirectory)GetPip(directoryNode); if (sealDirectory.Kind == SealDirectoryKind.Opaque) { m_writer.WriteLine("Directory Producer:"); m_producers.Add(PipGraph.GetProducer(containingDirectory)); m_writer.WriteLine(GetDescription(GetPip(PipGraph.GetProducer(containingDirectory)))); m_writer.WriteLine("Directory: " + GetDescription(sealDirectory)); m_writer.WriteLine(); } } path = path.GetParent(PathTable); } }
private ReadOnlyArray <FileArtifact> GetContents(DirectoryArtifact directoryArtifact) { var sealId = PipGraph.GetSealedDirectoryNode(directoryArtifact).ToPipId(); var sealKind = PipTable.GetSealDirectoryKind(sealId); if (sealKind == SealDirectoryKind.Full || sealKind == SealDirectoryKind.Partial) { return(PipGraph.ListSealedDirectoryContents(directoryArtifact)); } else if (sealKind.IsOpaqueOutput()) { return(m_dynamicContents[directoryArtifact]); } return(ReadOnlyArray <FileArtifact> .Empty); }
public override int Analyze() { var rootExpander = new RootExpander(PathTable); HashSet <ContentHash> hashes = new HashSet <ContentHash>(); hashes.Add(ContentHashingUtilities.ZeroHash); HashSet <FileArtifact> files = new HashSet <FileArtifact>(); foreach (var root in Roots) { rootExpander.Add(AbsolutePath.Create(PathTable, root.Key), root.Value); } Func <AbsolutePath, string> expandRoot = absPath => absPath.ToString(PathTable, rootExpander); var orderedPips = CachedGraph.PipGraph.RetrievePipReferencesOfType(PipType.Process) .Where(lazyPip => TargetSemiStableHash == null || TargetSemiStableHash == lazyPip.SemiStableHash) .Select(lazyPip => (Process)lazyPip.HydratePip()) .ToLookup(process => process.Provenance.Token.Path) .OrderBy(grouping => grouping.Key.ToString(PathTable, rootExpander)) .ToList(); using (var fingerprintStream = File.Create(DumpFilePath, bufferSize: 64 << 10 /* 64 KB */)) using (var hashWriter = new StreamWriter(DumpFilePath + ".hashes.txt")) { using ( var fingerprintArchive = CompressFile ? new ZipArchive(fingerprintStream, ZipArchiveMode.Create) : null) { using ( var writer = XmlWriter.Create( CompressFile ? fingerprintArchive.CreateEntry("dump.xml", CompressionLevel.Fastest).Open() : fingerprintStream, new XmlWriterSettings() { Indent = true })) { int doneProcesses = 0; var t = new Timer( o => { var done = doneProcesses; Console.WriteLine("Processes Done: {0} of {1}", done, orderedPips.Count); }, null, 5000, 5000); try { writer.WriteStartElement("ProcessDump"); writer.WriteAttributeString("Count", orderedPips.Count.ToString(CultureInfo.InvariantCulture)); foreach (var specPipGroup in orderedPips) { writer.WriteStartElement("SpecFile"); writer.WriteAttributeString("Path", specPipGroup.Key.ToString(PathTable, rootExpander)); foreach (var pip in specPipGroup) { doneProcesses++; writer.WriteStartElement("Process"); writer.WriteAttributeString("Name", pip.Executable.Path.ToString(PathTable, rootExpander)); writer.WriteAttributeString("CMD", RenderProcessArguments(pip)); writer.WriteElementString("Description", pip.GetDescription(PipGraph.Context)); writer.WriteStartElement("EnvironmentVariables"); foreach (var environmentVariable in pip.EnvironmentVariables) { writer.WriteStartElement("Environment"); writer.WriteAttributeString("Name", environmentVariable.Name.ToString(PathTable.StringTable)); if (environmentVariable.Value.IsValid) { writer.WriteAttributeString("Value", environmentVariable.Value.ToString(expandRoot, PathTable.StringTable, PipData.MaxMonikerRenderer)); } else { writer.WriteAttributeString("Value", "Unset"); } writer.WriteEndElement(); } writer.WriteEndElement(); writer.WriteStartElement("Dependencies"); foreach (var input in pip.Dependencies) { writer.WriteStartElement("Item"); writer.WriteAttributeString("Path", input.Path.ToString(PathTable, rootExpander)); writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(input, ContentHashingUtilities.ZeroHash).Item.Value.ToString()); writer.WriteAttributeString("RewriteCount", input.RewriteCount.ToString()); writer.WriteEndElement(); } writer.WriteEndElement(); writer.WriteStartElement("DirectoryDependencies"); foreach (var input in pip.DirectoryDependencies) { writer.WriteStartElement("Item"); writer.WriteAttributeString("Path", input.Path.ToString(PathTable, rootExpander)); var kind = PipTable.GetSealDirectoryKind(PipGraph.GetSealedDirectoryNode(input).ToPipId()); writer.WriteAttributeString("Kind", kind.ToString()); // Print directory dependency file details when dumping a specific process if (TargetSemiStableHash != null && (kind == SealDirectoryKind.Full || kind == SealDirectoryKind.Partial)) { foreach (var file in PipGraph.ListSealedDirectoryContents(input)) { writer.WriteStartElement("Item"); writer.WriteAttributeString("Path", file.Path.ToString(PathTable, rootExpander)); writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(file, ContentHashingUtilities.ZeroHash).Item.Value.ToString()); writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString()); writer.WriteEndElement(); } } else if (m_contents.TryGetValue(input, out var contents)) { m_observedInputs.TryGetValue(pip.PipId.Value, out var observedInputs); foreach (var file in contents) { // skip the files that were not accessed if (observedInputs != null && !observedInputs.Contains(file.Path)) { continue; } writer.WriteStartElement("Item"); writer.WriteAttributeString("Path", file.Path.ToString(PathTable, rootExpander)); writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(file, ContentHashingUtilities.ZeroHash).Item.Value.ToString()); writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString()); writer.WriteEndElement(); } } writer.WriteEndElement(); } writer.WriteEndElement(); writer.WriteStartElement("Outputs"); foreach (var input in pip.FileOutputs) { writer.WriteStartElement("Item"); if (input.RewriteCount > 1) { writer.WriteAttributeString("RewriteCount", input.RewriteCount.ToString()); } writer.WriteString(input.Path.ToString(PathTable, rootExpander)); writer.WriteEndElement(); } writer.WriteEndElement(); writer.WriteStartElement("DirectoryOutputs"); foreach (var output in pip.DirectoryOutputs) { writer.WriteStartElement("Directory"); { writer.WriteAttributeString("Path", output.Path.ToString(PathTable, rootExpander)); if (m_contents.TryGetValue(output, out var contents)) { writer.WriteStartElement("Contents"); { foreach (var file in contents) { writer.WriteStartElement("Item"); if (file.RewriteCount > 1) { writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString()); } writer.WriteString(file.Path.ToString(PathTable, rootExpander)); writer.WriteEndElement(); } } writer.WriteEndElement(); } } writer.WriteEndElement(); } writer.WriteEndElement(); if (pip.TempDirectory.IsValid) { writer.WriteElementString("TempDirectory", pip.TempDirectory.ToString(PathTable, rootExpander)); } writer.WriteStartElement("AdditionalTempDirectories"); foreach (var item in pip.AdditionalTempDirectories) { writer.WriteElementString("Item", item.ToString(PathTable, rootExpander)); } writer.WriteEndElement(); writer.WriteEndElement(); // Process } writer.WriteEndElement(); // SpecFile } writer.WriteEndElement(); // ProcessDump } finally { // kill and wait for the status timer to die... using (var e = new AutoResetEvent(false)) { t.Dispose(e); e.WaitOne(); } } } } } return(0); }