public bool IsSourceOnlySeal(PipId pipId)
{
return(m_isSourceOnlySeal.GetOrAdd(pipId, true, (k, v) =>
{
var sealKind = PipTable.GetSealDirectoryKind(k);
if (sealKind == SealDirectoryKind.SourceAllDirectories || sealKind == SealDirectoryKind.SourceTopDirectoryOnly)
{
return true;
}
if (sealKind == SealDirectoryKind.Full || sealKind == SealDirectoryKind.Partial)
{
var isSourceOnly = true;
var sd = (SealDirectory)GetPip(k);
foreach (var file in sd.Contents)
{
if (file.IsOutputFile)
{
isSourceOnly = false;
}
}
return isSourceOnly;
}
return false;
}).Item.Value);
}
private AbsolutePath Convert(AbsolutePath path)
{
if (AreGraphsSame)
{
return(path);
}
var result = m_pathMap.GetOrAdd(path, this, (p, m) =>
{
try
{
return(AbsolutePath.Create(m.OldModel.PathTable, p.ToString(m.ConvertedNewModel.PathTable)));
}
#pragma warning disable ERP022 // TODO: This should really handle specific errors
catch
{
return(AbsolutePath.Invalid);
}
#pragma warning restore ERP022 // Unobserved exception in generic exception handler
});
var convertedPath = result.Item.Value;
return(convertedPath);
}
private void AddEdges()
{
var nodes = MutableDataflowGraph.Nodes.ToArray();
int count = 0;
Parallel.ForEach(nodes, node =>
{
var consumes = m_pipConsumes[node.Value];
if (consumes != null)
{
foreach (var consumed in consumes)
{
var producer = m_pipProducers.GetOrAdd(consumed, MutableDataflowGraph, (consumed0, graph) =>
{
var sourceNode = graph.CreateNode();
PipTypes[sourceNode] = PipType.HashSourceFile;
return(sourceNode);
}).Item.Value;
MutableDataflowGraph.AddEdge(producer, node);
}
int incrementedCount = Interlocked.Increment(ref count);
if ((incrementedCount % 10000) == 0)
{
Console.WriteLine("Added edges: {0} / {1} ({2} %)", incrementedCount, nodes.Length, ((double)incrementedCount * 100) / nodes.Length);
}
}
});
}
/// <summary>
/// Add directory artifacts to the directory consumer map
/// </summary>
private void AddToDirectoryConsumerMap(Utilities.DirectoryArtifact directoryArtifact, PipId pipId)
{
var consumers = m_directoryConsumerMap.GetOrAdd(directoryArtifact, new HashSet <uint>()).Item.Value;
lock (consumers)
{
consumers.Add(pipId.Value);
}
}
/// <summary>
/// Add file artifacts to the file consumer map
/// </summary>
private void AddToFileConsumerMap(Utilities.FileArtifact fileArtifact, PipId pipId)
{
var consumers = m_fileConsumerMap.GetOrAdd(fileArtifact, new HashSet <uint>()).Item.Value;
lock (consumers)
{
consumers.Add(pipId.Value);
}
}
/// <summary>
/// Adds a file node (and any needed parent directory nodes) at the VFS root relative path
/// </summary>
public VfsFileNode AddFileNode(string relativePath, VfsFilePlacementData data, string realPath)
{
var timestamp = DateTime.UtcNow;
if (_nodeMap.TryGetValue(relativePath, out var node))
{
return((VfsFileNode)node);
}
else
{
var parent = GetOrAddDirectoryNode(Path.GetDirectoryName(relativePath), allowAdd: true);
var result = _nodeMap.GetOrAdd(relativePath, (parent, timestamp, data), (l_relativePath, l_data) =>
{
return(new VfsFileNode(Path.GetFileName(l_relativePath), l_data.timestamp, l_data.parent, l_data.data, RealPath.Create(PathTable, realPath)));
});
node = result.Item.Value;
return((VfsFileNode)node);
}
}
private PipEntry GetEntry(PipId pipId)
{
if (!pipId.IsValid)
{
return(null);
}
var result = m_pipEntries.GetOrAdd(pipId, this, (key, _this) => _this.CreateEntry(key));
var entry = result.Item.Value;
return(entry);
}
private FileEntry GetEntry(FileArtifact file)
{
var result = m_fileEntries.GetOrAdd(file, this, (key, _this) => _this.CreateEntry(key));
var entry = result.Item.Value;
if (file.IsOutputFile && entry.Producer == null)
{
var producer = GetProducer(file);
if (producer.IsValid)
{
entry.Producer = GetEntry(producer);
}
}
return(entry);
}
private DirectoryEntry GetEntry(DirectoryArtifact dir)
{
if (!dir.IsValid)
{
return(null);
}
var result = m_directoryEntries.GetOrAdd(dir, this, (key, _this) => _this.CreateEntry(key));
var entry = result.Item.Value;
if (!result.IsFound)
{
var pipId = PipGraph.TryGetProducer(dir);
entry.Producer = GetEntry(pipId);
}
return(entry);
}
private int EnsurePath(AbsolutePath path)
{
if (!path.IsValid)
{
return(0);
}
var getResult = m_pathToParentIndexMap.TryGet(path);
if (getResult.IsFound)
{
return(getResult.Index);
}
var parentIndex = EnsurePath(path.GetParent(m_pathTable));
var addResult = m_pathToParentIndexMap.GetOrAdd(path, parentIndex);
Contract.Assert(!addResult.IsFound);
return(addResult.Index);
}
public void TestConcurrentBigMapOperations()
{
var map = new ConcurrentBigMap <int, string>();
XAssert.IsTrue(map.TryAdd(0, "value"));
XAssert.IsFalse(map.TryAdd(0, "not added value"));
XAssert.AreEqual("value", map[0]);
map[0] = "newValue";
map[1] = "value1";
var value0 = "newValue";
XAssert.AreEqual(value0, map[0]);
XAssert.IsTrue(map.ContainsKey(0));
XAssert.IsTrue(map.ContainsKey(1));
XAssert.IsFalse(map.ContainsKey(12));
XAssert.AreEqual(2, map.Count);
// Test TryGetValue
string value1;
XAssert.IsTrue(map.TryGetValue(1, out value1));
XAssert.AreEqual("value1", value1);
string value31;
XAssert.IsFalse(map.TryGetValue(31, out value31));
// Test update
XAssert.IsFalse(map.TryUpdate(1, "notUpdatedValue1", "notActualValue1"));
XAssert.AreEqual("value1", map[1]);
XAssert.IsTrue(map.TryUpdate(1, "updatedValue1", "value1"));
value1 = map[1];
XAssert.AreEqual("updatedValue1", value1);
// Test remove
int beforeFailedRemoveCount = map.Count;
string value23;
XAssert.IsFalse(map.TryRemove(23, out value23));
XAssert.AreEqual(beforeFailedRemoveCount, map.Count);
map.Add(23, "value23");
XAssert.AreEqual(beforeFailedRemoveCount + 1, map.Count);
XAssert.IsTrue(map.TryRemove(23, out value23));
XAssert.AreEqual("value23", value23);
XAssert.AreEqual(beforeFailedRemoveCount, map.Count);
Assert.Equal(new int[] { 0, 1 }, map.Keys.ToArray());
Assert.Equal(new string[] { value0, value1 }, map.Values.ToArray());
XAssert.AreEqual(2, map.Count);
string addedData = "added data";
string notAddedData = "not added data";
var result = map.GetOrAdd(2, addedData, (key, data0) => data0);
XAssert.IsFalse(result.IsFound);
XAssert.AreEqual(addedData, result.Item.Value);
XAssert.AreEqual(addedData, map[2]);
// Ensure entry is not updated for get or add
result = map.GetOrAdd(2, notAddedData, (key, data0) => data0);
XAssert.IsTrue(result.IsFound);
XAssert.AreEqual(addedData, result.Item.Value);
XAssert.AreEqual(addedData, map[2]);
Func <int, string, string, string> updateFunction =
(key, data0, currentValue) => "updated " + currentValue;
var updatedData = updateFunction(2, notAddedData, addedData);
result = map.AddOrUpdate(2, notAddedData, (key, data0) => data0, updateFunction);
XAssert.IsTrue(result.IsFound);
XAssert.AreEqual(addedData, result.OldItem.Value);
XAssert.AreEqual(updatedData, result.Item.Value);
XAssert.AreEqual(updatedData, map[2]);
result = map.AddOrUpdate(3, addedData, (key, data0) => data0, updateFunction);
XAssert.IsFalse(result.IsFound);
XAssert.AreEqual(addedData, result.Item.Value);
XAssert.AreEqual(addedData, map[3]);
TestOperationsHelper(parallel: false);
}
public override int Analyze()
{
var rootExpander = new RootExpander(PathTable);
HashSet <ContentHash> hashes = new HashSet <ContentHash>();
hashes.Add(ContentHashingUtilities.ZeroHash);
HashSet <FileArtifact> files = new HashSet <FileArtifact>();
foreach (var root in Roots)
{
rootExpander.Add(AbsolutePath.Create(PathTable, root.Key), root.Value);
}
Func <AbsolutePath, string> expandRoot = absPath => absPath.ToString(PathTable, rootExpander);
var orderedPips = CachedGraph.PipGraph.RetrievePipReferencesOfType(PipType.Process)
.Where(lazyPip => TargetSemiStableHash == null || TargetSemiStableHash == lazyPip.SemiStableHash)
.Select(lazyPip => (Process)lazyPip.HydratePip())
.ToLookup(process => process.Provenance.Token.Path)
.OrderBy(grouping => grouping.Key.ToString(PathTable, rootExpander))
.ToList();
using (var fingerprintStream = File.Create(DumpFilePath, bufferSize: 64 << 10 /* 64 KB */))
using (var hashWriter = new StreamWriter(DumpFilePath + ".hashes.txt"))
{
using (
var fingerprintArchive = CompressFile
? new ZipArchive(fingerprintStream, ZipArchiveMode.Create)
: null)
{
using (
var writer =
XmlWriter.Create(
CompressFile
? fingerprintArchive.CreateEntry("dump.xml", CompressionLevel.Fastest).Open()
: fingerprintStream, new XmlWriterSettings()
{
Indent = true
}))
{
int doneProcesses = 0;
var t = new Timer(
o =>
{
var done = doneProcesses;
Console.WriteLine("Processes Done: {0} of {1}", done, orderedPips.Count);
},
null,
5000,
5000);
try
{
writer.WriteStartElement("ProcessDump");
writer.WriteAttributeString("Count", orderedPips.Count.ToString(CultureInfo.InvariantCulture));
foreach (var specPipGroup in orderedPips)
{
writer.WriteStartElement("SpecFile");
writer.WriteAttributeString("Path", specPipGroup.Key.ToString(PathTable, rootExpander));
foreach (var pip in specPipGroup)
{
doneProcesses++;
writer.WriteStartElement("Process");
writer.WriteAttributeString("Name", pip.Executable.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("CMD", RenderProcessArguments(pip));
writer.WriteElementString("Description", pip.GetDescription(PipGraph.Context));
writer.WriteStartElement("EnvironmentVariables");
foreach (var environmentVariable in pip.EnvironmentVariables)
{
writer.WriteStartElement("Environment");
writer.WriteAttributeString("Name", environmentVariable.Name.ToString(PathTable.StringTable));
if (environmentVariable.Value.IsValid)
{
writer.WriteAttributeString("Value", environmentVariable.Value.ToString(expandRoot, PathTable.StringTable, PipData.MaxMonikerRenderer));
}
else
{
writer.WriteAttributeString("Value", "Unset");
}
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("Dependencies");
foreach (var input in pip.Dependencies)
{
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", input.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(input, ContentHashingUtilities.ZeroHash).Item.Value.ToString());
writer.WriteAttributeString("RewriteCount", input.RewriteCount.ToString());
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("DirectoryDependencies");
foreach (var input in pip.DirectoryDependencies)
{
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", input.Path.ToString(PathTable, rootExpander));
var kind = PipTable.GetSealDirectoryKind(PipGraph.GetSealedDirectoryNode(input).ToPipId());
writer.WriteAttributeString("Kind", kind.ToString());
// Print directory dependency file details when dumping a specific process
if (TargetSemiStableHash != null && (kind == SealDirectoryKind.Full || kind == SealDirectoryKind.Partial))
{
foreach (var file in PipGraph.ListSealedDirectoryContents(input))
{
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", file.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(file, ContentHashingUtilities.ZeroHash).Item.Value.ToString());
writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString());
writer.WriteEndElement();
}
}
else if (m_contents.TryGetValue(input, out var contents))
{
m_observedInputs.TryGetValue(pip.PipId.Value, out var observedInputs);
foreach (var file in contents)
{
// skip the files that were not accessed
if (observedInputs != null && !observedInputs.Contains(file.Path))
{
continue;
}
writer.WriteStartElement("Item");
writer.WriteAttributeString("Path", file.Path.ToString(PathTable, rootExpander));
writer.WriteAttributeString("Hash", m_fileHashes.GetOrAdd(file, ContentHashingUtilities.ZeroHash).Item.Value.ToString());
writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString());
writer.WriteEndElement();
}
}
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("Outputs");
foreach (var input in pip.FileOutputs)
{
writer.WriteStartElement("Item");
if (input.RewriteCount > 1)
{
writer.WriteAttributeString("RewriteCount", input.RewriteCount.ToString());
}
writer.WriteString(input.Path.ToString(PathTable, rootExpander));
writer.WriteEndElement();
}
writer.WriteEndElement();
writer.WriteStartElement("DirectoryOutputs");
foreach (var output in pip.DirectoryOutputs)
{
writer.WriteStartElement("Directory");
{
writer.WriteAttributeString("Path", output.Path.ToString(PathTable, rootExpander));
if (m_contents.TryGetValue(output, out var contents))
{
writer.WriteStartElement("Contents");
{
foreach (var file in contents)
{
writer.WriteStartElement("Item");
if (file.RewriteCount > 1)
{
writer.WriteAttributeString("RewriteCount", file.RewriteCount.ToString());
}
writer.WriteString(file.Path.ToString(PathTable, rootExpander));
writer.WriteEndElement();
}
}
writer.WriteEndElement();
}
}
writer.WriteEndElement();
}
writer.WriteEndElement();
if (pip.TempDirectory.IsValid)
{
writer.WriteElementString("TempDirectory", pip.TempDirectory.ToString(PathTable, rootExpander));
}
writer.WriteStartElement("AdditionalTempDirectories");
foreach (var item in pip.AdditionalTempDirectories)
{
writer.WriteElementString("Item", item.ToString(PathTable, rootExpander));
}
writer.WriteEndElement();
writer.WriteEndElement(); // Process
}
writer.WriteEndElement(); // SpecFile
}
writer.WriteEndElement(); // ProcessDump
}
finally
{
// kill and wait for the status timer to die...
using (var e = new AutoResetEvent(false))
{
t.Dispose(e);
e.WaitOne();
}
}
}
}
}
return(0);
}