/// <summary>
/// Releases temporary resources and prevents the table from mutating from this point forward.
/// </summary>
/// <remarks>
/// This method is NOT thread-safe.
/// </remarks>
internal void Freeze()
{
Contract.Requires(IsValid());
// release this set to free memory
m_stringSet = null;
}
/// <summary>
/// Reads serialization state
/// </summary>
protected static SerializedState ReadSerializationState(BuildXLReader reader)
{
Contract.Requires(reader != null);
SerializedState result = new SerializedState();
result.NextId = reader.ReadInt32();
result.ByteBuffers = new byte[NumByteBuffers][];
int indexOfPartiallyFilledBuffer, lengthInPartiallyFilledBuffer;
GetBufferIndex(result.NextId, out indexOfPartiallyFilledBuffer, out lengthInPartiallyFilledBuffer);
for (int i = 0; i < NumByteBuffers; i++)
{
int arrayLength = reader.ReadInt32();
if (arrayLength == NullArrayMarker)
{
continue;
}
var buffer = new byte[arrayLength];
reader.Read(buffer, 0, i == indexOfPartiallyFilledBuffer ? lengthInPartiallyFilledBuffer : arrayLength);
result.ByteBuffers[i] = buffer;
}
result.StringSet = ConcurrentBigSet <StringId> .Deserialize(reader, () => reader.ReadStringId());
result.Count = reader.ReadInt32();
return(result);
}
/// <summary>
/// Writes <see cref="ConcurrentBigSet{TValue}"/>.
/// </summary>
public static void WriteTextSet <V>(TextWriter writer, ConcurrentBigSet <V> set, Func <V, string> valueToString)
{
Contract.Requires(writer != null);
Contract.Requires(set != null);
Contract.Requires(valueToString != null);
WriteTextList(writer, set.UnsafeGetList(), v => valueToString(v));
}
/// <summary>
/// Prevents further modification to graph and indicates that the graph
/// should switch to being read-optimized
/// </summary>
public void Seal()
{
using (m_globalLock.AcquireWriteLock())
{
m_state = MutableGraphState.Sealed;
// Remove the edge set which is only needed for deduplication when adding
m_edgeSet = null;
}
}
/// <summary>
/// Class constructor
/// </summary>
public MutableDirectedGraph()
{
m_edgeSetBuffer = new BigBuffer <LinkedEdgeSetItem>();
m_edgeSet = new ConcurrentBigSet <LinkedEdgeSetItem>(backingItemsBuffer: m_edgeSetBuffer);
// Create enough locks to ensure reasonable low contention even
// if all threads are accessing this data structure
m_locks = new ReadWriteLock[Environment.ProcessorCount * 4];
for (int i = 0; i < m_locks.Length; i++)
{
m_locks[i] = ReadWriteLock.Create();
}
}
/// <summary>
/// Private constructor. Please use CreateAndRegister factory method.
/// </summary>
private MountsTable(LoggingContext loggingContext, BuildXLContext context, MountPathExpander mountPathExpander = null)
{
m_parent = null;
m_loggingContext = loggingContext;
m_context = context;
MountPathExpander = mountPathExpander ?? new MountPathExpander(context.PathTable);
m_mountMapBuilder = new ConcurrentDictionary <AbsolutePath, IMount>();
m_mountsByName = new ConcurrentDictionary <string, IMount>(StringComparer.OrdinalIgnoreCase);
m_mountPathIdentifiersByMount = new ConcurrentDictionary <IMount, PathAtom>();
m_alternativeRoots = new ConcurrentDictionary <AbsolutePath, IMount>();
m_moduleDefinedMounts = new ConcurrentBigSet <IMount>();
}
private bool AddEdgeUnchecked(
NodeId source,
NodeId target,
BufferPointer <NodeEdgeListHeader> targetInEdges,
bool isLight,
bool bulkAddingTargetIncoming)
{
BufferPointer <NodeEdgeListHeader> outEdges = OutEdges.GetBufferPointer(source.Value);
var edgeSetItem = new LinkedEdgeSetItem(source, target, isLight);
int index = 0;
if (!bulkAddingTargetIncoming)
{
ConcurrentBigSet <LinkedEdgeSetItem> .GetAddOrUpdateResult result =
m_edgeSet.GetOrAdd(edgeSetItem);
if (result.IsFound)
{
// Edge already existed
return(false);
}
index = result.Index;
}
else
{
index = m_edgeSet.ReservedNextIndex(m_edgeSetBuffer);
m_edgeSetBuffer[index] = edgeSetItem;
}
// Update head index for in edges and out edges
int inEdgesNext = Interlocked.Exchange(ref targetInEdges.Buffer[targetInEdges.Index].FirstIndex, index);
int outEdgesNext = Interlocked.Exchange(ref outEdges.Buffer[outEdges.Index].FirstIndex, index);
var linkedEdgeSetItemPtr = m_edgeSetBuffer.GetBufferPointer(index);
// Update next pointers
linkedEdgeSetItemPtr.Buffer[linkedEdgeSetItemPtr.Index].NextTargetIncomingEdgeIndex = inEdgesNext;
linkedEdgeSetItemPtr.Buffer[linkedEdgeSetItemPtr.Index].NextSourceOutgoingEdgeIndex = outEdgesNext;
Interlocked.Increment(ref m_edgeCount);
// Update edge counts
targetInEdges.Buffer[targetInEdges.Index].InterlockedIncrementCount();
outEdges.Buffer[outEdges.Index].InterlockedIncrementCount();
return(true);
}
/// <nodoc />
internal StringTable(int initialCapacity = 0)
{
Contract.Requires(initialCapacity >= 0);
#if DebugStringTable
DebugRegisterStringTable(this);
#endif
CaseInsensitiveEqualityComparer = new CaseInsensitiveStringIdEqualityComparer(this);
OrdinalComparer = new OrdinalStringIdComparer(this);
m_stringSet = new ConcurrentBigSet <StringId>(capacity: initialCapacity);
// set up the initial buffer and consume the first byte so that StringId.Invalid's slot is consumed
m_byteBuffers[0] = new byte[BytesPerBuffer];
m_nextId = 1;
Empty = AddString(string.Empty);
}
/// <summary>
/// Constructor used for deserialized tables
/// </summary>
protected StringTable(SerializedState state)
{
Contract.Requires(state != null);
#if DebugStringTable
DebugRegisterStringTable(this);
#endif
CaseInsensitiveEqualityComparer = new CaseInsensitiveStringIdEqualityComparer(this);
OrdinalComparer = new OrdinalStringIdComparer(this);
m_byteBuffers = state.ByteBuffers;
m_count = state.Count;
m_nextId = state.NextId;
m_stringSet = state.StringSet;
Empty = AddString(string.Empty);
}
/// <summary>
/// Creates a content tracking set for the pip graph
/// </summary>
public ContentTrackingSet(PipGraph graph)
{
m_graph = graph;
m_contentSet = new ConcurrentBitArray(graph.ContentCount);
m_serviceContentSet = new ConcurrentBigSet <PipId>();
}
private static void TestOperationsHelper(bool parallel)
{
var set = new ConcurrentBigSet <int>();
int length = 100000;
int expectedAddedCount = length;
var indexedItems = new int[length];
// Verify that all bits start off with the default value (false in this case)
For(length, i =>
{
XAssert.IsFalse(set.Contains(i));
}, parallel);
XAssert.AreEqual(0, set.Count);
int addedCount = 0;
// Verify setting bits
For(length, i =>
{
if ((i % 4) == 3)
{
// Introduce some contention for setting the same key.
i = i - 1;
Interlocked.Decrement(ref expectedAddedCount);
}
if ((i % 7) == 3)
{
// Introduce some concurrent read-only operations
// in the parallel case
set.Contains(i - 2);
}
ConcurrentBigSet <int> .GetAddOrUpdateResult result =
((i % 5) != 3) ?
set.GetOrAdd(i) :
// Test heterogeneous add in set
set.GetOrAddItem(new StringIntItem(i.ToString()));
if (!result.IsFound)
{
Interlocked.Increment(ref addedCount);
}
XAssert.AreEqual(i, set[result.Index]);
// Save where the result claims the index of the item
// to verify it later.
indexedItems[result.Index] = i;
}, parallel);
XAssert.AreEqual(expectedAddedCount, addedCount);
XAssert.AreEqual(expectedAddedCount, set.Count);
For(length, i =>
{
XAssert.AreEqual((i % 4) != 3, set.Contains(i));
// Test heterogeneous search in set
XAssert.AreEqual((i % 4) != 3, set.ContainsItem(new StringIntItem(i.ToString())));
if (i < expectedAddedCount)
{
// Verify the order of items doesn't change.
XAssert.AreEqual(indexedItems[i], set[i]);
}
}, parallel);
}
private static bool TryConstructGraph(
ProjectGraph projectGraph,
GraphBuilderReporter reporter,
ConcurrentDictionary <ProjectInstance, Project> projectInstanceToProjectCache,
IReadOnlyCollection <string> entryPointTargets,
IReadOnlyCollection <IProjectPredictor> projectPredictorsForTesting,
bool allowProjectsWithoutTargetProtocol,
out ProjectGraphWithPredictions projectGraphWithPredictions,
out string failure)
{
Contract.Assert(projectGraph != null);
var projectNodes = new ProjectWithPredictions[projectGraph.ProjectNodes.Count];
var nodes = projectGraph.ProjectNodes.ToArray();
// Compute the list of targets to run per project
reporter.ReportMessage("Computing targets to execute for each project...");
// This dictionary should be exclusively read only at this point, and therefore thread safe
var targetsPerProject = projectGraph.GetTargetLists(entryPointTargets.ToArray());
// Bidirectional access from nodes with predictions to msbuild nodes in order to compute node references in the second pass
// TODO: revisit the structures, since the projects are known upfront we might be able to use lock-free structures
var nodeWithPredictionsToMsBuildNodes = new ConcurrentDictionary <ProjectWithPredictions, ProjectGraphNode>(Environment.ProcessorCount, projectNodes.Length);
var msBuildNodesToNodeWithPredictionIndex = new ConcurrentDictionary <ProjectGraphNode, ProjectWithPredictions>(Environment.ProcessorCount, projectNodes.Length);
reporter.ReportMessage("Statically predicting inputs and outputs...");
// Create the registered predictors and initialize the prediction executor
// The prediction executor potentially initializes third-party predictors, which may contain bugs. So let's be very defensive here
IReadOnlyCollection <IProjectPredictor> predictors;
try
{
predictors = projectPredictorsForTesting ?? ProjectPredictors.AllPredictors;
}
catch (Exception ex)
{
failure = $"Cannot create standard predictors. An unexpected error occurred. Please contact BuildPrediction project owners with this stack trace: {ex.ToString()}";
projectGraphWithPredictions = new ProjectGraphWithPredictions(new ProjectWithPredictions <string>[] { });
return(false);
}
// Using single-threaded prediction since we're parallelizing on project nodes instead.
var predictionExecutor = new ProjectPredictionExecutor(predictors, new ProjectPredictionOptions {
MaxDegreeOfParallelism = 1
});
// Each predictor may return unexpected/incorrect results and targets may not be able to be predicted. We put those failures here for post-processing.
ConcurrentQueue <(string predictorName, string failure)> predictionFailures = new ConcurrentQueue <(string, string)>();
var predictedTargetFailures = new ConcurrentQueue <string>();
// The predicted targets to execute (per project) go here
var computedTargets = new ConcurrentBigMap <ProjectGraphNode, PredictedTargetsToExecute>();
// When projects are allowed to not implement the target protocol, its references need default targets as a post-processing step
var pendingAddDefaultTargets = new ConcurrentBigSet <ProjectGraphNode>();
// First pass
// Predict all projects in the graph in parallel and populate ProjectNodes
Parallel.For(0, projectNodes.Length, (int i) => {
ProjectGraphNode msBuildNode = nodes[i];
ProjectInstance projectInstance = msBuildNode.ProjectInstance;
Project project = projectInstanceToProjectCache[projectInstance];
var outputFolderPredictions = new List <string>();
var predictionCollector = new MsBuildOutputPredictionCollector(outputFolderPredictions, predictionFailures);
try
{
// Again, be defensive when using arbitrary predictors
predictionExecutor.PredictInputsAndOutputs(project, predictionCollector);
}
catch (Exception ex)
{
predictionFailures.Enqueue((
"Unknown predictor",
$"Cannot run static predictor on project '{project.FullPath ?? "Unknown project"}'. An unexpected error occurred. Please contact BuildPrediction project owners with this stack trace: {ex.ToString()}"));
}
public async Task Serialization_Bug695424()
{
var st = new StringTable();
var pt = new PathTable(st);
ConcurrentBigSet <AbsolutePath> paths = new ConcurrentBigSet <AbsolutePath>();
List <string> pathStrings = new List <string>();
int max = 32769;
StringBuilder builder = new StringBuilder();
builder.Append(A("c", "i"));
var length = builder.Length;
for (int i = 0; i < 100; i++)
{
builder.Length = length;
builder.Append(i);
builder.Append('\\');
var jLength = builder.Length;
for (int j = 0; j < 10; j++)
{
builder.Length = jLength;
builder.Append('j');
builder.Append(j);
builder.Append('\\');
var kLenght = builder.Length;
for (int k = 0; k < 66; k++)
{
builder.Length = kLenght;
builder.Append('k');
builder.Append(k);
builder.Append('\\');
if (pt.Count < max)
{
paths.Add(AbsolutePath.Create(pt, builder.ToString()));
}
else
{
pathStrings.Add(builder.ToString());
}
}
}
}
PathTable pt2;
using (MemoryStream ms = new MemoryStream())
{
using (BuildXLWriter writer = new BuildXLWriter(true, ms, true, logStats: true))
{
pt.Serialize(writer);
}
ms.Position = 0;
using (BuildXLReader reader = new BuildXLReader(true, ms, true))
{
pt2 = await PathTable.DeserializeAsync(reader, Task.FromResult(st));
}
}
foreach (var pathString in pathStrings)
{
AbsolutePath.Create(pt2, pathString);
}
foreach (var path in paths.UnsafeGetList())
{
var pathString = path.ToString(pt).ToUpperInvariant();
var path2 = AbsolutePath.Create(pt2, pathString);
XAssert.AreEqual(path, path2);
}
}
private bool RemoveExtraneousFilesAndDirectories(
Func <string, bool> isPathInBuild,
List <string> pathsToScrub,
HashSet <string> blockedPaths,
HashSet <string> nonDeletableRootDirectories,
MountPathExpander mountPathExpander,
bool logRemovedFiles,
string statisticIdentifier)
{
int directoriesEncountered = 0;
int filesEncountered = 0;
int filesRemoved = 0;
int directoriesRemovedRecursively = 0;
using (var pm = PerformanceMeasurement.Start(
m_loggingContext,
statisticIdentifier,
// The start of the scrubbing is logged before calling this function, since there are two sources of scrubbing (regular scrubbing and shared opaque scrubbing)
// with particular messages
(_ => {}),
loggingContext =>
{
Tracing.Logger.Log.ScrubbingFinished(loggingContext, directoriesEncountered, filesEncountered, filesRemoved, directoriesRemovedRecursively);
Logger.Log.BulkStatistic(
loggingContext,
new Dictionary <string, long>
{
[I($"{Category}.DirectoriesEncountered")] = directoriesEncountered,
[I($"{Category}.FilesEncountered")] = filesEncountered,
[I($"{Category}.FilesRemoved")] = filesRemoved,
[I($"{Category}.DirectoriesRemovedRecursively")] = directoriesRemovedRecursively,
});
}))
using (var timer = new Timer(
o =>
{
// We don't have a good proxy for how much scrubbing is left. Instead we use the file counters to at least show progress
Tracing.Logger.Log.ScrubbingStatus(m_loggingContext, filesEncountered);
},
null,
dueTime: m_loggingConfiguration.GetTimerUpdatePeriodInMs(),
period: m_loggingConfiguration.GetTimerUpdatePeriodInMs()))
{
var deletableDirectoryCandidates = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
var nondeletableDirectories = new ConcurrentDictionary <string, bool>(StringComparer.OrdinalIgnoreCase);
var directoriesToEnumerate = new BlockingCollection <string>();
var allEnumeratedDirectories = new ConcurrentBigSet <string>();
foreach (var path in pathsToScrub)
{
SemanticPathInfo foundSemanticPathInfo;
if (blockedPaths.Contains(path))
{
continue;
}
if (ValidateDirectory(mountPathExpander, path, out foundSemanticPathInfo))
{
if (!isPathInBuild(path))
{
directoriesToEnumerate.Add(path);
allEnumeratedDirectories.Add(path);
}
else
{
nondeletableDirectories.TryAdd(path, true);
}
}
else
{
string mountName = "Invalid";
string mountPath = "Invalid";
if (mountPathExpander != null && foundSemanticPathInfo.IsValid)
{
mountName = foundSemanticPathInfo.RootName.ToString(mountPathExpander.PathTable.StringTable);
mountPath = foundSemanticPathInfo.Root.ToString(mountPathExpander.PathTable);
}
Tracing.Logger.Log.ScrubbingFailedBecauseDirectoryIsNotScrubbable(pm.LoggingContext, path, mountName, mountPath);
}
}
var cleaningThreads = new Thread[m_maxDegreeParallelism];
int pending = directoriesToEnumerate.Count;
if (directoriesToEnumerate.Count == 0)
{
directoriesToEnumerate.CompleteAdding();
}
for (int i = 0; i < m_maxDegreeParallelism; i++)
{
var t = new Thread(() =>
{
while (!directoriesToEnumerate.IsCompleted && !m_cancellationToken.IsCancellationRequested)
{
string currentDirectory;
if (directoriesToEnumerate.TryTake(out currentDirectory, Timeout.Infinite))
{
Interlocked.Increment(ref directoriesEncountered);
bool shouldDeleteCurrentDirectory = true;
var result = FileUtilities.EnumerateDirectoryEntries(
currentDirectory,
false,
(dir, fileName, attributes) =>
{
string fullPath = Path.Combine(dir, fileName);
// Skip specifically blocked paths.
if (blockedPaths.Contains(fullPath))
{
shouldDeleteCurrentDirectory = false;
return;
}
string realPath = fullPath;
// If this is a symlinked directory, get the final real target directory that it points to, so we can track duplicate work properly
var isDirectorySymlink = FileUtilities.IsDirectorySymlinkOrJunction(fullPath);
if (isDirectorySymlink &&
FileUtilities.TryGetLastReparsePointTargetInChain(handle: null, sourcePath: fullPath) is var maybeRealPath &&
maybeRealPath.Succeeded)
{
realPath = maybeRealPath.Result;
}
// If the current path is a directory, only follow it if we haven't followed it before (making sure we use the real path in case of symlinks)
var shouldEnumerateDirectory = (attributes & FileAttributes.Directory) == FileAttributes.Directory && !allEnumeratedDirectories.GetOrAdd(realPath).IsFound;
if (shouldEnumerateDirectory)
{
if (nondeletableDirectories.ContainsKey(fullPath))
{
shouldDeleteCurrentDirectory = false;
}
if (!isPathInBuild(fullPath))
{
// Current directory is not in the build, then recurse to its members.
Interlocked.Increment(ref pending);
directoriesToEnumerate.Add(fullPath);
if (!nonDeletableRootDirectories.Contains(fullPath))
{
// Current directory can be deleted, then it is a candidate to be deleted.
deletableDirectoryCandidates.TryAdd(fullPath, true);
}
else
{
// Current directory can't be deleted (e.g., the root of a mount), then don't delete it.
// However, note that we recurse to its members to find all extraneous directories and files.
shouldDeleteCurrentDirectory = false;
}
}
else
{
// Current directory is in the build, i.e., directory is an output directory.
// Stop recursive directory traversal because none of its members should be deleted.
shouldDeleteCurrentDirectory = false;
}
}
// On Mac directory symlinks are treated like any files, and so we must delete them if
// when they happen to be marked as shared opaque directory output.
//
// When 'fullPath' is a directory symlink the 'if' right above this 'if' will add it to
// 'deletableDirectoryCandidates'; there is code that deletes all directories added to this
// list but that code expects a real directory and so might fail to delete a directory symlink.
if (!shouldEnumerateDirectory || (isDirectorySymlink && OperatingSystemHelper.IsMacOS))
{
Interlocked.Increment(ref filesEncountered);
if (!isPathInBuild(fullPath))
{
// File is not in the build, delete it.
if (TryDeleteFile(pm.LoggingContext, fullPath, logRemovedFiles))
{
Interlocked.Increment(ref filesRemoved);
}
}
else
{
// File is in the build, then don't delete it, but mark the current directory that
// it should not be deleted.
shouldDeleteCurrentDirectory = false;
}
}
});
if (!result.Succeeded)
{
// Different trace levels based on result.
if (result.Status != EnumerateDirectoryStatus.SearchDirectoryNotFound)
{
Tracing.Logger.Log.ScrubbingFailedToEnumerateDirectory(
pm.LoggingContext,
currentDirectory,
result.Status.ToString());
}
}
if (!shouldDeleteCurrentDirectory)
{
// If directory should not be deleted, then all of its parents should not be deleted.
int index;
string preservedDirectory = currentDirectory;
bool added;
do
{
added = nondeletableDirectories.TryAdd(preservedDirectory, true);
}while (added &&
(index = preservedDirectory.LastIndexOf(Path.DirectorySeparatorChar)) != -1 &&
!string.IsNullOrEmpty(preservedDirectory = preservedDirectory.Substring(0, index)));
}
Interlocked.Decrement(ref pending);
}
if (Volatile.Read(ref pending) == 0)
{
directoriesToEnumerate.CompleteAdding();
}
}
});
internal PipRetryInfo()
{
m_pipsSucceedingAfterUserRetry = new ConcurrentBigSet <string>();
m_pipsFailingAfterLastUserRetry = new ConcurrentBigSet <string>();
}
