/// <nodoc />
public ApiServer(
IIpcProvider ipcProvider,
string ipcMonikerId,
FileContentManager fileContentManager,
PipExecutionContext context,
IServerConfig config,
PipTwoPhaseCache pipTwoPhaseCache,
Tracing.IExecutionLogTarget executionLog,
Tracing.BuildManifestGenerator buildManifestGenerator,
ServiceManager serviceManger,
bool verifyFileContentOnBuildManifestHashComputation)
{
Contract.Requires(ipcMonikerId != null);
Contract.Requires(fileContentManager != null);
Contract.Requires(context != null);
Contract.Requires(config != null);
Contract.Requires(pipTwoPhaseCache != null);
Contract.Requires(executionLog != null);
m_fileContentManager = fileContentManager;
m_server = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config);
m_context = context;
m_executionLog = executionLog;
m_buildManifestGenerator = buildManifestGenerator;
m_serviceManger = serviceManger;
m_pipTwoPhaseCache = pipTwoPhaseCache;
m_inMemoryBuildManifestStore = new ConcurrentBigMap <ContentHash, IReadOnlyList <ContentHash> >();
m_receivedStatistics = new ConcurrentBigMap <string, long>();
m_verifyFileContentOnBuildManifestHashComputation = verifyFileContentOnBuildManifestHashComputation;
}
public Harness(string testOutputDirectory)
{
Context = BuildXLContext.CreateInstanceForTesting();
var config = ConfigurationHelpers.GetDefaultForTesting(Context.PathTable, AbsolutePath.Create(Context.PathTable, System.IO.Path.Combine(testOutputDirectory, "config.dc")));
m_env = new DummyPipExecutionEnvironment(
CreateLoggingContextForTest(),
Context,
config,
subst: FileUtilities.TryGetSubstSourceAndTarget(testOutputDirectory, out var substSource, out var substTarget)
? (substSource, substTarget)
: default((string, string)?),
sandboxConnection: GetSandboxConnection());
var sealContentsCache = new ConcurrentBigMap <DirectoryArtifact, int[]>();
Table = Context.PathTable;
// Create the paths in the PathTable before creating the CachedFileSystemView
Path(a);
Path(b);
Path(c);
Path(d);
Path(e);
Path(f);
Path(g);
Path(h);
Path(i);
Path(j);
Path(k);
Path(l);
m_fileSystem = new PipFileSystemView();
m_fileSystem.Initialize(Table);
}
private WeakContentFingerprint GenerateSaltedWeakFingerprint(ContentHash hash) => new WeakContentFingerprint(FingerprintUtilities.Hash($"Hash: '{hash.ToHex()}' Salt: '{m_buildManifestHashCacheSalt}'")); // Changes to this string will invalidate all existing cache entries
/// <nodoc />
public ApiServer(
IIpcProvider ipcProvider,
string ipcMonikerId,
FileContentManager fileContentManager,
PipExecutionContext context,
IServerConfig config,
EngineCache engineCache,
Tracing.IExecutionLogTarget executionLog,
Tracing.BuildManifestGenerator buildManifestGenerator)
{
Contract.Requires(ipcMonikerId != null);
Contract.Requires(fileContentManager != null);
Contract.Requires(context != null);
Contract.Requires(config != null);
Contract.Requires(engineCache != null);
Contract.Requires(executionLog != null);
m_fileContentManager = fileContentManager;
m_server = ipcProvider.GetServer(ipcProvider.LoadAndRenderMoniker(ipcMonikerId), config);
m_context = context;
m_engineCache = engineCache;
m_executionLog = executionLog;
m_buildManifestGenerator = buildManifestGenerator;
m_inMemoryBuildManifestStore = new ConcurrentBigMap <ContentHash, ContentHash>();
m_buildManifestHashCacheSalt = string.IsNullOrEmpty(Utilities.Configuration.EngineEnvironmentSettings.BuildManifestHashCacheSalt)
? string.Empty
: Utilities.Configuration.EngineEnvironmentSettings.BuildManifestHashCacheSalt;
}
/// <summary>
/// Creates a new binary logger to write to the given stream
/// </summary>
/// <param name="logStream">the stream to write events to</param>
/// <param name="context">the context containing the path table</param>
/// <param name="logId">the log id to place in the header of execution used to verify with other data structures on load.</param>
/// <param name="lastStaticAbsolutePathIndex">the last absolute path guaranteed to be in the serialized form of the corresponding path table</param>
/// <param name="closeStreamOnDispose">specifies whether the stream is closed on disposal of the logger</param>
/// <param name="onEventWritten">optional callback after each event is written to underlying stream</param>
public BinaryLogger(Stream logStream, PipExecutionContext context, Guid logId, int lastStaticAbsolutePathIndex = int.MinValue, bool closeStreamOnDispose = true, Action onEventWritten = null)
{
m_context = context;
LogId = logId;
m_lastStaticAbsolutePathIndex = lastStaticAbsolutePathIndex;
m_logStreamWriter = new BuildXLWriter(debug: false, stream: logStream, leaveOpen: !closeStreamOnDispose, logStats: false);
m_watch = Stopwatch.StartNew();
m_capturedPaths = new ConcurrentBigMap <AbsolutePath, bool>();
m_capturedStrings = new ConcurrentBigMap <StringId, bool>();
m_capturedStrings.Add(StringId.Invalid, true);
m_writerPool = new ObjectPool <EventWriter>(
() => new EventWriter(this),
writer => { writer.Seek(0, SeekOrigin.Begin); return(writer); });
var logIdBytes = logId.ToByteArray();
Contract.Assert(logIdBytes.Length == LogIdByteLength);
logStream.Write(logIdBytes, 0, logIdBytes.Length);
LogStartTime(DateTime.UtcNow);
m_pendingEventsDrainingThread = new Thread(
() =>
{
foreach (PooledObjectWrapper <EventWriter> wrapper in m_pendingEvents.GetConsumingEnumerable())
{
var eventWriter = wrapper.Instance;
WriteEventData(eventWriter);
m_writerPool.PutInstance(wrapper);
onEventWritten?.Invoke();
}
});
m_pendingEventsDrainingThread.Start();
}
/// <summary>
/// Deserializes an instance of <see cref="PipProducers"/>.
/// </summary>
public static PipProducers Deserialize(BuildXLReader reader)
{
Contract.Requires(reader != null);
var producedPaths = new ConcurrentBigMap <PipStableId, HashSet <AbsolutePath> >();
var pipProducers = ConcurrentBigMap <AbsolutePath, PipStableId> .Deserialize(
reader,
() =>
{
var path = reader.ReadAbsolutePath();
var producer = reader.ReadPipStableId();
producedPaths.AddOrUpdate(
producer,
path,
(pip, pathToAdd) => new HashSet <AbsolutePath> {
pathToAdd
},
(pip, pathToAdd, existingSet) =>
{
existingSet.Add(pathToAdd);
return(existingSet);
});
return(new KeyValuePair <AbsolutePath, PipStableId>(path, producer));
});
return(new PipProducers(pipProducers, producedPaths));
}
private static Task <ConcurrentBigMap <AbsolutePath, AbsolutePath> > LoadAsync(Stream stream, string fileName, PathTable pathTable)
{
return(ExceptionUtilities.HandleRecoverableIOException(
async() =>
{
s_fileEnvelope.ReadHeader(stream);
using (BuildXLReader reader = new BuildXLReader(debug: false, stream: stream, leaveOpen: true))
{
var stringTable = await StringTable.DeserializeAsync(reader);
var loadedPathTable = await PathTable.DeserializeAsync(reader, Task.FromResult(stringTable));
var importedPathIndex = pathTable.Import(loadedPathTable);
var pathMapping = new ConcurrentBigMap <AbsolutePath, AbsolutePath>();
var count = reader.ReadInt32();
for (int i = 0; i < count; i++)
{
var loadedKey = reader.ReadAbsolutePath();
var loadedValue = reader.ReadAbsolutePath();
var key = importedPathIndex[loadedKey.Value.Index];
var value = importedPathIndex[loadedValue.Value.Index];
pathMapping[key] = value;
}
return pathMapping;
}
},
ex => { throw new BuildXLException(I($"Failed to read '{fileName}'"), ex); }));
}
/// <nodoc/>
public MaterializationDaemon(
IParser parser,
DaemonConfig daemonConfig,
MaterializationDaemonConfig materializationConfig,
IIpcProvider rpcProvider = null,
Client bxlClient = null)
: base(parser,
daemonConfig,
!string.IsNullOrWhiteSpace(materializationConfig.LogDir) ? new FileLogger(materializationConfig.LogDir, LogFileName, daemonConfig.Moniker, logVerbose: true, MaterializationDaemonLogPrefix) : daemonConfig.Logger,
rpcProvider,
bxlClient)
{
m_config = materializationConfig;
m_actionQueue = new ActionQueue(m_config.MaxDegreeOfParallelism);
m_materializationStatus = new ConcurrentBigMap <string, bool>();
m_counters = new CounterCollection <MaterializationDaemonCounter>();
m_macros = new Dictionary <string, string>
{
["$(build.nttree)"] = Environment.GetEnvironmentVariable("_NTTREE")
};
m_logger.Info($"MaterializationDaemon config: {JsonConvert.SerializeObject(m_config)}");
m_logger.Info($"Defined macros (count={m_macros.Count}):{Environment.NewLine}{string.Join(Environment.NewLine, m_macros.Select(kvp => $"{kvp.Key}={kvp.Value}"))}");
}
/// <summary>
/// Creates an instance of <see cref="SymlinkDefinitions" />.
/// </summary>
public SymlinkDefinitions(PathTable pathTable, ConcurrentBigMap <AbsolutePath, AbsolutePath> symlinkDefinitionMap)
{
Contract.Requires(pathTable != null);
Contract.Requires(symlinkDefinitionMap != null);
Contract.Requires(pathTable != null);
m_pathTable = pathTable;
m_symlinkDefinitionMap = symlinkDefinitionMap;
m_directorySymlinkContents = new Dictionary <AbsolutePath, List <AbsolutePath> >();
m_directoriesContainingSymlinks = new HashSet <HierarchicalNameId>();
foreach (var symlink in m_symlinkDefinitionMap.Keys)
{
var directory = symlink.GetParent(m_pathTable);
foreach (var pathId in m_pathTable.EnumerateHierarchyBottomUp(directory.Value))
{
if (!m_directoriesContainingSymlinks.Add(pathId))
{
break;
}
}
List <AbsolutePath> paths;
if (!m_directorySymlinkContents.TryGetValue(directory, out paths))
{
paths = new List <AbsolutePath>();
m_directorySymlinkContents.Add(directory, paths);
}
paths.Add(symlink);
}
}
/// <summary>
/// Deserializes an instance of <see cref="IncrementalSchedulingPathMapping{T}"/> from a reader.
/// </summary>
public static IncrementalSchedulingPathMapping <T> Deserialize(BuildXLReader reader, Func <BinaryReader, T> readValue)
{
Contract.Requires(reader != null);
var valueToPathApproximation = new ConcurrentBigMap <T, List <AbsolutePath> >();
var pathToValue = ConcurrentBigMap <AbsolutePath, HashSet <T> > .Deserialize(
reader,
() =>
{
var path = reader.ReadAbsolutePath();
var valueCount = reader.ReadInt32Compact();
var values = new HashSet <T>();
for (int i = 0; i < valueCount; i++)
{
T value = readValue(reader);
// Reconstruct the approximation map.
AddHelper(valueToPathApproximation, value, path);
values.Add(value);
}
return(new KeyValuePair <AbsolutePath, HashSet <T> >(path, values));
});
return(new IncrementalSchedulingPathMapping <T>(pathToValue, valueToPathApproximation));
}
private static void TestOperationsHelper(bool parallel)
{
var map = new ConcurrentBigMap <int, string>();
int length = 100000;
int expectedAddedCount = length;
var expectedValues = new string[length];
// Verify that all bits start off with the default value (false in this case)
For(length, i =>
{
XAssert.IsFalse(map.ContainsKey(i));
}, parallel);
XAssert.AreEqual(0, map.Count);
int addedCount = 0;
// Verify setting bits
For(length, i =>
{
if ((i % 4) == 3)
{
// Introduce some contention for setting the same key.
i = i - 1;
Interlocked.Decrement(ref expectedAddedCount);
}
if ((i % 7) == 3)
{
// Introduce some concurrent read-only operations
// in the parallel case
map.ContainsKey(i - 2);
}
var stringRepresentation = i.ToString();
if (map.TryAdd(i, stringRepresentation))
{
expectedValues[i] = stringRepresentation;
Interlocked.Increment(ref addedCount);
}
XAssert.AreEqual(stringRepresentation, map[i]);
}, parallel);
XAssert.AreEqual(expectedAddedCount, addedCount);
XAssert.AreEqual(expectedAddedCount, map.Count);
For(length, i =>
{
XAssert.AreEqual((i % 4) != 3, map.ContainsKey(i));
var expectedValue = expectedValues[i];
if (expectedValue != null)
{
XAssert.AreEqual(expectedValue, map[i]);
}
}, parallel);
}
private PipProducers(ConcurrentBigMap <AbsolutePath, PipStableId> pipProducers, ConcurrentBigMap <PipStableId, HashSet <AbsolutePath> > producedPaths)
{
Contract.Requires(pipProducers != null);
Contract.Requires(producedPaths != null);
m_pipProducers = pipProducers;
m_producedPaths = producedPaths;
}
/// <summary>
/// Create an instance of <see cref="PathMapSerializer"/>.
/// </summary>
public PathMapSerializer(string filePath)
{
Contract.Requires(!string.IsNullOrWhiteSpace(filePath));
m_filePath = filePath;
m_pathTable = new PathTable();
m_pathMapping = new ConcurrentBigMap <AbsolutePath, AbsolutePath>();
}
/// <summary>
/// Needs to take the flushing lock. Called only from <see cref="FlushAsync(OperationContext)"/>. Refactored
/// out for clarity.
/// </summary>
private void PerformFlush(OperationContext context)
{
_database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCacheFlushes].Increment();
using (_database.Counters[ContentLocationDatabaseCounters.CacheFlush].Start())
{
using (_exchangeLock.AcquireWriteLock())
{
_flushingCache = _cache;
_cache = new ConcurrentBigMap <ShortHash, ContentLocationEntry>();
}
if (_configuration.FlushSingleTransaction)
{
_database.PersistBatch(context, _flushingCache);
}
else
{
var actionBlock = new ActionBlockSlim <KeyValuePair <ShortHash, ContentLocationEntry> >(_configuration.FlushDegreeOfParallelism, kv =>
{
// Do not lock on GetLock here, as it will cause a deadlock with
// SetMachineExistenceAndUpdateDatabase. It is correct not do take any locks as well, because
// no Store can happen while flush is running.
_database.Persist(context, kv.Key, kv.Value);
});
foreach (var kv in _flushingCache)
{
actionBlock.Post(kv);
}
actionBlock.Complete();
actionBlock.CompletionAsync().Wait();
}
_database.Counters[ContentLocationDatabaseCounters.NumberOfPersistedEntries].Add(_flushingCache.Count);
if (_configuration.FlushPreservePercentInMemory > 0)
{
int targetFlushingSize = (int)(_flushingCache.Count * _configuration.FlushPreservePercentInMemory);
int removeAmount = _flushingCache.Count - targetFlushingSize;
foreach (var key in _flushingCache.Keys.Take(removeAmount))
{
_flushingCache.RemoveKey(key);
}
}
else
{
using (_exchangeLock.AcquireWriteLock())
{
_flushingCache = new ConcurrentBigMap <ShortHash, ContentLocationEntry>();
}
}
_database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCompletedCacheFlushes].Increment();
}
}
/// <summary>
/// Creates a new binary logger to write to the given stream
/// </summary>
/// <param name="logStream">the stream to write events to</param>
/// <param name="context">the context containing the path table</param>
/// <param name="logId">the log id to place in the header of execution used to verify with other data structures on load.</param>
/// <param name="lastStaticAbsolutePathIndex">the last absolute path guaranteed to be in the serialized form of the corresponding path table</param>
/// <param name="closeStreamOnDispose">specifies whether the stream is closed on disposal of the logger</param>
/// <param name="onEventWritten">optional callback after each event is written to underlying stream</param>
public BinaryLogger(Stream logStream, PipExecutionContext context, Guid logId, int lastStaticAbsolutePathIndex = int.MinValue, bool closeStreamOnDispose = true, Action onEventWritten = null)
{
m_context = context;
LogId = logId;
m_lastStaticAbsolutePathIndex = lastStaticAbsolutePathIndex;
m_logStreamWriter = new BuildXLWriter(debug: false, stream: logStream, leaveOpen: !closeStreamOnDispose, logStats: false);
m_watch = Stopwatch.StartNew();
m_capturedPaths = new ConcurrentBigMap <AbsolutePath, bool>();
m_capturedStrings = new ConcurrentBigMap <StringId, bool>();
m_capturedStrings.Add(StringId.Invalid, true);
m_writerPool = new ObjectPool <EventWriter>(
() => new EventWriter(this),
writer => { writer.Seek(0, SeekOrigin.Begin); return(writer); });
m_onEventWritten = onEventWritten;
var logIdBytes = logId.ToByteArray();
Contract.Assert(logIdBytes.Length == LogIdByteLength);
logStream.Write(logIdBytes, 0, logIdBytes.Length);
LogStartTime(DateTime.UtcNow);
m_pendingEventsDrainingThread = new Thread(
() =>
{
// Keeps trying to drain the event queue as long as new events can be added
// Adding events and completing the queue are properly synchronized already (a write lock
// is taken before completing), but there is the slim chance we finish draining the queue
// and before we check for m_completeAdding here again, an Add + CompleteAdding happen, which
// could leave unprocessed events in the queue. So we also check here whether the queue is not empty
while (!m_completeAdding || !m_pendingEvents.IsEmpty)
{
// Wait until at least one event is available for consumption
m_eventsAvailable.WaitOne();
// If new events can still be added, let's give writers the chance to add more events.
// This allows for more efficient draining, since therefore the thread synchronization tax is only paid
// once for many events. A 100ms lag is acceptable for events to be sitting in the queue. Profiler shows a 10x
// aggregated time gain by doing this. Some consideration for why picking this time:
// - A greater didn't seem to make a difference (e.g. 500ms was tried with equivalent perf results)
// - There are temporal constraints that need events (in particular, the build manifest one) to get serialized
// and sent to workers before the pip completion event (which does not go through the binary logger) happens. So
// we don't want xlg events to sit in the queue for too long.
if (!m_completeAdding)
{
Thread.Sleep(100);
}
// Drain all events available on the queue
while (m_pendingEvents.TryDequeue(out var action))
{
action.Run();
}
}
});
m_pendingEventsDrainingThread.Start();
}
/// <summary>
/// Class constructor
/// </summary>
public SealedDirectoryTable(PathTable pathTable)
{
IsReadOnly = false;
m_patchingState = null;
m_pathTable = pathTable;
m_seals = new ConcurrentBigMap <DirectoryArtifact, Node>();
m_pathToSealInfo = new ConcurrentBigMap <HierarchicalNameId, SealInfo>();
}
/// <summary>
/// Writes <see cref="ConcurrentBigMap{TKey, TValue}"/>.
/// </summary>
public static void WriteTextMap <K, V>(TextWriter writer, ConcurrentBigMap <K, V> map, Func <K, string> keyToString, Func <V, string> valueToString)
{
Contract.Requires(writer != null);
Contract.Requires(map != null);
Contract.Requires(keyToString != null);
Contract.Requires(valueToString != null);
WriteTextList(writer, map, kvp => I($"{keyToString(kvp.Key)}: {valueToString(kvp.Value)}"));
}
private PipGraphStaticFingerprints(
ConcurrentBigMap <ContentFingerprint, PipId> staticFingerprintsToPips,
ConcurrentBigMap <PipId, ContentFingerprint> pipsToStaticFingerprints)
{
Contract.Requires(staticFingerprintsToPips != null);
Contract.Requires(pipsToStaticFingerprints != null);
m_staticFingerprintsToPips = staticFingerprintsToPips;
m_pipsToStaticFingerprints = pipsToStaticFingerprints;
}
private IncrementalSchedulingPathMapping(
ConcurrentBigMap <AbsolutePath, HashSet <T> > pathToValue,
ConcurrentBigMap <T, List <AbsolutePath> > valueToPathApproximation)
{
Contract.Requires(pathToValue != null);
Contract.Requires(valueToPathApproximation != null);
m_pathToValue = pathToValue;
m_valueToPathApproximation = valueToPathApproximation;
}
/// <summary>
/// Creates a new file system view
/// </summary>
public FileSystemView(
PathTable pathTable,
IPipGraphFileSystemView pipGraph,
ILocalDiskFileSystemView localDiskContentStore,
bool inferNonExistenceBasedOnParentPathInRealFileSystem = true)
{
PathTable = pathTable;
PipGraph = pipGraph;
LocalDiskFileSystem = localDiskContentStore;
PathExistenceCache = new ConcurrentBigMap <AbsolutePath, FileSystemEntry>();
m_inferNonExistenceBasedOnParentPathInRealFileSystem = inferNonExistenceBasedOnParentPathInRealFileSystem;
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="oldPipGraph">Old pip graph.</param>
/// <param name="oldPipTable">Old pip table.</param>
/// <param name="graphBuilder">Pip graph builder to which to delegate all "add pip" operations.</param>
/// <param name="maxDegreeOfParallelism">Max concurrency for graph reloading (<see cref="PartiallyReloadGraph"/>).</param>
public PatchablePipGraph(
IReadonlyDirectedGraph oldPipGraph,
PipTable oldPipTable,
PipGraph.Builder graphBuilder,
int maxDegreeOfParallelism)
{
m_oldPipGraph = oldPipGraph;
m_oldPipTable = oldPipTable;
m_builder = graphBuilder;
m_maxDegreeOfParallelism = maxDegreeOfParallelism;
m_reloadedSealDirectories = new ConcurrentBigMap <long, DirectoryArtifact>();
m_reloadedServicePips = new ConcurrentBigMap <long, PipId>();
m_pipIdMap = new ConcurrentBigMap <PipId, PipId>();
}
public PipQueueTestExecutionEnvironment(BuildXLContext context, IConfiguration configuration, PipTable pipTable, string tempDirectory, ISandboxConnection SandboxConnection = null)
{
Contract.Requires(context != null);
Contract.Requires(configuration != null);
Context = context;
LoggingContext = CreateLoggingContextForTest();
Configuration = configuration;
FileContentTable = FileContentTable.CreateNew();
ContentFingerprinter = new PipContentFingerprinter(
context.PathTable,
artifact => State.FileContentManager.GetInputContent(artifact).FileContentInfo,
ExtraFingerprintSalts.Default(),
pathExpander: PathExpander);
PipTable = pipTable;
PipFragmentRenderer = this.CreatePipFragmentRenderer();
IpcProvider = IpcFactory.GetProvider();
var tracker = FileChangeTracker.CreateDisabledTracker(LoggingContext);
Cache = InMemoryCacheFactory.Create();
LocalDiskContentStore = new LocalDiskContentStore(LoggingContext, context.PathTable, FileContentTable, tracker);
m_sandboxConnectionKext = SandboxConnection;
m_expectedWrittenContent = new ConcurrentDictionary <FileArtifact, ContentHash>();
m_wellKnownFiles = new ConcurrentDictionary <FileArtifact, ContentHash>();
m_producers = new ConcurrentDictionary <FileArtifact, Pip>();
m_filesystemView = new TestPipGraphFilesystemView(Context.PathTable);
var fileSystemView = new FileSystemView(Context.PathTable, m_filesystemView, LocalDiskContentStore);
TempCleaner = new TestMoveDeleteCleaner(tempDirectory);
State = new PipExecutionState(
configuration,
cache: new PipTwoPhaseCache(LoggingContext, Cache, context, PathExpander),
unsafeConfiguration: configuration.Sandbox.UnsafeSandboxConfiguration,
preserveOutputsSalt: ContentHashingUtilities.CreateRandom(),
fileAccessWhitelist: FileAccessWhitelist,
directoryMembershipFingerprinter: this,
pathExpander: PathExpander,
executionLog: null,
fileSystemView: fileSystemView,
fileContentManager: new FileContentManager(this, new NullOperationTracker()),
directoryMembershipFinterprinterRuleSet: null);
m_sealContentsById = new ConcurrentBigMap <DirectoryArtifact, int[]>();
ProcessInContainerManager = new ProcessInContainerManager(LoggingContext, context.PathTable);
}
/// <nodoc />
public void UnsafeClear()
{
// Order is important here, inverse order could cause deadlock.
using (_flushMutex.AcquireSemaphore())
using (_exchangeLock.AcquireWriteLock())
{
if (_cache.Count != 0)
{
// Nothing guarantees that some number of updates couldn't have happened in between the last flush
// and this reset, because acquisition of the write lock happens after the flush lock. The only way
// to deal with this situation is to force a flush before this assignment and after locks have been
// acquired. However, this isn't required by our code right now; although it is a simple change.
_cache = new ConcurrentBigMap <ShortHash, ContentLocationEntry>();
}
}
}
/// <summary>
/// Creates a new binary logger to write to the given stream
/// </summary>
/// <param name="logStream">the stream to write events to</param>
/// <param name="context">the context containing the path table</param>
/// <param name="logId">the log id to place in the header of execution used to verify with other data structures on load.</param>
/// <param name="lastStaticAbsolutePathIndex">the last absolute path guaranteed to be in the serialized form of the corresponding path table</param>
/// <param name="closeStreamOnDispose">specifies whether the stream is closed on disposal of the logger</param>
/// <param name="onEventWritten">optional callback after each event is written to underlying stream</param>
public BinaryLogger(Stream logStream, PipExecutionContext context, Guid logId, int lastStaticAbsolutePathIndex = int.MinValue, bool closeStreamOnDispose = true, Action onEventWritten = null)
{
m_context = context;
LogId = logId;
m_lastStaticAbsolutePathIndex = lastStaticAbsolutePathIndex;
m_logStreamWriter = new BuildXLWriter(debug: false, stream: logStream, leaveOpen: !closeStreamOnDispose, logStats: false);
m_watch = Stopwatch.StartNew();
m_capturedPaths = new ConcurrentBigMap <AbsolutePath, bool>();
m_capturedStrings = new ConcurrentBigMap <StringId, bool>();
m_capturedStrings.Add(StringId.Invalid, true);
m_writerPool = new ObjectPool <EventWriter>(
() => new EventWriter(this),
writer => { writer.Seek(0, SeekOrigin.Begin); return(writer); });
var logIdBytes = logId.ToByteArray();
Contract.Assert(logIdBytes.Length == LogIdByteLength);
logStream.Write(logIdBytes, 0, logIdBytes.Length);
LogStartTime(DateTime.UtcNow);
m_pendingEventsDrainingThread = new Thread(
() =>
{
try
{
foreach (PooledObjectWrapper <EventWriter> wrapper in m_pendingEvents.GetConsumingEnumerable())
{
var eventWriter = wrapper.Instance;
WriteEventData(eventWriter);
m_writerPool.PutInstance(wrapper);
onEventWritten?.Invoke();
}
}
catch (InvalidOperationException)
{
// InvalidOperationException is thrown when calling Take() for a marked-as-completed blocking collection.
// However, GetConsumingEnumerable throws an InvalidOperationException here, which is unusual.
// In further investigations, we discovered that it might throw one if the collection in BlockingCollection
// is passed in the constructor and we externally modify that collection outside of BlockingCollection.
// Even we do not do that, we rarely have InvalidOperationException here, which is a NetCore bug.
// We reported the bug; but for now, we swallow that exception and we treat it as a signal for completion.
return;
}
});
m_pendingEventsDrainingThread.Start();
}
/// <summary>
/// Class constructor
/// </summary>
public PipExecutionState(
IConfiguration configuration,
LoggingContext loggingContext,
PipTwoPhaseCache cache,
FileAccessWhitelist fileAccessWhitelist,
IDirectoryMembershipFingerprinter directoryMembershipFingerprinter,
SemanticPathExpander pathExpander,
IExecutionLogTarget executionLog,
DirectoryMembershipFingerprinterRuleSet directoryMembershipFinterprinterRuleSet,
FileContentManager fileContentManager,
IUnsafeSandboxConfiguration unsafeConfiguration,
PreserveOutputsInfo preserveOutputsSalt,
FileSystemView fileSystemView,
bool lazyDeletionOfSharedOpaqueOutputsEnabled,
ServiceManager serviceManager = null)
{
Contract.Requires(fileContentManager != null);
Contract.Requires(directoryMembershipFingerprinter != null);
Contract.Requires(pathExpander != null);
Cache = cache;
m_fileAccessWhitelist = fileAccessWhitelist;
DirectoryMembershipFingerprinter = directoryMembershipFingerprinter;
ResourceManager = new ProcessResourceManager(loggingContext);
m_pathExpander = new FileContentManagerSemanticPathExpander(fileContentManager, pathExpander);
ExecutionLog = executionLog;
m_rootModuleConfiguration = configuration;
m_directoryMembershipFingerprinterRuleSet = directoryMembershipFinterprinterRuleSet;
PathExistenceCache = new ConcurrentBigMap <AbsolutePath, PathExistence>();
FileContentManager = fileContentManager;
ServiceManager = serviceManager ?? ServiceManager.Default;
PipEnvironment = new PipEnvironment(loggingContext);
FileSystemView = fileSystemView;
m_unsafeConfiguration = unsafeConfiguration;
m_preserveOutputsSalt = preserveOutputsSalt;
LazyDeletionOfSharedOpaqueOutputsEnabled = lazyDeletionOfSharedOpaqueOutputsEnabled;
if (fileSystemView != null)
{
fileContentManager.SetLocalDiskFileSystemExistenceView(fileSystemView);
}
}
private void WriteFileDependencies(string property, ConcurrentBigMap <AbsolutePath, CompactSet <PipId> > fileToConsumerMap, JsonWriter writer)
{
writer.WritePropertyName(property);
{
writer.WriteStartArray();
foreach (var fileEntry in fileToConsumerMap)
{
var file = fileEntry.Key;
var consumers = fileEntry.Value;
var path = ToDisplayFilePath(file);
if (path != null)
{
var fileArtifactContentDecidedEventData = m_fileContentMap[file];
// Print : { "Path": "value", "Type": "source/output", "FailPipReferenceCount": "count", "TotalPipsReferenceCount": "count", "Hash": "value", [pip array]}
writer.WriteStartObject();
WritePropertyAndValue(writer, "Path", path);
var typeOfFileArtifact = fileArtifactContentDecidedEventData.FileArtifact.IsSourceFile ? "SourceFile" : "OutputFile";
WritePropertyAndValue(writer, "Type", typeOfFileArtifact);
WritePropertyAndValue(writer, "FailPipReferenceCount", GetCountOfFailPips(consumers).ToString());
WritePropertyAndValue(writer, "TotalPipsReferenceCount", consumers.Count.ToString());
WritePropertyAndValue(writer, "Hash", fileArtifactContentDecidedEventData.FileContentInfo.Hash.ToString());
writer.WritePropertyName("Pips", true);
{
writer.WriteStartArray();
foreach (var consumer in consumers)
{
writer.WriteValue(ToDisplayString(consumer));
}
writer.WriteEndArray();
}
writer.WriteEndObject();
writer.WriteWhitespace(Environment.NewLine);
}
}
writer.WriteEndArray();
}
}
/// <summary>
/// Helper to add a key-value to the set of map
/// </summary>
private static void AddHelper <TKey, TValue, TCollection>(
ConcurrentBigMap <TKey, TCollection> map,
TKey key,
TValue value)
where TCollection : ICollection <TValue>, new()
{
map.AddOrUpdate(
key,
value,
(k, v) =>
{
var s = new TCollection {
v
};
return(s);
},
(k, v, s) =>
{
s.Add(v);
return(s);
});
}
/// <summary>
/// Deserialize symlink definitions serialized using <see cref="Serialize(BuildXLWriter, SymlinkDefinitions)"/>
/// </summary>
public static Possible <SymlinkDefinitions> Deserialize(LoggingContext loggingContext, PathTable pathTable, BuildXLReader reader)
{
try
{
bool isNull = reader.ReadBoolean();
if (isNull)
{
return((SymlinkDefinitions)null);
}
var pathMap = ConcurrentBigMap <AbsolutePath, AbsolutePath> .Deserialize(
reader,
() => new KeyValuePair <AbsolutePath, AbsolutePath>(
key : reader.ReadAbsolutePath(),
value : reader.ReadAbsolutePath()));
return(new SymlinkDefinitions(pathTable, pathMap));
}
catch (Exception ex)
{
Logger.Log.FailedLoadSymlinkFile(loggingContext, ex.GetLogEventMessage());
return(new Failure <string>("Failed loading symlink definition file"));
}
}
/// <summary>
/// Removes cache entries. This method is not thread safe and for testing only.
/// </summary>
public void Forget()
{
m_entries = new ConcurrentBigMap <WeakContentFingerprint, Node>();
}
public void TestConcurrentBigMapOperations()
{
var map = new ConcurrentBigMap <int, string>();
XAssert.IsTrue(map.TryAdd(0, "value"));
XAssert.IsFalse(map.TryAdd(0, "not added value"));
XAssert.AreEqual("value", map[0]);
map[0] = "newValue";
map[1] = "value1";
var value0 = "newValue";
XAssert.AreEqual(value0, map[0]);
XAssert.IsTrue(map.ContainsKey(0));
XAssert.IsTrue(map.ContainsKey(1));
XAssert.IsFalse(map.ContainsKey(12));
XAssert.AreEqual(2, map.Count);
// Test TryGetValue
string value1;
XAssert.IsTrue(map.TryGetValue(1, out value1));
XAssert.AreEqual("value1", value1);
string value31;
XAssert.IsFalse(map.TryGetValue(31, out value31));
// Test update
XAssert.IsFalse(map.TryUpdate(1, "notUpdatedValue1", "notActualValue1"));
XAssert.AreEqual("value1", map[1]);
XAssert.IsTrue(map.TryUpdate(1, "updatedValue1", "value1"));
value1 = map[1];
XAssert.AreEqual("updatedValue1", value1);
// Test remove
int beforeFailedRemoveCount = map.Count;
string value23;
XAssert.IsFalse(map.TryRemove(23, out value23));
XAssert.AreEqual(beforeFailedRemoveCount, map.Count);
map.Add(23, "value23");
XAssert.AreEqual(beforeFailedRemoveCount + 1, map.Count);
XAssert.IsTrue(map.TryRemove(23, out value23));
XAssert.AreEqual("value23", value23);
XAssert.AreEqual(beforeFailedRemoveCount, map.Count);
Assert.Equal(new int[] { 0, 1 }, map.Keys.ToArray());
Assert.Equal(new string[] { value0, value1 }, map.Values.ToArray());
XAssert.AreEqual(2, map.Count);
string addedData = "added data";
string notAddedData = "not added data";
var result = map.GetOrAdd(2, addedData, (key, data0) => data0);
XAssert.IsFalse(result.IsFound);
XAssert.AreEqual(addedData, result.Item.Value);
XAssert.AreEqual(addedData, map[2]);
// Ensure entry is not updated for get or add
result = map.GetOrAdd(2, notAddedData, (key, data0) => data0);
XAssert.IsTrue(result.IsFound);
XAssert.AreEqual(addedData, result.Item.Value);
XAssert.AreEqual(addedData, map[2]);
Func <int, string, string, string> updateFunction =
(key, data0, currentValue) => "updated " + currentValue;
var updatedData = updateFunction(2, notAddedData, addedData);
result = map.AddOrUpdate(2, notAddedData, (key, data0) => data0, updateFunction);
XAssert.IsTrue(result.IsFound);
XAssert.AreEqual(addedData, result.OldItem.Value);
XAssert.AreEqual(updatedData, result.Item.Value);
XAssert.AreEqual(updatedData, map[2]);
result = map.AddOrUpdate(3, addedData, (key, data0) => data0, updateFunction);
XAssert.IsFalse(result.IsFound);
XAssert.AreEqual(addedData, result.Item.Value);
XAssert.AreEqual(addedData, map[3]);
TestOperationsHelper(parallel: false);
}
/// <summary>
/// Needs to take the flushing lock. Called only from <see cref="FlushAsync(OperationContext)"/>. Refactored
/// out for clarity.
/// </summary>
private CounterCollection <FlushableCacheCounters> PerformFlush(OperationContext context)
{
_database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCacheFlushes].Increment();
var counters = new CounterCollection <FlushableCacheCounters>();
using (_database.Counters[ContentLocationDatabaseCounters.CacheFlush].Start())
{
using (_exchangeLock.AcquireWriteLock())
{
_flushingCache = _cache;
_cache = new ConcurrentBigMap <ShortHash, ContentLocationEntry>();
}
using (counters[FlushableCacheCounters.FlushingTime].Start()) {
var threads = _configuration.FlushDegreeOfParallelism;
if (threads <= 0)
{
threads = Environment.ProcessorCount;
}
if (_configuration.FlushSingleTransaction)
{
if (_configuration.FlushDegreeOfParallelism == 1 || _flushingCache.Count <= _configuration.FlushTransactionSize)
{
_database.PersistBatch(context, _flushingCache);
}
else
{
var actionBlock = new ActionBlockSlim <IEnumerable <KeyValuePair <ShortHash, ContentLocationEntry> > >(threads, kvs =>
{
_database.PersistBatch(context, kvs);
});
foreach (var kvs in _flushingCache.GetPages(_configuration.FlushTransactionSize))
{
actionBlock.Post(kvs);
}
actionBlock.Complete();
actionBlock.CompletionAsync().Wait();
}
}
else
{
var actionBlock = new ActionBlockSlim <KeyValuePair <ShortHash, ContentLocationEntry> >(threads, kv =>
{
// Do not lock on GetLock here, as it will cause a deadlock with
// SetMachineExistenceAndUpdateDatabase. It is correct not do take any locks as well, because
// no Store can happen while flush is running.
_database.Persist(context, kv.Key, kv.Value);
});
foreach (var kv in _flushingCache)
{
actionBlock.Post(kv);
}
actionBlock.Complete();
actionBlock.CompletionAsync().Wait();
}
}
counters[FlushableCacheCounters.Persisted].Add(_flushingCache.Count);
_database.Counters[ContentLocationDatabaseCounters.NumberOfPersistedEntries].Add(_flushingCache.Count);
using (counters[FlushableCacheCounters.CleanupTime].Start())
{
if (_configuration.FlushPreservePercentInMemory > 0)
{
int targetFlushingSize = (int)(_flushingCache.Count * _configuration.FlushPreservePercentInMemory);
int removeAmount = _flushingCache.Count - targetFlushingSize;
foreach (var key in _flushingCache.Keys.Take(removeAmount))
{
_flushingCache.RemoveKey(key);
}
}
else
{
using (_exchangeLock.AcquireWriteLock())
{
_flushingCache = new ConcurrentBigMap <ShortHash, ContentLocationEntry>();
}
}
}
counters[FlushableCacheCounters.Leftover].Add(_flushingCache.Count);
_database.Counters[ContentLocationDatabaseCounters.TotalNumberOfCompletedCacheFlushes].Increment();
}
counters[FlushableCacheCounters.Growth].Add(_cache.Count);
return(counters);
}
