/// <summary>
/// Checks whether an exception is a checksum mismatch exception
/// </summary>
public bool IsChecksumMismatchException(Exception ex)
{
if (ex is RpcException && ex.Message.Contains(ChecksumMismatchExceptionMessage))
{
Counters.IncrementCounter(DistributionCounter.ClientChecksumMismatchCount);
return(true);
}
return(false);
}
private void StoreBuildManifestHashes(ContentHash hash, IReadOnlyList <ContentHash> manifestHashes)
{
using (ManifestCounters.StartStopwatch(BuildManifestCounters.InternalHashToHashCacheWriteDuration))
{
foreach (var manifestHash in manifestHashes)
{
ManifestCounters.IncrementCounter(BuildManifestCounters.InternalHashToHashCacheWriteCount);
m_pipTwoPhaseCache.TryStoreBuildManifestHash(hash, manifestHash);
}
}
}
/// <summary>
/// Stores cache descriptor metadata for pip
/// </summary>
public virtual async Task <Possible <ContentHash> > TryStoreMetadataAsync(PipCacheDescriptorV2Metadata metadata)
{
BoxRef <long> metadataSize = new BoxRef <long>();
var result = await ArtifactContentCache.TrySerializeAndStoreContent(metadata.ToEntry(), metadataSize);
if (result.Succeeded)
{
Counters.IncrementCounter(PipCachingCounter.StoredMetadataCount);
Counters.AddToCounter(PipCachingCounter.StoredMetadataSize, metadataSize.Value);
}
return(result);
}
private async Task StoreBuildManifestHashAsync(ContentHash hash, ContentHash manifestHash)
{
using (ManifestCounters.StartStopwatch(BuildManifestCounters.InternalHashToHashCacheWriteDuration))
{
ManifestCounters.IncrementCounter(BuildManifestCounters.InternalHashToHashCacheWriteCount);
(var wf, var sf) = GetBuildManifestHashKey(hash);
var result = await m_engineCache.TwoPhaseFingerprintStore.TryPublishCacheEntryAsync(wf, hash, sf, new CacheEntry(manifestHash, "", ArrayView <ContentHash> .Empty));
if (!result.Succeeded)
{
Tracing.Logger.Log.ApiServerStoreBuildManifestHashToCacheFailed(m_loggingContext, hash.Serialize(), manifestHash.Serialize(), result.Failure.DescribeIncludingInnerFailures());
}
}
}
/// <summary>
/// Tries to create copy-on-write by calling <see cref="IFileUtilities.CloneFile(string, string, bool)"/>.
/// </summary>
/// <param name="source">Source of copy.</param>
/// <param name="destination">Destination path.</param>
/// <param name="followSymlink">Flag indicating whether to follow source symlink or not.</param>
public static Possible <Unit> TryCreateCopyOnWrite(string source, string destination, bool followSymlink)
{
try
{
using (Counters?.StartStopwatch(StorageCounters.CopyOnWriteDuration))
{
Counters?.IncrementCounter(StorageCounters.CopyOnWriteCount);
s_fileUtilities.CloneFile(source, destination, followSymlink);
Counters?.IncrementCounter(StorageCounters.SuccessfulCopyOnWriteCount);
return(Unit.Void);
}
}
catch (NativeWin32Exception ex)
{
return(NativeFailure.CreateFromException(ex));
}
}
/// <inheritdoc />
public async Task <AddDebugEntryResult> AddFileAsync(SymbolFile symbolFile)
{
Contract.Requires(symbolFile.IsIndexed, "File has not been indexed.");
m_counters.IncrementCounter(SymbolClientCounter.NumAddFileRequests);
if (symbolFile.DebugEntries.Count == 0)
{
// If there are no debug entries, ask bxl to log a message and return early.
Analysis.IgnoreResult(await m_apiClient.LogMessage(I($"File '{symbolFile.FullFilePath}' does not contain symbols and will not be added to '{RequestName}'."), isWarning: false));
m_counters.IncrementCounter(SymbolClientCounter.NumFilesWithoutDebugEntries);
return(AddDebugEntryResult.NoSymbolData);
}
m_logger.Verbose($"Queued file '{symbolFile}'");
var batchedFile = new BatchedSymbolFile(symbolFile);
m_nagleQueue.Enqueue(batchedFile);
return(await batchedFile.ResultTaskSource.Task);
}
internal void UpdatePipRetryInfo(ProcessRunnablePip processRunnable, ExecutionResult executionResult, CounterCollection <PipExecutorCounter> pipExecutionCounters)
{
if (executionResult.HasUserRetries)
{
if (executionResult.Result == PipResultStatus.Succeeded)
{
pipExecutionCounters.IncrementCounter(PipExecutorCounter.ProcessUserRetriesSucceededPipsCount);
if (m_pipsSucceedingAfterUserRetry.Count < MaxListOfPipIdsForTelemetry)
{
m_pipsSucceedingAfterUserRetry.Add(processRunnable.Process.FormattedSemiStableHash);
}
}
else if (executionResult.Result == PipResultStatus.Failed)
{
pipExecutionCounters.IncrementCounter(PipExecutorCounter.ProcessUserRetriesFailedPipsCount);
if (m_pipsFailingAfterLastUserRetry.Count < MaxListOfPipIdsForTelemetry)
{
m_pipsFailingAfterLastUserRetry.Add(processRunnable.Process.FormattedSemiStableHash);
}
}
}
}
/// <summary>
/// Entry point for an I/O task which creates a process.
/// </summary>
/// <remarks>
/// This is a separate function and not inlined as an anonymous delegate, as VS seems to have trouble with those when
/// measuring code coverage
/// </remarks>
private static ISandboxedProcess ProcessStart(object state)
{
Counters.IncrementCounter(SandboxedProcessCounters.SandboxedProcessCount);
var stateTuple = (Tuple <SandboxedProcessInfo, bool>)state;
SandboxedProcessInfo info = stateTuple.Item1;
ISandboxedProcess result = null;
try
{
result = Create(info, forceSandboxing: stateTuple.Item2);
result.Start(); // this can take a while; performs I/O
}
catch
{
result?.Dispose();
throw;
}
return(result);
}
/// <summary>
/// Check if an exception is trancient and is worth retrying.
/// </summary>
/// <param name="ex">The exception thrown by Bond code.</param>
/// <param name="verifierCounters">counters use to track transient error type occurrences</param>
/// <returns>True is the retry makes sense; false otherwise.</returns>
internal static bool IsTransientBondException(Exception ex, CounterCollection <DistributionCounter> verifierCounters)
{
// Unwrap if its an aggregate exception
AggregateException aggregateException = ex as AggregateException;
if (aggregateException != null && aggregateException.InnerExceptions.Count == 1)
{
ex = aggregateException.InnerExceptions[0];
}
// SocketException is thrown when something goes wrong in the TCP channel.
// InvalidOperationException is thrown when a previous call has closed the connection.
// In the second case a retry will reopen the connection.
// BondTcpClient.OnConnectionComplete throws IOException on failure instead of SocketException.
// Sometimes we get TimeoutException
if (ex is SocketException || ex is InvalidOperationException || ex is IOException || ex is TimeoutException)
{
return(true);
}
#if !DISABLE_FEATURE_BOND_RPC
// 'No such method' probably means the buffer was corrupted somehow
// Retry and see if it succeeds next time
if (ex is Microsoft.Bond.RpcException && ex.Message.Contains("No such method"))
{
verifierCounters.IncrementCounter(DistributionCounter.ClientNoSuchMethodErrorCount);
return(true);
}
#endif
// If the connection gets broken while waiting for response, we can see a bug in the NetlibConnection
// where it tries to initialize an array section with null. It throws an unhandled and unwrapped
// ArgumentNullException.
if ((ex is ArgumentNullException || ex is NullReferenceException) && ex.StackTrace.Contains("Netlib"))
{
return(true);
}
return(false);
}
/// <inheritdoc />
public async Task <AddDebugEntryResult> AddFileAsync(SymbolFile symbolFile)
{
Contract.Requires(symbolFile.IsIndexed, "File has not been indexed.");
m_counters.IncrementCounter(SymbolClientCounter.NumAddFileRequests);
if (symbolFile.DebugEntries.Count == 0)
{
// If there are no debug entries, ask bxl to log a message and return early.
Analysis.IgnoreResult(await m_apiClient.LogMessage(I($"File '{symbolFile.FullFilePath}' does not contain symbols and will not be added to '{RequestName}'."), isWarning: false));
m_counters.IncrementCounter(SymbolClientCounter.NumFilesWithoutDebugEntries);
return(AddDebugEntryResult.NoSymbolData);
}
await EnsureRequestIdAndDomainIdAreInitalizedAsync();
List <DebugEntry> result;
using (m_counters.StartStopwatch(SymbolClientCounter.TotalAssociateTime))
{
try
{
result = await m_symbolClient.CreateRequestDebugEntriesAsync(
RequestId,
symbolFile.DebugEntries.Select(e => CreateDebugEntry(e, m_domainId)),
// First, we create debug entries with ThrowIfExists behavior not to silence the collision errors.
DebugEntryCreateBehavior.ThrowIfExists,
CancellationToken);
}
catch (DebugEntryExistsException)
{
string message = $"[SymbolDaemon] File: '{symbolFile.FullFilePath}' caused collision. " +
(m_debugEntryCreateBehavior == DebugEntryCreateBehavior.ThrowIfExists
? string.Empty
: $"SymbolDaemon will retry creating debug entry with {m_debugEntryCreateBehavior} behavior");
if (m_debugEntryCreateBehavior == DebugEntryCreateBehavior.ThrowIfExists)
{
// Log an error message in SymbolDaemon log file
m_logger.Error(message);
throw new DebugEntryExistsException(message);
}
// Log a message in SymbolDaemon log file
m_logger.Verbose(message);
result = await m_symbolClient.CreateRequestDebugEntriesAsync(
RequestId,
symbolFile.DebugEntries.Select(e => CreateDebugEntry(e, m_domainId)),
m_debugEntryCreateBehavior,
CancellationToken);
}
}
var entriesWithMissingBlobs = result.Where(e => e.Status == DebugEntryStatus.BlobMissing).ToList();
if (entriesWithMissingBlobs.Count > 0)
{
// All the entries are based on the same file, so we need to call upload only once.
// make sure that the file is on disk (it might not be on disk if we got DebugEntries from cache/metadata file)
var file = await symbolFile.EnsureMaterializedAsync();
BlobIdentifierWithBlocks uploadResult;
using (m_counters.StartStopwatch(SymbolClientCounter.TotalUploadTime))
{
uploadResult = await m_symbolClient.UploadFileAsync(
m_domainId,
// uploading to the location set by the symbol service
entriesWithMissingBlobs[0].BlobUri,
RequestId,
symbolFile.FullFilePath,
entriesWithMissingBlobs[0].BlobIdentifier,
CancellationToken);
}
m_counters.IncrementCounter(SymbolClientCounter.NumFilesUploaded);
m_counters.AddToCounter(SymbolClientCounter.TotalUploadSize, file.Length);
m_logger.Info($"File: '{symbolFile.FullFilePath}' -- upload result: {uploadResult}");
entriesWithMissingBlobs.ForEach(entry => entry.BlobDetails = uploadResult);
using (m_counters.StartStopwatch(SymbolClientCounter.TotalAssociateAfterUploadTime))
{
entriesWithMissingBlobs = await m_symbolClient.CreateRequestDebugEntriesAsync(
RequestId,
entriesWithMissingBlobs,
m_debugEntryCreateBehavior,
CancellationToken);
}
Contract.Assert(entriesWithMissingBlobs.All(e => e.Status != DebugEntryStatus.BlobMissing), "Entries with non-success code are present.");
return(AddDebugEntryResult.UploadedAndAssociated);
}
m_counters.IncrementCounter(SymbolClientCounter.NumFilesAssociated);
return(AddDebugEntryResult.Associated);
}
/// <summary>
/// Takes a list of sealed directories, searches their contents for special manifest files, parses each of them to obtain
/// a list of files to materialize, and finally requests from BuildXL's ApiServer to materialize them.
/// </summary>
private async Task <IIpcResult> RegisterManifestInternalAsync(ConfiguredCommand conf)
{
m_counters.IncrementCounter(MaterializationDaemonCounter.RegisterManifestRequestCount);
var directoryPaths = Directory.GetValues(conf.Config).ToArray();
var directoryIds = DirectoryId.GetValues(conf.Config).ToArray();
var directoryFilters = DirectoryContentFilter.GetValues(conf.Config).ToArray();
if (directoryPaths.Length != directoryIds.Length || directoryPaths.Length != directoryFilters.Length)
{
return(new IpcResult(
IpcResultStatus.GenericError,
I($"Directory counts don't match: #directories = {directoryPaths.Length}, #directoryIds = {directoryIds.Length}, #directoryFilters = {directoryFilters.Length}")));
}
if (ApiClient == null)
{
return(new IpcResult(IpcResultStatus.GenericError, "ApiClient is not initialized"));
}
var possibleFilters = InitializeFilters(directoryFilters);
if (!possibleFilters.Succeeded)
{
return(new IpcResult(IpcResultStatus.ExecutionError, possibleFilters.Failure.Describe()));
}
var initializedFilters = possibleFilters.Result;
var possibleManifestFiles = await GetUniqueFilteredDirectoryContentAsync(directoryPaths, directoryIds, initializedFilters);
if (!possibleManifestFiles.Succeeded)
{
return(new IpcResult(IpcResultStatus.ExecutionError, possibleManifestFiles.Failure.Describe()));
}
var manifestFiles = possibleManifestFiles.Result;
var sw = Stopwatch.StartNew();
List <string> filesToMaterialize = null;
try
{
// ensure that the manifest files are actually present on disk
using (m_counters.StartStopwatch(MaterializationDaemonCounter.RegisterManifestFileMaterializationDuration))
{
await m_actionQueue.ForEachAsync(
manifestFiles,
async (manifest, i) =>
{
m_counters.AddToCounter(MaterializationDaemonCounter.RegisterManifestFileMaterializationQueueDuration, sw.ElapsedMilliseconds);
await MaterializeFileAsync(manifest.Artifact, manifest.FileName, ignoreMaterializationFailures: false);
});
}
Possible <List <string> > possibleFiles;
using (m_counters.StartStopwatch(MaterializationDaemonCounter.ManifestParsingOuterDuration))
{
m_counters.AddToCounter(MaterializationDaemonCounter.ManifestParsingTotalFiles, possibleManifestFiles.Result.Count);
possibleFiles = await ParseManifestFilesAsync(manifestFiles);
if (!possibleFiles.Succeeded)
{
return(new IpcResult(IpcResultStatus.ExecutionError, possibleFiles.Failure.Describe()));
}
m_counters.AddToCounter(MaterializationDaemonCounter.ManifestParsingReferencedTotalFiles, possibleFiles.Result.Count);
}
filesToMaterialize = possibleFiles.Result;
sw = Stopwatch.StartNew();
using (m_counters.StartStopwatch(MaterializationDaemonCounter.RegisterManifestReferencedFileMaterializationDuration))
{
await m_actionQueue.ForEachAsync(
filesToMaterialize,
async (file, i) =>
{
// Since these file paths are not real build artifacts (i.e., just strings constructed by a parser),
// they might not be "known" to BuildXL, and because of that, BuildXL won't be able to materialize them.
// Manifests are known to contain references to files that are not produced during a build, so we are
// ignoring materialization failures for such files. We are doing this so we won't fail IPC pips and
// a build could succeed.
// If there are real materialization errors (e.g., cache failure), the daemon relies on BuildXL logging
// the appropriate error events and failing the build.
m_counters.AddToCounter(MaterializationDaemonCounter.RegisterManifestReferencedFileMaterializationQueueDuration, sw.ElapsedMilliseconds);
await MaterializeFileAsync(FileArtifact.Invalid, file, ignoreMaterializationFailures: true);
});
}
}
catch (Exception e)
{
return(new IpcResult(
IpcResultStatus.GenericError,
e.ToStringDemystified()));
}
// Note: we are not claiming here that all the files in filesToMaterialize were materialized cause we are ignoring materialization failures.
// The real materialization status will be logged during execution of the Finalize command.
return(IpcResult.Success(
$"Processed paths ({filesToMaterialize.Count}):{Environment.NewLine}{string.Join(Environment.NewLine, filesToMaterialize)}"));
}
private static Task <IIpcResult> ExecuteCommandWithStats <TCommand>(Func <TCommand, Task <IIpcResult> > executor, TCommand cmd, ApiServerCounters totalCounter)
where TCommand : Command
{
Counters.IncrementCounter(totalCounter);
return(executor(cmd));
}
