/// <summary>
/// Executes and, if necessary, retries an operation
/// </summary>
protected async Task <U> RetryAsync <U>(
string operationName,
Func <T, CancellationToken, Task <U> > fn,
CancellationToken cancellationToken,
IEnumerator <TimeSpan> retryIntervalEnumerator = null,
bool reloadFirst = false,
Guid?operationId = null,
TimeSpan?timeout = null)
{
operationId = operationId ?? Guid.NewGuid();
retryIntervalEnumerator = retryIntervalEnumerator ?? m_retryIntervals.GetEnumerator();
timeout = timeout ?? s_defaultOperationTimeout;
try
{
using (CancellationTokenSource timeoutCancellationSource = new CancellationTokenSource())
using (CancellationTokenSource innerCancellationSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCancellationSource.Token))
{
var instance = GetCurrentVersionedValue();
if (reloadFirst)
{
var reloaded = Reloader.Reload(instance.Version);
m_logger.Warning("[{2}] Service client reloaded; new instance created: {0}, new client version: {1}", reloaded, Reloader.CurrentVersion, operationId.Value);
}
m_logger.Verbose("[{2}] Invoking '{0}' against instance version {1}", operationName, instance.Version, operationId.Value);
return(await WithTimeoutAsync(fn(instance.Value, innerCancellationSource.Token), timeout.Value, timeoutCancellationSource));
}
}
catch (Exception e) when(m_nonRetryableExceptions.Contains(e.GetType()))
{
// We should not retry exceptions of this type.
throw;
}
catch (Exception e)
{
if (e is TimeoutException)
{
m_logger.Warning("Timeout ({0}sec) happened while waiting {1}.", timeout.Value.TotalSeconds, operationName);
}
if (retryIntervalEnumerator.MoveNext())
{
m_logger.Warning("[{2}] Waiting {1} before retrying on exception: {0}", e.ToString(), retryIntervalEnumerator.Current, operationId.Value);
await Task.Delay(retryIntervalEnumerator.Current);
return(await RetryAsync(operationName, fn, cancellationToken, retryIntervalEnumerator, reloadFirst : true, operationId : operationId));
}
else
{
m_logger.Error("[{1}] Failing because number of retries were exhausted. Final exception: {0};", e.ToString(), operationId.Value);
throw;
}
}
}
/// <summary>
/// Starts to listen for client connections. As soon as a connection is received,
/// it is placed in an action block from which it is picked up and handled asynchronously
/// (in the <see cref="ParseAndExecuteCommandAsync"/> method).
/// </summary>
public void Start()
{
m_server.Start(this);
// some tests run without API Client, so there is no one to notify that the service is ready
if (ApiClient != null)
{
var process = Process.GetCurrentProcess();
m_logger.Verbose($"Reporting to BuildXL that the service is ready (pid: {process.Id}, processName: '{process.ProcessName}')");
var possibleResult = ApiClient.ReportServicePipIsReady(process.Id, process.ProcessName).GetAwaiter().GetResult();
if (!possibleResult.Succeeded)
{
m_logger.Error("Failed to notify BuildXL that the service is ready.");
}
else
{
m_logger.Verbose("Successfully notified BuildXL that the service is ready.");
}
}
}
/// <inheritdoc />
public async Task <AddDebugEntryResult> AddFileAsync(SymbolFile symbolFile)
{
Contract.Requires(symbolFile.IsIndexed, "File has not been indexed.");
m_counters.IncrementCounter(SymbolClientCounter.NumAddFileRequests);
if (symbolFile.DebugEntries.Count == 0)
{
// If there are no debug entries, ask bxl to log a message and return early.
Analysis.IgnoreResult(await m_apiClient.LogMessage(I($"File '{symbolFile.FullFilePath}' does not contain symbols and will not be added to '{RequestName}'."), isWarning: false));
m_counters.IncrementCounter(SymbolClientCounter.NumFilesWithoutDebugEntries);
return(AddDebugEntryResult.NoSymbolData);
}
await EnsureRequestIdAndDomainIdAreInitalizedAsync();
List <DebugEntry> result;
using (m_counters.StartStopwatch(SymbolClientCounter.TotalAssociateTime))
{
try
{
result = await m_symbolClient.CreateRequestDebugEntriesAsync(
RequestId,
symbolFile.DebugEntries.Select(e => CreateDebugEntry(e, m_domainId)),
// First, we create debug entries with ThrowIfExists behavior not to silence the collision errors.
DebugEntryCreateBehavior.ThrowIfExists,
CancellationToken);
}
catch (DebugEntryExistsException)
{
string message = $"[SymbolDaemon] File: '{symbolFile.FullFilePath}' caused collision. " +
(m_debugEntryCreateBehavior == DebugEntryCreateBehavior.ThrowIfExists
? string.Empty
: $"SymbolDaemon will retry creating debug entry with {m_debugEntryCreateBehavior} behavior");
if (m_debugEntryCreateBehavior == DebugEntryCreateBehavior.ThrowIfExists)
{
// Log an error message in SymbolDaemon log file
m_logger.Error(message);
throw new DebugEntryExistsException(message);
}
// Log a message in SymbolDaemon log file
m_logger.Verbose(message);
result = await m_symbolClient.CreateRequestDebugEntriesAsync(
RequestId,
symbolFile.DebugEntries.Select(e => CreateDebugEntry(e, m_domainId)),
m_debugEntryCreateBehavior,
CancellationToken);
}
}
var entriesWithMissingBlobs = result.Where(e => e.Status == DebugEntryStatus.BlobMissing).ToList();
if (entriesWithMissingBlobs.Count > 0)
{
// All the entries are based on the same file, so we need to call upload only once.
// make sure that the file is on disk (it might not be on disk if we got DebugEntries from cache/metadata file)
var file = await symbolFile.EnsureMaterializedAsync();
BlobIdentifierWithBlocks uploadResult;
using (m_counters.StartStopwatch(SymbolClientCounter.TotalUploadTime))
{
uploadResult = await m_symbolClient.UploadFileAsync(
m_domainId,
// uploading to the location set by the symbol service
entriesWithMissingBlobs[0].BlobUri,
RequestId,
symbolFile.FullFilePath,
entriesWithMissingBlobs[0].BlobIdentifier,
CancellationToken);
}
m_counters.IncrementCounter(SymbolClientCounter.NumFilesUploaded);
m_counters.AddToCounter(SymbolClientCounter.TotalUploadSize, file.Length);
m_logger.Info($"File: '{symbolFile.FullFilePath}' -- upload result: {uploadResult}");
entriesWithMissingBlobs.ForEach(entry => entry.BlobDetails = uploadResult);
using (m_counters.StartStopwatch(SymbolClientCounter.TotalAssociateAfterUploadTime))
{
entriesWithMissingBlobs = await m_symbolClient.CreateRequestDebugEntriesAsync(
RequestId,
entriesWithMissingBlobs,
m_debugEntryCreateBehavior,
CancellationToken);
}
Contract.Assert(entriesWithMissingBlobs.All(e => e.Status != DebugEntryStatus.BlobMissing), "Entries with non-success code are present.");
return(AddDebugEntryResult.UploadedAndAssociated);
}
m_counters.IncrementCounter(SymbolClientCounter.NumFilesAssociated);
return(AddDebugEntryResult.Associated);
}