/// <nodoc />
public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient)
{
m_logger = logger;
m_apiClient = apiClient;
m_config = config;
m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior;
m_cancellationSource = new CancellationTokenSource();
m_counters = new CounterCollection <SymbolClientCounter>();
m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}"));
m_credentialFactory = new VssCredentialsFactory(pat: null, new CredentialProviderHelper(m => m_logger.Verbose(m)), m => m_logger.Verbose(m));
m_symbolClient = new ReloadingSymbolClient(
logger: logger,
clientConstructor: CreateSymbolServiceClient);
m_nagleQueue = NagleQueue <BatchedSymbolFile> .Create(
maxDegreeOfParallelism : m_config.MaxParallelUploads,
batchSize : m_config.BatchSize,
interval : m_config.NagleTime,
processBatch : ProcessBatchedFilesAsync);
m_fileUploadQueue = new ActionQueue(m_config.MaxParallelUploads);
}
/// <nodoc />
public VsoClient(IIpcLogger logger, DropDaemon dropDaemon)
{
Contract.Requires(dropDaemon?.DropConfig != null);
m_logger = logger;
m_dropDaemon = dropDaemon;
m_config = dropDaemon.DropConfig;
m_cancellationSource = new CancellationTokenSource();
logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config));
Stats = new DropStatistics();
// instantiate drop client
m_dropClient = new ReloadingDropServiceClient(
logger: logger,
clientConstructor: CreateDropServiceClient);
m_nagleQueue = NagleQueue <AddFileItem> .Create(
maxDegreeOfParallelism : m_config.MaxParallelUploads,
batchSize : m_config.BatchSize,
interval : m_config.NagleTime,
processBatch : ProcessAddFilesAsync);
if (m_config.ArtifactLogName != null)
{
DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose);
Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName));
}
}
/// <nodoc />
public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient)
{
m_logger = logger;
m_apiClient = apiClient;
m_config = config;
m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior;
m_cancellationSource = new CancellationTokenSource();
m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}"));
m_symbolClient = CreateSymbolServiceClient();
}
/// <nodoc />
public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient)
{
m_logger = logger;
m_apiClient = apiClient;
m_config = config;
m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior;
m_cancellationSource = new CancellationTokenSource();
m_counters = new CounterCollection <SymbolClientCounter>();
m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}"));
m_symbolClient = new ReloadingSymbolClient(
logger: logger,
clientConstructor: CreateSymbolServiceClient);
}
/// <nodoc />
public VsoClient(IIpcLogger logger, DropDaemon dropDaemon)
{
Contract.Requires(dropDaemon?.DropConfig != null);
m_logger = logger;
m_dropDaemon = dropDaemon;
m_config = dropDaemon.DropConfig;
m_cancellationSource = new CancellationTokenSource();
logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config));
Stats = new DropStatistics();
// instantiate drop client
m_dropClient = new ReloadingDropServiceClient(
logger: logger,
clientConstructor: CreateDropServiceClient);
// create dataflow blocks
var groupingOptions = new GroupingDataflowBlockOptions {
Greedy = true
};
var actionOptions = new ExecutionDataflowBlockOptions {
MaxDegreeOfParallelism = m_config.MaxParallelUploads
};
var linkOptions = new DataflowLinkOptions {
PropagateCompletion = true
};
m_batchBlock = new BatchBlock <AddFileItem>(m_config.BatchSize, groupingOptions);
m_bufferBlock = new BufferBlock <AddFileItem[]>(); // per http://blog.stephencleary.com/2012/11/async-producerconsumer-queue-using.html, good to have buffer when throttling
m_actionBlock = new ActionBlock <AddFileItem[]>(ProcessAddFilesAsync, actionOptions);
m_batchBlock.LinkTo(m_bufferBlock, linkOptions);
m_bufferBlock.LinkTo(m_actionBlock, linkOptions);
// create and set up timer for triggering the batch block
TimeSpan timerInterval = m_config.NagleTime;
m_batchTimer = new Timer(FlushBatchBlock, null, timerInterval, timerInterval);
if (m_config.ArtifactLogName != null)
{
DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose);
Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName));
}
}
private async Task ProcessBatchedFilesAsync(BatchedSymbolFile[] batch)
{
var batchNumber = Interlocked.Increment(ref m_batchCount);
try
{
m_logger.Info($"Started processing batch #{batchNumber} ({batch.Length} files).");
await EnsureRequestIdAndDomainIdAreInitalizedAsync();
var debugEntriesToAssociate = batch.SelectMany(
file => file.File.DebugEntries.Select(entry => CreateDebugEntry(entry, DomainId)))
.ToList();
var resultOfAssociateCall = await AssociateAsync(debugEntriesToAssociate);
var entriesWithMissingBlobs = resultOfAssociateCall.Where(e => e.Status == DebugEntryStatus.BlobMissing).ToList();
var missingBlobsToFilesMap = SetResultForAssociatedFiles(batch, entriesWithMissingBlobs);
// materialize all files that we need to upload
var materializedFiles = await TaskUtilities.SafeWhenAll(missingBlobsToFilesMap.Values.Select(static file => file.File.EnsureMaterializedAsync()));
/// <nodoc />
public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient)
{
m_logger = logger;
m_apiClient = apiClient;
m_config = config;
m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior;
m_cancellationSource = new CancellationTokenSource();
m_counters = new CounterCollection <SymbolClientCounter>();
m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}"));
m_symbolClient = new ReloadingSymbolClient(
logger: logger,
clientConstructor: CreateSymbolServiceClient);
m_batchBlock = new TimedBatchBlock <BatchedSymbolFile>(
maxDegreeOfParallelism: m_config.MaxParallelUploads,
batchSize: m_config.BatchSize,
nagleInterval: m_config.NagleTime,
batchProcessor: ProcessBatchedFilesAsync);
m_fileUploadQueue = new ActionQueue(m_config.MaxParallelUploads);
}
/// <inheritdoc />
public async Task <AddDebugEntryResult> AddFileAsync(SymbolFile symbolFile)
{
Contract.Requires(symbolFile.IsIndexed, "File has not been indexed.");
m_counters.IncrementCounter(SymbolClientCounter.NumAddFileRequests);
if (symbolFile.DebugEntries.Count == 0)
{
// If there are no debug entries, ask bxl to log a message and return early.
Analysis.IgnoreResult(await m_apiClient.LogMessage(I($"File '{symbolFile.FullFilePath}' does not contain symbols and will not be added to '{RequestName}'."), isWarning: false));
m_counters.IncrementCounter(SymbolClientCounter.NumFilesWithoutDebugEntries);
return(AddDebugEntryResult.NoSymbolData);
}
await EnsureRequestIdAndDomainIdAreInitalizedAsync();
List <DebugEntry> result;
using (m_counters.StartStopwatch(SymbolClientCounter.TotalAssociateTime))
{
try
{
result = await m_symbolClient.CreateRequestDebugEntriesAsync(
RequestId,
symbolFile.DebugEntries.Select(e => CreateDebugEntry(e, m_domainId)),
// First, we create debug entries with ThrowIfExists behavior not to silence the collision errors.
DebugEntryCreateBehavior.ThrowIfExists,
CancellationToken);
}
catch (DebugEntryExistsException)
{
string message = $"[SymbolDaemon] File: '{symbolFile.FullFilePath}' caused collision. " +
(m_debugEntryCreateBehavior == DebugEntryCreateBehavior.ThrowIfExists
? string.Empty
: $"SymbolDaemon will retry creating debug entry with {m_debugEntryCreateBehavior} behavior");
if (m_debugEntryCreateBehavior == DebugEntryCreateBehavior.ThrowIfExists)
{
// Log an error message in SymbolDaemon log file
m_logger.Error(message);
throw new DebugEntryExistsException(message);
}
// Log a message in SymbolDaemon log file
m_logger.Verbose(message);
result = await m_symbolClient.CreateRequestDebugEntriesAsync(
RequestId,
symbolFile.DebugEntries.Select(e => CreateDebugEntry(e, m_domainId)),
m_debugEntryCreateBehavior,
CancellationToken);
}
}
var entriesWithMissingBlobs = result.Where(e => e.Status == DebugEntryStatus.BlobMissing).ToList();
if (entriesWithMissingBlobs.Count > 0)
{
// All the entries are based on the same file, so we need to call upload only once.
// make sure that the file is on disk (it might not be on disk if we got DebugEntries from cache/metadata file)
var file = await symbolFile.EnsureMaterializedAsync();
BlobIdentifierWithBlocks uploadResult;
using (m_counters.StartStopwatch(SymbolClientCounter.TotalUploadTime))
{
uploadResult = await m_symbolClient.UploadFileAsync(
m_domainId,
// uploading to the location set by the symbol service
entriesWithMissingBlobs[0].BlobUri,
RequestId,
symbolFile.FullFilePath,
entriesWithMissingBlobs[0].BlobIdentifier,
CancellationToken);
}
m_counters.IncrementCounter(SymbolClientCounter.NumFilesUploaded);
m_counters.AddToCounter(SymbolClientCounter.TotalUploadSize, file.Length);
m_logger.Info($"File: '{symbolFile.FullFilePath}' -- upload result: {uploadResult}");
entriesWithMissingBlobs.ForEach(entry => entry.BlobDetails = uploadResult);
using (m_counters.StartStopwatch(SymbolClientCounter.TotalAssociateAfterUploadTime))
{
entriesWithMissingBlobs = await m_symbolClient.CreateRequestDebugEntriesAsync(
RequestId,
entriesWithMissingBlobs,
m_debugEntryCreateBehavior,
CancellationToken);
}
Contract.Assert(entriesWithMissingBlobs.All(e => e.Status != DebugEntryStatus.BlobMissing), "Entries with non-success code are present.");
return(AddDebugEntryResult.UploadedAndAssociated);
}
m_counters.IncrementCounter(SymbolClientCounter.NumFilesAssociated);
return(AddDebugEntryResult.Associated);
}
private void LogDropEventLocally(DropOperationBaseEvent e)
{
m_localLogger.Info("Logging {0}Event(dropUrl: {1}, succeeded: {2})", e.Kind, e.DropUrl, e.Succeeded);
}
/// <summary>
/// Implements 'drop addfile' by first calling <see cref="IDropServiceClient.AssociateAsync"/>,
/// and then <see cref="IDropServiceClient.UploadAndAssociateAsync"/>.
/// </summary>
/// <remarks>
/// This method is called concurrently.
/// </remarks>
private async Task ProcessAddFilesAsync(AddFileItem[] batch)
{
if (batch.Length == 0)
{
return;
}
Interlocked.Increment(ref Stats.NumBatches);
if (batch.Length == m_config.BatchSize)
{
Interlocked.Increment(ref Stats.NumCompleteBatches);
}
else
{
Interlocked.Increment(ref Stats.NumIncompleteBatches);
}
FileBlobDescriptor[] blobsForAssociate = new FileBlobDescriptor[0];
try
{
var dedupedBatch = SkipFilesWithTheSameDropPathAndContent(batch);
var numSkipped = batch.Length - dedupedBatch.Length;
m_logger.Info("Processing a batch of {0} drop files after skipping {1} files.", dedupedBatch.Length, numSkipped);
Task <HashSet <string> > registerFilesForBuildManifestTask = null;
// Register files for Build Manifest
if (m_config.GenerateBuildManifest)
{
BuildManifestEntry[] buildManifestEntries = dedupedBatch
.Where(dropItem => dropItem.BlobIdentifier != null && dropItem.FileId.HasValue)
.Select(dropItem => new BuildManifestEntry(dropItem.RelativeDropFilePath, dropItem.BlobIdentifier.ToContentHash(), dropItem.FullFilePath, dropItem.FileId.Value))
.ToArray();
if (buildManifestEntries.Length > 0) // dropItem.BlobIdentifier = null for files generated in the DropDaemon
{
registerFilesForBuildManifestTask = Task.Run(() => RegisterFilesForBuildManifestAsync(buildManifestEntries));
}
}
// compute blobs for associate
var startTime = DateTime.UtcNow;
blobsForAssociate = await Task.WhenAll(dedupedBatch.Select(item => item.FileBlobDescriptorForAssociateAsync(m_config.EnableChunkDedup, Token)));
Interlocked.Add(ref Stats.TotalComputeFileBlobDescriptorForAssociateMs, ElapsedMillis(startTime));
// run 'Associate' on all items from the batch; the result will indicate which files were associated and which need to be uploaded
AssociationsStatus associateStatus = await AssociateAsync(blobsForAssociate);
IReadOnlyList <AddFileItem> itemsLeftToUpload = await SetResultForAssociatedNonMissingItemsAsync(dedupedBatch, associateStatus, m_config.EnableChunkDedup, Token);
// compute blobs for upload
startTime = DateTime.UtcNow;
FileBlobDescriptor[] blobsForUpload = await Task.WhenAll(itemsLeftToUpload.Select(item => item.FileBlobDescriptorForUploadAsync(m_config.EnableChunkDedup, Token)));
Interlocked.Add(ref Stats.TotalComputeFileBlobDescriptorForUploadMs, ElapsedMillis(startTime));
// run 'UploadAndAssociate' for the missing files.
await UploadAndAssociateAsync(associateStatus, blobsForUpload);
SetResultForUploadedMissingItems(itemsLeftToUpload);
Interlocked.Add(ref Stats.TotalUploadSizeBytes, blobsForUpload.Sum(b => b.FileSize ?? 0));
startTime = DateTime.UtcNow;
foreach (var file in dedupedBatch)
{
RegisterFileForBuildManifestResult result = registerFilesForBuildManifestTask == null
? RegisterFileForBuildManifestResult.Skipped
: (await registerFilesForBuildManifestTask).Contains(file.RelativeDropFilePath)
? RegisterFileForBuildManifestResult.Failed
: RegisterFileForBuildManifestResult.Registered;
file.BuildManifestTaskSource.TrySetResult(result);
if (result == RegisterFileForBuildManifestResult.Failed)
{
Interlocked.Increment(ref Stats.TotalBuildManifestRegistrationFailures);
m_logger.Info($"Build Manifest File registration failed for file at RelativePath '{file.RelativeDropFilePath}' with VSO '{file.BlobIdentifier.AlgorithmResultString}'.");
}
}
Interlocked.Add(ref Stats.TotalBuildManifestRegistrationDurationMs, ElapsedMillis(startTime));
m_logger.Info("Done processing AddFile batch.");
}
catch (Exception e)
{
m_logger.Verbose($"Failed ProcessAddFilesAsync (batch size:{batch.Length}, blobsForAssociate size:{blobsForAssociate.Length}){Environment.NewLine}"
+ string.Join(
Environment.NewLine,
batch.Select(item => $"'{item.FullFilePath}', '{item.RelativeDropFilePath}', BlobId:'{item.BlobIdentifier?.ToString() ?? ""}', Task.IsCompleted:{item.DropResultTaskSource.Task.IsCompleted}")));
foreach (AddFileItem item in batch)
{
if (!item.DropResultTaskSource.Task.IsCompleted)
{
item.DropResultTaskSource.SetException(e);
}
// No exceptions are thrown by RegisterFilesForBuildManifestAsync
item.BuildManifestTaskSource.TrySetResult(RegisterFileForBuildManifestResult.Skipped);
}
}
}
private void LogDropEventLocally(DropOperationBaseEvent e)
{
var enabled = ETWLogger.Log.IsEnabled(EventLevel.Verbose, Keywords.CloudBuild) ? "ENABLED" : "DISABLED";
m_localLogger.Info("Logging {0}Event(dropUrl: {1}, succeeded: {2}): {3}", e.Kind, e.DropUrl, e.Succeeded, enabled);
}