/// <nodoc /> public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient) { m_logger = logger; m_apiClient = apiClient; m_config = config; m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior; m_cancellationSource = new CancellationTokenSource(); m_counters = new CounterCollection <SymbolClientCounter>(); m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}")); m_credentialFactory = new VssCredentialsFactory(pat: null, new CredentialProviderHelper(m => m_logger.Verbose(m)), m => m_logger.Verbose(m)); m_symbolClient = new ReloadingSymbolClient( logger: logger, clientConstructor: CreateSymbolServiceClient); m_nagleQueue = NagleQueue <BatchedSymbolFile> .Create( maxDegreeOfParallelism : m_config.MaxParallelUploads, batchSize : m_config.BatchSize, interval : m_config.NagleTime, processBatch : ProcessBatchedFilesAsync); m_fileUploadQueue = new ActionQueue(m_config.MaxParallelUploads); }
/// <nodoc /> public VsoClient(IIpcLogger logger, DropDaemon dropDaemon) { Contract.Requires(dropDaemon?.DropConfig != null); m_logger = logger; m_dropDaemon = dropDaemon; m_config = dropDaemon.DropConfig; m_cancellationSource = new CancellationTokenSource(); logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config)); Stats = new DropStatistics(); // instantiate drop client m_dropClient = new ReloadingDropServiceClient( logger: logger, clientConstructor: CreateDropServiceClient); m_nagleQueue = NagleQueue <AddFileItem> .Create( maxDegreeOfParallelism : m_config.MaxParallelUploads, batchSize : m_config.BatchSize, interval : m_config.NagleTime, processBatch : ProcessAddFilesAsync); if (m_config.ArtifactLogName != null) { DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose); Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName)); } }
/// <nodoc /> public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient) { m_logger = logger; m_apiClient = apiClient; m_config = config; m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior; m_cancellationSource = new CancellationTokenSource(); m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}")); m_symbolClient = CreateSymbolServiceClient(); }
/// <nodoc /> public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient) { m_logger = logger; m_apiClient = apiClient; m_config = config; m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior; m_cancellationSource = new CancellationTokenSource(); m_counters = new CounterCollection <SymbolClientCounter>(); m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}")); m_symbolClient = new ReloadingSymbolClient( logger: logger, clientConstructor: CreateSymbolServiceClient); }
/// <nodoc /> public VsoClient(IIpcLogger logger, DropDaemon dropDaemon) { Contract.Requires(dropDaemon?.DropConfig != null); m_logger = logger; m_dropDaemon = dropDaemon; m_config = dropDaemon.DropConfig; m_cancellationSource = new CancellationTokenSource(); logger.Info("Using drop config: " + JsonConvert.SerializeObject(m_config)); Stats = new DropStatistics(); // instantiate drop client m_dropClient = new ReloadingDropServiceClient( logger: logger, clientConstructor: CreateDropServiceClient); // create dataflow blocks var groupingOptions = new GroupingDataflowBlockOptions { Greedy = true }; var actionOptions = new ExecutionDataflowBlockOptions { MaxDegreeOfParallelism = m_config.MaxParallelUploads }; var linkOptions = new DataflowLinkOptions { PropagateCompletion = true }; m_batchBlock = new BatchBlock <AddFileItem>(m_config.BatchSize, groupingOptions); m_bufferBlock = new BufferBlock <AddFileItem[]>(); // per http://blog.stephencleary.com/2012/11/async-producerconsumer-queue-using.html, good to have buffer when throttling m_actionBlock = new ActionBlock <AddFileItem[]>(ProcessAddFilesAsync, actionOptions); m_batchBlock.LinkTo(m_bufferBlock, linkOptions); m_bufferBlock.LinkTo(m_actionBlock, linkOptions); // create and set up timer for triggering the batch block TimeSpan timerInterval = m_config.NagleTime; m_batchTimer = new Timer(FlushBatchBlock, null, timerInterval, timerInterval); if (m_config.ArtifactLogName != null) { DropAppTraceSource.SingleInstance.SetSourceLevel(System.Diagnostics.SourceLevels.Verbose); Tracer.AddFileTraceListener(Path.Combine(m_config.LogDir, m_config.ArtifactLogName)); } }
private async Task ProcessBatchedFilesAsync(BatchedSymbolFile[] batch) { var batchNumber = Interlocked.Increment(ref m_batchCount); try { m_logger.Info($"Started processing batch #{batchNumber} ({batch.Length} files)."); await EnsureRequestIdAndDomainIdAreInitalizedAsync(); var debugEntriesToAssociate = batch.SelectMany( file => file.File.DebugEntries.Select(entry => CreateDebugEntry(entry, DomainId))) .ToList(); var resultOfAssociateCall = await AssociateAsync(debugEntriesToAssociate); var entriesWithMissingBlobs = resultOfAssociateCall.Where(e => e.Status == DebugEntryStatus.BlobMissing).ToList(); var missingBlobsToFilesMap = SetResultForAssociatedFiles(batch, entriesWithMissingBlobs); // materialize all files that we need to upload var materializedFiles = await TaskUtilities.SafeWhenAll(missingBlobsToFilesMap.Values.Select(static file => file.File.EnsureMaterializedAsync()));
/// <nodoc /> public VsoSymbolClient(IIpcLogger logger, SymbolConfig config, Client apiClient) { m_logger = logger; m_apiClient = apiClient; m_config = config; m_debugEntryCreateBehavior = config.DebugEntryCreateBehavior; m_cancellationSource = new CancellationTokenSource(); m_counters = new CounterCollection <SymbolClientCounter>(); m_logger.Info(I($"[{nameof(VsoSymbolClient)}] Using symbol config: {JsonConvert.SerializeObject(m_config)}")); m_symbolClient = new ReloadingSymbolClient( logger: logger, clientConstructor: CreateSymbolServiceClient); m_batchBlock = new TimedBatchBlock <BatchedSymbolFile>( maxDegreeOfParallelism: m_config.MaxParallelUploads, batchSize: m_config.BatchSize, nagleInterval: m_config.NagleTime, batchProcessor: ProcessBatchedFilesAsync); m_fileUploadQueue = new ActionQueue(m_config.MaxParallelUploads); }
/// <inheritdoc /> public async Task <AddDebugEntryResult> AddFileAsync(SymbolFile symbolFile) { Contract.Requires(symbolFile.IsIndexed, "File has not been indexed."); m_counters.IncrementCounter(SymbolClientCounter.NumAddFileRequests); if (symbolFile.DebugEntries.Count == 0) { // If there are no debug entries, ask bxl to log a message and return early. Analysis.IgnoreResult(await m_apiClient.LogMessage(I($"File '{symbolFile.FullFilePath}' does not contain symbols and will not be added to '{RequestName}'."), isWarning: false)); m_counters.IncrementCounter(SymbolClientCounter.NumFilesWithoutDebugEntries); return(AddDebugEntryResult.NoSymbolData); } await EnsureRequestIdAndDomainIdAreInitalizedAsync(); List <DebugEntry> result; using (m_counters.StartStopwatch(SymbolClientCounter.TotalAssociateTime)) { try { result = await m_symbolClient.CreateRequestDebugEntriesAsync( RequestId, symbolFile.DebugEntries.Select(e => CreateDebugEntry(e, m_domainId)), // First, we create debug entries with ThrowIfExists behavior not to silence the collision errors. DebugEntryCreateBehavior.ThrowIfExists, CancellationToken); } catch (DebugEntryExistsException) { string message = $"[SymbolDaemon] File: '{symbolFile.FullFilePath}' caused collision. " + (m_debugEntryCreateBehavior == DebugEntryCreateBehavior.ThrowIfExists ? string.Empty : $"SymbolDaemon will retry creating debug entry with {m_debugEntryCreateBehavior} behavior"); if (m_debugEntryCreateBehavior == DebugEntryCreateBehavior.ThrowIfExists) { // Log an error message in SymbolDaemon log file m_logger.Error(message); throw new DebugEntryExistsException(message); } // Log a message in SymbolDaemon log file m_logger.Verbose(message); result = await m_symbolClient.CreateRequestDebugEntriesAsync( RequestId, symbolFile.DebugEntries.Select(e => CreateDebugEntry(e, m_domainId)), m_debugEntryCreateBehavior, CancellationToken); } } var entriesWithMissingBlobs = result.Where(e => e.Status == DebugEntryStatus.BlobMissing).ToList(); if (entriesWithMissingBlobs.Count > 0) { // All the entries are based on the same file, so we need to call upload only once. // make sure that the file is on disk (it might not be on disk if we got DebugEntries from cache/metadata file) var file = await symbolFile.EnsureMaterializedAsync(); BlobIdentifierWithBlocks uploadResult; using (m_counters.StartStopwatch(SymbolClientCounter.TotalUploadTime)) { uploadResult = await m_symbolClient.UploadFileAsync( m_domainId, // uploading to the location set by the symbol service entriesWithMissingBlobs[0].BlobUri, RequestId, symbolFile.FullFilePath, entriesWithMissingBlobs[0].BlobIdentifier, CancellationToken); } m_counters.IncrementCounter(SymbolClientCounter.NumFilesUploaded); m_counters.AddToCounter(SymbolClientCounter.TotalUploadSize, file.Length); m_logger.Info($"File: '{symbolFile.FullFilePath}' -- upload result: {uploadResult}"); entriesWithMissingBlobs.ForEach(entry => entry.BlobDetails = uploadResult); using (m_counters.StartStopwatch(SymbolClientCounter.TotalAssociateAfterUploadTime)) { entriesWithMissingBlobs = await m_symbolClient.CreateRequestDebugEntriesAsync( RequestId, entriesWithMissingBlobs, m_debugEntryCreateBehavior, CancellationToken); } Contract.Assert(entriesWithMissingBlobs.All(e => e.Status != DebugEntryStatus.BlobMissing), "Entries with non-success code are present."); return(AddDebugEntryResult.UploadedAndAssociated); } m_counters.IncrementCounter(SymbolClientCounter.NumFilesAssociated); return(AddDebugEntryResult.Associated); }
private void LogDropEventLocally(DropOperationBaseEvent e) { m_localLogger.Info("Logging {0}Event(dropUrl: {1}, succeeded: {2})", e.Kind, e.DropUrl, e.Succeeded); }
/// <summary> /// Implements 'drop addfile' by first calling <see cref="IDropServiceClient.AssociateAsync"/>, /// and then <see cref="IDropServiceClient.UploadAndAssociateAsync"/>. /// </summary> /// <remarks> /// This method is called concurrently. /// </remarks> private async Task ProcessAddFilesAsync(AddFileItem[] batch) { if (batch.Length == 0) { return; } Interlocked.Increment(ref Stats.NumBatches); if (batch.Length == m_config.BatchSize) { Interlocked.Increment(ref Stats.NumCompleteBatches); } else { Interlocked.Increment(ref Stats.NumIncompleteBatches); } FileBlobDescriptor[] blobsForAssociate = new FileBlobDescriptor[0]; try { var dedupedBatch = SkipFilesWithTheSameDropPathAndContent(batch); var numSkipped = batch.Length - dedupedBatch.Length; m_logger.Info("Processing a batch of {0} drop files after skipping {1} files.", dedupedBatch.Length, numSkipped); Task <HashSet <string> > registerFilesForBuildManifestTask = null; // Register files for Build Manifest if (m_config.GenerateBuildManifest) { BuildManifestEntry[] buildManifestEntries = dedupedBatch .Where(dropItem => dropItem.BlobIdentifier != null && dropItem.FileId.HasValue) .Select(dropItem => new BuildManifestEntry(dropItem.RelativeDropFilePath, dropItem.BlobIdentifier.ToContentHash(), dropItem.FullFilePath, dropItem.FileId.Value)) .ToArray(); if (buildManifestEntries.Length > 0) // dropItem.BlobIdentifier = null for files generated in the DropDaemon { registerFilesForBuildManifestTask = Task.Run(() => RegisterFilesForBuildManifestAsync(buildManifestEntries)); } } // compute blobs for associate var startTime = DateTime.UtcNow; blobsForAssociate = await Task.WhenAll(dedupedBatch.Select(item => item.FileBlobDescriptorForAssociateAsync(m_config.EnableChunkDedup, Token))); Interlocked.Add(ref Stats.TotalComputeFileBlobDescriptorForAssociateMs, ElapsedMillis(startTime)); // run 'Associate' on all items from the batch; the result will indicate which files were associated and which need to be uploaded AssociationsStatus associateStatus = await AssociateAsync(blobsForAssociate); IReadOnlyList <AddFileItem> itemsLeftToUpload = await SetResultForAssociatedNonMissingItemsAsync(dedupedBatch, associateStatus, m_config.EnableChunkDedup, Token); // compute blobs for upload startTime = DateTime.UtcNow; FileBlobDescriptor[] blobsForUpload = await Task.WhenAll(itemsLeftToUpload.Select(item => item.FileBlobDescriptorForUploadAsync(m_config.EnableChunkDedup, Token))); Interlocked.Add(ref Stats.TotalComputeFileBlobDescriptorForUploadMs, ElapsedMillis(startTime)); // run 'UploadAndAssociate' for the missing files. await UploadAndAssociateAsync(associateStatus, blobsForUpload); SetResultForUploadedMissingItems(itemsLeftToUpload); Interlocked.Add(ref Stats.TotalUploadSizeBytes, blobsForUpload.Sum(b => b.FileSize ?? 0)); startTime = DateTime.UtcNow; foreach (var file in dedupedBatch) { RegisterFileForBuildManifestResult result = registerFilesForBuildManifestTask == null ? RegisterFileForBuildManifestResult.Skipped : (await registerFilesForBuildManifestTask).Contains(file.RelativeDropFilePath) ? RegisterFileForBuildManifestResult.Failed : RegisterFileForBuildManifestResult.Registered; file.BuildManifestTaskSource.TrySetResult(result); if (result == RegisterFileForBuildManifestResult.Failed) { Interlocked.Increment(ref Stats.TotalBuildManifestRegistrationFailures); m_logger.Info($"Build Manifest File registration failed for file at RelativePath '{file.RelativeDropFilePath}' with VSO '{file.BlobIdentifier.AlgorithmResultString}'."); } } Interlocked.Add(ref Stats.TotalBuildManifestRegistrationDurationMs, ElapsedMillis(startTime)); m_logger.Info("Done processing AddFile batch."); } catch (Exception e) { m_logger.Verbose($"Failed ProcessAddFilesAsync (batch size:{batch.Length}, blobsForAssociate size:{blobsForAssociate.Length}){Environment.NewLine}" + string.Join( Environment.NewLine, batch.Select(item => $"'{item.FullFilePath}', '{item.RelativeDropFilePath}', BlobId:'{item.BlobIdentifier?.ToString() ?? ""}', Task.IsCompleted:{item.DropResultTaskSource.Task.IsCompleted}"))); foreach (AddFileItem item in batch) { if (!item.DropResultTaskSource.Task.IsCompleted) { item.DropResultTaskSource.SetException(e); } // No exceptions are thrown by RegisterFilesForBuildManifestAsync item.BuildManifestTaskSource.TrySetResult(RegisterFileForBuildManifestResult.Skipped); } } }
private void LogDropEventLocally(DropOperationBaseEvent e) { var enabled = ETWLogger.Log.IsEnabled(EventLevel.Verbose, Keywords.CloudBuild) ? "ENABLED" : "DISABLED"; m_localLogger.Info("Logging {0}Event(dropUrl: {1}, succeeded: {2}): {3}", e.Kind, e.DropUrl, e.Succeeded, enabled); }