public async Task CloseBatch( string batchLocation, SymbolUploadBatch batch, CancellationToken token) { // get logger factory and create a logger for symsorter var symsorterOutput = Path.Combine(_symsorterOutputPath, batch.BatchId.ToString()); Directory.CreateDirectory(symsorterOutput); if (SorterSymbols(batchLocation, batch, symsorterOutput)) { return; } // TODO: Turn into a job. var trimDown = symsorterOutput + "/"; async Task UploadToGoogle(string filePath, CancellationToken token) { var destinationName = filePath.Replace(trimDown, string.Empty); await using var file = File.OpenRead(filePath); await _gcsWriter.WriteAsync(destinationName, file, token); } var counter = 0; var groups = from directory in Directory.GetDirectories(symsorterOutput, "*", SearchOption.AllDirectories) from file in Directory.GetFiles(directory) let c = counter++ group file by c / 20 // TODO: config into fileGroup select fileGroup.ToList(); try { foreach (var group in groups) { await Task.WhenAll(group.Select(file => UploadToGoogle(file, token))); } } catch (Exception e) { _logger.LogError(e, "Failed uploading files to GCS."); throw; } }
public Task CloseBatch( string batchLocation, SymbolUploadBatch batch, CancellationToken token) { // TODO: Turn into a job. var stopwatch = Stopwatch.StartNew(); var gcsUploadCancellation = CancellationToken.None; // Since we'll run the closing of the batch on a background thread, trigger an event // (that will be dropped since Debug events are not captured in prod) // in order to get the Sentry SDK to read the request data and add to the Scope. In case there's an error // when closing the batch, the request data will already be available to add to outgoing events. SentrySdk.CaptureMessage("To read Request data on the request thread", SentryLevel.Debug); // TODO: Create it from current open transaction? (trace-parent) // TODO: Why isn't this optional? var closeBatchTransaction = _hub.StartTransaction("CloseBatch", "batch.close"); var handle = _metrics.BeginGcsBatchUpload(); _ = Task.Run(async() => { // get logger factory and create a logger for symsorter var symsorterOutput = Path.Combine(_symsorterOutputPath, batch.BatchId.ToString()); try { var symsorterSpan = closeBatchTransaction.StartChild("symsorter"); Directory.CreateDirectory(symsorterOutput); if (SortSymbols(batchLocation, batch, symsorterOutput, symsorterSpan)) { symsorterSpan.Finish(SpanStatus.UnknownError); return; } symsorterSpan.Finish(); if (_options.DeleteDoneDirectory) { Directory.Delete(batchLocation, true); } var trimDown = symsorterOutput + "/"; async Task UploadToGoogle(string filePath) { var destinationName = filePath.Replace(trimDown !, string.Empty); await using var file = File.OpenRead(filePath); await _gcsWriter.WriteAsync(destinationName, file, // The client disconnecting at this point shouldn't affect closing this batch. // This should anyway be a background job queued by the batch finalizer gcsUploadCancellation); } var counter = 0; var groups = from directory in Directory.GetDirectories(symsorterOutput, "*", SearchOption.AllDirectories) from file in Directory.GetFiles(directory) let c = counter++ group file by c / 20 // TODO: config into fileGroup select fileGroup.ToList(); symsorterSpan.SetExtra("batch_size", counter); var gcpUploadSpan = closeBatchTransaction.StartChild("GcpUpload"); try { foreach (var group in groups) { var gcpUploadSpanGroup = gcpUploadSpan.StartChild("GcpUploadBatch"); gcpUploadSpanGroup.SetExtra("Count", group.Count); try { await Task.WhenAll(group.Select(g => UploadToGoogle(g))); } catch (Exception e) { gcpUploadSpanGroup.Finish(e); throw; } } gcpUploadSpan.Finish(); } catch (Exception e) { gcpUploadSpan.Finish(e); _logger.LogError(e, "Failed uploading files to GCS."); throw; } SentrySdk.CaptureMessage($"Batch {batch.BatchId} with name {batch.FriendlyName} completed in {stopwatch.Elapsed}"); } catch (Exception e) { // TODO: Assign ex to Span closeBatchTransaction.Finish(SpanStatus.InternalError); _logger.LogError(e, "Batch {batchId} with name {friendlyName} completed in {stopwatch}.", batch.BatchId, batch.FriendlyName, stopwatch.Elapsed); throw; } finally { try { if (_options.DeleteSymsortedDirectory) { Directory.Delete(symsorterOutput, true); _logger.LogInformation( "Batch {batchId} with name {friendlyName} deleted sorted directory {symsorterOutput}.", batch.BatchId, batch.FriendlyName, symsorterOutput); } } catch (Exception e) { _logger.LogError(e, "Failed attempting to delete symsorter directory."); } handle.Dispose(); } }, gcsUploadCancellation) .ContinueWith(t => { _logger.LogInformation("Batch {batchId} with name {friendlyName} completed in {stopwatch}.", batch.BatchId, batch.FriendlyName, stopwatch.Elapsed); if (t.IsFaulted) { closeBatchTransaction.Finish(SpanStatus.InternalError); _logger.LogError(t.Exception, "GCS upload Task failed."); } else { closeBatchTransaction.Finish(); } }, gcsUploadCancellation); return(Task.CompletedTask); }