public async Task CloseBatch( string batchLocation, SymbolUploadBatch batch, CancellationToken token) { // get logger factory and create a logger for symsorter var symsorterOutput = Path.Combine(_symsorterOutputPath, batch.BatchId.ToString()); Directory.CreateDirectory(symsorterOutput); if (SorterSymbols(batchLocation, batch, symsorterOutput)) { return; } // TODO: Turn into a job. var trimDown = symsorterOutput + "/"; async Task UploadToGoogle(string filePath, CancellationToken token) { var destinationName = filePath.Replace(trimDown, string.Empty); await using var file = File.OpenRead(filePath); await _gcsWriter.WriteAsync(destinationName, file, token); } var counter = 0; var groups = from directory in Directory.GetDirectories(symsorterOutput, "*", SearchOption.AllDirectories) from file in Directory.GetFiles(directory) let c = counter++ group file by c / 20 // TODO: config into fileGroup select fileGroup.ToList(); try { foreach (var group in groups) { await Task.WhenAll(group.Select(file => UploadToGoogle(file, token))); } } catch (Exception e) { _logger.LogError(e, "Failed uploading files to GCS."); throw; } }
public Task Start(Guid batchId, string friendlyName, BatchType batchType, CancellationToken token) { if (_batches.ContainsKey(batchId)) { throw new ArgumentException($"Batch Id {batchId} was already used."); } _batches[batchId] = new SymbolUploadBatch(batchId, friendlyName, batchType); var batchIdString = batchId.ToString(); var processingDir = Path.Combine(_processingPath, batchType.ToSymsorterPrefix(), batchIdString); Directory.CreateDirectory(processingDir); _logger.LogInformation("Started batch {batchId} with friendly name {friendlyName} and type {batchType}", batchIdString, friendlyName, batchType); return(Task.CompletedTask); }
private bool SorterSymbols(string batchLocation, SymbolUploadBatch batch, string symsorterOutput) { var bundleId = ToBundleId(batch.FriendlyName); var symsorterPrefix = batch.BatchType.ToSymsorterPrefix(); var args = $"-zz -o {symsorterOutput} --prefix {symsorterPrefix} --bundle-id {bundleId} {batchLocation}"; var process = new Process { StartInfo = new ProcessStartInfo(_options.SymsorterPath, args) { UseShellExecute = false, RedirectStandardOutput = true, CreateNoWindow = true } }; string?lastLine = null; var sw = Stopwatch.StartNew(); if (!process.Start()) { throw new InvalidOperationException("symsorter failed to start"); } while (!process.StandardOutput.EndOfStream) { var line = process.StandardOutput.ReadLine(); _logger.LogInformation(line); lastLine = line; } const int waitUpToMs = 500_000; process.WaitForExit(waitUpToMs); sw.Stop(); if (!process.HasExited) { throw new InvalidOperationException($"Timed out waiting for {batch.BatchId}. Symsorter args: {args}"); } lastLine ??= string.Empty; if (process.ExitCode != 0) { throw new InvalidOperationException($"Symsorter exit code: {process.ExitCode}. Args: {args}"); } _logger.LogInformation("Symsorter finished in {timespan} and logged last: {lastLine}", sw.Elapsed, lastLine); var match = Regex.Match(lastLine, "Done: sorted (?<count>\\d+) debug files"); if (!match.Success) { _logger.LogError("Last line didn't match success: {lastLine}", lastLine); return(true); } _logger.LogInformation("Symsorter processed: {count}", match.Groups["count"].Value); return(false); string ToBundleId(string friendlyName) { var invalids = Path.GetInvalidFileNameChars().Concat(" ").ToArray(); return(string.Join("_", friendlyName.Split(invalids, StringSplitOptions.RemoveEmptyEntries) .Append(_generator.Generate())) .TrimEnd('.')); } }
public Task CloseBatch( string batchLocation, SymbolUploadBatch batch, CancellationToken token) { // TODO: Turn into a job. var stopwatch = Stopwatch.StartNew(); var gcsUploadCancellation = CancellationToken.None; // Since we'll run the closing of the batch on a background thread, trigger an event // (that will be dropped since Debug events are not captured in prod) // in order to get the Sentry SDK to read the request data and add to the Scope. In case there's an error // when closing the batch, the request data will already be available to add to outgoing events. SentrySdk.CaptureMessage("To read Request data on the request thread", SentryLevel.Debug); // TODO: Create it from current open transaction? (trace-parent) // TODO: Why isn't this optional? var closeBatchTransaction = _hub.StartTransaction("CloseBatch", "batch.close"); var handle = _metrics.BeginGcsBatchUpload(); _ = Task.Run(async() => { // get logger factory and create a logger for symsorter var symsorterOutput = Path.Combine(_symsorterOutputPath, batch.BatchId.ToString()); try { var symsorterSpan = closeBatchTransaction.StartChild("symsorter"); Directory.CreateDirectory(symsorterOutput); if (SortSymbols(batchLocation, batch, symsorterOutput, symsorterSpan)) { symsorterSpan.Finish(SpanStatus.UnknownError); return; } symsorterSpan.Finish(); if (_options.DeleteDoneDirectory) { Directory.Delete(batchLocation, true); } var trimDown = symsorterOutput + "/"; async Task UploadToGoogle(string filePath) { var destinationName = filePath.Replace(trimDown !, string.Empty); await using var file = File.OpenRead(filePath); await _gcsWriter.WriteAsync(destinationName, file, // The client disconnecting at this point shouldn't affect closing this batch. // This should anyway be a background job queued by the batch finalizer gcsUploadCancellation); } var counter = 0; var groups = from directory in Directory.GetDirectories(symsorterOutput, "*", SearchOption.AllDirectories) from file in Directory.GetFiles(directory) let c = counter++ group file by c / 20 // TODO: config into fileGroup select fileGroup.ToList(); symsorterSpan.SetExtra("batch_size", counter); var gcpUploadSpan = closeBatchTransaction.StartChild("GcpUpload"); try { foreach (var group in groups) { var gcpUploadSpanGroup = gcpUploadSpan.StartChild("GcpUploadBatch"); gcpUploadSpanGroup.SetExtra("Count", group.Count); try { await Task.WhenAll(group.Select(g => UploadToGoogle(g))); } catch (Exception e) { gcpUploadSpanGroup.Finish(e); throw; } } gcpUploadSpan.Finish(); } catch (Exception e) { gcpUploadSpan.Finish(e); _logger.LogError(e, "Failed uploading files to GCS."); throw; } SentrySdk.CaptureMessage($"Batch {batch.BatchId} with name {batch.FriendlyName} completed in {stopwatch.Elapsed}"); } catch (Exception e) { // TODO: Assign ex to Span closeBatchTransaction.Finish(SpanStatus.InternalError); _logger.LogError(e, "Batch {batchId} with name {friendlyName} completed in {stopwatch}.", batch.BatchId, batch.FriendlyName, stopwatch.Elapsed); throw; } finally { try { if (_options.DeleteSymsortedDirectory) { Directory.Delete(symsorterOutput, true); _logger.LogInformation( "Batch {batchId} with name {friendlyName} deleted sorted directory {symsorterOutput}.", batch.BatchId, batch.FriendlyName, symsorterOutput); } } catch (Exception e) { _logger.LogError(e, "Failed attempting to delete symsorter directory."); } handle.Dispose(); } }, gcsUploadCancellation) .ContinueWith(t => { _logger.LogInformation("Batch {batchId} with name {friendlyName} completed in {stopwatch}.", batch.BatchId, batch.FriendlyName, stopwatch.Elapsed); if (t.IsFaulted) { closeBatchTransaction.Finish(SpanStatus.InternalError); _logger.LogError(t.Exception, "GCS upload Task failed."); } else { closeBatchTransaction.Finish(); } }, gcsUploadCancellation); return(Task.CompletedTask); }
private bool SortSymbols(string batchLocation, SymbolUploadBatch batch, string symsorterOutput, ISpan symsorterSpan) { var bundleId = _bundleIdGenerator.CreateBundleId(batch.FriendlyName); var symsorterPrefix = batch.BatchType.ToSymsorterPrefix(); var args = $"--ignore-errors -zz -o {symsorterOutput} --prefix {symsorterPrefix} --bundle-id {bundleId} {batchLocation}"; symsorterSpan.SetExtra("args", args); var process = new Process { StartInfo = new ProcessStartInfo(_options.SymsorterPath, args) { UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true, CreateNoWindow = true, Environment = { { "RUST_BACKTRACE", "full" } } } }; string?lastLine = null; var sw = Stopwatch.StartNew(); if (!process.Start()) { throw new InvalidOperationException("symsorter failed to start"); } while (!process.StandardOutput.EndOfStream) { var line = process.StandardOutput.ReadLine(); if (string.IsNullOrWhiteSpace(line)) { continue; } _logger.LogInformation(line); lastLine = line; } while (!process.StandardError.EndOfStream) { var line = process.StandardError.ReadLine(); if (string.IsNullOrWhiteSpace(line)) { continue; } _logger.LogWarning(line); lastLine = line; } const int waitUpToMs = 500_000; process.WaitForExit(waitUpToMs); sw.Stop(); if (!process.HasExited) { throw new InvalidOperationException($"Timed out waiting for {batch.BatchId}. Symsorter args: {args}"); } lastLine ??= string.Empty; if (process.ExitCode != 0) { throw new InvalidOperationException($"Symsorter exit code: {process.ExitCode}. Args: {args}"); } _logger.LogInformation("Symsorter finished in {timespan} and logged last: {lastLine}", sw.Elapsed, lastLine); var match = Regex.Match(lastLine, "Sorted (?<count>\\d+) debug files"); if (!match.Success) { _logger.LogError("Last line didn't match success: {lastLine}", lastLine); return(true); } _logger.LogInformation("Symsorter processed: {count}", match.Groups["count"].Value); return(false); }