public async Task CloseBatch(
string batchLocation,
SymbolUploadBatch batch,
CancellationToken token)
{
// get logger factory and create a logger for symsorter
var symsorterOutput = Path.Combine(_symsorterOutputPath, batch.BatchId.ToString());
Directory.CreateDirectory(symsorterOutput);
if (SorterSymbols(batchLocation, batch, symsorterOutput))
{
return;
}
// TODO: Turn into a job.
var trimDown = symsorterOutput + "/";
async Task UploadToGoogle(string filePath, CancellationToken token)
{
var destinationName = filePath.Replace(trimDown, string.Empty);
await using var file = File.OpenRead(filePath);
await _gcsWriter.WriteAsync(destinationName, file, token);
}
var counter = 0;
var groups =
from directory in Directory.GetDirectories(symsorterOutput, "*", SearchOption.AllDirectories)
from file in Directory.GetFiles(directory)
let c = counter++
group file by c / 20 // TODO: config
into fileGroup
select fileGroup.ToList();
try
{
foreach (var group in groups)
{
await Task.WhenAll(group.Select(file => UploadToGoogle(file, token)));
}
}
catch (Exception e)
{
_logger.LogError(e, "Failed uploading files to GCS.");
throw;
}
}
public Task Start(Guid batchId, string friendlyName, BatchType batchType, CancellationToken token)
{
if (_batches.ContainsKey(batchId))
{
throw new ArgumentException($"Batch Id {batchId} was already used.");
}
_batches[batchId] = new SymbolUploadBatch(batchId, friendlyName, batchType);
var batchIdString = batchId.ToString();
var processingDir = Path.Combine(_processingPath, batchType.ToSymsorterPrefix(), batchIdString);
Directory.CreateDirectory(processingDir);
_logger.LogInformation("Started batch {batchId} with friendly name {friendlyName} and type {batchType}",
batchIdString, friendlyName, batchType);
return(Task.CompletedTask);
}
private bool SorterSymbols(string batchLocation, SymbolUploadBatch batch, string symsorterOutput)
{
var bundleId = ToBundleId(batch.FriendlyName);
var symsorterPrefix = batch.BatchType.ToSymsorterPrefix();
var args = $"-zz -o {symsorterOutput} --prefix {symsorterPrefix} --bundle-id {bundleId} {batchLocation}";
var process = new Process
{
StartInfo = new ProcessStartInfo(_options.SymsorterPath, args)
{
UseShellExecute = false,
RedirectStandardOutput = true,
CreateNoWindow = true
}
};
string?lastLine = null;
var sw = Stopwatch.StartNew();
if (!process.Start())
{
throw new InvalidOperationException("symsorter failed to start");
}
while (!process.StandardOutput.EndOfStream)
{
var line = process.StandardOutput.ReadLine();
_logger.LogInformation(line);
lastLine = line;
}
const int waitUpToMs = 500_000;
process.WaitForExit(waitUpToMs);
sw.Stop();
if (!process.HasExited)
{
throw new InvalidOperationException($"Timed out waiting for {batch.BatchId}. Symsorter args: {args}");
}
lastLine ??= string.Empty;
if (process.ExitCode != 0)
{
throw new InvalidOperationException($"Symsorter exit code: {process.ExitCode}. Args: {args}");
}
_logger.LogInformation("Symsorter finished in {timespan} and logged last: {lastLine}",
sw.Elapsed, lastLine);
var match = Regex.Match(lastLine, "Done: sorted (?<count>\\d+) debug files");
if (!match.Success)
{
_logger.LogError("Last line didn't match success: {lastLine}", lastLine);
return(true);
}
_logger.LogInformation("Symsorter processed: {count}", match.Groups["count"].Value);
return(false);
string ToBundleId(string friendlyName)
{
var invalids = Path.GetInvalidFileNameChars().Concat(" ").ToArray();
return(string.Join("_",
friendlyName.Split(invalids, StringSplitOptions.RemoveEmptyEntries)
.Append(_generator.Generate()))
.TrimEnd('.'));
}
}
public Task CloseBatch(
string batchLocation,
SymbolUploadBatch batch,
CancellationToken token)
{
// TODO: Turn into a job.
var stopwatch = Stopwatch.StartNew();
var gcsUploadCancellation = CancellationToken.None;
// Since we'll run the closing of the batch on a background thread, trigger an event
// (that will be dropped since Debug events are not captured in prod)
// in order to get the Sentry SDK to read the request data and add to the Scope. In case there's an error
// when closing the batch, the request data will already be available to add to outgoing events.
SentrySdk.CaptureMessage("To read Request data on the request thread", SentryLevel.Debug);
// TODO: Create it from current open transaction? (trace-parent)
// TODO: Why isn't this optional?
var closeBatchTransaction = _hub.StartTransaction("CloseBatch", "batch.close");
var handle = _metrics.BeginGcsBatchUpload();
_ = Task.Run(async() =>
{
// get logger factory and create a logger for symsorter
var symsorterOutput = Path.Combine(_symsorterOutputPath, batch.BatchId.ToString());
try
{
var symsorterSpan = closeBatchTransaction.StartChild("symsorter");
Directory.CreateDirectory(symsorterOutput);
if (SortSymbols(batchLocation, batch, symsorterOutput, symsorterSpan))
{
symsorterSpan.Finish(SpanStatus.UnknownError);
return;
}
symsorterSpan.Finish();
if (_options.DeleteDoneDirectory)
{
Directory.Delete(batchLocation, true);
}
var trimDown = symsorterOutput + "/";
async Task UploadToGoogle(string filePath)
{
var destinationName = filePath.Replace(trimDown !, string.Empty);
await using var file = File.OpenRead(filePath);
await _gcsWriter.WriteAsync(destinationName, file,
// The client disconnecting at this point shouldn't affect closing this batch.
// This should anyway be a background job queued by the batch finalizer
gcsUploadCancellation);
}
var counter = 0;
var groups =
from directory in Directory.GetDirectories(symsorterOutput, "*", SearchOption.AllDirectories)
from file in Directory.GetFiles(directory)
let c = counter++
group file by c / 20 // TODO: config
into fileGroup
select fileGroup.ToList();
symsorterSpan.SetExtra("batch_size", counter);
var gcpUploadSpan = closeBatchTransaction.StartChild("GcpUpload");
try
{
foreach (var group in groups)
{
var gcpUploadSpanGroup = gcpUploadSpan.StartChild("GcpUploadBatch");
gcpUploadSpanGroup.SetExtra("Count", group.Count);
try
{
await Task.WhenAll(group.Select(g => UploadToGoogle(g)));
}
catch (Exception e)
{
gcpUploadSpanGroup.Finish(e);
throw;
}
}
gcpUploadSpan.Finish();
}
catch (Exception e)
{
gcpUploadSpan.Finish(e);
_logger.LogError(e, "Failed uploading files to GCS.");
throw;
}
SentrySdk.CaptureMessage($"Batch {batch.BatchId} with name {batch.FriendlyName} completed in {stopwatch.Elapsed}");
}
catch (Exception e)
{
// TODO: Assign ex to Span
closeBatchTransaction.Finish(SpanStatus.InternalError);
_logger.LogError(e, "Batch {batchId} with name {friendlyName} completed in {stopwatch}.",
batch.BatchId, batch.FriendlyName, stopwatch.Elapsed);
throw;
}
finally
{
try
{
if (_options.DeleteSymsortedDirectory)
{
Directory.Delete(symsorterOutput, true);
_logger.LogInformation(
"Batch {batchId} with name {friendlyName} deleted sorted directory {symsorterOutput}.",
batch.BatchId, batch.FriendlyName, symsorterOutput);
}
}
catch (Exception e)
{
_logger.LogError(e, "Failed attempting to delete symsorter directory.");
}
handle.Dispose();
}
}, gcsUploadCancellation)
.ContinueWith(t =>
{
_logger.LogInformation("Batch {batchId} with name {friendlyName} completed in {stopwatch}.",
batch.BatchId, batch.FriendlyName, stopwatch.Elapsed);
if (t.IsFaulted)
{
closeBatchTransaction.Finish(SpanStatus.InternalError);
_logger.LogError(t.Exception, "GCS upload Task failed.");
}
else
{
closeBatchTransaction.Finish();
}
}, gcsUploadCancellation);
return(Task.CompletedTask);
}
private bool SortSymbols(string batchLocation, SymbolUploadBatch batch, string symsorterOutput, ISpan symsorterSpan)
{
var bundleId = _bundleIdGenerator.CreateBundleId(batch.FriendlyName);
var symsorterPrefix = batch.BatchType.ToSymsorterPrefix();
var args = $"--ignore-errors -zz -o {symsorterOutput} --prefix {symsorterPrefix} --bundle-id {bundleId} {batchLocation}";
symsorterSpan.SetExtra("args", args);
var process = new Process
{
StartInfo = new ProcessStartInfo(_options.SymsorterPath, args)
{
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true,
Environment = { { "RUST_BACKTRACE", "full" } }
}
};
string?lastLine = null;
var sw = Stopwatch.StartNew();
if (!process.Start())
{
throw new InvalidOperationException("symsorter failed to start");
}
while (!process.StandardOutput.EndOfStream)
{
var line = process.StandardOutput.ReadLine();
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
_logger.LogInformation(line);
lastLine = line;
}
while (!process.StandardError.EndOfStream)
{
var line = process.StandardError.ReadLine();
if (string.IsNullOrWhiteSpace(line))
{
continue;
}
_logger.LogWarning(line);
lastLine = line;
}
const int waitUpToMs = 500_000;
process.WaitForExit(waitUpToMs);
sw.Stop();
if (!process.HasExited)
{
throw new InvalidOperationException($"Timed out waiting for {batch.BatchId}. Symsorter args: {args}");
}
lastLine ??= string.Empty;
if (process.ExitCode != 0)
{
throw new InvalidOperationException($"Symsorter exit code: {process.ExitCode}. Args: {args}");
}
_logger.LogInformation("Symsorter finished in {timespan} and logged last: {lastLine}",
sw.Elapsed, lastLine);
var match = Regex.Match(lastLine, "Sorted (?<count>\\d+) debug files");
if (!match.Success)
{
_logger.LogError("Last line didn't match success: {lastLine}", lastLine);
return(true);
}
_logger.LogInformation("Symsorter processed: {count}", match.Groups["count"].Value);
return(false);
}
