private async Task ProcessWebConsoleLinesQueueAsync(bool runOnce = false)
{
while (!_jobCompletionSource.Task.IsCompleted || runOnce)
{
List <List <string> > batchedLines = new List <List <string> >();
List <string> currentBatch = new List <string>();
string line;
while (_webConsoleLineQueue.TryDequeue(out line))
{
if (!string.IsNullOrEmpty(line) && line.Length > 1024)
{
Trace.Verbose("Web console line is more than 1024 chars, truncate to first 1024 chars");
line = $"{line.Substring(0, 1024)}...";
}
currentBatch.Add(line);
// choose 100 lines since the whole web console UI will only shows about 40 lines in a 15" monitor.
if (currentBatch.Count > 100)
{
batchedLines.Add(currentBatch.ToList());
currentBatch.Clear();
}
// process at most about 500 lines of web console line during regular timer dequeue task.
if (!runOnce && batchedLines.Count > 5)
{
break;
}
}
if (currentBatch.Count > 0)
{
batchedLines.Add(currentBatch.ToList());
currentBatch.Clear();
}
if (batchedLines.Count > 0)
{
List <Exception> webConsoleLinePostExceptions = new List <Exception>();
foreach (var batch in batchedLines)
{
try
{
// we will not requeue failed batch, since the web console lines are time sensitive.
await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, batch, default(CancellationToken));
}
catch (Exception ex)
{
Trace.Info("Catch exception during append web console line, keep going since the process is best effort. Due with exception when all batches finish.");
webConsoleLinePostExceptions.Add(ex);
}
}
Trace.Info("Try to append {0} batches web console lines, success rate: {1}/{0}.", batchedLines.Count, batchedLines.Count - webConsoleLinePostExceptions.Count);
if (webConsoleLinePostExceptions.Count > 0)
{
AggregateException ex = new AggregateException("Catch exception during append web console line.", webConsoleLinePostExceptions);
if (!runOnce)
{
Trace.Verbose("Catch exception during process web console line queue, keep going since the process is best effort.");
Trace.Error(ex);
}
else
{
Trace.Error("Catch exception during drain web console line queue. throw aggregate exception to caller.");
throw ex;
}
}
}
if (runOnce)
{
break;
}
else
{
await Task.Delay(_delayForWebConsoleLineDequeue);
}
}
}
private async Task ProcessWebConsoleLinesQueueAsync(bool runOnce = false)
{
while (!_jobCompletionSource.Task.IsCompleted || runOnce)
{
if (_webConsoleLineAggressiveDequeue && ++_webConsoleLineAggressiveDequeueCount > _webConsoleLineAggressiveDequeueLimit)
{
Trace.Info("Stop aggressive process web console line queue.");
_webConsoleLineAggressiveDequeue = false;
}
// Group consolelines by timeline record of each step
Dictionary <Guid, List <TimelineRecordLogLine> > stepsConsoleLines = new Dictionary <Guid, List <TimelineRecordLogLine> >();
List <Guid> stepRecordIds = new List <Guid>(); // We need to keep lines in order
int linesCounter = 0;
ConsoleLineInfo lineInfo;
while (_webConsoleLineQueue.TryDequeue(out lineInfo))
{
if (!stepsConsoleLines.ContainsKey(lineInfo.StepRecordId))
{
stepsConsoleLines[lineInfo.StepRecordId] = new List <TimelineRecordLogLine>();
stepRecordIds.Add(lineInfo.StepRecordId);
}
if (lineInfo.Line?.Length > 1024)
{
Trace.Verbose("Web console line is more than 1024 chars, truncate to first 1024 chars");
lineInfo.Line = $"{lineInfo.Line.Substring(0, 1024)}...";
}
stepsConsoleLines[lineInfo.StepRecordId].Add(new TimelineRecordLogLine(lineInfo.Line, lineInfo.LineNumber));
linesCounter++;
// process at most about 500 lines of web console line during regular timer dequeue task.
// Send the first line of output to the customer right away
// It might take a while to reach 500 line outputs, which would cause delays before customers see the first line
if ((!runOnce && linesCounter > 500) || _firstConsoleOutputs)
{
break;
}
}
// Batch post consolelines for each step timeline record
foreach (var stepRecordId in stepRecordIds)
{
// Split consolelines into batch, each batch will container at most 100 lines.
int batchCounter = 0;
List <List <TimelineRecordLogLine> > batchedLines = new List <List <TimelineRecordLogLine> >();
foreach (var line in stepsConsoleLines[stepRecordId])
{
var currentBatch = batchedLines.ElementAtOrDefault(batchCounter);
if (currentBatch == null)
{
batchedLines.Add(new List <TimelineRecordLogLine>());
currentBatch = batchedLines.ElementAt(batchCounter);
}
currentBatch.Add(line);
if (currentBatch.Count >= 100)
{
batchCounter++;
}
}
if (batchedLines.Count > 0)
{
// When job finish, web console lines becomes less interesting to customer
// We batch and produce 500 lines of web console output every 500ms
// If customer's task produce massive of outputs, then the last queue drain run might take forever.
// So we will only upload the last 200 lines of each step from all buffered web console lines.
if (runOnce && batchedLines.Count > 2)
{
Trace.Info($"Skip {batchedLines.Count - 2} batches web console lines for last run");
batchedLines = batchedLines.TakeLast(2).ToList();
}
int errorCount = 0;
foreach (var batch in batchedLines)
{
try
{
// we will not requeue failed batch, since the web console lines are time sensitive.
await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(x => x.Line).ToList(), batch[0].LineNumber, default(CancellationToken));
if (_firstConsoleOutputs)
{
_firstConsoleOutputs = false;
HostContext.WritePerfCounter("WorkerJobServerQueueAppendFirstConsoleOutput");
}
}
catch (Exception ex)
{
Trace.Info("Catch exception during append web console line, keep going since the process is best effort.");
Trace.Error(ex);
errorCount++;
}
}
Trace.Info("Try to append {0} batches web console lines for record '{2}', success rate: {1}/{0}.", batchedLines.Count, batchedLines.Count - errorCount, stepRecordId);
}
}
if (runOnce)
{
break;
}
else
{
_webConsoleLinesDequeueNow = new TaskCompletionSource <object>();
await Task.WhenAny(
Task.Delay(_webConsoleLineAggressiveDequeue ? _aggressiveDelayForWebConsoleLineDequeue : TimeSpan.FromMilliseconds(_webConsoleLineUpdateRate)),
_webConsoleLinesDequeueNow.Task);
}
}
}
private async Task ProcessWebConsoleLinesQueueAsync(bool runOnce = false)
{
while (!_jobCompletionSource.Task.IsCompleted || runOnce)
{
if (_webConsoleLineAggressiveDequeue && ++_webConsoleLineAggressiveDequeueCount > _webConsoleLineAggressiveDequeueLimit)
{
Trace.Info("Stop aggressive process web console line queue.");
_webConsoleLineAggressiveDequeue = false;
}
List <List <string> > batchedLines = new List <List <string> >();
List <string> currentBatch = new List <string>();
string line;
while (_webConsoleLineQueue.TryDequeue(out line))
{
if (!string.IsNullOrEmpty(line) && line.Length > 1024)
{
Trace.Verbose("Web console line is more than 1024 chars, truncate to first 1024 chars");
line = $"{line.Substring(0, 1024)}...";
}
currentBatch.Add(line);
// choose 100 lines since the whole web console UI will only shows about 40 lines in a 15" monitor.
if (currentBatch.Count > 100)
{
batchedLines.Add(currentBatch.ToList());
currentBatch.Clear();
}
// process at most about 500 lines of web console line during regular timer dequeue task.
if (!runOnce && batchedLines.Count > 5)
{
break;
}
}
if (currentBatch.Count > 0)
{
batchedLines.Add(currentBatch.ToList());
currentBatch.Clear();
}
if (batchedLines.Count > 0)
{
// When job finish, web console lines becomes less interesting to customer
// We batch and produce 600 lines of web console output every 500ms
// If customer's task produce massive of outputs, then the last queue drain run might take forever.
// So we will only upload the last 200 lines of all buffered web console lines.
if (runOnce && batchedLines.Count > 2)
{
Trace.Info($"Skip {batchedLines.Count - 2} batches web console lines for last run");
batchedLines = batchedLines.TakeLast(2).ToList();
batchedLines[0].Insert(0, "...");
}
int errorCount = 0;
foreach (var batch in batchedLines)
{
try
{
// we will not requeue failed batch, since the web console lines are time sensitive.
await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, batch, default(CancellationToken));
}
catch (Exception ex)
{
Trace.Info("Catch exception during append web console line, keep going since the process is best effort.");
Trace.Error(ex);
errorCount++;
}
}
Trace.Info("Try to append {0} batches web console lines, success rate: {1}/{0}.", batchedLines.Count, batchedLines.Count - errorCount);
}
if (runOnce)
{
break;
}
else
{
await Task.Delay(_webConsoleLineAggressiveDequeue?_aggressiveDelayForWebConsoleLineDequeue : _delayForWebConsoleLineDequeue);
}
}
}