private async Task ProcessWebConsoleLinesQueueAsync(bool runOnce = false) { while (!_jobCompletionSource.Task.IsCompleted || runOnce) { List <List <string> > batchedLines = new List <List <string> >(); List <string> currentBatch = new List <string>(); string line; while (_webConsoleLineQueue.TryDequeue(out line)) { if (!string.IsNullOrEmpty(line) && line.Length > 1024) { Trace.Verbose("Web console line is more than 1024 chars, truncate to first 1024 chars"); line = $"{line.Substring(0, 1024)}..."; } currentBatch.Add(line); // choose 100 lines since the whole web console UI will only shows about 40 lines in a 15" monitor. if (currentBatch.Count > 100) { batchedLines.Add(currentBatch.ToList()); currentBatch.Clear(); } // process at most about 500 lines of web console line during regular timer dequeue task. if (!runOnce && batchedLines.Count > 5) { break; } } if (currentBatch.Count > 0) { batchedLines.Add(currentBatch.ToList()); currentBatch.Clear(); } if (batchedLines.Count > 0) { List <Exception> webConsoleLinePostExceptions = new List <Exception>(); foreach (var batch in batchedLines) { try { // we will not requeue failed batch, since the web console lines are time sensitive. await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, batch, default(CancellationToken)); } catch (Exception ex) { Trace.Info("Catch exception during append web console line, keep going since the process is best effort. Due with exception when all batches finish."); webConsoleLinePostExceptions.Add(ex); } } Trace.Info("Try to append {0} batches web console lines, success rate: {1}/{0}.", batchedLines.Count, batchedLines.Count - webConsoleLinePostExceptions.Count); if (webConsoleLinePostExceptions.Count > 0) { AggregateException ex = new AggregateException("Catch exception during append web console line.", webConsoleLinePostExceptions); if (!runOnce) { Trace.Verbose("Catch exception during process web console line queue, keep going since the process is best effort."); Trace.Error(ex); } else { Trace.Error("Catch exception during drain web console line queue. throw aggregate exception to caller."); throw ex; } } } if (runOnce) { break; } else { await Task.Delay(_delayForWebConsoleLineDequeue); } } }
private async Task ProcessWebConsoleLinesQueueAsync(bool runOnce = false) { while (!_jobCompletionSource.Task.IsCompleted || runOnce) { if (_webConsoleLineAggressiveDequeue && ++_webConsoleLineAggressiveDequeueCount > _webConsoleLineAggressiveDequeueLimit) { Trace.Info("Stop aggressive process web console line queue."); _webConsoleLineAggressiveDequeue = false; } // Group consolelines by timeline record of each step Dictionary <Guid, List <TimelineRecordLogLine> > stepsConsoleLines = new Dictionary <Guid, List <TimelineRecordLogLine> >(); List <Guid> stepRecordIds = new List <Guid>(); // We need to keep lines in order int linesCounter = 0; ConsoleLineInfo lineInfo; while (_webConsoleLineQueue.TryDequeue(out lineInfo)) { if (!stepsConsoleLines.ContainsKey(lineInfo.StepRecordId)) { stepsConsoleLines[lineInfo.StepRecordId] = new List <TimelineRecordLogLine>(); stepRecordIds.Add(lineInfo.StepRecordId); } if (lineInfo.Line?.Length > 1024) { Trace.Verbose("Web console line is more than 1024 chars, truncate to first 1024 chars"); lineInfo.Line = $"{lineInfo.Line.Substring(0, 1024)}..."; } stepsConsoleLines[lineInfo.StepRecordId].Add(new TimelineRecordLogLine(lineInfo.Line, lineInfo.LineNumber)); linesCounter++; // process at most about 500 lines of web console line during regular timer dequeue task. // Send the first line of output to the customer right away // It might take a while to reach 500 line outputs, which would cause delays before customers see the first line if ((!runOnce && linesCounter > 500) || _firstConsoleOutputs) { break; } } // Batch post consolelines for each step timeline record foreach (var stepRecordId in stepRecordIds) { // Split consolelines into batch, each batch will container at most 100 lines. int batchCounter = 0; List <List <TimelineRecordLogLine> > batchedLines = new List <List <TimelineRecordLogLine> >(); foreach (var line in stepsConsoleLines[stepRecordId]) { var currentBatch = batchedLines.ElementAtOrDefault(batchCounter); if (currentBatch == null) { batchedLines.Add(new List <TimelineRecordLogLine>()); currentBatch = batchedLines.ElementAt(batchCounter); } currentBatch.Add(line); if (currentBatch.Count >= 100) { batchCounter++; } } if (batchedLines.Count > 0) { // When job finish, web console lines becomes less interesting to customer // We batch and produce 500 lines of web console output every 500ms // If customer's task produce massive of outputs, then the last queue drain run might take forever. // So we will only upload the last 200 lines of each step from all buffered web console lines. if (runOnce && batchedLines.Count > 2) { Trace.Info($"Skip {batchedLines.Count - 2} batches web console lines for last run"); batchedLines = batchedLines.TakeLast(2).ToList(); } int errorCount = 0; foreach (var batch in batchedLines) { try { // we will not requeue failed batch, since the web console lines are time sensitive. await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(x => x.Line).ToList(), batch[0].LineNumber, default(CancellationToken)); if (_firstConsoleOutputs) { _firstConsoleOutputs = false; HostContext.WritePerfCounter("WorkerJobServerQueueAppendFirstConsoleOutput"); } } catch (Exception ex) { Trace.Info("Catch exception during append web console line, keep going since the process is best effort."); Trace.Error(ex); errorCount++; } } Trace.Info("Try to append {0} batches web console lines for record '{2}', success rate: {1}/{0}.", batchedLines.Count, batchedLines.Count - errorCount, stepRecordId); } } if (runOnce) { break; } else { _webConsoleLinesDequeueNow = new TaskCompletionSource <object>(); await Task.WhenAny( Task.Delay(_webConsoleLineAggressiveDequeue ? _aggressiveDelayForWebConsoleLineDequeue : TimeSpan.FromMilliseconds(_webConsoleLineUpdateRate)), _webConsoleLinesDequeueNow.Task); } } }
private async Task ProcessWebConsoleLinesQueueAsync(bool runOnce = false) { while (!_jobCompletionSource.Task.IsCompleted || runOnce) { if (_webConsoleLineAggressiveDequeue && ++_webConsoleLineAggressiveDequeueCount > _webConsoleLineAggressiveDequeueLimit) { Trace.Info("Stop aggressive process web console line queue."); _webConsoleLineAggressiveDequeue = false; } List <List <string> > batchedLines = new List <List <string> >(); List <string> currentBatch = new List <string>(); string line; while (_webConsoleLineQueue.TryDequeue(out line)) { if (!string.IsNullOrEmpty(line) && line.Length > 1024) { Trace.Verbose("Web console line is more than 1024 chars, truncate to first 1024 chars"); line = $"{line.Substring(0, 1024)}..."; } currentBatch.Add(line); // choose 100 lines since the whole web console UI will only shows about 40 lines in a 15" monitor. if (currentBatch.Count > 100) { batchedLines.Add(currentBatch.ToList()); currentBatch.Clear(); } // process at most about 500 lines of web console line during regular timer dequeue task. if (!runOnce && batchedLines.Count > 5) { break; } } if (currentBatch.Count > 0) { batchedLines.Add(currentBatch.ToList()); currentBatch.Clear(); } if (batchedLines.Count > 0) { // When job finish, web console lines becomes less interesting to customer // We batch and produce 600 lines of web console output every 500ms // If customer's task produce massive of outputs, then the last queue drain run might take forever. // So we will only upload the last 200 lines of all buffered web console lines. if (runOnce && batchedLines.Count > 2) { Trace.Info($"Skip {batchedLines.Count - 2} batches web console lines for last run"); batchedLines = batchedLines.TakeLast(2).ToList(); batchedLines[0].Insert(0, "..."); } int errorCount = 0; foreach (var batch in batchedLines) { try { // we will not requeue failed batch, since the web console lines are time sensitive. await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, batch, default(CancellationToken)); } catch (Exception ex) { Trace.Info("Catch exception during append web console line, keep going since the process is best effort."); Trace.Error(ex); errorCount++; } } Trace.Info("Try to append {0} batches web console lines, success rate: {1}/{0}.", batchedLines.Count, batchedLines.Count - errorCount); } if (runOnce) { break; } else { await Task.Delay(_webConsoleLineAggressiveDequeue?_aggressiveDelayForWebConsoleLineDequeue : _delayForWebConsoleLineDequeue); } } }