private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false)
{
while (!_jobCompletionSource.Task.IsCompleted || runOnce)
{
List <PendingTimelineRecord> pendingUpdates = new List <PendingTimelineRecord>();
foreach (var timeline in _allTimelines)
{
ConcurrentQueue <TimelineRecord> recordQueue;
if (_timelineUpdateQueue.TryGetValue(timeline, out recordQueue))
{
List <TimelineRecord> records = new List <TimelineRecord>();
TimelineRecord record;
while (recordQueue.TryDequeue(out record))
{
records.Add(record);
// process at most 25 timeline records update for each timeline.
if (!runOnce && records.Count > 25)
{
break;
}
}
if (records.Count > 0)
{
pendingUpdates.Add(new PendingTimelineRecord()
{
TimelineId = timeline, PendingRecords = records.ToList()
});
}
}
}
// we need track whether we have new sub-timeline been created on the last run.
// if so, we need continue update timeline record even we on the last run.
bool pendingSubtimelineUpdate = false;
List <Exception> mainTimelineRecordsUpdateErrors = new List <Exception>();
if (pendingUpdates.Count > 0)
{
foreach (var update in pendingUpdates)
{
List <TimelineRecord> bufferedRecords;
if (_bufferedRetryRecords.TryGetValue(update.TimelineId, out bufferedRecords))
{
update.PendingRecords.InsertRange(0, bufferedRecords);
}
update.PendingRecords = MergeTimelineRecords(update.PendingRecords);
foreach (var detailTimeline in update.PendingRecords.Where(r => r.Details != null))
{
if (!_allTimelines.Contains(detailTimeline.Details.Id))
{
try
{
Timeline newTimeline = await _jobServer.CreateTimelineAsync(_scopeIdentifier, _hubName, _planId, detailTimeline.Details.Id, default(CancellationToken));
_allTimelines.Add(newTimeline.Id);
pendingSubtimelineUpdate = true;
}
catch (TimelineExistsException)
{
Trace.Info("Catch TimelineExistsException during timeline creation. Ignore the error since server already had this timeline.");
_allTimelines.Add(detailTimeline.Details.Id);
}
catch (Exception ex)
{
Trace.Error(ex);
}
}
}
try
{
await _jobServer.UpdateTimelineRecordsAsync(_scopeIdentifier, _hubName, _planId, update.TimelineId, update.PendingRecords, default(CancellationToken));
if (_bufferedRetryRecords.Remove(update.TimelineId))
{
Trace.Verbose("Cleanup buffered timeline record for timeline: {0}.", update.TimelineId);
}
}
catch (Exception ex)
{
Trace.Info("Catch exception during update timeline records, try to update these timeline records next time.");
Trace.Error(ex);
_bufferedRetryRecords[update.TimelineId] = update.PendingRecords.ToList();
if (update.TimelineId == _jobTimelineId)
{
mainTimelineRecordsUpdateErrors.Add(ex);
}
}
}
}
if (runOnce)
{
// continue process timeline records update,
// we might have more records need update,
// since we just create a new sub-timeline
if (pendingSubtimelineUpdate)
{
continue;
}
else
{
if (mainTimelineRecordsUpdateErrors.Count > 0 &&
_bufferedRetryRecords.ContainsKey(_jobTimelineId) &&
_bufferedRetryRecords[_jobTimelineId] != null &&
_bufferedRetryRecords[_jobTimelineId].Any(r => r.Variables.Count > 0))
{
Trace.Info("Fail to update timeline records with output variables. Throw exception to fail the job since output variables are critical to downstream jobs.");
throw new AggregateException(StringUtil.Loc("OutputVariablePublishFailed"), mainTimelineRecordsUpdateErrors);
}
else
{
break;
}
}
}
else
{
await Task.Delay(_delayForTimelineUpdateDequeue);
}
}
}
private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false)
{
while (!_jobCompletionSource.Task.IsCompleted || runOnce)
{
List <PendingTimelineRecord> pendingUpdates = new List <PendingTimelineRecord>();
foreach (var timeline in _allTimelines)
{
ConcurrentQueue <TimelineRecord> recordQueue;
if (_timelineUpdateQueue.TryGetValue(timeline, out recordQueue))
{
List <TimelineRecord> records = new List <TimelineRecord>();
TimelineRecord record;
while (recordQueue.TryDequeue(out record))
{
records.Add(record);
// process at most 25 timeline records update for each timeline.
if (!runOnce && records.Count > 25)
{
break;
}
}
if (records.Count > 0)
{
pendingUpdates.Add(new PendingTimelineRecord()
{
TimelineId = timeline, PendingRecords = records.ToList()
});
}
}
}
if (pendingUpdates.Count > 0)
{
foreach (var update in pendingUpdates)
{
List <TimelineRecord> bufferedRecords;
if (_bufferedRetryRecords.TryGetValue(update.TimelineId, out bufferedRecords))
{
update.PendingRecords.InsertRange(0, bufferedRecords);
}
update.PendingRecords = MergeTimelineRecords(update.PendingRecords);
foreach (var detailTimeline in update.PendingRecords.Where(r => r.Details != null))
{
if (!_allTimelines.Contains(detailTimeline.Details.Id))
{
try
{
Timeline newTimeline = await _jobServer.CreateTimelineAsync(_scopeIdentifier, _hubName, _planId, detailTimeline.Details.Id, default(CancellationToken));
_allTimelines.Add(newTimeline.Id);
}
catch (TimelineExistsException)
{
Trace.Info("Catch TimelineExistsException during timeline creation. Ignore the error since server already had this timeline.");
_allTimelines.Add(detailTimeline.Details.Id);
}
catch (Exception ex)
{
Trace.Error(ex);
}
}
}
try
{
await _jobServer.UpdateTimelineRecordsAsync(_scopeIdentifier, _hubName, _planId, update.TimelineId, update.PendingRecords, default(CancellationToken));
if (_bufferedRetryRecords.Remove(update.TimelineId))
{
Trace.Verbose("Cleanup buffered timeline record for timeline: {0}.", update.TimelineId);
}
}
catch (Exception ex)
{
Trace.Info("Catch exception during update timeline records, try to update these timeline records next time.");
Trace.Error(ex);
_bufferedRetryRecords[update.TimelineId] = update.PendingRecords.ToList();
}
}
}
if (runOnce)
{
break;
}
else
{
await Task.Delay(_delayForTimelineUpdateDequeue);
}
}
}
