private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false) { while (!_jobCompletionSource.Task.IsCompleted || runOnce) { List <PendingTimelineRecord> pendingUpdates = new List <PendingTimelineRecord>(); foreach (var timeline in _allTimelines) { ConcurrentQueue <TimelineRecord> recordQueue; if (_timelineUpdateQueue.TryGetValue(timeline, out recordQueue)) { List <TimelineRecord> records = new List <TimelineRecord>(); TimelineRecord record; while (recordQueue.TryDequeue(out record)) { records.Add(record); // process at most 25 timeline records update for each timeline. if (!runOnce && records.Count > 25) { break; } } if (records.Count > 0) { pendingUpdates.Add(new PendingTimelineRecord() { TimelineId = timeline, PendingRecords = records.ToList() }); } } } // we need track whether we have new sub-timeline been created on the last run. // if so, we need continue update timeline record even we on the last run. bool pendingSubtimelineUpdate = false; List <Exception> mainTimelineRecordsUpdateErrors = new List <Exception>(); if (pendingUpdates.Count > 0) { foreach (var update in pendingUpdates) { List <TimelineRecord> bufferedRecords; if (_bufferedRetryRecords.TryGetValue(update.TimelineId, out bufferedRecords)) { update.PendingRecords.InsertRange(0, bufferedRecords); } update.PendingRecords = MergeTimelineRecords(update.PendingRecords); foreach (var detailTimeline in update.PendingRecords.Where(r => r.Details != null)) { if (!_allTimelines.Contains(detailTimeline.Details.Id)) { try { Timeline newTimeline = await _jobServer.CreateTimelineAsync(_scopeIdentifier, _hubName, _planId, detailTimeline.Details.Id, default(CancellationToken)); _allTimelines.Add(newTimeline.Id); pendingSubtimelineUpdate = true; } catch (TimelineExistsException) { Trace.Info("Catch TimelineExistsException during timeline creation. Ignore the error since server already had this timeline."); _allTimelines.Add(detailTimeline.Details.Id); } catch (Exception ex) { Trace.Error(ex); } } } try { await _jobServer.UpdateTimelineRecordsAsync(_scopeIdentifier, _hubName, _planId, update.TimelineId, update.PendingRecords, default(CancellationToken)); if (_bufferedRetryRecords.Remove(update.TimelineId)) { Trace.Verbose("Cleanup buffered timeline record for timeline: {0}.", update.TimelineId); } } catch (Exception ex) { Trace.Info("Catch exception during update timeline records, try to update these timeline records next time."); Trace.Error(ex); _bufferedRetryRecords[update.TimelineId] = update.PendingRecords.ToList(); if (update.TimelineId == _jobTimelineId) { mainTimelineRecordsUpdateErrors.Add(ex); } } } } if (runOnce) { // continue process timeline records update, // we might have more records need update, // since we just create a new sub-timeline if (pendingSubtimelineUpdate) { continue; } else { if (mainTimelineRecordsUpdateErrors.Count > 0 && _bufferedRetryRecords.ContainsKey(_jobTimelineId) && _bufferedRetryRecords[_jobTimelineId] != null && _bufferedRetryRecords[_jobTimelineId].Any(r => r.Variables.Count > 0)) { Trace.Info("Fail to update timeline records with output variables. Throw exception to fail the job since output variables are critical to downstream jobs."); throw new AggregateException(StringUtil.Loc("OutputVariablePublishFailed"), mainTimelineRecordsUpdateErrors); } else { break; } } } else { await Task.Delay(_delayForTimelineUpdateDequeue); } } }
private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false) { while (!_jobCompletionSource.Task.IsCompleted || runOnce) { List <PendingTimelineRecord> pendingUpdates = new List <PendingTimelineRecord>(); foreach (var timeline in _allTimelines) { ConcurrentQueue <TimelineRecord> recordQueue; if (_timelineUpdateQueue.TryGetValue(timeline, out recordQueue)) { List <TimelineRecord> records = new List <TimelineRecord>(); TimelineRecord record; while (recordQueue.TryDequeue(out record)) { records.Add(record); // process at most 25 timeline records update for each timeline. if (!runOnce && records.Count > 25) { break; } } if (records.Count > 0) { pendingUpdates.Add(new PendingTimelineRecord() { TimelineId = timeline, PendingRecords = records.ToList() }); } } } if (pendingUpdates.Count > 0) { foreach (var update in pendingUpdates) { List <TimelineRecord> bufferedRecords; if (_bufferedRetryRecords.TryGetValue(update.TimelineId, out bufferedRecords)) { update.PendingRecords.InsertRange(0, bufferedRecords); } update.PendingRecords = MergeTimelineRecords(update.PendingRecords); foreach (var detailTimeline in update.PendingRecords.Where(r => r.Details != null)) { if (!_allTimelines.Contains(detailTimeline.Details.Id)) { try { Timeline newTimeline = await _jobServer.CreateTimelineAsync(_scopeIdentifier, _hubName, _planId, detailTimeline.Details.Id, default(CancellationToken)); _allTimelines.Add(newTimeline.Id); } catch (TimelineExistsException) { Trace.Info("Catch TimelineExistsException during timeline creation. Ignore the error since server already had this timeline."); _allTimelines.Add(detailTimeline.Details.Id); } catch (Exception ex) { Trace.Error(ex); } } } try { await _jobServer.UpdateTimelineRecordsAsync(_scopeIdentifier, _hubName, _planId, update.TimelineId, update.PendingRecords, default(CancellationToken)); if (_bufferedRetryRecords.Remove(update.TimelineId)) { Trace.Verbose("Cleanup buffered timeline record for timeline: {0}.", update.TimelineId); } } catch (Exception ex) { Trace.Info("Catch exception during update timeline records, try to update these timeline records next time."); Trace.Error(ex); _bufferedRetryRecords[update.TimelineId] = update.PendingRecords.ToList(); } } } if (runOnce) { break; } else { await Task.Delay(_delayForTimelineUpdateDequeue); } } }