void ProcessAll(Guid id, ConcurrentQueue <CoordinationRequest> q)
{
_eventStream.Publish <JobCoordinator>(EventType.Activity,
EventProperty.ActivityName("CoordinatedEventProcessingCycleStarted"));
while (true)
{
CoordinationRequest request;
if (q.TryDequeue(out request))
{
_recoverableAction.Run(request.Action, () => Run(request.Job, request.Action));
}
else
{
lock (_latch)
{
if (q.IsEmpty)
{
_coordinationQueues.Remove(id);
break;
}
}
}
}
_eventStream.Publish <JobCoordinator>(EventType.Activity,
EventProperty.ActivityName("CoordinatedEventProcessingCycleFinished"));
}
void PublishQueueInitializedEvent(int activityCount, int suspendedCount, Type activityType = null)
{
_eventStream.Publish <JobQueueFactory>(
EventType.Activity,
EventProperty.ActivityName("JobQueueInitialized"),
EventProperty.Named("ActivityType", activityType == null ? "All" : activityType.FullName),
EventProperty.Named("ActivityCount", activityCount),
EventProperty.Named("SuspendedCount", suspendedCount));
}
public void Add(Job job)
{
_eventStream.Publish <FailedJobQueue>(
EventType.Activity,
EventProperty.ActivityName("NewFailedItem"),
EventProperty.JobSnapshot(job));
_jobFailures.Add(Tuple.Create(job.Id, job.RetryOn));
}
void OnTick(object state)
{
var ready = new Collection <Tuple <Guid, DateTime?> >();
var notReady = new Collection <Tuple <Guid, DateTime?> >();
try
{
Tuple <Guid, DateTime?> item;
_eventStream.Publish <FailedJobQueue>(
EventType.TimerActivity,
EventProperty.ActivityName("RescheduleFailedJobs"),
EventProperty.Named("FailedItemsQueueLength", _jobFailures.Count));
while (_jobFailures.TryTake(out item))
{
if (item.Item2 < _now())
{
ready.Add(item);
}
else
{
notReady.Add(item);
}
}
foreach (var job in ready.Select(i => _persistenceStore.Load(i.Item1)))
{
_router.Route(job);
}
ready.Clear();
}
catch (Exception e)
{
if (e.IsFatal())
{
throw;
}
_eventStream.Publish <FailedJobQueue>(e);
}
finally
{
foreach (var item in ready.Concat(notReady))
{
_jobFailures.Add(item);
}
_timer.Change(_configuration.RetryTimerInterval, Timeout.InfiniteTimeSpan);
}
}
async Task <Job[]> LoadSuspended()
{
var list = new List <Job>();
while (list.Count == 0)
{
try
{
var max = Configuration.MaxQueueLength;
var items =
(Configuration.Type != null ?
_persistenceStore.LoadSuspended(Configuration.Type, max) :
_persistenceStore.LoadSuspended(_allActivityConfiguration.Select(c => c.Type), max))
.ToArray();
_eventStream.Publish <JobQueue>(EventType.Activity,
EventProperty.ActivityName("LoadSuspendedItemsStarted"),
EventProperty.Named("NumberOfItems", items.Length));
list.AddRange(items.Select(item => _jobMutator.Mutate <JobQueue>(item, suspended: false)));
}
catch (Exception e)
{
if (e.IsFatal())
{
throw;
}
_eventStream.Publish <JobQueue>(e);
}
if (!list.Any())
{
await Task.Delay(Configuration.RetryDelay);
}
}
_eventStream.Publish <JobQueue>(EventType.Activity,
EventProperty.ActivityName("LoadSuspendedItemsFinished"),
EventProperty.Named("NumberOfItems", list.Count));
return(list.ToArray());
}
void OnTick(object state)
{
try
{
var maxItemsToRetry = _itemsToRecover.Count;
_eventStream.Publish <RecoverableAction>(
EventType.TimerActivity,
EventProperty.ActivityName("RecoveryTimer"),
new KeyValuePair <string, object>(
"NumberOfItemsToProcess",
maxItemsToRetry));
for (var i = 0; i < maxItemsToRetry; i++)
{
RecoverableActionRequest request;
if (!_itemsToRecover.TryDequeue(out request))
{
break;
}
Task.Run(() => RunCore(request)).FailFastOnException();
}
}
catch (Exception e)
{
if (e.IsFatal())
{
throw;
}
_eventStream.Publish <RecoverableAction>(e);
}
finally
{
_timer.Change(_configuration.RetryTimerInterval, Timeout.InfiniteTimeSpan);
}
}
