/// <summary>
/// Performs the main processing work.
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
protected override async Task RunAsync(CancellationToken cancellationToken)
{
try
{
IReliableQueue <ReportProcessingStep> processQueue
= await this.StateManager.GetOrAddAsync <IReliableQueue <ReportProcessingStep> >(ProcessingQueueName);
IReliableDictionary <string, ReportStatus> statusDictionary
= await this.StateManager.GetOrAddAsync <IReliableDictionary <string, ReportStatus> >(StatusDictionaryName);
// First time setup: queue up all the processing steps and create an initial processing status if one doesn't exist already
// Note that this will execute every time a replica is promoted to primary,
// so we need to check to see if it's already been done because we only want to initialize once.
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <ReportStatus> tryGetResult
= await statusDictionary.TryGetValueAsync(tx, this.reportContext.Name, LockMode.Update);
if (!tryGetResult.HasValue)
{
await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Creating"));
await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Evaluating"));
await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Reticulating"));
for (int i = 0; i < processingMultiplier * queueLengthMultiplier; ++i)
{
cancellationToken.ThrowIfCancellationRequested();
await processQueue.EnqueueAsync(tx, new ReportProcessingStep($"Processing {i}"));
}
await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Sanitizing"));
await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Mystery Step"));
await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Finalizing"));
await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Complete"));
await statusDictionary.AddAsync(tx, this.reportContext.Name, new ReportStatus(0, processingMultiplier *queueLengthMultiplier + 7, "Not started.", String.Empty));
}
await tx.CommitAsync();
}
// start processing and checkpoint between each step so we don't lose any progress in the event of a fail-over
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
// Get the next step from the queue with a peek.
// This keeps the item on the queue in case processing fails.
ConditionalValue <ReportProcessingStep> dequeueResult;
using (ITransaction tx = this.StateManager.CreateTransaction())
{
dequeueResult = await processQueue.TryPeekAsync(tx, LockMode.Default);
}
if (!dequeueResult.HasValue)
{
// all done!
break;
}
ReportProcessingStep currentProcessingStep = dequeueResult.Value;
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"Processing step: {currentProcessingStep.Name}");
// Get the current processing step
ConditionalValue <ReportStatus> dictionaryGetResult;
using (ITransaction tx = this.StateManager.CreateTransaction())
{
dictionaryGetResult = await statusDictionary.TryGetValueAsync(tx, this.reportContext.Name, LockMode.Default);
}
// Perform the next processing step.
// This is potentially a long-running operation, therefore it is not executed within a transaction.
ReportStatus currentStatus = dictionaryGetResult.Value;
ReportStatus newStatus = await this.ProcessReport(currentStatus, currentProcessingStep, cancellationToken);
// Once processing is done, save the results and dequeue the processing step.
// If an exception or other failure occurs at this point, the processing step will run again.
using (ITransaction tx = this.StateManager.CreateTransaction())
{
dequeueResult = await processQueue.TryDequeueAsync(tx);
await statusDictionary.SetAsync(tx, this.reportContext.Name, newStatus);
await tx.CommitAsync();
}
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"Processing step: {currentProcessingStep.Name} complete.");
}
catch (TimeoutException)
{
// transient error. Retry.
ServiceEventSource.Current.ServiceMessage(this.Context, "TimeoutException in RunAsync.");
}
catch (FabricTransientException fte)
{
// transient error. Retry.
ServiceEventSource.Current.ServiceMessage(this.Context, "FabricTransientException in RunAsync: {0}", fte.Message);
}
catch (FabricNotReadableException)
{
// retry or wait until not primary
ServiceEventSource.Current.ServiceMessage(this.Context, "FabricNotReadableException in RunAsync.");
}
// delay between each to step to prevent starving other processing service instances.
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
}
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"Processing complete!");
}
catch (OperationCanceledException)
{
// time to quit
throw;
}
catch (FabricNotPrimaryException)
{
// time to quit
return;
}
catch (Exception ex)
{
// all other exceptions: log and re-throw.
ServiceEventSource.Current.ServiceMessage(this.Context, "Exception in RunAsync: {0}", ex.Message);
throw;
}
}
