/// <summary> /// "Processes" a report /// </summary> /// <param name="currentStatus"></param> /// <param name="currentProcessingStep"></param> /// <param name="cancellationToken"></param> /// <returns></returns> internal async Task <ReportStatus> ProcessReport(ReportStatus currentStatus, ReportProcessingStep currentProcessingStep, CancellationToken cancellationToken) { const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; TimeSpan ioDelay = TimeSpan.FromMilliseconds(this.random.Next(100, 200) / this.processingMultiplier); int randomLength = this.random.Next(1000, 2000) * this.processingMultiplier; string randomJunk = ""; for (int i = 0; i < (10 * this.processingMultiplier); ++i) { cancellationToken.ThrowIfCancellationRequested(); // pointless inefficient string processing to drive up CPU and memory use randomJunk += new string(Enumerable.Repeat(chars, randomLength).Select(s => s[this.random.Next(s.Length)]).ToArray()); int numberOfAs = 0; for (int j = 0; j < randomJunk.Length; ++j) { cancellationToken.ThrowIfCancellationRequested(); if (randomJunk[j] == 'A') { ++numberOfAs; } } await Task.Delay(ioDelay, cancellationToken); } return(new ReportStatus(currentStatus.Step + 1, currentStatus.Remaining - 1, currentProcessingStep.Name, randomJunk)); }
protected override async Task RunAsync(CancellationToken cancellationToken) { IReliableConcurrentQueue <ReportProcessingStep> processQueue = await this.StateManager.GetOrAddAsync <IReliableConcurrentQueue <ReportProcessingStep> >(ProcessingQueueName); IReliableDictionary <string, ReportStatus> statusDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, ReportStatus> >(StatusDictionaryName); // queue up all the processing steps and create an initial processing status if one doesn't exist already using (ITransaction tx = this.StateManager.CreateTransaction()) { ConditionalValue <ReportStatus> tryGetResult = await statusDictionary.TryGetValueAsync(tx, this.reportContext.Name, LockMode.Update); if (!tryGetResult.HasValue) { foreach (string processingStep in processingSteps) { cancellationToken.ThrowIfCancellationRequested(); await processQueue.EnqueueAsync(tx, new ReportProcessingStep(processingStep)); } await statusDictionary.AddAsync(tx, this.reportContext.Name, new ReportStatus(0, "Not started.")); } await tx.CommitAsync(); } // start processing and checkpoint between each step so we don't lose any progress in the event of a fail-over while (true) { cancellationToken.ThrowIfCancellationRequested(); try { using (ITransaction tx = this.StateManager.CreateTransaction()) { ConditionalValue <ReportProcessingStep> dequeueResult = await processQueue.TryDequeueAsync(tx, cancellationToken); if (!dequeueResult.HasValue) { // all done! break; } ReportProcessingStep currentProcessingStep = dequeueResult.Value; ServiceEventSource.Current.ServiceMessage( this.Context, $"Processing step: {currentProcessingStep.Name}"); // This takes a shared lock rather than an update lock // because this is the only place the row is written to. // If there were other writers, then this should be an update lock. ConditionalValue <ReportStatus> dictionaryGetResult = await statusDictionary.TryGetValueAsync(tx, this.reportContext.Name, LockMode.Default); ReportStatus currentStatus = dictionaryGetResult.Value; ReportStatus newStatus = await this.ProcessReport(currentStatus, currentProcessingStep, cancellationToken); await statusDictionary.SetAsync(tx, this.reportContext.Name, newStatus); await tx.CommitAsync(); } } catch (TimeoutException) { // transient error. Retry. ServiceEventSource.Current.ServiceMessage(this.Context, "TimeoutException in RunAsync."); } catch (FabricTransientException fte) { // transient error. Retry. ServiceEventSource.Current.ServiceMessage(this.Context, "FabricTransientException in RunAsync: {0}", fte.Message); } catch (FabricNotPrimaryException) { // not primary any more, time to quit. return; } catch (FabricNotReadableException) { // retry or wait until not primary ServiceEventSource.Current.ServiceMessage(this.Context, "FabricNotReadableException in RunAsync."); } catch (Exception ex) { // all other exceptions: log and re-throw. ServiceEventSource.Current.ServiceMessage(this.Context, "Exception in RunAsync: {0}", ex.Message); throw; } // delay between each to step to prevent starving other processing service instances. await Task.Delay(TimeSpan.FromSeconds(10), cancellationToken); } ServiceEventSource.Current.ServiceMessage( this.Context, $"Processing complete!"); }
/// <summary> /// Performs the main processing work. /// </summary> /// <param name="cancellationToken"></param> /// <returns></returns> protected override async Task RunAsync(CancellationToken cancellationToken) { try { IReliableQueue <ReportProcessingStep> processQueue = await this.StateManager.GetOrAddAsync <IReliableQueue <ReportProcessingStep> >(ProcessingQueueName); IReliableDictionary <string, ReportStatus> statusDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, ReportStatus> >(StatusDictionaryName); // First time setup: queue up all the processing steps and create an initial processing status if one doesn't exist already // Note that this will execute every time a replica is promoted to primary, // so we need to check to see if it's already been done because we only want to initialize once. using (ITransaction tx = this.StateManager.CreateTransaction()) { ConditionalValue <ReportStatus> tryGetResult = await statusDictionary.TryGetValueAsync(tx, this.reportContext.Name, LockMode.Update); if (!tryGetResult.HasValue) { await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Creating")); await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Evaluating")); await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Reticulating")); for (int i = 0; i < processingMultiplier * queueLengthMultiplier; ++i) { cancellationToken.ThrowIfCancellationRequested(); await processQueue.EnqueueAsync(tx, new ReportProcessingStep($"Processing {i}")); } await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Sanitizing")); await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Mystery Step")); await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Finalizing")); await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Complete")); await statusDictionary.AddAsync(tx, this.reportContext.Name, new ReportStatus(0, processingMultiplier *queueLengthMultiplier + 7, "Not started.", String.Empty)); } await tx.CommitAsync(); } // start processing and checkpoint between each step so we don't lose any progress in the event of a fail-over while (true) { cancellationToken.ThrowIfCancellationRequested(); try { // Get the next step from the queue with a peek. // This keeps the item on the queue in case processing fails. ConditionalValue <ReportProcessingStep> dequeueResult; using (ITransaction tx = this.StateManager.CreateTransaction()) { dequeueResult = await processQueue.TryPeekAsync(tx, LockMode.Default); } if (!dequeueResult.HasValue) { // all done! break; } ReportProcessingStep currentProcessingStep = dequeueResult.Value; ServiceEventSource.Current.ServiceMessage( this.Context, $"Processing step: {currentProcessingStep.Name}"); // Get the current processing step ConditionalValue <ReportStatus> dictionaryGetResult; using (ITransaction tx = this.StateManager.CreateTransaction()) { dictionaryGetResult = await statusDictionary.TryGetValueAsync(tx, this.reportContext.Name, LockMode.Default); } // Perform the next processing step. // This is potentially a long-running operation, therefore it is not executed within a transaction. ReportStatus currentStatus = dictionaryGetResult.Value; ReportStatus newStatus = await this.ProcessReport(currentStatus, currentProcessingStep, cancellationToken); // Once processing is done, save the results and dequeue the processing step. // If an exception or other failure occurs at this point, the processing step will run again. using (ITransaction tx = this.StateManager.CreateTransaction()) { dequeueResult = await processQueue.TryDequeueAsync(tx); await statusDictionary.SetAsync(tx, this.reportContext.Name, newStatus); await tx.CommitAsync(); } ServiceEventSource.Current.ServiceMessage( this.Context, $"Processing step: {currentProcessingStep.Name} complete."); } catch (TimeoutException) { // transient error. Retry. ServiceEventSource.Current.ServiceMessage(this.Context, "TimeoutException in RunAsync."); } catch (FabricTransientException fte) { // transient error. Retry. ServiceEventSource.Current.ServiceMessage(this.Context, "FabricTransientException in RunAsync: {0}", fte.Message); } catch (FabricNotReadableException) { // retry or wait until not primary ServiceEventSource.Current.ServiceMessage(this.Context, "FabricNotReadableException in RunAsync."); } // delay between each to step to prevent starving other processing service instances. await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); } ServiceEventSource.Current.ServiceMessage( this.Context, $"Processing complete!"); } catch (OperationCanceledException) { // time to quit throw; } catch (FabricNotPrimaryException) { // time to quit return; } catch (Exception ex) { // all other exceptions: log and re-throw. ServiceEventSource.Current.ServiceMessage(this.Context, "Exception in RunAsync: {0}", ex.Message); throw; } }
/// <summary> /// "Processes" a report /// </summary> /// <param name="currentStatus"></param> /// <param name="currentProcessingStep"></param> /// <param name="cancellationToken"></param> /// <returns></returns> internal async Task <ReportStatus> ProcessReport(ReportStatus currentStatus, ReportProcessingStep currentProcessingStep, CancellationToken cancellationToken) { await Task.Delay(TimeSpan.FromSeconds(new Random().Next(5, 15)), cancellationToken); return(new ReportStatus(currentStatus.Step + 1, currentProcessingStep.Name)); }