public async Task <ITask> GetTask(CancellationToken token)
{
await EnsureInit();
ITransaction tx = null;
ITask ret = null;
try
{
tx = ServiceFabricUtils.CreateTransaction();
await ServiceFabricUtils.DoWithTimeoutRetry(async() =>
{
var result = await m_queue.TryDequeueAsync(tx, TimeSpan.FromSeconds(10), m_cancel);
if (result.HasValue)
{
ret = Utils.Deserialize <ITask>(result.Value);
}
});
if (ret == null)
{
tx.Dispose(); return(null);
}
m_tx[ret] = tx;
return(ret);
}
catch
{
ReleaseTx(ret, tx);
throw;
}
}
public static async Task ClearEntriesAsync <T>(this IReliableQueue <T> queue, ITransaction tx, int count)
{
while (count-- > 0)
{
await queue.TryDequeueAsync(tx).ConfigureAwait(false);
}
}
private async Task <IEnumerable <QueueMessage> > ReceiveMessagesAsync(int count)
{
IReliableQueue <byte[]> collection = await _stateManager.GetOrAddAsync <IReliableQueue <byte[]> >(_queueName);
var result = new List <QueueMessage>();
using (var tx = new ServiceFabricTransaction(_stateManager, null))
{
while (result.Count < count)
{
ConditionalValue <byte[]> message = await collection.TryDequeueAsync(tx.Tx);
if (message.HasValue)
{
QueueMessage qm = QueueMessage.FromByteArray(message.Value);
result.Add(qm);
}
else
{
break;
}
}
await tx.CommitAsync();
}
return(result.Count == 0 ? null : result);
}
internal async Task <bool> TryDequeueAndProcessAsync(CancellationToken cancellationToken)
{
IReliableQueue <Guid> queue =
await this.StateManager.GetOrAddAsync <IReliableQueue <Guid> >(QueueName);
IReliableDictionary <Guid, ApplicationDeployment> dictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, ApplicationDeployment> >(DictionaryName);
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <Guid> workItem = await queue.TryDequeueAsync(tx, this.transactionTimeout, cancellationToken);
if (!workItem.HasValue)
{
ServiceEventSource.Current.ServiceMessage(this, "No new application deployment requests.");
return(false);
}
Guid workItemId = workItem.Value;
ConditionalValue <ApplicationDeployment> appDeployment =
await dictionary.TryGetValueAsync(tx, workItemId, LockMode.Update, this.transactionTimeout, cancellationToken);
if (!appDeployment.HasValue)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Found queued application deployment request with no associated deployment information. Discarding.");
return(true);
}
ApplicationDeployment processedDeployment = await this.ProcessApplicationDeployment(appDeployment.Value, cancellationToken);
if (processedDeployment.Status == ApplicationDeployStatus.Complete ||
processedDeployment.Status == ApplicationDeployStatus.Failed)
{
// Remove deployments that completed or failed
await dictionary.TryRemoveAsync(tx, workItemId, this.transactionTimeout, cancellationToken);
}
else
{
// The deployment hasn't completed or failed, so queue up the next stage of deployment
await queue.EnqueueAsync(tx, workItemId, this.transactionTimeout, cancellationToken);
// And update the deployment record with the new status
await dictionary.SetAsync(tx, workItemId, processedDeployment, this.transactionTimeout, cancellationToken);
ServiceEventSource.Current.ServiceMessage(
this,
"Application deployment request successfully processed. Cluster: {0}. Status: {1}",
processedDeployment.Cluster,
processedDeployment.Status);
}
await tx.CommitAsync();
}
return(true);
}
public async Task <Guid> GetNextConsumerInLineAsync(string appApiLicenseCode)
{
try
{
IReliableQueue <Guid> queue = await StateManager.GetOrAddAsync <IReliableQueue <Guid> >(appApiLicenseCode).ConfigureAwait(false);
using (ITransaction tx = StateManager.CreateTransaction())
{
ConditionalValue <Guid> queueItem = await queue.TryDequeueAsync(tx);
if (queueItem.HasValue)
{
await tx.CommitAsync();
return(queueItem.Value);
}
}
Guid actorId = await CreateConsumerAsync(appApiLicenseCode).ConfigureAwait(false);
return(actorId);
}
catch (Exception ex)
{
Debug.WriteLine(ex);
Debugger.Break();
throw;
}
}
private async Task SendMailTaskCode(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
ConditionalValue <MailData> mailData;
using (var trx = this.StateManager.CreateTransaction())
{
mailData = await this.mailQueue.TryPeekAsync(trx);
await trx.CommitAsync();
}
if (!mailData.HasValue)
{
await Task.Delay(this.DelayBetweenMailSend, cancellationToken);
continue;
}
var sendResult = await this.mailAdapter.SendMailAsync(mailData.Value, cancellationToken);
await SendCallbackToCallerActorAsync(mailData.Value, sendResult, cancellationToken);
using (var trx = this.StateManager.CreateTransaction())
{
mailData = await mailQueue.TryDequeueAsync(trx);
await trx.CommitAsync();
}
}
}
public async Task SingleEvent()
{
MockApplicationLifetime appLifetime = new MockApplicationLifetime();
MockReliableStateManager stateManager = new MockReliableStateManager();
IReliableDictionary <string, DeviceEvent> store =
await stateManager.GetOrAddAsync <IReliableDictionary <string, DeviceEvent> >(DataService.EventDictionaryName);
IReliableQueue <DeviceEventSeries> queue =
await stateManager.GetOrAddAsync <IReliableQueue <DeviceEventSeries> >(DataService.EventQueueName);
string expectedDeviceId = "some-device";
DeviceEvent expectedDeviceEvent = new DeviceEvent(new DateTimeOffset(1, TimeSpan.Zero));
EventsController target = new EventsController(stateManager, statefulServiceContext, appLifetime);
IActionResult result = await target.Post(expectedDeviceId, new[] { expectedDeviceEvent });
Assert.True(result is OkResult);
using (ITransaction tx = stateManager.CreateTransaction())
{
ConditionalValue <DeviceEvent> actualStoredEvent = await store.TryGetValueAsync(tx, expectedDeviceId);
ConditionalValue <DeviceEventSeries> actualQueuedEvent = await queue.TryDequeueAsync(tx);
Assert.True(actualStoredEvent.HasValue);
Assert.Equal(expectedDeviceEvent.Timestamp, actualStoredEvent.Value.Timestamp);
Assert.True(actualQueuedEvent.HasValue);
Assert.Equal(expectedDeviceEvent.Timestamp, actualQueuedEvent.Value.Events.First().Timestamp);
await tx.CommitAsync();
}
}
protected override async Task <ConditionalValue <byte[]> > TryDequeueAsync(ServiceFabricTransaction tx, IReliableState collectionBase, CancellationToken cancellationToken)
{
IReliableQueue <byte[]> collection = (IReliableQueue <byte[]>)collectionBase;
ConditionalValue <byte[]> message = await collection.TryDequeueAsync(tx.Tx, TimeSpan.FromSeconds(4), cancellationToken).ConfigureAwait(false);
return(message);
}
protected override async Task RunAsync(CancellationToken cancellationToken)
{
cancellationToken.Register(() => this._webApiCancellationSource.Cancel());
IReliableDictionary <string, RateAggregation> citiesDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, RateAggregation> >(RateCitiesDictionaryName);
IReliableQueue <RateRequest> queue = await this.StateManager.GetOrAddAsync <IReliableQueue <RateRequest> >(RateQueueName);
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
using (var tx = this.StateManager.CreateTransaction())
{
var result = await queue.TryDequeueAsync(tx);
if (result.HasValue)
{
RateRequest request = result.Value;
// TODO: Process the request
// TODO: Go against the reservation provider to pick up the rate
// TODO: Determine the reservation provider per tenant from the configuration parameters
string providerName = GetParameterValue(ParamatersSection, "ProviderName");
int nights = (request.CheckOutDate - request.CheckInDate).Days;
int netAmount = _random.Next(500) * nights;
var newAggregation = new RateAggregation();
newAggregation.Transactions = 1;
newAggregation.Nights = nights;
newAggregation.Amount = (double)netAmount;
await citiesDictionary.AddOrUpdateAsync(tx, $"{request.City}/{request.Country}/{providerName}", newAggregation, (key, currentValue) =>
{
currentValue.Transactions += newAggregation.Transactions;
currentValue.Nights += newAggregation.Nights;
currentValue.Amount += newAggregation.Amount;
return(currentValue);
});
// This commits the add to dictionary and the dequeue operation.
await tx.CommitAsync();
}
}
}
catch (Exception e)
{
}
await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken);
}
}
public async Task <ConditionalValue <T> > TryDequeueAsync(ITransaction tx, CancellationToken token)
{
await signal.WaitAsync(token).ConfigureAwait(false);
var result = await queue.TryDequeueAsync(tx).ConfigureAwait(false);
var countDiff = await GetCountDiff(tx);
if (countDiff > 0)
{
signal.Release(countDiff);
}
return(result);
}
private async Task ReceiveMessagesAsync(Func <IEnumerable <QueueMessage>, Task> onMessage, int maxBatchSize, CancellationToken cancellationToken)
{
var messages = new List <QueueMessage>();
while (!cancellationToken.IsCancellationRequested && !_disposed)
{
try
{
using (var tx = new ServiceFabricTransaction(_stateManager, null))
{
IReliableQueue <byte[]> collection = await GetCollectionAsync();
while (messages.Count < maxBatchSize)
{
ConditionalValue <byte[]> message = await collection.TryDequeueAsync(tx.Tx, TimeSpan.FromSeconds(4), cancellationToken);
if (message.HasValue)
{
QueueMessage qm = QueueMessage.FromByteArray(message.Value);
messages.Add(qm);
}
else
{
break;
}
}
//make the call before committing the transaction
if (messages.Count > 0)
{
await onMessage(messages);
messages.Clear();
}
await tx.CommitAsync();
}
}
catch (Exception ex)
{
Trace.Fail($"failed to listen to messages on queue '{_queueName}'", ex.ToString());
}
await Task.Delay(_scanInterval);
}
}
public async Task AddMostRecentEvent()
{
MockApplicationLifetime appLifetime = new MockApplicationLifetime();
MockReliableStateManager stateManager = new MockReliableStateManager();
IReliableDictionary <string, DeviceEvent> store =
await stateManager.GetOrAddAsync <IReliableDictionary <string, DeviceEvent> >(DataService.EventDictionaryName);
IReliableQueue <DeviceEventSeries> queue =
await stateManager.GetOrAddAsync <IReliableQueue <DeviceEventSeries> >(DataService.EventQueueName);
string expectedDeviceId = "some-device";
List <DeviceEvent> expectedDeviceList = new List <DeviceEvent>();
DeviceEvent expectedDeviceEvent = new DeviceEvent(new DateTimeOffset(100, TimeSpan.Zero));
for (int i = 0; i < 10; ++i)
{
expectedDeviceList.Add(new DeviceEvent(new DateTimeOffset(i, TimeSpan.Zero)));
}
expectedDeviceList.Insert(4, expectedDeviceEvent);
EventsController target = new EventsController(stateManager, statefulServiceContext, appLifetime);
IActionResult result = await target.Post(expectedDeviceId, expectedDeviceList);
Assert.True(result is OkResult);
using (ITransaction tx = stateManager.CreateTransaction())
{
ConditionalValue <DeviceEvent> actualStoredEvent = await store.TryGetValueAsync(tx, expectedDeviceId);
ConditionalValue <DeviceEventSeries> actualQueuedEvent = await queue.TryDequeueAsync(tx);
Assert.True(actualStoredEvent.HasValue);
Assert.Equal(expectedDeviceEvent.Timestamp, actualStoredEvent.Value.Timestamp);
Assert.True(actualQueuedEvent.HasValue);
Assert.True(actualQueuedEvent.Value.Events.Select(x => x.Timestamp).SequenceEqual(expectedDeviceList.Select(x => x.Timestamp)));
await tx.CommitAsync();
}
}
public static async Task <List <T> > TryReadEntriesAsync <T>(this IReliableQueue <T> queue, IReliableStateManager stateManager, int maxCount)
{
List <T> results = null;
using (var tx = stateManager.CreateTransaction())
{
do
{
var entry = await queue.TryDequeueAsync(tx).ConfigureAwait(false);
if (!entry.HasValue)
{
break;
}
results = results ?? new List <T>(maxCount);
results.Add(entry.Value);
} while (results.Count < maxCount);
tx.Abort();
}
return(results);
}
/// <summary>
/// Drains the queue of completed restock requests sends them to InventoryService.
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
protected override async Task RunAsync(CancellationToken cancellationToken)
{
IReliableQueue <RestockRequest> completedRequests = await this.StateManager.GetOrAddAsync <IReliableQueue <RestockRequest> >(CompletedRequestsQueueName);
while (!cancellationToken.IsCancellationRequested)
{
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <RestockRequest> result = await completedRequests.TryDequeueAsync(tx, TxTimeout, cancellationToken);
if (result.HasValue)
{
ServiceUriBuilder builder = new ServiceUriBuilder(InventoryServiceName);
IInventoryService inventoryService = ServiceProxy.Create <IInventoryService>(builder.ToUri(), result.Value.ItemId.GetPartitionKey());
await inventoryService.AddStockAsync(result.Value.ItemId, result.Value.Quantity);
ServiceEventSource.Current.ServiceMessage(
this,
"Adding stock to inventory service. ID: {0}. Quantity: {1}",
result.Value.ItemId,
result.Value.Quantity);
}
// This commits the dequeue operations.
// If the request to add the stock to the inventory service throws, this commit will not execute
// and the items will remain on the queue, so we can be sure that we didn't dequeue items
// that didn't get saved successfully in the inventory service.
// However there is a very small chance that the stock was added to the inventory service successfully,
// but service execution stopped before reaching this commit (machine crash, for example).
await tx.CommitAsync();
}
await Task.Delay(CompletedRequestsBatchInterval, cancellationToken);
}
}
protected override async Task RunAsync(CancellationToken cancellationToken)
{
ServiceEventSource.Current.RunAsyncInvoked(ServiceEventSourceName);
IReliableQueue <string> inputQueue = await this.StateManager.GetOrAddAsync <IReliableQueue <string> >("inputQueue");
IReliableDictionary <string, long> wordCountDictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <string, long> >("wordCountDictionary");
IReliableDictionary <string, long> statsDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, long> >("statsDictionary");
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <string> dequeuReply = await inputQueue.TryDequeueAsync(tx);
if (dequeuReply.HasValue)
{
string word = dequeuReply.Value;
long count = await wordCountDictionary.AddOrUpdateAsync(
tx,
word, /* key */
1, /* value to add if key is absent */
(key, oldValue) => oldValue + 1);
long numberOfProcessedWords = await statsDictionary.AddOrUpdateAsync(
tx,
"Number of Words Processed",
1,
(key, oldValue) => oldValue + 1);
long queueLength = await inputQueue.GetCountAsync(tx);
await tx.CommitAsync();
ServiceEventSource.Current.RunAsyncStatus(
this.Partition.PartitionInfo.Id,
numberOfProcessedWords,
queueLength,
word,
count);
}
}
await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);
}
catch (TimeoutException)
{
//Service Fabric uses timeouts on collection operations to prevent deadlocks.
//If this exception is thrown, it means that this transaction was waiting the default
//amount of time (4 seconds) but was unable to acquire the lock. In this case we simply
//retry after a random backoff interval. You can also control the timeout via a parameter
//on the collection operation.
Thread.Sleep(TimeSpan.FromSeconds(new Random().Next(100, 300)));
continue;
}
catch (Exception exception)
{
//For sample code only: simply trace the exception.
ServiceEventSource.Current.MessageEvent(exception.ToString());
}
}
}
private async Task workLoopAsync()
{
int nLongDequeueWaitTimeMs = 20 * 1000;
int nShortDequeueWaitTimeMs = 2 * 1000;
int nNoQueueWaitTimeMS = 5 * 1000;
int nPauseCheckMs = 5 * 1000;
while (this.m_KeepWorking)
{
// pause check
while (this.m_Pause)
{
await Task.Delay(nPauseCheckMs);
}
// take the queue
KeyValuePair <string, IReliableQueue <Wi> > kvp = this.m_WorkManager.m_QueueManager.TakeQueueAsync();
if (null == kvp.Value) // no queue to work on.
{
// this will only happen if executers # are > than queues
// usually a situation that should resolve it self.
// well by the following logic
this.m_WorkManager.m_TraceWriter.TraceMessage(
string.Format("Executer {0} found no q and will sleep for {1}", this.m_WorkerExecuterId, nNoQueueWaitTimeMS));
await this.FinilizeQueueWork(0, null, null); // check removal
await Task.Delay(nNoQueueWaitTimeMS); // sleep as there is no point of retrying right away.
continue;
}
// got Q
IReliableQueue <Wi> q = kvp.Value;
string qName = kvp.Key;
int nCurrentMessage = 0;
try
{
while (this.m_KeepWorking & !this.m_Pause)
{
nCurrentMessage++;
// processed the # of messages?
if (nCurrentMessage > this.m_WorkManager.YieldQueueAfter)
{
break; //-> to finally
}
// as long as we have other queues. we need to have a short wait time
int ActualTimeOut = this.m_WorkManager.m_QueueManager.Count > this.m_WorkManager.m_Executers.Count
? nShortDequeueWaitTimeMs
: nLongDequeueWaitTimeMs;
using (ITransaction tx = this.m_WorkManager.StateManager.CreateTransaction())
{
ConditionalResult <Wi> cResults = await q.TryDequeueAsync(
tx,
TimeSpan.FromMilliseconds(ActualTimeOut),
CancellationToken.None);
if (cResults.HasValue)
{
Handler handler = this.m_WorkManager.GetHandlerForQueue(qName);
Wi wi = await handler.HandleWorkItem(cResults.Value);
if (null != wi) // do we have an enqueue request?
{
await q.EnqueueAsync(tx, wi);
}
await tx.CommitAsync();
this.m_WorkManager.DecreaseBufferedWorkItems();
}
else
{
break; // -> to finally
}
}
}
}
catch (TimeoutException to)
{
/* Queue is locked for enqueues */
this.m_WorkManager.m_TraceWriter.TraceMessage(
string.Format("Executer Dequeue Timeout after {0}: {1}", nLongDequeueWaitTimeMs, to.Message));
break; //-> to finally
}
catch (AggregateException aex)
{
AggregateException ae = aex.Flatten();
this.m_WorkManager.m_TraceWriter.TraceMessage(
string.Format(
"Executer encountered fatel error and will exit E:{0} StackTrace:{1}",
ae.GetCombinedExceptionMessage(),
ae.GetCombinedExceptionStackTrace()));
throw;
}
catch (Exception E)
{
this.m_WorkManager.m_TraceWriter.TraceMessage(
string.Format("Executer encountered fatel error and will exit E:{0} StackTrace:{1}", E.Message, E.StackTrace));
throw;
}
finally
{
await this.FinilizeQueueWork(nCurrentMessage, qName, q);
}
}
this.m_WorkManager.m_TraceWriter.TraceMessage(string.Format("Worker {0} exited loop", this.m_WorkerExecuterId));
}
protected async override Task ExecuteAsync(CancellationToken stoppingToken)
{
bool queueEmpty = false;
var delayString = configurationPackage.Settings.Sections["Timing"]
.Parameters["MessageMaxDelaySeconds"].Value;
var delay = int.Parse(delayString);
var filter = await IdempotencyFilter.NewIdempotencyFilterAsync(
"logMessages", delay, stateManager);
var store = await
stateManager.GetOrAddAsync <IReliableDictionary <string, RunningTotal> >("partialCount");
while (!stoppingToken.IsCancellationRequested)
{
while (!queueEmpty && !stoppingToken.IsCancellationRequested)
{
RunningTotal finalDayTotal = null;
using (ITransaction tx = stateManager.CreateTransaction())
{
var result = await queue.TryDequeueAsync(tx);
if (!result.HasValue)
{
queueEmpty = true;
}
else
{
var item = await filter.NewMessage <PurchaseInfo>(result.Value);
if (item != null)
{
var counter = await store.TryGetValueAsync(tx, item.Location);
var newCounter = counter.HasValue ?
new RunningTotal
{
Count = counter.Value.Count,
Day = counter.Value.Day
}
: new RunningTotal();
finalDayTotal = newCounter.Update(item.Time, item.Cost);
if (counter.HasValue)
{
await store.TryUpdateAsync(tx, item.Location,
newCounter, counter.Value);
}
else
{
await store.TryAddAsync(tx, item.Location, newCounter);
}
}
await tx.CommitAsync();
if (finalDayTotal != null)
{
await SendTotal(finalDayTotal, item.Location);
}
}
}
}
await Task.Delay(100, stoppingToken);
queueEmpty = false;
}
}
protected override async Task RunAsync(CancellationToken cancellationToken)
{
IReliableQueue <DeviceEventSeries> queue = await this.StateManager.GetOrAddAsync <IReliableQueue <DeviceEventSeries> >(EventQueueName);
int iteration = 0;
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
using (ITransaction tx = this.StateManager.CreateTransaction())
{
// When the number of items in the queue reaches a certain size..
int count = (int)await queue.GetCountAsync(tx);
ServiceEventSource.Current.ServiceMessage(
this.Context,
"Current queue size: {0} in iteration {1}",
count, iteration);
// if the queue size reaches the batch size, start draining the queue
// always drain the queue every nth iteration so that nothing sits in the queue indefinitely
if (count >= OffloadBatchSize || iteration == DrainIteration)
{
ServiceEventSource.Current.ServiceMessage(
this.Context,
"Starting batch offload..");
// Dequeue the items into a batch
List <DeviceEventSeries> batch = new List <DeviceEventSeries>(count);
for (int i = 0; i < count; ++i)
{
cancellationToken.ThrowIfCancellationRequested();
ConditionalValue <DeviceEventSeries> result = await queue.TryDequeueAsync(tx);
if (result.HasValue)
{
batch.Add(result.Value);
}
}
//TODO: Process the data or send to a storage location.
// Commit the dequeue operations
await tx.CommitAsync();
ServiceEventSource.Current.ServiceMessage(
this.Context,
"Batch offloaded {0} events.",
count);
// skip the delay and move on to the next batch.
iteration = 0;
continue;
}
else if (count > 0)
{
iteration++;
}
}
}
catch (TimeoutException)
{
// transient error. Retry.
ServiceEventSource.Current.ServiceMessage(this.Context, "TimeoutException in RunAsync.");
}
catch (FabricTransientException fte)
{
// transient error. Retry.
ServiceEventSource.Current.ServiceMessage(this.Context, "FabricTransientException in RunAsync: {0}", fte.Message);
}
catch (FabricNotPrimaryException)
{
// not primary any more, time to quit.
return;
}
await Task.Delay(this.OffloadBatchInterval, cancellationToken);
}
}
protected override async Task RunAsync(CancellationToken cancellationToken)
{
IReliableQueue <string> queue = await this.StateManager.GetOrAddAsync <IReliableQueue <string> >("jobQueue");
IReliableDictionary <string, Job> dictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, Job> >("jobs");
try
{
// need to restart any existing jobs after failover
using (ITransaction tx = this.StateManager.CreateTransaction())
{
var enumerable = await dictionary.CreateEnumerableAsync(tx);
var enumerator = enumerable.GetAsyncEnumerator();
while (await enumerator.MoveNextAsync(cancellationToken))
{
cancellationToken.ThrowIfCancellationRequested();
Job job = enumerator.Current.Value;
this.runningJobs.Add(this.StartJob(job, cancellationToken));
}
}
// start processing new jobs from the queue.
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <string> dequeueResult = await queue.TryDequeueAsync(tx);
if (!dequeueResult.HasValue)
{
await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken);
continue;
}
string jobName = dequeueResult.Value;
ConditionalValue <Job> getResult = await dictionary.TryGetValueAsync(tx, jobName, LockMode.Update);
if (getResult.HasValue)
{
Job job = getResult.Value;
this.runningJobs.Add(this.StartJob(job, cancellationToken));
await dictionary.SetAsync(tx, jobName, new Job(job.Name, job.Parameters, job.Running));
}
await tx.CommitAsync();
}
}
catch (FabricTransientException)
{
await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken);
}
catch (TimeoutException)
{
await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken);
}
}
}
catch (OperationCanceledException)
{
await Task.WhenAll(this.runningJobs);
throw;
}
}
/// <summary>
/// Performs the main processing work.
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
protected override async Task RunAsync(CancellationToken cancellationToken)
{
try
{
IReliableQueue <ReportProcessingStep> processQueue
= await this.StateManager.GetOrAddAsync <IReliableQueue <ReportProcessingStep> >(ProcessingQueueName);
IReliableDictionary <string, ReportStatus> statusDictionary
= await this.StateManager.GetOrAddAsync <IReliableDictionary <string, ReportStatus> >(StatusDictionaryName);
// First time setup: queue up all the processing steps and create an initial processing status if one doesn't exist already
// Note that this will execute every time a replica is promoted to primary,
// so we need to check to see if it's already been done because we only want to initialize once.
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <ReportStatus> tryGetResult
= await statusDictionary.TryGetValueAsync(tx, this.reportContext.Name, LockMode.Update);
if (!tryGetResult.HasValue)
{
await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Creating"));
await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Evaluating"));
await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Reticulating"));
for (int i = 0; i < processingMultiplier * queueLengthMultiplier; ++i)
{
cancellationToken.ThrowIfCancellationRequested();
await processQueue.EnqueueAsync(tx, new ReportProcessingStep($"Processing {i}"));
}
await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Sanitizing"));
await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Mystery Step"));
await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Finalizing"));
await processQueue.EnqueueAsync(tx, new ReportProcessingStep("Complete"));
await statusDictionary.AddAsync(tx, this.reportContext.Name, new ReportStatus(0, processingMultiplier *queueLengthMultiplier + 7, "Not started.", String.Empty));
}
await tx.CommitAsync();
}
// start processing and checkpoint between each step so we don't lose any progress in the event of a fail-over
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
// Get the next step from the queue with a peek.
// This keeps the item on the queue in case processing fails.
ConditionalValue <ReportProcessingStep> dequeueResult;
using (ITransaction tx = this.StateManager.CreateTransaction())
{
dequeueResult = await processQueue.TryPeekAsync(tx, LockMode.Default);
}
if (!dequeueResult.HasValue)
{
// all done!
break;
}
ReportProcessingStep currentProcessingStep = dequeueResult.Value;
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"Processing step: {currentProcessingStep.Name}");
// Get the current processing step
ConditionalValue <ReportStatus> dictionaryGetResult;
using (ITransaction tx = this.StateManager.CreateTransaction())
{
dictionaryGetResult = await statusDictionary.TryGetValueAsync(tx, this.reportContext.Name, LockMode.Default);
}
// Perform the next processing step.
// This is potentially a long-running operation, therefore it is not executed within a transaction.
ReportStatus currentStatus = dictionaryGetResult.Value;
ReportStatus newStatus = await this.ProcessReport(currentStatus, currentProcessingStep, cancellationToken);
// Once processing is done, save the results and dequeue the processing step.
// If an exception or other failure occurs at this point, the processing step will run again.
using (ITransaction tx = this.StateManager.CreateTransaction())
{
dequeueResult = await processQueue.TryDequeueAsync(tx);
await statusDictionary.SetAsync(tx, this.reportContext.Name, newStatus);
await tx.CommitAsync();
}
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"Processing step: {currentProcessingStep.Name} complete.");
}
catch (TimeoutException)
{
// transient error. Retry.
ServiceEventSource.Current.ServiceMessage(this.Context, "TimeoutException in RunAsync.");
}
catch (FabricTransientException fte)
{
// transient error. Retry.
ServiceEventSource.Current.ServiceMessage(this.Context, "FabricTransientException in RunAsync: {0}", fte.Message);
}
catch (FabricNotReadableException)
{
// retry or wait until not primary
ServiceEventSource.Current.ServiceMessage(this.Context, "FabricNotReadableException in RunAsync.");
}
// delay between each to step to prevent starving other processing service instances.
await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken);
}
ServiceEventSource.Current.ServiceMessage(
this.Context,
$"Processing complete!");
}
catch (OperationCanceledException)
{
// time to quit
throw;
}
catch (FabricNotPrimaryException)
{
// time to quit
return;
}
catch (Exception ex)
{
// all other exceptions: log and re-throw.
ServiceEventSource.Current.ServiceMessage(this.Context, "Exception in RunAsync: {0}", ex.Message);
throw;
}
}
