public async Task <IActionResult> GetQueueLengthAsync()
{
IReliableQueue <DeviceEventSeries> queue =
await this.stateManager.GetOrAddAsync <IReliableQueue <DeviceEventSeries> >(DataService.EventQueueName);
using (ITransaction tx = this.stateManager.CreateTransaction())
{
long count = await queue.GetCountAsync(tx);
return(this.Ok(count));
}
}
protected override async Task RunAsync(CancellationToken cancellationToken)
{
cancellationToken.Register(() => this._webApiCancellationSource.Cancel());
IReliableDictionary <string, RateAggregation> citiesDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, RateAggregation> >(RateCitiesDictionaryName);
IReliableQueue <RateRequest> queue = await this.StateManager.GetOrAddAsync <IReliableQueue <RateRequest> >(RateQueueName);
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
using (var tx = this.StateManager.CreateTransaction())
{
var result = await queue.TryDequeueAsync(tx);
if (result.HasValue)
{
RateRequest request = result.Value;
// TODO: Process the request
// TODO: Go against the reservation provider to pick up the rate
// TODO: Determine the reservation provider per tenant from the configuration parameters
string providerName = GetParameterValue(ParamatersSection, "ProviderName");
int nights = (request.CheckOutDate - request.CheckInDate).Days;
int netAmount = _random.Next(500) * nights;
var newAggregation = new RateAggregation();
newAggregation.Transactions = 1;
newAggregation.Nights = nights;
newAggregation.Amount = (double)netAmount;
await citiesDictionary.AddOrUpdateAsync(tx, $"{request.City}/{request.Country}/{providerName}", newAggregation, (key, currentValue) =>
{
currentValue.Transactions += newAggregation.Transactions;
currentValue.Nights += newAggregation.Nights;
currentValue.Amount += newAggregation.Amount;
return(currentValue);
});
// This commits the add to dictionary and the dequeue operation.
await tx.CommitAsync();
}
}
}
catch (Exception e)
{
}
await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken);
}
}
private async Task LoadNumOfBufferedItems()
{
long buffered = 0;
foreach (string qName in this.m_QueueManager.QueueNames)
{
IReliableQueue <Wi> q = await this.m_QueueManager.GetOrAddQueueAsync(qName);
buffered += await q.GetCountAsync();
}
this.m_NumOfBufferedWorkItems = buffered;
}
public async Task <IHttpActionResult> AddWord(string word)
{
IReliableQueue <string> queue = await this.stateManager.GetOrAddAsync <IReliableQueue <string> >("inputQueue");
using (ITransaction tx = this.stateManager.CreateTransaction())
{
await queue.EnqueueAsync(tx, word);
await tx.CommitAsync();
}
return(this.Ok());
}
/// <summary>
/// This method uses an IReliableQueue to store completed RestockRequests which are later sent to the client using batch processing.
/// We could send the request immediately but we prefer to minimize traffic back to the Inventory Service by batching multiple requests
/// in one trip.
/// </summary>
/// <param name="actorId"></param>
/// <param name="request"></param>
public async void RestockRequestCompleted(ActorId actorId, RestockRequest request)
{
IReliableQueue <RestockRequest> completedRequests = await this.StateManager.GetOrAddAsync <IReliableQueue <RestockRequest> >(CompletedRequestsQueueName);
using (ITransaction tx = this.StateManager.CreateTransaction())
{
await completedRequests.EnqueueAsync(tx, request);
await tx.CommitAsync();
}
IRestockRequestActor restockRequestActor = ActorProxy.Create <IRestockRequestActor>(actorId, this.ApplicationName);
await restockRequestActor.UnsubscribeAsync <IRestockRequestEvents>(this); //QUESTION:What does this method do?
}
public async Task ResetAsync()
{
IReliableDictionary <string, int> states = await StateManager.GetOrAddAsync <IReliableDictionary <string, int> >("states");
IReliableQueue <string> events = await StateManager.GetOrAddAsync <IReliableQueue <string> >("events");
using (ITransaction transaction = StateManager.CreateTransaction())
{
await states.SetAsync(transaction, "Counter", 0);
await events.EnqueueAsync(transaction, $"{DateTime.UtcNow:O} The Counter is reset.");
await transaction.CommitAsync();
}
}
private async Task LoadNumOfBufferedItems()
{
long buffered = 0;
using (ITransaction tx = this.StateManager.CreateTransaction())
{
foreach (string qName in this.m_QueueManager.QueueNames)
{
IReliableQueue <Wi> q = await this.m_QueueManager.GetOrAddQueueAsync(qName);
buffered += await q.GetCountAsync(tx);
}
}
this.m_NumOfBufferedWorkItems = buffered;
}
public async Task PutMessagesAsync(IReadOnlyCollection <QueueMessage> messages, CancellationToken cancellationToken)
{
IReliableQueue <byte[]> collection = await _stateManager.GetOrAddAsync <IReliableQueue <byte[]> >(_queueName);
using (var tx = new ServiceFabricTransaction(_stateManager, null))
{
foreach (QueueMessage message in messages)
{
byte[] data = message.ToByteArray();
await collection.EnqueueAsync(tx.Tx, data, _timeout, cancellationToken);
}
await tx.CommitAsync();
}
}
private async Task ReceiveMessagesAsync(Func <IEnumerable <QueueMessage>, Task> onMessage, int maxBatchSize, CancellationToken cancellationToken)
{
var messages = new List <QueueMessage>();
while (!cancellationToken.IsCancellationRequested && !_disposed)
{
try
{
using (var tx = new ServiceFabricTransaction(_stateManager, null))
{
IReliableQueue <byte[]> collection = await GetCollectionAsync();
while (messages.Count < maxBatchSize)
{
ConditionalValue <byte[]> message = await collection.TryDequeueAsync(tx.Tx, TimeSpan.FromSeconds(4), cancellationToken);
if (message.HasValue)
{
QueueMessage qm = QueueMessage.FromByteArray(message.Value);
messages.Add(qm);
}
else
{
break;
}
}
//make the call before committing the transaction
if (messages.Count > 0)
{
await onMessage(messages);
messages.Clear();
}
await tx.CommitAsync();
}
}
catch (Exception ex)
{
Trace.Fail($"failed to listen to messages on queue '{_queueName}'", ex.ToString());
}
await Task.Delay(_scanInterval);
}
}
public async Task <string> CountAsync()
{
await Task.Delay(TimeSpan.FromSeconds(3));
IReliableDictionary <string, int> states = await StateManager.GetOrAddAsync <IReliableDictionary <string, int> >("states");
IReliableQueue <string> events = await StateManager.GetOrAddAsync <IReliableQueue <string> >("events");
int number;
//using (ITransaction transaction = StateManager.CreateTransaction())
//{
// ConditionalValue<int> counter = await states.TryGetValueAsync(transaction, "Counter");
// number = counter.HasValue ? counter.Value : 0;
// number++;
//}
//await Task.Delay(TimeSpan.FromSeconds(3));
//using (ITransaction transaction = StateManager.CreateTransaction())
//{
// await states.SetAsync(transaction, "Counter", number);
// await events.EnqueueAsync(transaction, $"{DateTime.UtcNow:O} The Counter is {number}.");
// await transaction.CommitAsync();
//}
using (ITransaction transaction = StateManager.CreateTransaction())
{
ConditionalValue <int> counter = await states.TryGetValueAsync(transaction, "Counter");
number = counter.HasValue ? counter.Value : 0;
number++;
await Task.Delay(TimeSpan.FromSeconds(3));
await states.SetAsync(transaction, "Counter", number);
await events.EnqueueAsync(transaction, $"{DateTime.UtcNow:O} The Counter is {number}.");
await transaction.CommitAsync();
}
return($"Current number is {number}, from partition {Context.PartitionId} and replica {Context.ReplicaId}");
}
internal static async Task <WorkItemQueue> CreateOrGetWorkItemQueue(StatefulService statefulService)
{
if (workItemQueue == null)
{
IReliableQueue <WorkItemProcessInfo> workItemReliableQueue =
await statefulService.StateManager.GetOrAddAsync <IReliableQueue <WorkItemProcessInfo> >(Constants.WorkItemQueue);
lock (syncRoot)
{
if (workItemQueue == null)
{
workItemQueue = new WorkItemQueue(workItemReliableQueue, 0, WorkItemQueueRunType.WorkItemQueue, statefulService);
}
}
}
return(workItemQueue);
}
public async Task IgnoreOldEvent()
{
MockApplicationLifetime appLifetime = new MockApplicationLifetime();
MockReliableStateManager stateManager = new MockReliableStateManager();
IReliableDictionary <string, DeviceEvent> store =
await stateManager.GetOrAddAsync <IReliableDictionary <string, DeviceEvent> >(DataService.EventDictionaryName);
IReliableQueue <DeviceEventSeries> queue =
await stateManager.GetOrAddAsync <IReliableQueue <DeviceEventSeries> >(DataService.EventQueueName);
string expectedDeviceId = "some-device";
DeviceEvent expectedDeviceEvent = new DeviceEvent(new DateTimeOffset(100, TimeSpan.Zero));
EventsController target = new EventsController(stateManager, statefulServiceContext, appLifetime);
IActionResult result = await target.Post(expectedDeviceId, new[] { expectedDeviceEvent });
Assert.True(result is OkResult);
using (ITransaction tx = stateManager.CreateTransaction())
{
ConditionalValue <DeviceEvent> actualStoredEvent = await store.TryGetValueAsync(tx, expectedDeviceId);
Assert.True(actualStoredEvent.HasValue);
Assert.Equal(expectedDeviceEvent.Timestamp, actualStoredEvent.Value.Timestamp);
await tx.CommitAsync();
}
DeviceEvent oldEvent = new DeviceEvent(new DateTimeOffset(10, TimeSpan.Zero));
result = await target.Post(expectedDeviceId, new[] { oldEvent });
Assert.True(result is OkResult);
using (ITransaction tx = stateManager.CreateTransaction())
{
ConditionalValue <DeviceEvent> actualStoredEvent = await store.TryGetValueAsync(tx, expectedDeviceId);
Assert.True(actualStoredEvent.HasValue);
Assert.Equal(expectedDeviceEvent.Timestamp, actualStoredEvent.Value.Timestamp);
await tx.CommitAsync();
}
}
public async Task AddMessageAsync(string roomName, ChatMessage chatMessage, CancellationToken cancellationToken)
{
IReliableQueue <ChatMessage> room = await GetRoomQueue(roomName);
using (var tx = stateManager.CreateTransaction())
{
long msgCount = await room.GetCountAsync(tx);
if (msgCount == 0)
{
// new room, increment counter
await IncrementRoomCountAsync(1, cancellationToken, tx);
}
await room.EnqueueAsync(tx, chatMessage);
await tx.CommitAsync();
}
}
public async Task AddMostRecentEvent()
{
MockApplicationLifetime appLifetime = new MockApplicationLifetime();
MockReliableStateManager stateManager = new MockReliableStateManager();
IReliableDictionary <string, DeviceEvent> store =
await stateManager.GetOrAddAsync <IReliableDictionary <string, DeviceEvent> >(DataService.EventDictionaryName);
IReliableQueue <DeviceEventSeries> queue =
await stateManager.GetOrAddAsync <IReliableQueue <DeviceEventSeries> >(DataService.EventQueueName);
string expectedDeviceId = "some-device";
List <DeviceEvent> expectedDeviceList = new List <DeviceEvent>();
DeviceEvent expectedDeviceEvent = new DeviceEvent(new DateTimeOffset(100, TimeSpan.Zero));
for (int i = 0; i < 10; ++i)
{
expectedDeviceList.Add(new DeviceEvent(new DateTimeOffset(i, TimeSpan.Zero)));
}
expectedDeviceList.Insert(4, expectedDeviceEvent);
EventsController target = new EventsController(stateManager, statefulServiceContext, appLifetime);
IActionResult result = await target.Post(expectedDeviceId, expectedDeviceList);
Assert.True(result is OkResult);
using (ITransaction tx = stateManager.CreateTransaction())
{
ConditionalValue <DeviceEvent> actualStoredEvent = await store.TryGetValueAsync(tx, expectedDeviceId);
ConditionalValue <DeviceEventSeries> actualQueuedEvent = await queue.TryDequeueAsync(tx);
Assert.True(actualStoredEvent.HasValue);
Assert.Equal(expectedDeviceEvent.Timestamp, actualStoredEvent.Value.Timestamp);
Assert.True(actualQueuedEvent.HasValue);
Assert.True(actualQueuedEvent.Value.Events.Select(x => x.Timestamp).SequenceEqual(expectedDeviceList.Select(x => x.Timestamp)));
await tx.CommitAsync();
}
}
public async Task <ChatRoom> GetRoomAsync(string roomName, CancellationToken cancellationToken)
{
IReliableQueue <ChatMessage> room = await GetRoomQueue(roomName);
ChatRoom chatRoom = new ChatRoom(roomName);
using (var tx = stateManager.CreateTransaction())
{
var roomIterator = (await room.CreateEnumerableAsync(tx)).GetAsyncEnumerator();
while (await roomIterator.MoveNextAsync(cancellationToken))
{
var msg = roomIterator.Current;
chatRoom.messages.Add(msg);
}
}
return(chatRoom);
}
public async Task <IReliableQueue <W> > GetOrAddQueueAsync(string qName)
{
using (ITransaction tx = this.m_WorkManager.StateManager.CreateTransaction())
{
if (!await this.m_dictListOfQueues.ContainsKeyAsync(tx, qName, TimeSpan.FromSeconds(5), CancellationToken.None))
{
IReliableQueue <W> reliableQ = await this.m_WorkManager.StateManager.GetOrAddAsync <IReliableQueue <W> >(tx, qName);
await this.m_dictListOfQueues.AddAsync(tx, qName, qName, TimeSpan.FromSeconds(5), CancellationToken.None);
await tx.CommitAsync();
this.m_QueueRefs.TryAdd(qName, reliableQ);
this.qOfq.Enqueue(qName);
}
}
return(this.m_QueueRefs[qName]);
}
protected override async Task RunAsync(CancellationToken cancellationToken)
{
IReliableQueue <string> messageQueue = await this.StateManager.GetOrAddAsync <IReliableQueue <string> >("messageQueue");
const string baseAddress = "http://localhost:8864";
using (Microsoft.Owin.Hosting.WebApp.Start <Startup>(baseAddress))
{
HttpClient client = new HttpClient();
HttpResponseMessage res = client.GetAsync(baseAddress + "api/values/5").Result;
res.EnsureSuccessStatusCode();
var results = res.Content.ReadAsStringAsync().Result;
using (ITransaction tx = this.StateManager.CreateTransaction())
{
await messageQueue.EnqueueAsync(tx, results);
await tx.CommitAsync();
}
}
}
public async Task GetQueueLength()
{
MockApplicationLifetime appLifetime = new MockApplicationLifetime();
MockReliableStateManager stateManager = new MockReliableStateManager();
IReliableQueue <DeviceEventSeries> queue =
await stateManager.GetOrAddAsync <IReliableQueue <DeviceEventSeries> >(DataService.EventQueueName);
using (ITransaction tx = stateManager.CreateTransaction())
{
await queue.EnqueueAsync(tx, new DeviceEventSeries("", new DeviceEvent[0]));
}
DevicesController target = new DevicesController(stateManager, appLifetime);
IActionResult result = await target.GetQueueLengthAsync();
Assert.True(result is OkObjectResult);
long actual = (long)((OkObjectResult)result).Value;
Assert.Equal(1, actual);
}
public async Task PostWorkItemAsync(Wi workItem)
{
if (this.WorkManagerStatus != WorkManagerStatus.Working)
{
throw new InvalidOperationException("Work Manager is not working state");
}
if (this.m_NumOfBufferedWorkItems >= this.m_MaxNumOfBufferedWorkItems)
{
throw new InvalidOperationException(string.Format("Work Manger is at maximum buffered work items:{0}", this.m_NumOfBufferedWorkItems));
}
try
{
// Which Q
IReliableQueue <Wi> targetQueue = await this.m_QueueManager.GetOrAddQueueAsync(workItem.QueueName);
// enqueue
using (ITransaction tx = this.StateManager.CreateTransaction())
{
await targetQueue.EnqueueAsync(tx, workItem, TimeSpan.FromSeconds(10), CancellationToken.None);
await tx.CommitAsync();
}
this.IncreaseBufferedWorkItems();
this.m_DeferedTaskExec.AddWork(this.TryIncreaseExecuters);
}
catch (AggregateException aex)
{
AggregateException ae = aex.Flatten();
this.m_TraceWriter.TraceMessage(
string.Format(
"Post to work manager failed, caller should retry E:{0} StackTrace:{1}",
ae.GetCombinedExceptionMessage(),
ae.GetCombinedExceptionStackTrace()));
throw;
}
}
/// <summary>
/// Queues deployment of application packages to the given cluster.
/// </summary>
/// <param name="clusterAddress"></param>
/// <param name="clusterPort"></param>
/// <returns></returns>
public async Task <IEnumerable <Guid> > QueueApplicationDeploymentAsync(string clusterAddress, int clusterPort)
{
IReliableQueue <Guid> queue =
await this.StateManager.GetOrAddAsync <IReliableQueue <Guid> >(QueueName);
IReliableDictionary <Guid, ApplicationDeployment> dictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, ApplicationDeployment> >(DictionaryName);
List <Guid> workIds = new List <Guid>(this.ApplicationPackages.Count());
using (ITransaction tx = this.StateManager.CreateTransaction())
{
// Grab each application package that's included with the service
// and create an ApplicationDeployment record of it.
// Then queue a job to begin processing each one.
foreach (ApplicationPackageInfo package in this.ApplicationPackages)
{
Guid id = Guid.NewGuid();
ApplicationDeployment applicationDeployment = new ApplicationDeployment(
cluster: GetClusterAddress(clusterAddress, clusterPort),
status: ApplicationDeployStatus.Copy,
imageStorePath: null,
applicationTypeName: package.ApplicationTypeName,
applicationTypeVersion: package.ApplicationTypeVersion,
applicationInstanceName: GetApplicationInstanceName(package.PackageFileName),
packageZipFilePath: Path.Combine(this.applicationPackageDataPath.FullName, package.PackageFileName),
timestamp: DateTimeOffset.UtcNow);
await dictionary.AddAsync(tx, id, applicationDeployment);
await queue.EnqueueAsync(tx, id);
workIds.Add(id);
}
await tx.CommitAsync();
return(workIds);
}
}
/// <summary>
/// This is the main entry point for your service replica.
/// This method executes when this replica of your service becomes primary and has write status.
/// </summary>
/// <param name="cancellationToken">Canceled when Service Fabric needs to shut down this service replica.</param>
protected override async Task RunAsync(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
LogQueue = await
this.StateManager
.GetOrAddAsync <IReliableQueue <IdempotentMessage <PurchaseInfo> > >("logQueue");
var configurationPackage = Context
.CodePackageActivationContext
.GetConfigurationPackageObject("Config");
var host = new HostBuilder()
.ConfigureServices((hostContext, services) =>
{
services.AddSingleton(this.StateManager);
services.AddSingleton(this.LogQueue);
services.AddSingleton(configurationPackage);
services.AddHostedService <ComputeStatistics>();
})
.Build();
await host.RunAsync(cancellationToken);
}
public async Task Post(string name, string parameters)
{
IReliableQueue <string> queue = await this.stateManager.GetOrAddAsync <IReliableQueue <string> >("jobQueue");
IReliableDictionary <string, Job> dictionary = await this.stateManager.GetOrAddAsync <IReliableDictionary <string, Job> >("jobs");
using (ITransaction tx = this.stateManager.CreateTransaction())
{
if (await dictionary.ContainsKeyAsync(tx, name, LockMode.Update))
{
throw new ArgumentException($"Job {name} already exists.");
}
Job job = new Job(name, parameters, false);
await queue.EnqueueAsync(tx, name);
await dictionary.SetAsync(tx, name, job);
await tx.CommitAsync();
}
}
public static async Task <List <T> > TryReadEntriesAsync <T>(this IReliableQueue <T> queue, IReliableStateManager stateManager, int maxCount)
{
List <T> results = null;
using (var tx = stateManager.CreateTransaction())
{
do
{
var entry = await queue.TryDequeueAsync(tx).ConfigureAwait(false);
if (!entry.HasValue)
{
break;
}
results = results ?? new List <T>(maxCount);
results.Add(entry.Value);
} while (results.Count < maxCount);
tx.Abort();
}
return(results);
}
internal BaseWorkItemQueue(IReliableQueue <WorkItemProcessInfo> workItemReliableQueue, int maxWaitTimeInMinutes,
WorkItemQueueRunType workItemQueueRunType, string traceType, StatefulService statefulService)
{
BackupRestoreTrace.TraceSource.WriteInfo(traceType, "Constructing BaseWorkItemQueue with following parameters - workItemReliableQueue: {0}, maxWaitTimeInMinutes: {1}, workItemQueueRunType: {2}",
workItemReliableQueue.Name,
maxWaitTimeInMinutes,
workItemQueueRunType);
this.WorkItemReliableQueue = workItemReliableQueue;
this.StatefulService = statefulService;
this.WorkItemStore = WorkItemStore.CreateOrGetWorkItemStore(this.StatefulService).GetAwaiter().GetResult();
this.WorkItemInProcessStore =
WorkItemInProcessStore.CreateOrGetWorkItemInProcessStore(this.StatefulService).GetAwaiter().GetResult();
this.MaxWaitTimeInMilliSeconds = maxWaitTimeInMinutes == 0 ? FastQueueWaitTime : maxWaitTimeInMinutes * 60 * 1000;
this.AllowedTicks = (this.MaxWaitTimeInMilliSeconds * TimeSpan.TicksPerMillisecond) / 20;
this.workItemQueueRunType = workItemQueueRunType;
this.TraceType = traceType;
this.emptyTraceType = string.Format("{0}.{1}", traceType, "Empty");
this.WorkItemHandlerTimer = new Timer(new TimerCallback(this.ProcessWorkItemHandler),
null,
this.MaxWaitTimeInMilliSeconds,
Timeout.Infinite);
}
/// <summary>
/// This is the main entry point for your service replica.
/// This method executes when this replica of your service becomes primary and has write status.
/// </summary>
/// <param name="cancellationToken">Canceled when Service Fabric needs to shut down this service replica.</param>
protected override async Task RunAsync(CancellationToken cancellationToken)
{
mailQueue = await this.StateManager.GetOrAddAsync <IReliableQueue <MailData> >(MailQueueName);
ConfigureService();
List <Task> taskList = new List <Task>();
taskList.Add(SendMailTaskCode(cancellationToken));
var taskResult = await Task.WhenAny(taskList);
if (!cancellationToken.IsCancellationRequested)
{
if (taskResult.IsFaulted)
{
throw taskResult.Exception;
}
else
{
throw new InvalidOperationException("One or more tasks completed in unexpected ways");
}
}
}
/// <summary>
/// Drains the queue of completed restock requests sends them to InventoryService.
/// </summary>
/// <param name="cancellationToken"></param>
/// <returns></returns>
protected override async Task RunAsync(CancellationToken cancellationToken)
{
IReliableQueue <RestockRequest> completedRequests = await this.StateManager.GetOrAddAsync <IReliableQueue <RestockRequest> >(CompletedRequestsQueueName);
while (!cancellationToken.IsCancellationRequested)
{
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <RestockRequest> result = await completedRequests.TryDequeueAsync(tx, TxTimeout, cancellationToken);
if (result.HasValue)
{
ServiceUriBuilder builder = new ServiceUriBuilder(InventoryServiceName);
IInventoryService inventoryService = ServiceProxy.Create <IInventoryService>(builder.ToUri(), result.Value.ItemId.GetPartitionKey());
await inventoryService.AddStockAsync(result.Value.ItemId, result.Value.Quantity);
ServiceEventSource.Current.ServiceMessage(
this,
"Adding stock to inventory service. ID: {0}. Quantity: {1}",
result.Value.ItemId,
result.Value.Quantity);
}
// This commits the dequeue operations.
// If the request to add the stock to the inventory service throws, this commit will not execute
// and the items will remain on the queue, so we can be sure that we didn't dequeue items
// that didn't get saved successfully in the inventory service.
// However there is a very small chance that the stock was added to the inventory service successfully,
// but service execution stopped before reaching this commit (machine crash, for example).
await tx.CommitAsync();
}
await Task.Delay(CompletedRequestsBatchInterval, cancellationToken);
}
}
public async Task <IActionResult> Post(string deviceId, [FromBody] IEnumerable <DeviceEvent> events)
{
if (String.IsNullOrEmpty(deviceId))
{
return(this.BadRequest());
}
if (events == null)
{
return(this.BadRequest());
}
ServiceEventSource.Current.ServiceMessage(
this.context,
"Received {0} events from device {1}",
events.Count(),
deviceId);
DeviceEvent max = events.FirstOrDefault();
if (max == null)
{
return(this.Ok());
}
DeviceEventSeries eventList = new DeviceEventSeries(deviceId, events);
IReliableDictionary <string, DeviceEvent> store =
await this.stateManager.GetOrAddAsync <IReliableDictionary <string, DeviceEvent> >(DataService.EventDictionaryName);
IReliableQueue <DeviceEventSeries> queue =
await this.stateManager.GetOrAddAsync <IReliableQueue <DeviceEventSeries> >(DataService.EventQueueName);
// determine the most recent event in the time series
foreach (DeviceEvent item in events)
{
this.appLifetime.ApplicationStopping.ThrowIfCancellationRequested();
if (item.Timestamp > max.Timestamp)
{
max = item;
}
}
using (ITransaction tx = this.stateManager.CreateTransaction())
{
// Update the current value if the max in the new set is more recent
// Or add the max in the current set if a value for this device doesn't exist.
await store.AddOrUpdateAsync(
tx,
deviceId,
max,
(key, currentValue) =>
{
return(max.Timestamp > currentValue.Timestamp
? max
: currentValue);
});
// Queue the time series for offload
await queue.EnqueueAsync(tx, eventList);
// Commit
await tx.CommitAsync();
}
return(this.Ok());
}
protected override async Task RunAsync(CancellationToken cancellationToken)
{
ServiceEventSource.Current.RunAsyncInvoked(ServiceEventSourceName);
IReliableQueue <string> inputQueue = await this.StateManager.GetOrAddAsync <IReliableQueue <string> >("inputQueue");
IReliableDictionary <string, long> wordCountDictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <string, long> >("wordCountDictionary");
IReliableDictionary <string, long> statsDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, long> >("statsDictionary");
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <string> dequeuReply = await inputQueue.TryDequeueAsync(tx);
if (dequeuReply.HasValue)
{
string word = dequeuReply.Value;
long count = await wordCountDictionary.AddOrUpdateAsync(
tx,
word, /* key */
1, /* value to add if key is absent */
(key, oldValue) => oldValue + 1);
long numberOfProcessedWords = await statsDictionary.AddOrUpdateAsync(
tx,
"Number of Words Processed",
1,
(key, oldValue) => oldValue + 1);
long queueLength = await inputQueue.GetCountAsync(tx);
await tx.CommitAsync();
ServiceEventSource.Current.RunAsyncStatus(
this.Partition.PartitionInfo.Id,
numberOfProcessedWords,
queueLength,
word,
count);
}
}
await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken);
}
catch (TimeoutException)
{
//Service Fabric uses timeouts on collection operations to prevent deadlocks.
//If this exception is thrown, it means that this transaction was waiting the default
//amount of time (4 seconds) but was unable to acquire the lock. In this case we simply
//retry after a random backoff interval. You can also control the timeout via a parameter
//on the collection operation.
Thread.Sleep(TimeSpan.FromSeconds(new Random().Next(100, 300)));
continue;
}
catch (Exception exception)
{
//For sample code only: simply trace the exception.
ServiceEventSource.Current.MessageEvent(exception.ToString());
}
}
}
internal async Task <bool> TryDequeueAndProcessAsync(CancellationToken cancellationToken)
{
IReliableQueue <Guid> queue =
await this.StateManager.GetOrAddAsync <IReliableQueue <Guid> >(QueueName);
IReliableDictionary <Guid, ApplicationDeployment> dictionary =
await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, ApplicationDeployment> >(DictionaryName);
Stopwatch sw = Stopwatch.StartNew();
using (ITransaction tx = this.StateManager.CreateTransaction())
{
ConditionalValue <Guid> workItem = await queue.TryDequeueAsync(tx, this.transactionTimeout, cancellationToken);
if (!workItem.HasValue)
{
ServiceEventSource.Current.ServiceMessage(this, "No new application deployment requests.");
return(false);
}
Guid workItemId = workItem.Value;
ConditionalValue <ApplicationDeployment> appDeployment =
await dictionary.TryGetValueAsync(tx, workItemId, LockMode.Update, this.transactionTimeout, cancellationToken);
if (!appDeployment.HasValue)
{
ServiceEventSource.Current.ServiceMessage(
this,
"Found queued application deployment request with no associated deployment information. Discarding.");
return(true);
}
ApplicationDeployment processedDeployment = await this.ProcessApplicationDeployment(appDeployment.Value, cancellationToken);
if (processedDeployment.Status == ApplicationDeployStatus.Complete ||
processedDeployment.Status == ApplicationDeployStatus.Failed)
{
// Remove deployments that completed or failed
await dictionary.TryRemoveAsync(tx, workItemId, this.transactionTimeout, cancellationToken);
// Log completion.
ServiceEventSource.Current.ApplicationDeploymentCompleted(
sw.ElapsedMilliseconds,
ApplicationDeployStatus.Complete == processedDeployment.Status,
processedDeployment.Cluster,
processedDeployment.ApplicationTypeName,
processedDeployment.ApplicationTypeVersion,
processedDeployment.ApplicationInstanceName);
}
else
{
// The deployment hasn't completed or failed, so queue up the next stage of deployment
await queue.EnqueueAsync(tx, workItemId, this.transactionTimeout, cancellationToken);
// And update the deployment record with the new status
await dictionary.SetAsync(tx, workItemId, processedDeployment, this.transactionTimeout, cancellationToken);
ServiceEventSource.Current.ApplicationDeploymentSuccessStatus(
processedDeployment.Cluster,
processedDeployment.ApplicationTypeName,
processedDeployment.ApplicationTypeVersion,
processedDeployment.ApplicationInstanceName,
Enum.GetName(typeof(ApplicationDeployStatus), processedDeployment.Status));
}
await tx.CommitAsync();
}
return(true);
}
private async Task workLoopAsync()
{
int nLongDequeueWaitTimeMs = 20 * 1000;
int nShortDequeueWaitTimeMs = 2 * 1000;
int nNoQueueWaitTimeMS = 5 * 1000;
int nPauseCheckMs = 5 * 1000;
while (this.m_KeepWorking)
{
// pause check
while (this.m_Pause)
{
await Task.Delay(nPauseCheckMs);
}
// take the queue
KeyValuePair <string, IReliableQueue <Wi> > kvp = this.m_WorkManager.m_QueueManager.TakeQueueAsync();
if (null == kvp.Value) // no queue to work on.
{
// this will only happen if executers # are > than queues
// usually a situation that should resolve it self.
// well by the following logic
this.m_WorkManager.m_TraceWriter.TraceMessage(
string.Format("Executer {0} found no q and will sleep for {1}", this.m_WorkerExecuterId, nNoQueueWaitTimeMS));
await this.FinilizeQueueWork(0, null, null); // check removal
await Task.Delay(nNoQueueWaitTimeMS); // sleep as there is no point of retrying right away.
continue;
}
// got Q
IReliableQueue <Wi> q = kvp.Value;
string qName = kvp.Key;
int nCurrentMessage = 0;
try
{
while (this.m_KeepWorking & !this.m_Pause)
{
nCurrentMessage++;
// processed the # of messages?
if (nCurrentMessage > this.m_WorkManager.YieldQueueAfter)
{
break; //-> to finally
}
// as long as we have other queues. we need to have a short wait time
int ActualTimeOut = this.m_WorkManager.m_QueueManager.Count > this.m_WorkManager.m_Executers.Count
? nShortDequeueWaitTimeMs
: nLongDequeueWaitTimeMs;
using (ITransaction tx = this.m_WorkManager.StateManager.CreateTransaction())
{
ConditionalResult <Wi> cResults = await q.TryDequeueAsync(
tx,
TimeSpan.FromMilliseconds(ActualTimeOut),
CancellationToken.None);
if (cResults.HasValue)
{
Handler handler = this.m_WorkManager.GetHandlerForQueue(qName);
Wi wi = await handler.HandleWorkItem(cResults.Value);
if (null != wi) // do we have an enqueue request?
{
await q.EnqueueAsync(tx, wi);
}
await tx.CommitAsync();
this.m_WorkManager.DecreaseBufferedWorkItems();
}
else
{
break; // -> to finally
}
}
}
}
catch (TimeoutException to)
{
/* Queue is locked for enqueues */
this.m_WorkManager.m_TraceWriter.TraceMessage(
string.Format("Executer Dequeue Timeout after {0}: {1}", nLongDequeueWaitTimeMs, to.Message));
break; //-> to finally
}
catch (AggregateException aex)
{
AggregateException ae = aex.Flatten();
this.m_WorkManager.m_TraceWriter.TraceMessage(
string.Format(
"Executer encountered fatel error and will exit E:{0} StackTrace:{1}",
ae.GetCombinedExceptionMessage(),
ae.GetCombinedExceptionStackTrace()));
throw;
}
catch (Exception E)
{
this.m_WorkManager.m_TraceWriter.TraceMessage(
string.Format("Executer encountered fatel error and will exit E:{0} StackTrace:{1}", E.Message, E.StackTrace));
throw;
}
finally
{
await this.FinilizeQueueWork(nCurrentMessage, qName, q);
}
}
this.m_WorkManager.m_TraceWriter.TraceMessage(string.Format("Worker {0} exited loop", this.m_WorkerExecuterId));
}
