public async Task <IActionResult> GetQueueLengthAsync() { IReliableQueue <DeviceEventSeries> queue = await this.stateManager.GetOrAddAsync <IReliableQueue <DeviceEventSeries> >(DataService.EventQueueName); using (ITransaction tx = this.stateManager.CreateTransaction()) { long count = await queue.GetCountAsync(tx); return(this.Ok(count)); } }
protected override async Task RunAsync(CancellationToken cancellationToken) { cancellationToken.Register(() => this._webApiCancellationSource.Cancel()); IReliableDictionary <string, RateAggregation> citiesDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, RateAggregation> >(RateCitiesDictionaryName); IReliableQueue <RateRequest> queue = await this.StateManager.GetOrAddAsync <IReliableQueue <RateRequest> >(RateQueueName); while (true) { cancellationToken.ThrowIfCancellationRequested(); try { using (var tx = this.StateManager.CreateTransaction()) { var result = await queue.TryDequeueAsync(tx); if (result.HasValue) { RateRequest request = result.Value; // TODO: Process the request // TODO: Go against the reservation provider to pick up the rate // TODO: Determine the reservation provider per tenant from the configuration parameters string providerName = GetParameterValue(ParamatersSection, "ProviderName"); int nights = (request.CheckOutDate - request.CheckInDate).Days; int netAmount = _random.Next(500) * nights; var newAggregation = new RateAggregation(); newAggregation.Transactions = 1; newAggregation.Nights = nights; newAggregation.Amount = (double)netAmount; await citiesDictionary.AddOrUpdateAsync(tx, $"{request.City}/{request.Country}/{providerName}", newAggregation, (key, currentValue) => { currentValue.Transactions += newAggregation.Transactions; currentValue.Nights += newAggregation.Nights; currentValue.Amount += newAggregation.Amount; return(currentValue); }); // This commits the add to dictionary and the dequeue operation. await tx.CommitAsync(); } } } catch (Exception e) { } await Task.Delay(TimeSpan.FromMilliseconds(500), cancellationToken); } }
private async Task LoadNumOfBufferedItems() { long buffered = 0; foreach (string qName in this.m_QueueManager.QueueNames) { IReliableQueue <Wi> q = await this.m_QueueManager.GetOrAddQueueAsync(qName); buffered += await q.GetCountAsync(); } this.m_NumOfBufferedWorkItems = buffered; }
public async Task <IHttpActionResult> AddWord(string word) { IReliableQueue <string> queue = await this.stateManager.GetOrAddAsync <IReliableQueue <string> >("inputQueue"); using (ITransaction tx = this.stateManager.CreateTransaction()) { await queue.EnqueueAsync(tx, word); await tx.CommitAsync(); } return(this.Ok()); }
/// <summary> /// This method uses an IReliableQueue to store completed RestockRequests which are later sent to the client using batch processing. /// We could send the request immediately but we prefer to minimize traffic back to the Inventory Service by batching multiple requests /// in one trip. /// </summary> /// <param name="actorId"></param> /// <param name="request"></param> public async void RestockRequestCompleted(ActorId actorId, RestockRequest request) { IReliableQueue <RestockRequest> completedRequests = await this.StateManager.GetOrAddAsync <IReliableQueue <RestockRequest> >(CompletedRequestsQueueName); using (ITransaction tx = this.StateManager.CreateTransaction()) { await completedRequests.EnqueueAsync(tx, request); await tx.CommitAsync(); } IRestockRequestActor restockRequestActor = ActorProxy.Create <IRestockRequestActor>(actorId, this.ApplicationName); await restockRequestActor.UnsubscribeAsync <IRestockRequestEvents>(this); //QUESTION:What does this method do? }
public async Task ResetAsync() { IReliableDictionary <string, int> states = await StateManager.GetOrAddAsync <IReliableDictionary <string, int> >("states"); IReliableQueue <string> events = await StateManager.GetOrAddAsync <IReliableQueue <string> >("events"); using (ITransaction transaction = StateManager.CreateTransaction()) { await states.SetAsync(transaction, "Counter", 0); await events.EnqueueAsync(transaction, $"{DateTime.UtcNow:O} The Counter is reset."); await transaction.CommitAsync(); } }
private async Task LoadNumOfBufferedItems() { long buffered = 0; using (ITransaction tx = this.StateManager.CreateTransaction()) { foreach (string qName in this.m_QueueManager.QueueNames) { IReliableQueue <Wi> q = await this.m_QueueManager.GetOrAddQueueAsync(qName); buffered += await q.GetCountAsync(tx); } } this.m_NumOfBufferedWorkItems = buffered; }
public async Task PutMessagesAsync(IReadOnlyCollection <QueueMessage> messages, CancellationToken cancellationToken) { IReliableQueue <byte[]> collection = await _stateManager.GetOrAddAsync <IReliableQueue <byte[]> >(_queueName); using (var tx = new ServiceFabricTransaction(_stateManager, null)) { foreach (QueueMessage message in messages) { byte[] data = message.ToByteArray(); await collection.EnqueueAsync(tx.Tx, data, _timeout, cancellationToken); } await tx.CommitAsync(); } }
private async Task ReceiveMessagesAsync(Func <IEnumerable <QueueMessage>, Task> onMessage, int maxBatchSize, CancellationToken cancellationToken) { var messages = new List <QueueMessage>(); while (!cancellationToken.IsCancellationRequested && !_disposed) { try { using (var tx = new ServiceFabricTransaction(_stateManager, null)) { IReliableQueue <byte[]> collection = await GetCollectionAsync(); while (messages.Count < maxBatchSize) { ConditionalValue <byte[]> message = await collection.TryDequeueAsync(tx.Tx, TimeSpan.FromSeconds(4), cancellationToken); if (message.HasValue) { QueueMessage qm = QueueMessage.FromByteArray(message.Value); messages.Add(qm); } else { break; } } //make the call before committing the transaction if (messages.Count > 0) { await onMessage(messages); messages.Clear(); } await tx.CommitAsync(); } } catch (Exception ex) { Trace.Fail($"failed to listen to messages on queue '{_queueName}'", ex.ToString()); } await Task.Delay(_scanInterval); } }
public async Task <string> CountAsync() { await Task.Delay(TimeSpan.FromSeconds(3)); IReliableDictionary <string, int> states = await StateManager.GetOrAddAsync <IReliableDictionary <string, int> >("states"); IReliableQueue <string> events = await StateManager.GetOrAddAsync <IReliableQueue <string> >("events"); int number; //using (ITransaction transaction = StateManager.CreateTransaction()) //{ // ConditionalValue<int> counter = await states.TryGetValueAsync(transaction, "Counter"); // number = counter.HasValue ? counter.Value : 0; // number++; //} //await Task.Delay(TimeSpan.FromSeconds(3)); //using (ITransaction transaction = StateManager.CreateTransaction()) //{ // await states.SetAsync(transaction, "Counter", number); // await events.EnqueueAsync(transaction, $"{DateTime.UtcNow:O} The Counter is {number}."); // await transaction.CommitAsync(); //} using (ITransaction transaction = StateManager.CreateTransaction()) { ConditionalValue <int> counter = await states.TryGetValueAsync(transaction, "Counter"); number = counter.HasValue ? counter.Value : 0; number++; await Task.Delay(TimeSpan.FromSeconds(3)); await states.SetAsync(transaction, "Counter", number); await events.EnqueueAsync(transaction, $"{DateTime.UtcNow:O} The Counter is {number}."); await transaction.CommitAsync(); } return($"Current number is {number}, from partition {Context.PartitionId} and replica {Context.ReplicaId}"); }
internal static async Task <WorkItemQueue> CreateOrGetWorkItemQueue(StatefulService statefulService) { if (workItemQueue == null) { IReliableQueue <WorkItemProcessInfo> workItemReliableQueue = await statefulService.StateManager.GetOrAddAsync <IReliableQueue <WorkItemProcessInfo> >(Constants.WorkItemQueue); lock (syncRoot) { if (workItemQueue == null) { workItemQueue = new WorkItemQueue(workItemReliableQueue, 0, WorkItemQueueRunType.WorkItemQueue, statefulService); } } } return(workItemQueue); }
public async Task IgnoreOldEvent() { MockApplicationLifetime appLifetime = new MockApplicationLifetime(); MockReliableStateManager stateManager = new MockReliableStateManager(); IReliableDictionary <string, DeviceEvent> store = await stateManager.GetOrAddAsync <IReliableDictionary <string, DeviceEvent> >(DataService.EventDictionaryName); IReliableQueue <DeviceEventSeries> queue = await stateManager.GetOrAddAsync <IReliableQueue <DeviceEventSeries> >(DataService.EventQueueName); string expectedDeviceId = "some-device"; DeviceEvent expectedDeviceEvent = new DeviceEvent(new DateTimeOffset(100, TimeSpan.Zero)); EventsController target = new EventsController(stateManager, statefulServiceContext, appLifetime); IActionResult result = await target.Post(expectedDeviceId, new[] { expectedDeviceEvent }); Assert.True(result is OkResult); using (ITransaction tx = stateManager.CreateTransaction()) { ConditionalValue <DeviceEvent> actualStoredEvent = await store.TryGetValueAsync(tx, expectedDeviceId); Assert.True(actualStoredEvent.HasValue); Assert.Equal(expectedDeviceEvent.Timestamp, actualStoredEvent.Value.Timestamp); await tx.CommitAsync(); } DeviceEvent oldEvent = new DeviceEvent(new DateTimeOffset(10, TimeSpan.Zero)); result = await target.Post(expectedDeviceId, new[] { oldEvent }); Assert.True(result is OkResult); using (ITransaction tx = stateManager.CreateTransaction()) { ConditionalValue <DeviceEvent> actualStoredEvent = await store.TryGetValueAsync(tx, expectedDeviceId); Assert.True(actualStoredEvent.HasValue); Assert.Equal(expectedDeviceEvent.Timestamp, actualStoredEvent.Value.Timestamp); await tx.CommitAsync(); } }
public async Task AddMessageAsync(string roomName, ChatMessage chatMessage, CancellationToken cancellationToken) { IReliableQueue <ChatMessage> room = await GetRoomQueue(roomName); using (var tx = stateManager.CreateTransaction()) { long msgCount = await room.GetCountAsync(tx); if (msgCount == 0) { // new room, increment counter await IncrementRoomCountAsync(1, cancellationToken, tx); } await room.EnqueueAsync(tx, chatMessage); await tx.CommitAsync(); } }
public async Task AddMostRecentEvent() { MockApplicationLifetime appLifetime = new MockApplicationLifetime(); MockReliableStateManager stateManager = new MockReliableStateManager(); IReliableDictionary <string, DeviceEvent> store = await stateManager.GetOrAddAsync <IReliableDictionary <string, DeviceEvent> >(DataService.EventDictionaryName); IReliableQueue <DeviceEventSeries> queue = await stateManager.GetOrAddAsync <IReliableQueue <DeviceEventSeries> >(DataService.EventQueueName); string expectedDeviceId = "some-device"; List <DeviceEvent> expectedDeviceList = new List <DeviceEvent>(); DeviceEvent expectedDeviceEvent = new DeviceEvent(new DateTimeOffset(100, TimeSpan.Zero)); for (int i = 0; i < 10; ++i) { expectedDeviceList.Add(new DeviceEvent(new DateTimeOffset(i, TimeSpan.Zero))); } expectedDeviceList.Insert(4, expectedDeviceEvent); EventsController target = new EventsController(stateManager, statefulServiceContext, appLifetime); IActionResult result = await target.Post(expectedDeviceId, expectedDeviceList); Assert.True(result is OkResult); using (ITransaction tx = stateManager.CreateTransaction()) { ConditionalValue <DeviceEvent> actualStoredEvent = await store.TryGetValueAsync(tx, expectedDeviceId); ConditionalValue <DeviceEventSeries> actualQueuedEvent = await queue.TryDequeueAsync(tx); Assert.True(actualStoredEvent.HasValue); Assert.Equal(expectedDeviceEvent.Timestamp, actualStoredEvent.Value.Timestamp); Assert.True(actualQueuedEvent.HasValue); Assert.True(actualQueuedEvent.Value.Events.Select(x => x.Timestamp).SequenceEqual(expectedDeviceList.Select(x => x.Timestamp))); await tx.CommitAsync(); } }
public async Task <ChatRoom> GetRoomAsync(string roomName, CancellationToken cancellationToken) { IReliableQueue <ChatMessage> room = await GetRoomQueue(roomName); ChatRoom chatRoom = new ChatRoom(roomName); using (var tx = stateManager.CreateTransaction()) { var roomIterator = (await room.CreateEnumerableAsync(tx)).GetAsyncEnumerator(); while (await roomIterator.MoveNextAsync(cancellationToken)) { var msg = roomIterator.Current; chatRoom.messages.Add(msg); } } return(chatRoom); }
public async Task <IReliableQueue <W> > GetOrAddQueueAsync(string qName) { using (ITransaction tx = this.m_WorkManager.StateManager.CreateTransaction()) { if (!await this.m_dictListOfQueues.ContainsKeyAsync(tx, qName, TimeSpan.FromSeconds(5), CancellationToken.None)) { IReliableQueue <W> reliableQ = await this.m_WorkManager.StateManager.GetOrAddAsync <IReliableQueue <W> >(tx, qName); await this.m_dictListOfQueues.AddAsync(tx, qName, qName, TimeSpan.FromSeconds(5), CancellationToken.None); await tx.CommitAsync(); this.m_QueueRefs.TryAdd(qName, reliableQ); this.qOfq.Enqueue(qName); } } return(this.m_QueueRefs[qName]); }
protected override async Task RunAsync(CancellationToken cancellationToken) { IReliableQueue <string> messageQueue = await this.StateManager.GetOrAddAsync <IReliableQueue <string> >("messageQueue"); const string baseAddress = "http://localhost:8864"; using (Microsoft.Owin.Hosting.WebApp.Start <Startup>(baseAddress)) { HttpClient client = new HttpClient(); HttpResponseMessage res = client.GetAsync(baseAddress + "api/values/5").Result; res.EnsureSuccessStatusCode(); var results = res.Content.ReadAsStringAsync().Result; using (ITransaction tx = this.StateManager.CreateTransaction()) { await messageQueue.EnqueueAsync(tx, results); await tx.CommitAsync(); } } }
public async Task GetQueueLength() { MockApplicationLifetime appLifetime = new MockApplicationLifetime(); MockReliableStateManager stateManager = new MockReliableStateManager(); IReliableQueue <DeviceEventSeries> queue = await stateManager.GetOrAddAsync <IReliableQueue <DeviceEventSeries> >(DataService.EventQueueName); using (ITransaction tx = stateManager.CreateTransaction()) { await queue.EnqueueAsync(tx, new DeviceEventSeries("", new DeviceEvent[0])); } DevicesController target = new DevicesController(stateManager, appLifetime); IActionResult result = await target.GetQueueLengthAsync(); Assert.True(result is OkObjectResult); long actual = (long)((OkObjectResult)result).Value; Assert.Equal(1, actual); }
public async Task PostWorkItemAsync(Wi workItem) { if (this.WorkManagerStatus != WorkManagerStatus.Working) { throw new InvalidOperationException("Work Manager is not working state"); } if (this.m_NumOfBufferedWorkItems >= this.m_MaxNumOfBufferedWorkItems) { throw new InvalidOperationException(string.Format("Work Manger is at maximum buffered work items:{0}", this.m_NumOfBufferedWorkItems)); } try { // Which Q IReliableQueue <Wi> targetQueue = await this.m_QueueManager.GetOrAddQueueAsync(workItem.QueueName); // enqueue using (ITransaction tx = this.StateManager.CreateTransaction()) { await targetQueue.EnqueueAsync(tx, workItem, TimeSpan.FromSeconds(10), CancellationToken.None); await tx.CommitAsync(); } this.IncreaseBufferedWorkItems(); this.m_DeferedTaskExec.AddWork(this.TryIncreaseExecuters); } catch (AggregateException aex) { AggregateException ae = aex.Flatten(); this.m_TraceWriter.TraceMessage( string.Format( "Post to work manager failed, caller should retry E:{0} StackTrace:{1}", ae.GetCombinedExceptionMessage(), ae.GetCombinedExceptionStackTrace())); throw; } }
/// <summary> /// Queues deployment of application packages to the given cluster. /// </summary> /// <param name="clusterAddress"></param> /// <param name="clusterPort"></param> /// <returns></returns> public async Task <IEnumerable <Guid> > QueueApplicationDeploymentAsync(string clusterAddress, int clusterPort) { IReliableQueue <Guid> queue = await this.StateManager.GetOrAddAsync <IReliableQueue <Guid> >(QueueName); IReliableDictionary <Guid, ApplicationDeployment> dictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, ApplicationDeployment> >(DictionaryName); List <Guid> workIds = new List <Guid>(this.ApplicationPackages.Count()); using (ITransaction tx = this.StateManager.CreateTransaction()) { // Grab each application package that's included with the service // and create an ApplicationDeployment record of it. // Then queue a job to begin processing each one. foreach (ApplicationPackageInfo package in this.ApplicationPackages) { Guid id = Guid.NewGuid(); ApplicationDeployment applicationDeployment = new ApplicationDeployment( cluster: GetClusterAddress(clusterAddress, clusterPort), status: ApplicationDeployStatus.Copy, imageStorePath: null, applicationTypeName: package.ApplicationTypeName, applicationTypeVersion: package.ApplicationTypeVersion, applicationInstanceName: GetApplicationInstanceName(package.PackageFileName), packageZipFilePath: Path.Combine(this.applicationPackageDataPath.FullName, package.PackageFileName), timestamp: DateTimeOffset.UtcNow); await dictionary.AddAsync(tx, id, applicationDeployment); await queue.EnqueueAsync(tx, id); workIds.Add(id); } await tx.CommitAsync(); return(workIds); } }
/// <summary> /// This is the main entry point for your service replica. /// This method executes when this replica of your service becomes primary and has write status. /// </summary> /// <param name="cancellationToken">Canceled when Service Fabric needs to shut down this service replica.</param> protected override async Task RunAsync(CancellationToken cancellationToken) { cancellationToken.ThrowIfCancellationRequested(); LogQueue = await this.StateManager .GetOrAddAsync <IReliableQueue <IdempotentMessage <PurchaseInfo> > >("logQueue"); var configurationPackage = Context .CodePackageActivationContext .GetConfigurationPackageObject("Config"); var host = new HostBuilder() .ConfigureServices((hostContext, services) => { services.AddSingleton(this.StateManager); services.AddSingleton(this.LogQueue); services.AddSingleton(configurationPackage); services.AddHostedService <ComputeStatistics>(); }) .Build(); await host.RunAsync(cancellationToken); }
public async Task Post(string name, string parameters) { IReliableQueue <string> queue = await this.stateManager.GetOrAddAsync <IReliableQueue <string> >("jobQueue"); IReliableDictionary <string, Job> dictionary = await this.stateManager.GetOrAddAsync <IReliableDictionary <string, Job> >("jobs"); using (ITransaction tx = this.stateManager.CreateTransaction()) { if (await dictionary.ContainsKeyAsync(tx, name, LockMode.Update)) { throw new ArgumentException($"Job {name} already exists."); } Job job = new Job(name, parameters, false); await queue.EnqueueAsync(tx, name); await dictionary.SetAsync(tx, name, job); await tx.CommitAsync(); } }
public static async Task <List <T> > TryReadEntriesAsync <T>(this IReliableQueue <T> queue, IReliableStateManager stateManager, int maxCount) { List <T> results = null; using (var tx = stateManager.CreateTransaction()) { do { var entry = await queue.TryDequeueAsync(tx).ConfigureAwait(false); if (!entry.HasValue) { break; } results = results ?? new List <T>(maxCount); results.Add(entry.Value); } while (results.Count < maxCount); tx.Abort(); } return(results); }
internal BaseWorkItemQueue(IReliableQueue <WorkItemProcessInfo> workItemReliableQueue, int maxWaitTimeInMinutes, WorkItemQueueRunType workItemQueueRunType, string traceType, StatefulService statefulService) { BackupRestoreTrace.TraceSource.WriteInfo(traceType, "Constructing BaseWorkItemQueue with following parameters - workItemReliableQueue: {0}, maxWaitTimeInMinutes: {1}, workItemQueueRunType: {2}", workItemReliableQueue.Name, maxWaitTimeInMinutes, workItemQueueRunType); this.WorkItemReliableQueue = workItemReliableQueue; this.StatefulService = statefulService; this.WorkItemStore = WorkItemStore.CreateOrGetWorkItemStore(this.StatefulService).GetAwaiter().GetResult(); this.WorkItemInProcessStore = WorkItemInProcessStore.CreateOrGetWorkItemInProcessStore(this.StatefulService).GetAwaiter().GetResult(); this.MaxWaitTimeInMilliSeconds = maxWaitTimeInMinutes == 0 ? FastQueueWaitTime : maxWaitTimeInMinutes * 60 * 1000; this.AllowedTicks = (this.MaxWaitTimeInMilliSeconds * TimeSpan.TicksPerMillisecond) / 20; this.workItemQueueRunType = workItemQueueRunType; this.TraceType = traceType; this.emptyTraceType = string.Format("{0}.{1}", traceType, "Empty"); this.WorkItemHandlerTimer = new Timer(new TimerCallback(this.ProcessWorkItemHandler), null, this.MaxWaitTimeInMilliSeconds, Timeout.Infinite); }
/// <summary> /// This is the main entry point for your service replica. /// This method executes when this replica of your service becomes primary and has write status. /// </summary> /// <param name="cancellationToken">Canceled when Service Fabric needs to shut down this service replica.</param> protected override async Task RunAsync(CancellationToken cancellationToken) { mailQueue = await this.StateManager.GetOrAddAsync <IReliableQueue <MailData> >(MailQueueName); ConfigureService(); List <Task> taskList = new List <Task>(); taskList.Add(SendMailTaskCode(cancellationToken)); var taskResult = await Task.WhenAny(taskList); if (!cancellationToken.IsCancellationRequested) { if (taskResult.IsFaulted) { throw taskResult.Exception; } else { throw new InvalidOperationException("One or more tasks completed in unexpected ways"); } } }
/// <summary> /// Drains the queue of completed restock requests sends them to InventoryService. /// </summary> /// <param name="cancellationToken"></param> /// <returns></returns> protected override async Task RunAsync(CancellationToken cancellationToken) { IReliableQueue <RestockRequest> completedRequests = await this.StateManager.GetOrAddAsync <IReliableQueue <RestockRequest> >(CompletedRequestsQueueName); while (!cancellationToken.IsCancellationRequested) { using (ITransaction tx = this.StateManager.CreateTransaction()) { ConditionalValue <RestockRequest> result = await completedRequests.TryDequeueAsync(tx, TxTimeout, cancellationToken); if (result.HasValue) { ServiceUriBuilder builder = new ServiceUriBuilder(InventoryServiceName); IInventoryService inventoryService = ServiceProxy.Create <IInventoryService>(builder.ToUri(), result.Value.ItemId.GetPartitionKey()); await inventoryService.AddStockAsync(result.Value.ItemId, result.Value.Quantity); ServiceEventSource.Current.ServiceMessage( this, "Adding stock to inventory service. ID: {0}. Quantity: {1}", result.Value.ItemId, result.Value.Quantity); } // This commits the dequeue operations. // If the request to add the stock to the inventory service throws, this commit will not execute // and the items will remain on the queue, so we can be sure that we didn't dequeue items // that didn't get saved successfully in the inventory service. // However there is a very small chance that the stock was added to the inventory service successfully, // but service execution stopped before reaching this commit (machine crash, for example). await tx.CommitAsync(); } await Task.Delay(CompletedRequestsBatchInterval, cancellationToken); } }
public async Task <IActionResult> Post(string deviceId, [FromBody] IEnumerable <DeviceEvent> events) { if (String.IsNullOrEmpty(deviceId)) { return(this.BadRequest()); } if (events == null) { return(this.BadRequest()); } ServiceEventSource.Current.ServiceMessage( this.context, "Received {0} events from device {1}", events.Count(), deviceId); DeviceEvent max = events.FirstOrDefault(); if (max == null) { return(this.Ok()); } DeviceEventSeries eventList = new DeviceEventSeries(deviceId, events); IReliableDictionary <string, DeviceEvent> store = await this.stateManager.GetOrAddAsync <IReliableDictionary <string, DeviceEvent> >(DataService.EventDictionaryName); IReliableQueue <DeviceEventSeries> queue = await this.stateManager.GetOrAddAsync <IReliableQueue <DeviceEventSeries> >(DataService.EventQueueName); // determine the most recent event in the time series foreach (DeviceEvent item in events) { this.appLifetime.ApplicationStopping.ThrowIfCancellationRequested(); if (item.Timestamp > max.Timestamp) { max = item; } } using (ITransaction tx = this.stateManager.CreateTransaction()) { // Update the current value if the max in the new set is more recent // Or add the max in the current set if a value for this device doesn't exist. await store.AddOrUpdateAsync( tx, deviceId, max, (key, currentValue) => { return(max.Timestamp > currentValue.Timestamp ? max : currentValue); }); // Queue the time series for offload await queue.EnqueueAsync(tx, eventList); // Commit await tx.CommitAsync(); } return(this.Ok()); }
protected override async Task RunAsync(CancellationToken cancellationToken) { ServiceEventSource.Current.RunAsyncInvoked(ServiceEventSourceName); IReliableQueue <string> inputQueue = await this.StateManager.GetOrAddAsync <IReliableQueue <string> >("inputQueue"); IReliableDictionary <string, long> wordCountDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, long> >("wordCountDictionary"); IReliableDictionary <string, long> statsDictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <string, long> >("statsDictionary"); while (true) { cancellationToken.ThrowIfCancellationRequested(); try { using (ITransaction tx = this.StateManager.CreateTransaction()) { ConditionalValue <string> dequeuReply = await inputQueue.TryDequeueAsync(tx); if (dequeuReply.HasValue) { string word = dequeuReply.Value; long count = await wordCountDictionary.AddOrUpdateAsync( tx, word, /* key */ 1, /* value to add if key is absent */ (key, oldValue) => oldValue + 1); long numberOfProcessedWords = await statsDictionary.AddOrUpdateAsync( tx, "Number of Words Processed", 1, (key, oldValue) => oldValue + 1); long queueLength = await inputQueue.GetCountAsync(tx); await tx.CommitAsync(); ServiceEventSource.Current.RunAsyncStatus( this.Partition.PartitionInfo.Id, numberOfProcessedWords, queueLength, word, count); } } await Task.Delay(TimeSpan.FromMilliseconds(100), cancellationToken); } catch (TimeoutException) { //Service Fabric uses timeouts on collection operations to prevent deadlocks. //If this exception is thrown, it means that this transaction was waiting the default //amount of time (4 seconds) but was unable to acquire the lock. In this case we simply //retry after a random backoff interval. You can also control the timeout via a parameter //on the collection operation. Thread.Sleep(TimeSpan.FromSeconds(new Random().Next(100, 300))); continue; } catch (Exception exception) { //For sample code only: simply trace the exception. ServiceEventSource.Current.MessageEvent(exception.ToString()); } } }
internal async Task <bool> TryDequeueAndProcessAsync(CancellationToken cancellationToken) { IReliableQueue <Guid> queue = await this.StateManager.GetOrAddAsync <IReliableQueue <Guid> >(QueueName); IReliableDictionary <Guid, ApplicationDeployment> dictionary = await this.StateManager.GetOrAddAsync <IReliableDictionary <Guid, ApplicationDeployment> >(DictionaryName); Stopwatch sw = Stopwatch.StartNew(); using (ITransaction tx = this.StateManager.CreateTransaction()) { ConditionalValue <Guid> workItem = await queue.TryDequeueAsync(tx, this.transactionTimeout, cancellationToken); if (!workItem.HasValue) { ServiceEventSource.Current.ServiceMessage(this, "No new application deployment requests."); return(false); } Guid workItemId = workItem.Value; ConditionalValue <ApplicationDeployment> appDeployment = await dictionary.TryGetValueAsync(tx, workItemId, LockMode.Update, this.transactionTimeout, cancellationToken); if (!appDeployment.HasValue) { ServiceEventSource.Current.ServiceMessage( this, "Found queued application deployment request with no associated deployment information. Discarding."); return(true); } ApplicationDeployment processedDeployment = await this.ProcessApplicationDeployment(appDeployment.Value, cancellationToken); if (processedDeployment.Status == ApplicationDeployStatus.Complete || processedDeployment.Status == ApplicationDeployStatus.Failed) { // Remove deployments that completed or failed await dictionary.TryRemoveAsync(tx, workItemId, this.transactionTimeout, cancellationToken); // Log completion. ServiceEventSource.Current.ApplicationDeploymentCompleted( sw.ElapsedMilliseconds, ApplicationDeployStatus.Complete == processedDeployment.Status, processedDeployment.Cluster, processedDeployment.ApplicationTypeName, processedDeployment.ApplicationTypeVersion, processedDeployment.ApplicationInstanceName); } else { // The deployment hasn't completed or failed, so queue up the next stage of deployment await queue.EnqueueAsync(tx, workItemId, this.transactionTimeout, cancellationToken); // And update the deployment record with the new status await dictionary.SetAsync(tx, workItemId, processedDeployment, this.transactionTimeout, cancellationToken); ServiceEventSource.Current.ApplicationDeploymentSuccessStatus( processedDeployment.Cluster, processedDeployment.ApplicationTypeName, processedDeployment.ApplicationTypeVersion, processedDeployment.ApplicationInstanceName, Enum.GetName(typeof(ApplicationDeployStatus), processedDeployment.Status)); } await tx.CommitAsync(); } return(true); }
private async Task workLoopAsync() { int nLongDequeueWaitTimeMs = 20 * 1000; int nShortDequeueWaitTimeMs = 2 * 1000; int nNoQueueWaitTimeMS = 5 * 1000; int nPauseCheckMs = 5 * 1000; while (this.m_KeepWorking) { // pause check while (this.m_Pause) { await Task.Delay(nPauseCheckMs); } // take the queue KeyValuePair <string, IReliableQueue <Wi> > kvp = this.m_WorkManager.m_QueueManager.TakeQueueAsync(); if (null == kvp.Value) // no queue to work on. { // this will only happen if executers # are > than queues // usually a situation that should resolve it self. // well by the following logic this.m_WorkManager.m_TraceWriter.TraceMessage( string.Format("Executer {0} found no q and will sleep for {1}", this.m_WorkerExecuterId, nNoQueueWaitTimeMS)); await this.FinilizeQueueWork(0, null, null); // check removal await Task.Delay(nNoQueueWaitTimeMS); // sleep as there is no point of retrying right away. continue; } // got Q IReliableQueue <Wi> q = kvp.Value; string qName = kvp.Key; int nCurrentMessage = 0; try { while (this.m_KeepWorking & !this.m_Pause) { nCurrentMessage++; // processed the # of messages? if (nCurrentMessage > this.m_WorkManager.YieldQueueAfter) { break; //-> to finally } // as long as we have other queues. we need to have a short wait time int ActualTimeOut = this.m_WorkManager.m_QueueManager.Count > this.m_WorkManager.m_Executers.Count ? nShortDequeueWaitTimeMs : nLongDequeueWaitTimeMs; using (ITransaction tx = this.m_WorkManager.StateManager.CreateTransaction()) { ConditionalResult <Wi> cResults = await q.TryDequeueAsync( tx, TimeSpan.FromMilliseconds(ActualTimeOut), CancellationToken.None); if (cResults.HasValue) { Handler handler = this.m_WorkManager.GetHandlerForQueue(qName); Wi wi = await handler.HandleWorkItem(cResults.Value); if (null != wi) // do we have an enqueue request? { await q.EnqueueAsync(tx, wi); } await tx.CommitAsync(); this.m_WorkManager.DecreaseBufferedWorkItems(); } else { break; // -> to finally } } } } catch (TimeoutException to) { /* Queue is locked for enqueues */ this.m_WorkManager.m_TraceWriter.TraceMessage( string.Format("Executer Dequeue Timeout after {0}: {1}", nLongDequeueWaitTimeMs, to.Message)); break; //-> to finally } catch (AggregateException aex) { AggregateException ae = aex.Flatten(); this.m_WorkManager.m_TraceWriter.TraceMessage( string.Format( "Executer encountered fatel error and will exit E:{0} StackTrace:{1}", ae.GetCombinedExceptionMessage(), ae.GetCombinedExceptionStackTrace())); throw; } catch (Exception E) { this.m_WorkManager.m_TraceWriter.TraceMessage( string.Format("Executer encountered fatel error and will exit E:{0} StackTrace:{1}", E.Message, E.StackTrace)); throw; } finally { await this.FinilizeQueueWork(nCurrentMessage, qName, q); } } this.m_WorkManager.m_TraceWriter.TraceMessage(string.Format("Worker {0} exited loop", this.m_WorkerExecuterId)); }