public async Task <long> CountAsync() { IReliableConcurrentQueue <Trade> transactions = await this.stateManager.GetOrAddAsync <IReliableConcurrentQueue <Trade> >(queueName); return(transactions.Count); }
public async Task <long> ActiveTradeCountAsync() { IReliableConcurrentQueue <Trade> trades = await this.StateManager.GetOrAddAsync <IReliableConcurrentQueue <Trade> >(QueueName); return(trades.Count); }
private async static Task ScrapeRepositoryAsync( ScrapingTask scrapingTask, IGitHubClient gitHubClient, ITransaction tx, IUserRepoSearchActor userRepoSearchActor, IReliableConcurrentQueue <ScrapingTask> secondTaskQueue) { ScheduledRepository scheduledRepository = scrapingTask.ScheduledRepositories[0]; Result <Repository> result = await gitHubClient .ScrapeRepositoryAsync( authToken : scrapingTask.AuthToken, owner : scheduledRepository.Owner, name : scheduledRepository.Name); if (scrapingTask.ScheduledRepositories.Count > 1) { ScrapingTask nextScrapingTask = Serializer.DeepCopy(scrapingTask); nextScrapingTask.ScheduledRepositories = nextScrapingTask .ScheduledRepositories.Skip(1).ToList(); await secondTaskQueue.EnqueueAsync(tx, nextScrapingTask); } if (result.Success) { result.Data.IsNew = false; await userRepoSearchActor.SetRepositoryAsync(result.Data); } }
public async static Task PerformTaskAsync( ScrapingTask scrapingTask, IGitHubClient gitHubClient, ITransaction tx, IUserRepoSearchActor userRepoSearchActor, IReliableConcurrentQueue <ScrapingTask> firstTaskQueue, IReliableConcurrentQueue <ScrapingTask> secondTaskQueue) { switch (scrapingTask.Type) { case ScrapingTaskType.Repository: await ScrapeRepositoryAsync( scrapingTask : scrapingTask, gitHubClient : gitHubClient, tx : tx, userRepoSearchActor : userRepoSearchActor, secondTaskQueue : secondTaskQueue); break; case ScrapingTaskType.UserInfo: await ScrapeUserInfoAsync( scrapingTask : scrapingTask, gitHubClient : gitHubClient, tx : tx, userRepoSearchActor : userRepoSearchActor, secondTaskQueue : secondTaskQueue); break; case ScrapingTaskType.FollowingRepositories: await ScrapeFollowingRepositoriesAsync( scrapingTask : scrapingTask, gitHubClient : gitHubClient, tx : tx, userRepoSearchActor : userRepoSearchActor, secondTaskQueue : secondTaskQueue); break; } }
protected override async Task <ConditionalValue <byte[]> > TryDequeueAsync(ServiceFabricTransaction tx, IReliableState collectionBase, CancellationToken cancellationToken) { IReliableConcurrentQueue <byte[]> collection = (IReliableConcurrentQueue <byte[]>)collectionBase; ConditionalValue <byte[]> message = await collection.TryDequeueAsync(tx.Tx, cancellationToken, TimeSpan.FromSeconds(4)); return(message); }
private async Task ExecuteAddTradeAsync(Trade trade, IReliableConcurrentQueue <Trade> exportQueue, CancellationToken cancellationToken) { using (var tx = this.StateManager.CreateTransaction()) { await exportQueue.EnqueueAsync(tx, trade, cancellationToken); await tx.CommitAsync(); } }
/// <summary> /// See interface /// </summary> public async Task <int> GetMessageCountAsync() { using (var tx = new ServiceFabricTransaction(_stateManager, null)) { IReliableConcurrentQueue <byte[]> collection = await GetCollectionAsync(); return((int)collection.Count); } }
private async Task <string> ExecuteEnqueueAsync(Trade trade, IReliableConcurrentQueue <Trade> trades, CancellationToken cancellationToken) { using (var tx = this.stateManager.CreateTransaction()) { await trades.EnqueueAsync(tx, trade, cancellationToken); await tx.CommitAsync(); } return(trade.Id); }
public BatchedDataCache(IReliableStateManagerTransactionProvider transactionProvider, IReliableStateManagerProvider reliableStateManagerProvider, IPaymentLogger logger) { this.transactionProvider = transactionProvider ?? throw new ArgumentNullException(nameof(transactionProvider)); this.logger = logger; if (reliableStateManagerProvider == null) { throw new ArgumentNullException(nameof(reliableStateManagerProvider)); } queue = reliableStateManagerProvider.Current.GetOrAddAsync <IReliableConcurrentQueue <T> >("BatchedDataCacheQueue").Result; }
private static async Task <Trade> ExecuteDequeueAsync(ITransaction tx, IReliableConcurrentQueue <Trade> transactions, CancellationToken cancellationToken) { Trade trade = null; var result = await transactions.TryDequeueAsync(tx, cancellationToken); if (result.HasValue) { trade = result.Value; } return(trade); }
public async Task StartUpdateDependenciesAsync(int buildId, int channelId) { IReliableConcurrentQueue <DependencyUpdateItem> queue = await StateManager.GetOrAddAsync <IReliableConcurrentQueue <DependencyUpdateItem> >("queue"); using (ITransaction tx = StateManager.CreateTransaction()) { await queue.EnqueueAsync(tx, new DependencyUpdateItem { BuildId = buildId, ChannelId = channelId }); await tx.CommitAsync(); } }
public static void VerifyConcurrentQueueUnordered(IReliableConcurrentQueue <long> queue, IEnumerable <long> expectedValues, string message = "") { var expected = expectedValues.ToList(); expected.Sort((val1, val2) => val1.CompareTo(val2)); Assert.AreEqual(expected.Count, queue.Count, message); var values = ((ReliableConcurrentQueue <long>)queue).GetVisibleListElementsUnsafe().ToList(); values.Sort((val1, val2) => val1.CompareTo(val2)); Assert.IsTrue(expected.SequenceEqual(values)); }
public async Task <IActionResult> Get() { string statusValue = "Nothing to show yet."; long remainingValue = -1; try { ConditionalValue <IReliableDictionary <string, ReportStatus> > tryGetDictionaryResult = await this.stateManager.TryGetAsync <IReliableDictionary <string, ReportStatus> >(ReportProcessingService.StatusDictionaryName); if (tryGetDictionaryResult.HasValue) { IReliableDictionary <string, ReportStatus> dictionary = tryGetDictionaryResult.Value; using (ITransaction tx = this.stateManager.CreateTransaction()) { ConditionalValue <ReportStatus> getResult = await dictionary.TryGetValueAsync(tx, this.reportContext.Name); if (getResult.HasValue) { statusValue = getResult.Value.Status; } } } ConditionalValue <IReliableConcurrentQueue <ReportProcessingStep> > tryGetQueueResult = await this.stateManager.TryGetAsync <IReliableConcurrentQueue <ReportProcessingStep> >(ReportProcessingService.ProcessingQueueName); if (tryGetQueueResult.HasValue) { IReliableConcurrentQueue <ReportProcessingStep> queue = tryGetQueueResult.Value; remainingValue = queue.Count; } return(this.Json(new { status = statusValue, remaining = remainingValue })); } catch (FabricNotPrimaryException) { return(new ContentResult { StatusCode = 410, Content = "The primary replica has moved. Please re-resolve the service." }); } catch (Exception ex) when(ex is FabricTransientException || ex is TimeoutException) { return(new ContentResult { StatusCode = 503, Content = "The service was unable to process the request. Please try again." }); } }
private async Task <IReliableConcurrentQueue <string> > GetQueueAsync() { IReliableConcurrentQueue <string> queue = null; try { queue = await this.stateManager.GetOrAddAsync <IReliableConcurrentQueue <string> >(this.queueName); } catch (Exception) { } return(queue); }
/// <summary> /// this method keeps on dequeuing the payload from the queue and processes it.. /// </summary> /// <returns></returns> private async Task ProcessJobAsync(IReliableConcurrentQueue <string> processQueue) { using (var tx = StateManager.CreateTransaction()) { var result = await processQueue.TryDequeueAsync(tx); if (result.HasValue) { // Do the work } await tx.CommitAsync(); } }
public async Task PutMessagesAsync(IReadOnlyCollection <QueueMessage> messages, CancellationToken cancellationToken) { IReliableConcurrentQueue <byte[]> collection = await _stateManager.GetOrAddAsync <IReliableConcurrentQueue <byte[]> >(_queueName).ConfigureAwait(false); using (var tx = new ServiceFabricTransaction(_stateManager, null)) { foreach (QueueMessage message in messages) { byte[] data = message.ToByteArray(); await collection.EnqueueAsync(tx.Tx, data, cancellationToken, _timeout).ConfigureAwait(false); } await tx.CommitAsync().ConfigureAwait(false); } }
/// <summary> /// this service has two runtime aspects - <see cref="RunAsync"/> and <see cref="IBullfrogNotificationBackendService"/> /// as soon as listener is open, messaging can start flowing in and the same goes for <see cref="RunAsync"/> kicking in /// /// the listeners are not open to secondary replicas and the same goes for <see cref="RunAsync"/> not being invoked on those so ultimately the queue can be shared /// within the partition/primary replica instance (and we will likely run one partition anyway) /// </summary> private void InitInflightQueue() { if (_inflightQueue != null) { return; } lock (_inflightQueueInstanceLock) { if (_inflightQueue == null) { _inflightQueue = StateManager.GetOrAddAsync <IReliableConcurrentQueue <InflightMessage> >(QueueName).GetAwaiter().GetResult(); } } }
private async Task ReceiveMessagesAsync(Func <IEnumerable <QueueMessage>, Task> onMessage, int maxBatchSize, CancellationToken cancellationToken) { var messages = new List <QueueMessage>(); while (!cancellationToken.IsCancellationRequested && !_disposed) { try { using (var tx = new ServiceFabricTransaction(_stateManager, null)) { IReliableConcurrentQueue <byte[]> collection = await GetCollectionAsync(); while (messages.Count < maxBatchSize) { ConditionalValue <byte[]> message = await collection.TryDequeueAsync(tx.Tx, cancellationToken); if (message.HasValue) { QueueMessage qm = QueueMessage.FromByteArray(message.Value); messages.Add(qm); } else { break; } } //make the call before committing the transaction if (messages.Count > 0) { await onMessage(messages); messages.Clear(); } await tx.CommitAsync(); } } catch (Exception ex) { Trace.Fail($"failed to listen to messages on queue '{_queueName}'", ex.ToString()); } await Task.Delay(_scanInterval); } }
private async static Task ProcessScrapingTask( ScrapingTask scrapingTask, IGitHubClient gitHubClient, ITransaction tx, IReliableConcurrentQueue <ScrapingTask> firstTaskQueue, IReliableConcurrentQueue <ScrapingTask> secondTaskQueue, IUserRepoSearchActorProvider userRepoSearchActorProvider) { IUserRepoSearchActor userRepoSearchActor = userRepoSearchActorProvider.Provide(scrapingTask.UserLogin); await Scraper.PerformTaskAsync( scrapingTask : scrapingTask, gitHubClient : gitHubClient, tx : tx, userRepoSearchActor : userRepoSearchActor, firstTaskQueue : firstTaskQueue, secondTaskQueue : secondTaskQueue); }
public async Task StartAssociatedReleasePipelinesAsync(int buildId, int channelId) { IReliableConcurrentQueue <ReleasePipelineRunnerItem> queue = await StateManager.GetOrAddAsync <IReliableConcurrentQueue <ReleasePipelineRunnerItem> >("queue"); using (ITransaction tx = StateManager.CreateTransaction()) { await queue.EnqueueAsync( tx, new ReleasePipelineRunnerItem { BuildId = buildId, ChannelId = channelId }); await tx.CommitAsync(); } }
private async void EnqueueBuildStatusCheck(ReleasePipelineRunnerItem item, int newNumberOfRetriesMade) { await Task.Delay(TimeSpan.FromMinutes(ReleasePipelineRunner.DelayBetweenBuildStatusChecksInMinutes)); IReliableConcurrentQueue <ReleasePipelineRunnerItem> queue = await StateManager.GetOrAddAsync <IReliableConcurrentQueue <ReleasePipelineRunnerItem> >("queue"); using (ITransaction tx = StateManager.CreateTransaction()) { await queue.EnqueueAsync(tx, new ReleasePipelineRunnerItem { BuildId = item.BuildId, ChannelId = item.ChannelId, NumberOfRetriesMade = newNumberOfRetriesMade }); await tx.CommitAsync(); } }
public async Task RunAsync(CancellationToken cancellationToken) { IReliableConcurrentQueue <DependencyUpdateItem> queue = await StateManager.GetOrAddAsync <IReliableConcurrentQueue <DependencyUpdateItem> >("queue"); while (!cancellationToken.IsCancellationRequested) { try { using (ITransaction tx = StateManager.CreateTransaction()) { ConditionalValue <DependencyUpdateItem> maybeItem = await queue.TryDequeueAsync( tx, cancellationToken); if (maybeItem.HasValue) { DependencyUpdateItem item = maybeItem.Value; using (Logger.BeginScope( "Processing dependency update for build {buildId} in channel {channelId}", item.BuildId, item.ChannelId)) { await UpdateDependenciesAsync(item.BuildId, item.ChannelId); } } await tx.CommitAsync(); } await Task.Delay(1000, cancellationToken); } catch (TaskCanceledException tcex) when(tcex.CancellationToken == cancellationToken) { // ignore } catch (Exception ex) { Logger.LogError(ex, "Processing queue messages"); } } }
public async static Task RunAsync( CancellationToken cancellationToken, IGitHubClient gitHubClient, IReliableStateManager stateManager, IUserRepoSearchActorProvider userRepoSearchActorProvider) { IReliableConcurrentQueue <ScrapingTask> firstTaskQueue = await GetFirstTaskQueue(stateManager); IReliableConcurrentQueue <ScrapingTask> secondTaskQueue = await GetSecondTaskQueue(stateManager); while (true) { cancellationToken.ThrowIfCancellationRequested(); if (firstTaskQueue.Count > 0 || secondTaskQueue.Count > 0) { using (var tx = stateManager.CreateTransaction()) { var dequeued = await firstTaskQueue.TryDequeueAsync(tx, cancellationToken); if (!dequeued.HasValue) { dequeued = await secondTaskQueue.TryDequeueAsync(tx, cancellationToken); } await ProcessScrapingTask( scrapingTask : dequeued.Value, gitHubClient : gitHubClient, tx : tx, firstTaskQueue : firstTaskQueue, secondTaskQueue : secondTaskQueue, userRepoSearchActorProvider : userRepoSearchActorProvider); await tx.CommitAsync(); } } else { await Task.Delay(Constants.EMPTY_DELAY, cancellationToken); } } }
private async static Task ScrapeUserInfoAsync( ScrapingTask scrapingTask, IGitHubClient gitHubClient, ITransaction tx, IUserRepoSearchActor userRepoSearchActor, IReliableConcurrentQueue <ScrapingTask> secondTaskQueue) { Result <UserInfo> result = await gitHubClient .GetUserInfoAsync(scrapingTask.AuthToken); if (result.Success) { await userRepoSearchActor.SetUserInfoAsync(result.Data); await secondTaskQueue.EnqueueAsync(tx, new ScrapingTask { Type = ScrapingTaskType.FollowingRepositories, AuthToken = scrapingTask.AuthToken, UserLogin = scrapingTask.UserLogin }); } }
protected override async Task RunAsync(CancellationToken cancellationToken) { if (_queue == null) { // we can make an interesting assumption here _queue = await this.StateManager.GetOrAddAsync <IReliableConcurrentQueue <DataPoint> >(QueueName); } // not sure this works? while (!cancellationToken.IsCancellationRequested) { var buffer = new List <DataPoint>(); bool state = true; using (var tx = this.StateManager.CreateTransaction()) { // process the queue ConditionalValue <DataPoint> point; while ((point = await _queue.TryDequeueAsync(tx, cancellationToken)).HasValue) { buffer.Add(point.Value); if (buffer.Count >= 100) { state = state && Flush(buffer, cancellationToken); } } // if all the flushes succeed if (state && Flush(buffer, cancellationToken)) { await tx.CommitAsync(); } else { tx.Abort(); } } Thread.Sleep(TimeSpan.FromSeconds(10)); } }
protected override async Task RunAsync(CancellationToken cancellationToken) { ServiceEventSource.Current.ServiceMessage(Context, "Started"); _cancellationToken = cancellationToken; _queue = await StateManager.GetOrAddAsync <IReliableConcurrentQueue <MessageData> >(ServiceConstants.APPLICATION_LOG_QUEUE); while (true) { //Waiting for signal resolution await _signal.WaitAsync(cancellationToken) .ConfigureAwait(false); //Waiting for better condition to messages processig await WaitAsync(); cancellationToken.ThrowIfCancellationRequested(); await ProcessAsync(); } }
public async Task <TimeSpan> RunAsync(CancellationToken cancellationToken) { IReliableConcurrentQueue <ReleasePipelineRunnerItem> queue = await StateManager.GetOrAddAsync <IReliableConcurrentQueue <ReleasePipelineRunnerItem> >("queue"); try { using (ITransaction tx = StateManager.CreateTransaction()) { ConditionalValue <ReleasePipelineRunnerItem> maybeItem = await queue.TryDequeueAsync( tx, cancellationToken); if (maybeItem.HasValue) { ReleasePipelineRunnerItem item = maybeItem.Value; using (Logger.BeginScope( $"Triggering release pipelines associated with channel {item.ChannelId} for build {item.BuildId}.", item.BuildId, item.ChannelId)) { await RunAssociatedReleasePipelinesAsync(item.BuildId, item.ChannelId, cancellationToken); } } await tx.CommitAsync(); } } catch (TaskCanceledException tcex) when(tcex.CancellationToken == cancellationToken) { return(TimeSpan.MaxValue); } catch (Exception ex) { Logger.LogError(ex, "Processing queue messages"); } return(TimeSpan.FromSeconds(1)); }
public async Task <TimeSpan> RunAsync(CancellationToken cancellationToken) { IReliableConcurrentQueue <DependencyUpdateItem> queue = await StateManager.GetOrAddAsync <IReliableConcurrentQueue <DependencyUpdateItem> >("queue"); try { using (ITransaction tx = StateManager.CreateTransaction()) { ConditionalValue <DependencyUpdateItem> maybeItem = await queue.TryDequeueAsync( tx, cancellationToken); if (maybeItem.HasValue) { DependencyUpdateItem item = maybeItem.Value; using (Logger.BeginScope( "Processing dependency update for build {buildId} in channel {channelId}", item.BuildId, item.ChannelId)) { await UpdateDependenciesAsync(item.BuildId, item.ChannelId); } } await tx.CommitAsync(); } } catch (TaskCanceledException tcex) when(tcex.CancellationToken == cancellationToken) { return(TimeSpan.MaxValue); } catch (Exception ex) { Logger.LogError(ex, "Processing queue messages"); } return(TimeSpan.FromSeconds(1)); }
/// <summary> /// Adds a trade to the trade log /// </summary> /// <param name="trade"></param> /// <returns></returns> public async Task LogAsync(Trade trade, CancellationToken cancellationToken = default(CancellationToken)) { IReliableConcurrentQueue <Trade> trades = await this.StateManager.GetOrAddAsync <IReliableConcurrentQueue <Trade> >(QueueName); var executed = false; var retryCount = 0; List <Exception> exceptions = new List <Exception>(); while (!executed && retryCount < 3) { cancellationToken.ThrowIfCancellationRequested(); try { await ExecuteAddTradeAsync(trade, trades, cancellationToken); executed = true; } catch (TimeoutException ex) { exceptions.Add(ex); retryCount++; continue; } catch (TransactionFaultedException ex) { exceptions.Add(ex); retryCount++; continue; } } if (exceptions.Count > 0) { throw new AggregateException( "Encounted errors while trying to add trade", exceptions); } }
private async static Task ScrapeFollowingRepositoriesAsync( ScrapingTask scrapingTask, IGitHubClient gitHubClient, ITransaction tx, IUserRepoSearchActor userRepoSearchActor, IReliableConcurrentQueue <ScrapingTask> secondTaskQueue) { Result <UserInfo> userInfoResult = await gitHubClient .GetUserInfoAsync(scrapingTask.AuthToken); if (!userInfoResult.Success) { return; } Result <IEnumerable <Repository> > repositoriesResult = await gitHubClient .ScrapeFollowingRepositoriesAsync(scrapingTask.AuthToken); if (!repositoriesResult.Success) { return; } List <ScheduledRepository> scheduledRepositories = repositoriesResult.Data.Select( repository => new ScheduledRepository { Name = repository.Name, Owner = repository.Owner }).ToList(); await secondTaskQueue.EnqueueAsync(tx, new ScrapingTask { Type = ScrapingTaskType.Repository, AuthToken = scrapingTask.AuthToken, UserLogin = scrapingTask.UserLogin, ScheduledRepositories = scheduledRepositories }); }