public async Task <long> CountAsync()
{
IReliableConcurrentQueue <Trade> transactions =
await this.stateManager.GetOrAddAsync <IReliableConcurrentQueue <Trade> >(queueName);
return(transactions.Count);
}
public async Task <long> ActiveTradeCountAsync()
{
IReliableConcurrentQueue <Trade> trades =
await this.StateManager.GetOrAddAsync <IReliableConcurrentQueue <Trade> >(QueueName);
return(trades.Count);
}
private async static Task ScrapeRepositoryAsync(
ScrapingTask scrapingTask, IGitHubClient gitHubClient,
ITransaction tx, IUserRepoSearchActor userRepoSearchActor,
IReliableConcurrentQueue <ScrapingTask> secondTaskQueue)
{
ScheduledRepository scheduledRepository =
scrapingTask.ScheduledRepositories[0];
Result <Repository> result = await gitHubClient
.ScrapeRepositoryAsync(
authToken : scrapingTask.AuthToken,
owner : scheduledRepository.Owner,
name : scheduledRepository.Name);
if (scrapingTask.ScheduledRepositories.Count > 1)
{
ScrapingTask nextScrapingTask = Serializer.DeepCopy(scrapingTask);
nextScrapingTask.ScheduledRepositories = nextScrapingTask
.ScheduledRepositories.Skip(1).ToList();
await secondTaskQueue.EnqueueAsync(tx, nextScrapingTask);
}
if (result.Success)
{
result.Data.IsNew = false;
await userRepoSearchActor.SetRepositoryAsync(result.Data);
}
}
public async static Task PerformTaskAsync(
ScrapingTask scrapingTask,
IGitHubClient gitHubClient,
ITransaction tx,
IUserRepoSearchActor userRepoSearchActor,
IReliableConcurrentQueue <ScrapingTask> firstTaskQueue,
IReliableConcurrentQueue <ScrapingTask> secondTaskQueue)
{
switch (scrapingTask.Type)
{
case ScrapingTaskType.Repository:
await ScrapeRepositoryAsync(
scrapingTask : scrapingTask, gitHubClient : gitHubClient,
tx : tx, userRepoSearchActor : userRepoSearchActor,
secondTaskQueue : secondTaskQueue);
break;
case ScrapingTaskType.UserInfo:
await ScrapeUserInfoAsync(
scrapingTask : scrapingTask, gitHubClient : gitHubClient,
tx : tx, userRepoSearchActor : userRepoSearchActor,
secondTaskQueue : secondTaskQueue);
break;
case ScrapingTaskType.FollowingRepositories:
await ScrapeFollowingRepositoriesAsync(
scrapingTask : scrapingTask, gitHubClient : gitHubClient,
tx : tx, userRepoSearchActor : userRepoSearchActor,
secondTaskQueue : secondTaskQueue);
break;
}
}
protected override async Task <ConditionalValue <byte[]> > TryDequeueAsync(ServiceFabricTransaction tx, IReliableState collectionBase, CancellationToken cancellationToken)
{
IReliableConcurrentQueue <byte[]> collection = (IReliableConcurrentQueue <byte[]>)collectionBase;
ConditionalValue <byte[]> message = await collection.TryDequeueAsync(tx.Tx, cancellationToken, TimeSpan.FromSeconds(4));
return(message);
}
private async Task ExecuteAddTradeAsync(Trade trade, IReliableConcurrentQueue <Trade> exportQueue, CancellationToken cancellationToken)
{
using (var tx = this.StateManager.CreateTransaction())
{
await exportQueue.EnqueueAsync(tx, trade, cancellationToken);
await tx.CommitAsync();
}
}
/// <summary>
/// See interface
/// </summary>
public async Task <int> GetMessageCountAsync()
{
using (var tx = new ServiceFabricTransaction(_stateManager, null))
{
IReliableConcurrentQueue <byte[]> collection = await GetCollectionAsync();
return((int)collection.Count);
}
}
private async Task <string> ExecuteEnqueueAsync(Trade trade, IReliableConcurrentQueue <Trade> trades, CancellationToken cancellationToken)
{
using (var tx = this.stateManager.CreateTransaction())
{
await trades.EnqueueAsync(tx, trade, cancellationToken);
await tx.CommitAsync();
}
return(trade.Id);
}
public BatchedDataCache(IReliableStateManagerTransactionProvider transactionProvider, IReliableStateManagerProvider reliableStateManagerProvider, IPaymentLogger logger)
{
this.transactionProvider = transactionProvider ?? throw new ArgumentNullException(nameof(transactionProvider));
this.logger = logger;
if (reliableStateManagerProvider == null)
{
throw new ArgumentNullException(nameof(reliableStateManagerProvider));
}
queue = reliableStateManagerProvider.Current.GetOrAddAsync <IReliableConcurrentQueue <T> >("BatchedDataCacheQueue").Result;
}
private static async Task <Trade> ExecuteDequeueAsync(ITransaction tx, IReliableConcurrentQueue <Trade> transactions, CancellationToken cancellationToken)
{
Trade trade = null;
var result = await transactions.TryDequeueAsync(tx, cancellationToken);
if (result.HasValue)
{
trade = result.Value;
}
return(trade);
}
public async Task StartUpdateDependenciesAsync(int buildId, int channelId)
{
IReliableConcurrentQueue <DependencyUpdateItem> queue =
await StateManager.GetOrAddAsync <IReliableConcurrentQueue <DependencyUpdateItem> >("queue");
using (ITransaction tx = StateManager.CreateTransaction())
{
await queue.EnqueueAsync(tx, new DependencyUpdateItem { BuildId = buildId, ChannelId = channelId });
await tx.CommitAsync();
}
}
public static void VerifyConcurrentQueueUnordered(IReliableConcurrentQueue <long> queue, IEnumerable <long> expectedValues, string message = "")
{
var expected = expectedValues.ToList();
expected.Sort((val1, val2) => val1.CompareTo(val2));
Assert.AreEqual(expected.Count, queue.Count, message);
var values = ((ReliableConcurrentQueue <long>)queue).GetVisibleListElementsUnsafe().ToList();
values.Sort((val1, val2) => val1.CompareTo(val2));
Assert.IsTrue(expected.SequenceEqual(values));
}
public async Task <IActionResult> Get()
{
string statusValue = "Nothing to show yet.";
long remainingValue = -1;
try
{
ConditionalValue <IReliableDictionary <string, ReportStatus> > tryGetDictionaryResult =
await this.stateManager.TryGetAsync <IReliableDictionary <string, ReportStatus> >(ReportProcessingService.StatusDictionaryName);
if (tryGetDictionaryResult.HasValue)
{
IReliableDictionary <string, ReportStatus> dictionary = tryGetDictionaryResult.Value;
using (ITransaction tx = this.stateManager.CreateTransaction())
{
ConditionalValue <ReportStatus> getResult = await dictionary.TryGetValueAsync(tx, this.reportContext.Name);
if (getResult.HasValue)
{
statusValue = getResult.Value.Status;
}
}
}
ConditionalValue <IReliableConcurrentQueue <ReportProcessingStep> > tryGetQueueResult =
await this.stateManager.TryGetAsync <IReliableConcurrentQueue <ReportProcessingStep> >(ReportProcessingService.ProcessingQueueName);
if (tryGetQueueResult.HasValue)
{
IReliableConcurrentQueue <ReportProcessingStep> queue = tryGetQueueResult.Value;
remainingValue = queue.Count;
}
return(this.Json(new { status = statusValue, remaining = remainingValue }));
}
catch (FabricNotPrimaryException)
{
return(new ContentResult {
StatusCode = 410, Content = "The primary replica has moved. Please re-resolve the service."
});
}
catch (Exception ex) when(ex is FabricTransientException || ex is TimeoutException)
{
return(new ContentResult {
StatusCode = 503, Content = "The service was unable to process the request. Please try again."
});
}
}
private async Task <IReliableConcurrentQueue <string> > GetQueueAsync()
{
IReliableConcurrentQueue <string> queue = null;
try
{
queue = await this.stateManager.GetOrAddAsync <IReliableConcurrentQueue <string> >(this.queueName);
}
catch (Exception)
{
}
return(queue);
}
/// <summary>
/// this method keeps on dequeuing the payload from the queue and processes it..
/// </summary>
/// <returns></returns>
private async Task ProcessJobAsync(IReliableConcurrentQueue <string> processQueue)
{
using (var tx = StateManager.CreateTransaction())
{
var result = await processQueue.TryDequeueAsync(tx);
if (result.HasValue)
{
// Do the work
}
await tx.CommitAsync();
}
}
public async Task PutMessagesAsync(IReadOnlyCollection <QueueMessage> messages, CancellationToken cancellationToken)
{
IReliableConcurrentQueue <byte[]> collection = await _stateManager.GetOrAddAsync <IReliableConcurrentQueue <byte[]> >(_queueName).ConfigureAwait(false);
using (var tx = new ServiceFabricTransaction(_stateManager, null))
{
foreach (QueueMessage message in messages)
{
byte[] data = message.ToByteArray();
await collection.EnqueueAsync(tx.Tx, data, cancellationToken, _timeout).ConfigureAwait(false);
}
await tx.CommitAsync().ConfigureAwait(false);
}
}
/// <summary>
/// this service has two runtime aspects - <see cref="RunAsync"/> and <see cref="IBullfrogNotificationBackendService"/>
/// as soon as listener is open, messaging can start flowing in and the same goes for <see cref="RunAsync"/> kicking in
///
/// the listeners are not open to secondary replicas and the same goes for <see cref="RunAsync"/> not being invoked on those so ultimately the queue can be shared
/// within the partition/primary replica instance (and we will likely run one partition anyway)
/// </summary>
private void InitInflightQueue()
{
if (_inflightQueue != null)
{
return;
}
lock (_inflightQueueInstanceLock)
{
if (_inflightQueue == null)
{
_inflightQueue =
StateManager.GetOrAddAsync <IReliableConcurrentQueue <InflightMessage> >(QueueName).GetAwaiter().GetResult();
}
}
}
private async Task ReceiveMessagesAsync(Func <IEnumerable <QueueMessage>, Task> onMessage, int maxBatchSize, CancellationToken cancellationToken)
{
var messages = new List <QueueMessage>();
while (!cancellationToken.IsCancellationRequested && !_disposed)
{
try
{
using (var tx = new ServiceFabricTransaction(_stateManager, null))
{
IReliableConcurrentQueue <byte[]> collection = await GetCollectionAsync();
while (messages.Count < maxBatchSize)
{
ConditionalValue <byte[]> message = await collection.TryDequeueAsync(tx.Tx, cancellationToken);
if (message.HasValue)
{
QueueMessage qm = QueueMessage.FromByteArray(message.Value);
messages.Add(qm);
}
else
{
break;
}
}
//make the call before committing the transaction
if (messages.Count > 0)
{
await onMessage(messages);
messages.Clear();
}
await tx.CommitAsync();
}
}
catch (Exception ex)
{
Trace.Fail($"failed to listen to messages on queue '{_queueName}'", ex.ToString());
}
await Task.Delay(_scanInterval);
}
}
private async static Task ProcessScrapingTask(
ScrapingTask scrapingTask,
IGitHubClient gitHubClient,
ITransaction tx,
IReliableConcurrentQueue <ScrapingTask> firstTaskQueue,
IReliableConcurrentQueue <ScrapingTask> secondTaskQueue,
IUserRepoSearchActorProvider userRepoSearchActorProvider)
{
IUserRepoSearchActor userRepoSearchActor =
userRepoSearchActorProvider.Provide(scrapingTask.UserLogin);
await Scraper.PerformTaskAsync(
scrapingTask : scrapingTask, gitHubClient : gitHubClient,
tx : tx, userRepoSearchActor : userRepoSearchActor,
firstTaskQueue : firstTaskQueue,
secondTaskQueue : secondTaskQueue);
}
public async Task StartAssociatedReleasePipelinesAsync(int buildId, int channelId)
{
IReliableConcurrentQueue <ReleasePipelineRunnerItem> queue =
await StateManager.GetOrAddAsync <IReliableConcurrentQueue <ReleasePipelineRunnerItem> >("queue");
using (ITransaction tx = StateManager.CreateTransaction())
{
await queue.EnqueueAsync(
tx,
new ReleasePipelineRunnerItem
{
BuildId = buildId,
ChannelId = channelId
});
await tx.CommitAsync();
}
}
private async void EnqueueBuildStatusCheck(ReleasePipelineRunnerItem item, int newNumberOfRetriesMade)
{
await Task.Delay(TimeSpan.FromMinutes(ReleasePipelineRunner.DelayBetweenBuildStatusChecksInMinutes));
IReliableConcurrentQueue <ReleasePipelineRunnerItem> queue =
await StateManager.GetOrAddAsync <IReliableConcurrentQueue <ReleasePipelineRunnerItem> >("queue");
using (ITransaction tx = StateManager.CreateTransaction())
{
await queue.EnqueueAsync(tx, new ReleasePipelineRunnerItem
{
BuildId = item.BuildId,
ChannelId = item.ChannelId,
NumberOfRetriesMade = newNumberOfRetriesMade
});
await tx.CommitAsync();
}
}
public async Task RunAsync(CancellationToken cancellationToken)
{
IReliableConcurrentQueue <DependencyUpdateItem> queue =
await StateManager.GetOrAddAsync <IReliableConcurrentQueue <DependencyUpdateItem> >("queue");
while (!cancellationToken.IsCancellationRequested)
{
try
{
using (ITransaction tx = StateManager.CreateTransaction())
{
ConditionalValue <DependencyUpdateItem> maybeItem = await queue.TryDequeueAsync(
tx,
cancellationToken);
if (maybeItem.HasValue)
{
DependencyUpdateItem item = maybeItem.Value;
using (Logger.BeginScope(
"Processing dependency update for build {buildId} in channel {channelId}",
item.BuildId,
item.ChannelId))
{
await UpdateDependenciesAsync(item.BuildId, item.ChannelId);
}
}
await tx.CommitAsync();
}
await Task.Delay(1000, cancellationToken);
}
catch (TaskCanceledException tcex) when(tcex.CancellationToken == cancellationToken)
{
// ignore
}
catch (Exception ex)
{
Logger.LogError(ex, "Processing queue messages");
}
}
}
public async static Task RunAsync(
CancellationToken cancellationToken,
IGitHubClient gitHubClient,
IReliableStateManager stateManager,
IUserRepoSearchActorProvider userRepoSearchActorProvider)
{
IReliableConcurrentQueue <ScrapingTask> firstTaskQueue =
await GetFirstTaskQueue(stateManager);
IReliableConcurrentQueue <ScrapingTask> secondTaskQueue =
await GetSecondTaskQueue(stateManager);
while (true)
{
cancellationToken.ThrowIfCancellationRequested();
if (firstTaskQueue.Count > 0 || secondTaskQueue.Count > 0)
{
using (var tx = stateManager.CreateTransaction())
{
var dequeued = await firstTaskQueue.TryDequeueAsync(tx, cancellationToken);
if (!dequeued.HasValue)
{
dequeued = await secondTaskQueue.TryDequeueAsync(tx, cancellationToken);
}
await ProcessScrapingTask(
scrapingTask : dequeued.Value,
gitHubClient : gitHubClient, tx : tx,
firstTaskQueue : firstTaskQueue,
secondTaskQueue : secondTaskQueue,
userRepoSearchActorProvider : userRepoSearchActorProvider);
await tx.CommitAsync();
}
}
else
{
await Task.Delay(Constants.EMPTY_DELAY, cancellationToken);
}
}
}
private async static Task ScrapeUserInfoAsync(
ScrapingTask scrapingTask, IGitHubClient gitHubClient,
ITransaction tx, IUserRepoSearchActor userRepoSearchActor,
IReliableConcurrentQueue <ScrapingTask> secondTaskQueue)
{
Result <UserInfo> result = await gitHubClient
.GetUserInfoAsync(scrapingTask.AuthToken);
if (result.Success)
{
await userRepoSearchActor.SetUserInfoAsync(result.Data);
await secondTaskQueue.EnqueueAsync(tx, new ScrapingTask
{
Type = ScrapingTaskType.FollowingRepositories,
AuthToken = scrapingTask.AuthToken,
UserLogin = scrapingTask.UserLogin
});
}
}
protected override async Task RunAsync(CancellationToken cancellationToken)
{
if (_queue == null)
{
// we can make an interesting assumption here
_queue = await this.StateManager.GetOrAddAsync <IReliableConcurrentQueue <DataPoint> >(QueueName);
}
// not sure this works?
while (!cancellationToken.IsCancellationRequested)
{
var buffer = new List <DataPoint>();
bool state = true;
using (var tx = this.StateManager.CreateTransaction())
{
// process the queue
ConditionalValue <DataPoint> point;
while ((point = await _queue.TryDequeueAsync(tx, cancellationToken)).HasValue)
{
buffer.Add(point.Value);
if (buffer.Count >= 100)
{
state = state && Flush(buffer, cancellationToken);
}
}
// if all the flushes succeed
if (state && Flush(buffer, cancellationToken))
{
await tx.CommitAsync();
}
else
{
tx.Abort();
}
}
Thread.Sleep(TimeSpan.FromSeconds(10));
}
}
protected override async Task RunAsync(CancellationToken cancellationToken)
{
ServiceEventSource.Current.ServiceMessage(Context, "Started");
_cancellationToken = cancellationToken;
_queue = await StateManager.GetOrAddAsync <IReliableConcurrentQueue <MessageData> >(ServiceConstants.APPLICATION_LOG_QUEUE);
while (true)
{
//Waiting for signal resolution
await _signal.WaitAsync(cancellationToken)
.ConfigureAwait(false);
//Waiting for better condition to messages processig
await WaitAsync();
cancellationToken.ThrowIfCancellationRequested();
await ProcessAsync();
}
}
public async Task <TimeSpan> RunAsync(CancellationToken cancellationToken)
{
IReliableConcurrentQueue <ReleasePipelineRunnerItem> queue =
await StateManager.GetOrAddAsync <IReliableConcurrentQueue <ReleasePipelineRunnerItem> >("queue");
try
{
using (ITransaction tx = StateManager.CreateTransaction())
{
ConditionalValue <ReleasePipelineRunnerItem> maybeItem = await queue.TryDequeueAsync(
tx,
cancellationToken);
if (maybeItem.HasValue)
{
ReleasePipelineRunnerItem item = maybeItem.Value;
using (Logger.BeginScope(
$"Triggering release pipelines associated with channel {item.ChannelId} for build {item.BuildId}.",
item.BuildId,
item.ChannelId))
{
await RunAssociatedReleasePipelinesAsync(item.BuildId, item.ChannelId, cancellationToken);
}
}
await tx.CommitAsync();
}
}
catch (TaskCanceledException tcex) when(tcex.CancellationToken == cancellationToken)
{
return(TimeSpan.MaxValue);
}
catch (Exception ex)
{
Logger.LogError(ex, "Processing queue messages");
}
return(TimeSpan.FromSeconds(1));
}
public async Task <TimeSpan> RunAsync(CancellationToken cancellationToken)
{
IReliableConcurrentQueue <DependencyUpdateItem> queue =
await StateManager.GetOrAddAsync <IReliableConcurrentQueue <DependencyUpdateItem> >("queue");
try
{
using (ITransaction tx = StateManager.CreateTransaction())
{
ConditionalValue <DependencyUpdateItem> maybeItem = await queue.TryDequeueAsync(
tx,
cancellationToken);
if (maybeItem.HasValue)
{
DependencyUpdateItem item = maybeItem.Value;
using (Logger.BeginScope(
"Processing dependency update for build {buildId} in channel {channelId}",
item.BuildId,
item.ChannelId))
{
await UpdateDependenciesAsync(item.BuildId, item.ChannelId);
}
}
await tx.CommitAsync();
}
}
catch (TaskCanceledException tcex) when(tcex.CancellationToken == cancellationToken)
{
return(TimeSpan.MaxValue);
}
catch (Exception ex)
{
Logger.LogError(ex, "Processing queue messages");
}
return(TimeSpan.FromSeconds(1));
}
/// <summary>
/// Adds a trade to the trade log
/// </summary>
/// <param name="trade"></param>
/// <returns></returns>
public async Task LogAsync(Trade trade, CancellationToken cancellationToken = default(CancellationToken))
{
IReliableConcurrentQueue <Trade> trades =
await this.StateManager.GetOrAddAsync <IReliableConcurrentQueue <Trade> >(QueueName);
var executed = false;
var retryCount = 0;
List <Exception> exceptions = new List <Exception>();
while (!executed && retryCount < 3)
{
cancellationToken.ThrowIfCancellationRequested();
try
{
await ExecuteAddTradeAsync(trade, trades, cancellationToken);
executed = true;
}
catch (TimeoutException ex)
{
exceptions.Add(ex);
retryCount++;
continue;
}
catch (TransactionFaultedException ex)
{
exceptions.Add(ex);
retryCount++;
continue;
}
}
if (exceptions.Count > 0)
{
throw new AggregateException(
"Encounted errors while trying to add trade",
exceptions);
}
}
private async static Task ScrapeFollowingRepositoriesAsync(
ScrapingTask scrapingTask,
IGitHubClient gitHubClient,
ITransaction tx,
IUserRepoSearchActor userRepoSearchActor,
IReliableConcurrentQueue <ScrapingTask> secondTaskQueue)
{
Result <UserInfo> userInfoResult = await gitHubClient
.GetUserInfoAsync(scrapingTask.AuthToken);
if (!userInfoResult.Success)
{
return;
}
Result <IEnumerable <Repository> > repositoriesResult = await gitHubClient
.ScrapeFollowingRepositoriesAsync(scrapingTask.AuthToken);
if (!repositoriesResult.Success)
{
return;
}
List <ScheduledRepository> scheduledRepositories = repositoriesResult.Data.Select(
repository => new ScheduledRepository
{
Name = repository.Name,
Owner = repository.Owner
}).ToList();
await secondTaskQueue.EnqueueAsync(tx, new ScrapingTask
{
Type = ScrapingTaskType.Repository,
AuthToken = scrapingTask.AuthToken,
UserLogin = scrapingTask.UserLogin,
ScheduledRepositories = scheduledRepositories
});
}
