public void CanHaveMultipleWorkers()
{
const int workItemCount = 50;
var latch = new CountDownLatch(workItemCount);
int errorCount = 0;
int abandonCount = 0;
var queue = new InMemoryQueue <SimpleWorkItem>(retries: 1, workItemTimeoutMilliseconds: 50, retryDelayMilliseconds: 0);
Parallel.For(0, workItemCount, i => queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello",
Id = i
}));
Task.Factory.StartNew(() => queue.StartWorking(w => DoWork(w, latch, ref abandonCount, ref errorCount)));
Task.Factory.StartNew(() => queue.StartWorking(w => DoWork(w, latch, ref abandonCount, ref errorCount)));
Task.Factory.StartNew(() => queue.StartWorking(w => DoWork(w, latch, ref abandonCount, ref errorCount)));
bool success = latch.Wait(2000);
Assert.True(success, "Failed to receive all work items.");
Task.Delay(50).Wait();
Assert.Equal(workItemCount, queue.Completed + queue.DeadletterCount);
Assert.Equal(errorCount, queue.WorkerErrors);
Assert.Equal(abandonCount + errorCount, queue.Abandoned);
}
public static void Main(string[] args)
{
InMemoryQueue queue = new InMemoryQueue(new[]
{
new ProductInformation(123),
new ProductInformation(9)
});
Container container = new Container();
container.RegisterInstance(typeof(IQueue <ProductInformation>), queue);
container.Register(typeof(ICallback <ProductInformation>), typeof(Callback));
container.Register <ProposalGenerator>();
container.Register <IStoreProposals, ConsoleProposalWriter>();
container.Register <IFetchSuppliers, EmptySupplierSource>();
container.Collection.Append <IShouldRestockRule, SimpleStockQuantityRule>();
container.Collection.Append <IAutomationRule, AlwaysAutomatableRule>();
container.Verify();
ProposalGenerator generator = container
.GetInstance <ProposalGenerator>();
generator.Go();
Console.WriteLine("Done. Hit [enter] to end...");
Console.ReadLine();
}
public async Task TestAsyncEvents()
{
var q = new InMemoryQueue <SimpleWorkItem>(loggerFactory: Log);
q.Enqueuing.AddHandler(async(sender, args) => {
await Task.Delay(250);
_logger.Info("First Enqueuing.");
});
q.Enqueuing.AddHandler(async(sender, args) => {
await Task.Delay(250);
_logger.Info("Second Enqueuing.");
});
var e1 = q.Enqueued.AddHandler(async(sender, args) => {
await Task.Delay(250);
_logger.Info("First.");
});
q.Enqueued.AddHandler(async(sender, args) => {
await Task.Delay(250);
_logger.Info("Second.");
});
var sw = Stopwatch.StartNew();
await q.EnqueueAsync(new SimpleWorkItem());
sw.Stop();
_logger.Trace(sw.Elapsed.ToString());
e1.Dispose();
sw.Restart();
await q.EnqueueAsync(new SimpleWorkItem());
sw.Stop();
_logger.Trace(sw.Elapsed.ToString());
}
public async Task CanRunWorkItemWithClassHandler()
{
ServiceProvider.SetServiceProvider(typeof(MyBootstrappedServiceProvider));
var queue = new InMemoryQueue <WorkItemData>();
var messageBus = new InMemoryMessageBus();
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
handlerRegistry.Register <MyWorkItem, MyWorkItemHandler>();
var jobId = await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test"
}, true);
int statusCount = 0;
messageBus.Subscribe <WorkItemStatus>(status => {
_logger.Trace("Progress: {progress}", status.Progress);
Assert.Equal(jobId, status.WorkItemId);
statusCount++;
});
await job.RunUntilEmptyAsync();
Assert.Equal(11, statusCount);
}
public void CanRunWorkItem()
{
var queue = new InMemoryQueue <WorkItemData>();
var messageBus = new InMemoryMessageBus();
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry);
handlerRegistry.Register <MyWorkItem>(async ctx => {
var jobData = ctx.GetData <MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
for (int i = 0; i < 10; i++)
{
Thread.Sleep(100);
ctx.ReportProgress(10 * i);
}
});
var jobId = queue.Enqueue(new MyWorkItem {
SomeData = "Test"
}, true);
int statusCount = 0;
messageBus.Subscribe <WorkItemStatus>(status => {
Assert.Equal(jobId, status.WorkItemId);
statusCount++;
});
job.RunUntilEmpty();
Assert.Equal(12, statusCount);
}
public async Task CanRunBadWorkItem()
{
using (var queue = new InMemoryQueue <WorkItemData>(o => o.RetryDelay(TimeSpan.FromMilliseconds(500)).LoggerFactory(Log))) {
using (var messageBus = new InMemoryMessageBus(o => o.LoggerFactory(Log))) {
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
handlerRegistry.Register <MyWorkItem>(ctx => {
var jobData = ctx.GetData <MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
throw new Exception();
});
string jobId = await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test"
}, true);
var countdown = new AsyncCountdownEvent(1);
await messageBus.SubscribeAsync <WorkItemStatus>(status => {
if (_logger.IsEnabled(LogLevel.Trace))
{
_logger.LogTrace("Progress: {Progress}", status.Progress);
}
Assert.Equal(jobId, status.WorkItemId);
countdown.Signal();
});
await job.RunUntilEmptyAsync();
await countdown.WaitAsync(TimeSpan.FromSeconds(2));
Assert.Equal(0, countdown.CurrentCount);
}
}
}
public async Task TestAsyncEvents() {
var q = new InMemoryQueue<SimpleWorkItem>();
q.Enqueuing.AddHandler(async (sender, args) => {
await Task.Delay(250);
Logger.Info().Message("First Enqueuing.").Write();
});
q.Enqueuing.AddHandler(async(sender, args) => {
await Task.Delay(250);
Logger.Info().Message("Second Enqueuing.").Write();
});
var e1 = q.Enqueued.AddHandler(async (sender, args) => {
await Task.Delay(250);
Logger.Info().Message("First.").Write();
});
q.Enqueued.AddHandler(async(sender, args) => {
await Task.Delay(250);
Logger.Info().Message("Second.").Write();
});
var sw = Stopwatch.StartNew();
await q.EnqueueAsync(new SimpleWorkItem());
sw.Stop();
_writer.WriteLine(sw.Elapsed.ToString());
e1.Dispose();
sw.Restart();
await q.EnqueueAsync(new SimpleWorkItem());
sw.Stop();
_writer.WriteLine(sw.Elapsed.ToString());
}
public void CanRunWorkItemWithClassHandler()
{
ServiceProvider.SetServiceProvider(typeof(MyBootstrappedServiceProvider));
var queue = new InMemoryQueue <WorkItemData>();
var messageBus = new InMemoryMessageBus();
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry);
handlerRegistry.Register <MyWorkItem, MyWorkItemHandler>();
var jobId = queue.Enqueue(new MyWorkItem {
SomeData = "Test"
}, true);
int statusCount = 0;
messageBus.Subscribe <WorkItemStatus>(status => {
Assert.Equal(jobId, status.WorkItemId);
statusCount++;
});
job.RunUntilEmpty();
Assert.Equal(12, statusCount);
}
public void CanRunWorkItem() {
var queue = new InMemoryQueue<WorkItemData>();
var messageBus = new InMemoryMessageBus();
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry);
handlerRegistry.Register<MyWorkItem>(async ctx => {
var jobData = ctx.GetData<MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
for (int i = 0; i < 10; i++) {
await Task.Delay(100);
ctx.ReportProgress(10 * i);
}
});
var jobId = queue.Enqueue(new MyWorkItem { SomeData = "Test" }, true);
int statusCount = 0;
messageBus.Subscribe<WorkItemStatus>(status => {
Assert.Equal(jobId, status.WorkItemId);
statusCount++;
});
job.RunUntilEmpty();
Assert.Equal(12, statusCount);
}
public async Task CanHandleMultipleWorkItemInstances()
{
var metrics = new InMemoryMetricsClient();
var queue = new InMemoryQueue<WorkItemData>(retryDelay: TimeSpan.Zero, retries: 0);
queue.AttachBehavior(new MetricsQueueBehavior<WorkItemData>(metrics));
var messageBus = new InMemoryMessageBus();
var handlerRegistry = new WorkItemHandlers();
var j1 = new WorkItemJob(queue, messageBus, handlerRegistry);
var j2 = new WorkItemJob(queue, messageBus, handlerRegistry);
var j3 = new WorkItemJob(queue, messageBus, handlerRegistry);
int errors = 0;
var jobIds = new ConcurrentDictionary<string, int>();
handlerRegistry.Register<MyWorkItem>(ctx => {
var jobData = ctx.GetData<MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
jobIds.AddOrUpdate(ctx.JobId, 1, (key, value) => value + 1);
for (int i = 0; i < 10; i++)
ctx.ReportProgress(10 * i);
if (RandomData.GetBool(1))
{
Interlocked.Increment(ref errors);
throw new ApplicationException("Boom!");
}
return TaskHelper.Completed();
});
for (int i = 0; i < 100; i++)
queue.Enqueue(new MyWorkItem { SomeData = "Test", Index = i }, true);
var completedItems = new List<string>();
object completedItemsLock = new object();
messageBus.Subscribe<WorkItemStatus>(status =>
{
if (status.Progress < 100)
return;
lock (completedItemsLock)
completedItems.Add(status.WorkItemId);
});
var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10));
var token = cancellationTokenSource.Token;
var tasks = new List<Task>();
tasks.AddRange(new[] {
Task.Run(async () => await j1.RunUntilEmptyAsync(token), token),
Task.Run(async () => await j2.RunUntilEmptyAsync(token), token),
Task.Run(async () => await j3.RunUntilEmptyAsync(token), token),
});
await Task.WhenAll(tasks);
Thread.Sleep(10);
Assert.Equal(100, completedItems.Count + errors);
Assert.Equal(3, jobIds.Count);
Assert.Equal(100, jobIds.Sum(kvp => kvp.Value));
}
public void SubscribeTest()
{
var logger = NullLoggerFactory.Instance.CreateLogger <InMemoryQueue>();
var queue = new InMemoryQueue(logger);
queue.Subscribe("groupid", x => new TransportMessage(null, null), "test-topic");
}
public void SendTest()
{
var logger = NullLoggerFactory.Instance.CreateLogger <InMemoryQueue>();
var queue = new InMemoryQueue(logger);
var topic = "test-topic";
var content = "test content";
var headers = new Dictionary <string, string>();
var messageId = SnowflakeId.Default().NextId().ToString();
headers.Add(Headers.MessageId, messageId);
headers.Add(Headers.MessageName, topic);
headers.Add(Headers.Type, typeof(string).FullName);
headers.Add(Headers.SentTime, DateTimeOffset.Now.ToString());
if (!headers.ContainsKey(Headers.CorrelationId))
{
headers.Add(Headers.CorrelationId, messageId);
headers.Add(Headers.CorrelationSequence, 0.ToString());
}
var transportMsg = new TransportMessage(headers, Encoding.UTF8.GetBytes(content));
ManualResetEventSlim reset = new ManualResetEventSlim(false);
queue.Subscribe("groupid", x =>
{
Assert.Equal(content, Encoding.UTF8.GetString(x.Body));
reset.Set();
}, topic);
queue.Send(transportMsg);
reset.Wait();
}
public async Task CanRunWorkItem() {
using (var queue = new InMemoryQueue<WorkItemData>()) {
using (var messageBus = new InMemoryMessageBus()) {
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
handlerRegistry.Register<MyWorkItem>(async ctx => {
var jobData = ctx.GetData<MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
for (int i = 0; i < 10; i++) {
await SystemClock.SleepAsync(100);
await ctx.ReportProgressAsync(10 * i);
}
});
var jobId = await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test"
}, true);
int statusCount = 0;
messageBus.Subscribe<WorkItemStatus>(status => {
_logger.Trace("Progress: {progress}", status.Progress);
Assert.Equal(jobId, status.WorkItemId);
statusCount++;
});
await job.RunUntilEmptyAsync();
Assert.Equal(12, statusCount);
}
}
}
public async Task CanRunBadWorkItem()
{
var queue = new InMemoryQueue <WorkItemData>(retries: 2, retryDelay: TimeSpan.FromMilliseconds(500));
var messageBus = new InMemoryMessageBus();
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry);
handlerRegistry.Register <MyWorkItem>(ctx => {
var jobData = ctx.GetData <MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
throw new ApplicationException();
});
var jobId = await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test"
}, true);
int statusCount = 0;
messageBus.Subscribe <WorkItemStatus>(status => {
Logger.Trace().Message($"Progress: {status.Progress}").Write();
Assert.Equal(jobId, status.WorkItemId);
statusCount++;
});
await job.RunUntilEmptyAsync();
Assert.Equal(1, statusCount);
}
public void WorkItemsWillGetMovedToDeadletter() {
var queue = new InMemoryQueue<SimpleWorkItem>(retries: 1, workItemTimeoutMilliseconds: 10);
queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
});
var workItem = queue.DequeueAsync(0).Result;
Assert.Equal("Hello", workItem.Value.Data);
Assert.Equal(1, queue.Dequeued);
Assert.Equal(0, queue.Count);
// wait for the task to be auto abandoned
Task.Delay(100).Wait();
Assert.Equal(1, queue.Count);
Assert.Equal(1, queue.Abandoned);
// work item should be retried 1 time.
workItem = queue.DequeueAsync(0).Result;
Assert.Equal("Hello", workItem.Value.Data);
Assert.Equal(2, queue.Dequeued);
Task.Delay(100).Wait();
// work item should be moved to deadletter queue after retries.
Assert.Equal(1, queue.DeadletterCount);
Assert.Equal(2, queue.Abandoned);
}
public async Task CanRunWorkItem()
{
using (var queue = new InMemoryQueue <WorkItemData>()) {
using (var messageBus = new InMemoryMessageBus()) {
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
handlerRegistry.Register <MyWorkItem>(async ctx => {
var jobData = ctx.GetData <MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
for (int i = 0; i < 10; i++)
{
await SystemClock.SleepAsync(100);
await ctx.ReportProgressAsync(10 * i);
}
});
var jobId = await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test"
}, true);
int statusCount = 0;
messageBus.Subscribe <WorkItemStatus>(status => {
_logger.Trace("Progress: {progress}", status.Progress);
Assert.Equal(jobId, status.WorkItemId);
statusCount++;
});
await job.RunUntilEmptyAsync();
Assert.Equal(12, statusCount);
}
}
}
public async Task CanRunBadWorkItem()
{
using (var queue = new InMemoryQueue <WorkItemData>(new InMemoryQueueOptions <WorkItemData> {
RetryDelay = TimeSpan.FromMilliseconds(500), LoggerFactory = Log
})) {
using (var messageBus = new InMemoryMessageBus(new InMemoryMessageBusOptions {
LoggerFactory = Log
})) {
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
handlerRegistry.Register <MyWorkItem>(ctx => {
var jobData = ctx.GetData <MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
throw new Exception();
});
string jobId = await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test"
}, true);
int statusCount = 0;
await messageBus.SubscribeAsync <WorkItemStatus>(status => {
_logger.Trace("Progress: {progress}", status.Progress);
Assert.Equal(jobId, status.WorkItemId);
statusCount++;
});
await job.RunUntilEmptyAsync();
Assert.Equal(1, statusCount);
}
}
}
public async Task CanRunWorkItemJobUntilEmpty()
{
using (var queue = new InMemoryQueue <WorkItemData>(new InMemoryQueueOptions <WorkItemData> {
LoggerFactory = Log
})) {
using (var messageBus = new InMemoryMessageBus(new InMemoryMessageBusOptions {
LoggerFactory = Log
})) {
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
handlerRegistry.Register <MyWorkItem>(new MyWorkItemHandler(Log));
await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test"
}, true);
await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test"
}, true);
await job.RunUntilEmptyAsync();
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(2, stats.Enqueued);
Assert.Equal(2, stats.Dequeued);
Assert.Equal(2, stats.Completed);
}
}
}
public async Task CanRunWorkItemWithClassHandler()
{
using (var queue = new InMemoryQueue <WorkItemData>(new InMemoryQueueOptions <WorkItemData> {
LoggerFactory = Log
})) {
using (var messageBus = new InMemoryMessageBus(new InMemoryMessageBusOptions {
LoggerFactory = Log
})) {
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
handlerRegistry.Register <MyWorkItem>(new MyWorkItemHandler(Log));
string jobId = await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test"
}, true);
int statusCount = 0;
await messageBus.SubscribeAsync <WorkItemStatus>(status => {
_logger.Trace("Progress: {progress}", status.Progress);
Assert.Equal(jobId, status.WorkItemId);
statusCount++;
});
await job.RunUntilEmptyAsync();
Assert.Equal(11, statusCount);
}
}
}
public void WorkItemsWillGetMovedToDeadletter()
{
var queue = new InMemoryQueue <SimpleWorkItem>(retries: 1, workItemTimeoutMilliseconds: 10, retryDelayMilliseconds: 10);
queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
}).Wait();
var workItem = queue.DequeueAsync(0).Result;
Assert.Equal("Hello", workItem.Value.Data);
Assert.Equal(1, queue.Dequeued);
Assert.Equal(0, queue.Count);
// wait for the task to be auto abandoned
Task.Delay(100).Wait();
Assert.Equal(1, queue.Count);
Assert.Equal(1, queue.Abandoned);
// work item should be retried 1 time.
workItem = queue.DequeueAsync(0).Result;
Assert.Equal("Hello", workItem.Value.Data);
Assert.Equal(2, queue.Dequeued);
Task.Delay(100).Wait();
// work item should be moved to deadletter queue after retries.
Assert.Equal(1, queue.DeadletterCount);
Assert.Equal(2, queue.Abandoned);
}
/// <summary>
/// Waits on the specified queue for messages
/// </summary>
/// <param name="queue"></param>
/// <param name="token"></param>
/// <param name="onMessageReceived"></param>
private void GetMessages(InMemoryQueue queue, CancellationToken token, ManualResetEvent resetEvent, Action <RawMessage, Action, Action> onMessageReceived)
{
while (true)
{
if (token.IsCancellationRequested)
{
token.ThrowIfCancellationRequested();
}
resetEvent.WaitOne(TimeSpan.FromSeconds(15));
RawMessage message = null;
while (queue.Dequeue(ref message))
{
onMessageReceived(
message,
() => { /* Ack No Op for InMemory */ },
() =>
{
// Reject puts message back in queue
queue.Nack(message);
});
}
resetEvent.Reset();
}
}
public virtual async Task CanGetBufferedQueueMetrics() {
using (var metrics = GetMetricsClient(true) as IBufferedMetricsClient) {
var stats = metrics as IMetricsClientStats;
if (stats == null)
return;
using (var queue = new InMemoryQueue<SimpleWorkItem>(behaviors: new[] { new MetricsQueueBehavior<SimpleWorkItem>(metrics, loggerFactory: Log) }, loggerFactory: Log)) {
await queue.EnqueueAsync(new SimpleWorkItem { Id = 1, Data = "1" });
await SystemClock.SleepAsync(50);
var entry = await queue.DequeueAsync(TimeSpan.Zero);
await SystemClock.SleepAsync(30);
await entry.CompleteAsync();
await SystemClock.SleepAsync(500); // give queue metrics time
await metrics.FlushAsync();
var queueStats = await stats.GetQueueStatsAsync("simpleworkitem");
Assert.Equal(1, queueStats.Count.Max);
Assert.Equal(0, queueStats.Count.Last);
Assert.Equal(1, queueStats.Enqueued.Count);
Assert.Equal(1, queueStats.Dequeued.Count);
Assert.Equal(1, queueStats.Completed.Count);
Assert.InRange(queueStats.QueueTime.AverageDuration, 50, 200);
Assert.InRange(queueStats.ProcessTime.AverageDuration, 30, 200);
}
}
}
public async Task InMemoryQueue_PushAndGet(int batchCount)
{
var secondary = new InMemoryQueue <List <string> >(10);
var q = new AsyncBatchQueue <string>(1000,
new long[] { batchCount },
new Func <string, long>[] { s => 1 }, secondary);
await q.PushSecondaryAsync(Enumerable.Repeat("a", batchCount + 1).ToList());
var output = new List <string>();
var getTask = q.GetNextBatchAsync(output, 10 * 1000).AsTask();
Assert.True(getTask.Wait(1000));
Assert.Equal(batchCount, output.Count);
if (batchCount == 0)
{
return;
}
// second call should return the last item
output.Clear();
await q.GetNextBatchAsync(output, 100);
Assert.Single(output);
}
public async Task InMemoryQueue_ConcurrentRead(int readerCount, int itemCount)
{
var secondary = new InMemoryQueue <List <int> >(1000);
using var cts = new CancellationTokenSource();
using var semaphore = new SemaphoreSlim(0, readerCount);
var results = new List <int>();
var q = new AsyncBatchQueue <int>(10000,
new long[] { 100 },
new Func <int, long>[] { s => 1 }, secondary);
async Task readerTask()
{
var output = new List <int>();
await semaphore.WaitAsync();
// we're trying to test that the readers will 'eventually' read all the items, so we do several pulls here
await q.GetNextBatchAsync(output, 500);
await q.GetNextBatchAsync(output, 500);
await q.GetNextBatchAsync(output, 500);
await Task.Delay(100);
lock (results)
{
results.AddRange(output);
}
};
var readers = new Task[readerCount];
for (var i = 0; i < readerCount; i++)
{
readers[i] = readerTask();
}
for (var i = 0; i < itemCount; i++)
{
if (i % 2 == 0)
{
await q.PushAsync(i);
}
else
{
await q.PushSecondaryAsync(new List <int> {
i
});
}
}
semaphore.Release(readerCount);
await Task.WhenAll(readers);
_output.WriteLine(results.Count.ToString());
_output.WriteLine(q.EstimateSize().ToString());
_output.WriteLine(q.EstimateSecondaryQueueSize().ToString());
Assert.Equal(itemCount, results.Distinct().Count());
}
public async Task InMemoryQueue_PullBothQueues()
{
var secondary = new InMemoryQueue <List <int> >(10);
var q = new AsyncBatchQueue <int>(500,
new long[] { 500 },
new Func <int, long>[] { s => 1 }, secondary);
await q.PushSecondaryAsync(Enumerable.Range(0, 500).ToList());
for (var i = 500; i < 1000; i++)
{
await q.PushAsync(i);
}
// pull 3 times
var output = new List <int>();
await q.GetNextBatchAsync(output, 1000);
await q.GetNextBatchAsync(output, 1000);
await q.GetNextBatchAsync(output, 1000);
Assert.Equal(1000, output.Distinct().Count());
}
public void AddingNullHeaderKeyAlsoReturnsNullHeaderKey()
{
var queue = new InMemoryQueue();
var index1 = queue.AddMessage(null, null, new object());
queue.GetMessageHeader(index1).Should().BeNull();
queue.GetMessageKey(index1).Should().BeNull();
}
public void WhenCallingRemoveAndIsNotEmptyShouldRemove()
{
var queue = new InMemoryQueue();
queue.Add(new Message(null, null));
var actual = queue.Remove();
Assert.NotNull(actual);
Assert.Null(actual.Url);
Assert.Null(actual.Body);
}
public void AddingMessagesIncreasesIndex()
{
var queue = new InMemoryQueue();
var index1 = queue.AddMessage(null, null, new object());
var index2 = queue.AddMessage(null, null, new object());
var index3 = queue.AddMessage(null, null, new object());
index1.Should().Be(0);
index2.Should().Be(1);
index3.Should().Be(2);
}
public DefaultTransport(IAssemblyLocator assemblyLocator)
{
this.primaryResetEvent = new ManualResetEvent(false);
this.delayResetEvent = new ManualResetEvent(false);
this.primaryQueueName = assemblyLocator.GetEntryAssembly().GetName().Name;
this.delayQueueName = string.Format("{0}_Delay", this.primaryQueueName);
this.errorQueueName = string.Format("{0}_Error", this.primaryQueueName);
this.primaryQueue = new InMemoryQueue();
this.delayQueue = new InMemoryQueue();
this.errorQueue = new InMemoryQueue();
this.publishedMessages = new List <Object>();
}
public DefaultTransport(IAssemblyLocator assemblyLocator)
{
this.primaryResetEvent = new ManualResetEvent(false);
this.delayResetEvent = new ManualResetEvent(false);
this.primaryQueueName = assemblyLocator.GetEntryAssembly().GetName().Name;
this.delayQueueName = string.Format("{0}_Delay", this.primaryQueueName);
this.errorQueueName = string.Format("{0}_Error", this.primaryQueueName);
this.primaryQueue = new InMemoryQueue();
this.delayQueue = new InMemoryQueue();
this.errorQueue = new InMemoryQueue();
this.publishedMessages = new List<Object>();
}
public void WorkItemsWillTimeout() {
var queue = new InMemoryQueue<SimpleWorkItem>(workItemTimeoutMilliseconds: 10);
queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
});
var workItem = queue.DequeueAsync(0).Result;
Assert.Equal("Hello", workItem.Value.Data);
Assert.Equal(0, queue.Count);
// wait for the task to be auto abandoned
Task.Delay(100).Wait();
Assert.Equal(1, queue.Count);
Assert.Equal(1, queue.Abandoned);
}
public async Task BotService_AGivenCommand_EnqueueMessage()
{
var stock = @"Symbol,Date,Time,Open,High,Low,Close,Volume
AAPL.US,2019-04-26,22:00:20,204.9,205,202.12,204.3,18649102";
var fakeStockService = new FakeStockService(stock);
var messageQueue = new InMemoryQueue <Message>();
var botService = new BotService(fakeStockService, messageQueue);
await botService.QueryAndSend(stockCode : "aapl.us");
var lastMessage = await messageQueue.DequeueAsync();
Assert.IsNotNull(lastMessage);
}
public async Task TestAsyncEvents()
{
using (var q = new InMemoryQueue <SimpleWorkItem>(new InMemoryQueueOptions <SimpleWorkItem> {
LoggerFactory = Log
})) {
var disposables = new List <IDisposable>(5);
try {
disposables.Add(q.Enqueuing.AddHandler(async(sender, args) => {
await SystemClock.SleepAsync(250);
_logger.LogInformation("First Enqueuing.");
}));
disposables.Add(q.Enqueuing.AddHandler(async(sender, args) => {
await SystemClock.SleepAsync(250);
_logger.LogInformation("Second Enqueuing.");
}));
disposables.Add(q.Enqueued.AddHandler(async(sender, args) => {
await SystemClock.SleepAsync(250);
_logger.LogInformation("First.");
}));
disposables.Add(q.Enqueued.AddHandler(async(sender, args) => {
await SystemClock.SleepAsync(250);
_logger.LogInformation("Second.");
}));
var sw = Stopwatch.StartNew();
await q.EnqueueAsync(new SimpleWorkItem());
sw.Stop();
if (_logger.IsEnabled(LogLevel.Trace))
{
_logger.LogTrace("Time {Elapsed:g}", sw.Elapsed);
}
sw.Restart();
await q.EnqueueAsync(new SimpleWorkItem());
sw.Stop();
if (_logger.IsEnabled(LogLevel.Trace))
{
_logger.LogTrace("Time {Elapsed:g}", sw.Elapsed);
}
} finally {
foreach (var disposable in disposables)
{
disposable.Dispose();
}
}
}
}
public void AddedMessagesCanBeRetrieved()
{
var queue = new InMemoryQueue();
var message1 = new object();
var message2 = new object();
var message3 = new object();
var index1 = queue.AddMessage(new object(), null, message1);
var index2 = queue.AddMessage(null, new object(), message2);
var index3 = queue.AddMessage(null, new object(), message3);
queue.GetMessage(index1).Should().Be(message1);
queue.GetMessage(index2).Should().Be(message2);
queue.GetMessage(index3).Should().Be(message3);
}
public void CanQueueAndDequeueWorkItem() {
var queue = new InMemoryQueue<SimpleWorkItem>();
queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
});
Assert.Equal(1, queue.Count);
var workItem = queue.DequeueAsync(0).Result;
Assert.NotNull(workItem);
Assert.Equal("Hello", workItem.Value.Data);
Assert.Equal(0, queue.Count);
Assert.Equal(1, queue.Dequeued);
workItem.CompleteAsync().Wait();
Assert.Equal(1, queue.Completed);
}
public void WorkItemsWillTimeout()
{
var queue = new InMemoryQueue <SimpleWorkItem>(workItemTimeoutMilliseconds: 10, retryDelayMilliseconds: 0);
queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
});
var workItem = queue.DequeueAsync(0).Result;
Assert.Equal("Hello", workItem.Value.Data);
Assert.Equal(0, queue.Count);
// wait for the task to be auto abandoned
Task.Delay(100).Wait();
Assert.Equal(1, queue.Count);
Assert.Equal(1, queue.Abandoned);
}
public void WhenCallingAddShouldAddAndRaiseEvent()
{
var messageUri = new Uri("http://www.google.com");
var messageBody = string.Empty;
var queue = new InMemoryQueue();
var enqueueCounter = 0;
queue.MessageAdded += (sender, e) => { enqueueCounter++; };
queue.Add(new Message(messageUri, messageBody));
Assert.NotNull(queue.Get());
Assert.Equal(messageUri, queue.Get().Url);
Assert.Equal(messageBody, queue.Get().Body);
Assert.Equal(1, enqueueCounter);
}
public void When_adding_items_to_the_queue_they_can_later_be_retrieved_from_the_queue()
{
var firstItem = "not set";
var secondItem = "not set";
var inMemoryQueue = new InMemoryQueue();
inMemoryQueue.Put("first item");
inMemoryQueue.Put("second item");
Assert.That(firstItem, Is.EqualTo("not set"));
Assert.That(secondItem, Is.EqualTo("not set"));
inMemoryQueue.Pop(x => firstItem = x.ToString());
inMemoryQueue.Pop(x => secondItem = x.ToString());
Assert.That(firstItem, Is.EqualTo("first item"));
Assert.That(secondItem, Is.EqualTo("second item"));
}
public void When_adding_items_to_the_queue_they_can_later_be_retrieved_from_the_queue()
{
var firstItem = "not set";
var secondItem = "not set";
var inMemoryQueue = new InMemoryQueue();
inMemoryQueue.Put("first item");
inMemoryQueue.Put("second item");
Assert.That(firstItem, Is.EqualTo("not set"));
Assert.That(secondItem, Is.EqualTo("not set"));
inMemoryQueue.Pop(x => firstItem = x.ToString());
inMemoryQueue.Pop(x => secondItem = x.ToString());
Assert.That(firstItem, Is.EqualTo("first item"));
Assert.That(secondItem, Is.EqualTo("second item"));
}
public void CanUseQueueWorker() {
var resetEvent = new AutoResetEvent(false);
var queue = new InMemoryQueue<SimpleWorkItem>();
queue.StartWorking(w => {
Assert.Equal("Hello", w.Value.Data);
w.CompleteAsync().Wait();
resetEvent.Set();
});
queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
});
Assert.Equal(1, queue.Count);
bool success = resetEvent.WaitOne(250);
Assert.Equal(0, queue.Count);
Assert.Equal(1, queue.Completed);
Assert.True(success, "Failed to receive message.");
Assert.Equal(0, queue.WorkerErrors);
}
public void CanQueueAndDequeueWorkItem()
{
var queue = new InMemoryQueue <SimpleWorkItem>();
queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
});
Assert.Equal(1, queue.Count);
var workItem = queue.DequeueAsync(0).Result;
Assert.NotNull(workItem);
Assert.Equal("Hello", workItem.Value.Data);
Assert.Equal(0, queue.Count);
Assert.Equal(1, queue.Dequeued);
workItem.CompleteAsync().Wait();
Assert.Equal(1, queue.Completed);
}
public async Task CanRunWorkItemWithDelegateHandler()
{
using (var queue = new InMemoryQueue <WorkItemData>(new InMemoryQueueOptions <WorkItemData> {
LoggerFactory = Log
})) {
using (var messageBus = new InMemoryMessageBus(new InMemoryMessageBusOptions {
LoggerFactory = Log
})) {
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
handlerRegistry.Register <MyWorkItem>(async ctx => {
var jobData = ctx.GetData <MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
for (int i = 1; i < 10; i++)
{
await SystemClock.SleepAsync(100);
await ctx.ReportProgressAsync(10 * i);
}
}, Log.CreateLogger("MyWorkItem"));
string jobId = await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test"
}, true);
int statusCount = 0;
await messageBus.SubscribeAsync <WorkItemStatus>(status => {
if (_logger.IsEnabled(LogLevel.Trace))
{
_logger.LogTrace("Progress: {Progress}", status.Progress);
}
Assert.Equal(jobId, status.WorkItemId);
Interlocked.Increment(ref statusCount);
});
await job.RunUntilEmptyAsync();
Assert.Equal(11, statusCount);
}
}
}
public void CanRunQueueJob()
{
const int workItemCount = 1000;
var metrics = new InMemoryMetricsClient();
var queue = new InMemoryQueue <SampleQueueWorkItem>(0, TimeSpan.Zero, metrics: metrics);
for (int i = 0; i < workItemCount; i++)
{
queue.Enqueue(new SampleQueueWorkItem {
Created = DateTime.Now, Path = "somepath" + i
});
}
var job = new SampleQueueJob(queue, metrics);
job.RunUntilEmpty();
metrics.DisplayStats();
Assert.Equal(0, queue.GetQueueCount());
}
public async Task TestAsyncEvents() {
using (var q = new InMemoryQueue<SimpleWorkItem>(loggerFactory: Log)) {
var disposables = new List<IDisposable>(5);
try {
disposables.Add(q.Enqueuing.AddHandler(async (sender, args) => {
await SystemClock.SleepAsync(250);
_logger.Info("First Enqueuing.");
}));
disposables.Add(q.Enqueuing.AddHandler(async (sender, args) => {
await SystemClock.SleepAsync(250);
_logger.Info("Second Enqueuing.");
}));
disposables.Add(q.Enqueued.AddHandler(async (sender, args) => {
await SystemClock.SleepAsync(250);
_logger.Info("First.");
}));
disposables.Add(q.Enqueued.AddHandler(async (sender, args) => {
await SystemClock.SleepAsync(250);
_logger.Info("Second.");
}));
var sw = Stopwatch.StartNew();
await q.EnqueueAsync(new SimpleWorkItem());
sw.Stop();
_logger.Trace("Time {0}", sw.Elapsed);
sw.Restart();
await q.EnqueueAsync(new SimpleWorkItem());
sw.Stop();
_logger.Trace("Time {0}", sw.Elapsed);
} finally {
foreach (var disposable in disposables)
disposable.Dispose();
}
}
}
public virtual async Task CanRunWorkItemWithMetrics() {
int completedCount = 0;
//Log.MinimumLevel = LogLevel.Trace;
//Log.SetLogLevel<ScheduledTimer>(LogLevel.Information);
//Log.SetLogLevel<InMemoryCacheClient>(LogLevel.Information);
using (var metricsClient = new InMemoryMetricsClient(false, loggerFactory: Log)) {
var behavior = new MetricsQueueBehavior<WorkItemData>(metricsClient, "metric", loggerFactory: Log, reportCountsInterval: TimeSpan.Zero);
using (var queue = new InMemoryQueue<WorkItemData>(behaviors: new[] { behavior }, loggerFactory: Log)) {
Func<object, CompletedEventArgs<WorkItemData>, Task> handler = (sender, e) => {
completedCount++;
return Task.CompletedTask;
};
using (queue.Completed.AddHandler(handler)) {
_logger.Trace("Before enqueue");
await queue.EnqueueAsync(new SimpleWorkItem { Id = 1, Data = "Testing" });
await queue.EnqueueAsync(new SimpleWorkItem { Id = 2, Data = "Testing" });
await queue.EnqueueAsync(new SimpleWorkItem { Id = 3, Data = "Testing" });
await SystemClock.SleepAsync(100);
_logger.Trace("Before dequeue");
var item = await queue.DequeueAsync();
await item.CompleteAsync();
item = await queue.DequeueAsync();
await item.CompleteAsync();
item = await queue.DequeueAsync();
await item.AbandonAsync();
_logger.Trace("Before asserts");
Assert.Equal(2, completedCount);
await SystemClock.SleepAsync(100);
Assert.InRange((await metricsClient.GetGaugeStatsAsync("metric.workitemdata.count")).Max, 1, 3);
Assert.InRange((await metricsClient.GetGaugeStatsAsync("metric.workitemdata.working")).Max, 0, 1);
Assert.Equal(3, await metricsClient.GetCounterCountAsync("metric.workitemdata.simple.enqueued"));
Assert.Equal(3, await metricsClient.GetCounterCountAsync("metric.workitemdata.enqueued"));
Assert.Equal(3, await metricsClient.GetCounterCountAsync("metric.workitemdata.simple.dequeued"));
Assert.Equal(3, await metricsClient.GetCounterCountAsync("metric.workitemdata.dequeued"));
Assert.Equal(2, await metricsClient.GetCounterCountAsync("metric.workitemdata.simple.completed"));
Assert.Equal(2, await metricsClient.GetCounterCountAsync("metric.workitemdata.completed"));
Assert.Equal(1, await metricsClient.GetCounterCountAsync("metric.workitemdata.simple.abandoned"));
Assert.Equal(1, await metricsClient.GetCounterCountAsync("metric.workitemdata.abandoned"));
var queueTiming = await metricsClient.GetTimerStatsAsync("metric.workitemdata.simple.queuetime");
Assert.Equal(3, queueTiming.Count);
queueTiming = await metricsClient.GetTimerStatsAsync("metric.workitemdata.queuetime");
Assert.Equal(3, queueTiming.Count);
var processTiming = await metricsClient.GetTimerStatsAsync("metric.workitemdata.simple.processtime");
Assert.Equal(3, processTiming.Count);
processTiming = await metricsClient.GetTimerStatsAsync("metric.workitemdata.processtime");
Assert.Equal(3, processTiming.Count);
var queueStats = await metricsClient.GetQueueStatsAsync("metric.workitemdata");
Assert.Equal(3, queueStats.Enqueued.Count);
Assert.Equal(3, queueStats.Dequeued.Count);
Assert.Equal(2, queueStats.Completed.Count);
Assert.Equal(1, queueStats.Abandoned.Count);
Assert.InRange(queueStats.Count.Max, 1, 3);
Assert.InRange(queueStats.Working.Max, 0, 1);
var subQueueStats = await metricsClient.GetQueueStatsAsync("metric.workitemdata", "simple");
Assert.Equal(3, subQueueStats.Enqueued.Count);
Assert.Equal(3, subQueueStats.Dequeued.Count);
Assert.Equal(2, subQueueStats.Completed.Count);
Assert.Equal(1, subQueueStats.Abandoned.Count);
}
}
}
}
public void CanRunBadWorkItem() {
var queue = new InMemoryQueue<WorkItemData>();
var messageBus = new InMemoryMessageBus();
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry);
handlerRegistry.Register<MyWorkItem>(ctx => {
var jobData = ctx.GetData<MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
throw new ApplicationException();
});
var jobId = queue.Enqueue(new MyWorkItem { SomeData = "Test" }, true);
int statusCount = 0;
messageBus.Subscribe<WorkItemStatus>(status => {
Assert.Equal(jobId, status.WorkItemId);
statusCount++;
});
job.RunUntilEmpty();
Thread.Sleep(1);
Assert.Equal(1, statusCount);
handlerRegistry.Register<MyWorkItem>(ctx => {
var jobData = ctx.GetData<MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
return TaskHelper.Completed();
});
jobId = queue.Enqueue(new MyWorkItem { SomeData = "Test" }, true);
job.RunUntilEmpty();
Assert.Equal(2, statusCount);
}
public void CanRunWorkItemWithClassHandler() {
ServiceProvider.SetServiceProvider(typeof(MyBootstrappedServiceProvider));
var queue = new InMemoryQueue<WorkItemData>();
var messageBus = new InMemoryMessageBus();
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry);
handlerRegistry.Register<MyWorkItem, MyWorkItemHandler>();
var jobId = queue.Enqueue(new MyWorkItem { SomeData = "Test" }, true);
int statusCount = 0;
messageBus.Subscribe<WorkItemStatus>(status => {
Assert.Equal(jobId, status.WorkItemId);
statusCount++;
});
job.RunUntilEmpty();
Assert.Equal(12, statusCount);
}
public void CanAutoCompleteWorker() {
var resetEvent = new AutoResetEvent(false);
var queue = new InMemoryQueue<SimpleWorkItem>(workItemTimeoutMilliseconds: 100);
queue.StartWorking(w => {
Assert.Equal("Hello", w.Value.Data);
resetEvent.Set();
}, true);
queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello"
});
Assert.Equal(1, queue.Count);
bool success = resetEvent.WaitOne(100);
Assert.True(success, "Failed to receive message.");
Task.Delay(25).Wait();
Assert.Equal(0, queue.Count);
Assert.Equal(1, queue.Completed);
Assert.Equal(0, queue.WorkerErrors);
}
public void CanHaveMultipleWorkers() {
const int workItemCount = 50;
var latch = new CountDownLatch(workItemCount);
int errorCount = 0;
int abandonCount = 0;
var queue = new InMemoryQueue<SimpleWorkItem>(retries: 1, workItemTimeoutMilliseconds: 50);
Task.Factory.StartNew(() => queue.StartWorking(w => DoWork(w, latch, ref abandonCount, ref errorCount)));
Task.Factory.StartNew(() => queue.StartWorking(w => DoWork(w, latch, ref abandonCount, ref errorCount)));
Task.Factory.StartNew(() => queue.StartWorking(w => DoWork(w, latch, ref abandonCount, ref errorCount)));
Parallel.For(0, workItemCount, i => queue.EnqueueAsync(new SimpleWorkItem {
Data = "Hello",
Id = i
}));
bool success = latch.Wait(1000);
Assert.True(success, "Failed to receive all work items.");
Task.Delay(50).Wait();
Assert.Equal(workItemCount, queue.Completed + queue.DeadletterCount);
Assert.Equal(errorCount, queue.WorkerErrors);
Assert.Equal(abandonCount + errorCount, queue.Abandoned);
}
public void WhenCallingInitialiseShouldNotThrow()
{
InMemoryQueue q = new InMemoryQueue();
q.Initialise();
}
/// <summary>
/// Waits on the specified queue for messages
/// </summary>
/// <param name="queue"></param>
/// <param name="token"></param>
/// <param name="onMessageReceived"></param>
private void GetMessages(InMemoryQueue queue, CancellationToken token, ManualResetEvent resetEvent, Action<RawMessage, Action, Action> onMessageReceived)
{
while (true)
{
if (token.IsCancellationRequested)
{
token.ThrowIfCancellationRequested();
}
resetEvent.WaitOne(TimeSpan.FromSeconds(15));
RawMessage message = null;
while (queue.Dequeue(ref message))
{
onMessageReceived(
message,
() => { /* Ack No Op for InMemory */ },
() =>
{
// Reject puts message back in queue
queue.Nack(message);
});
}
resetEvent.Reset();
}
}
public async Task CanRunWorkItemWithClassHandler() {
ServiceProvider.SetServiceProvider(typeof(MyBootstrappedServiceProvider));
var queue = new InMemoryQueue<WorkItemData>();
var messageBus = new InMemoryMessageBus();
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry);
handlerRegistry.Register<MyWorkItem, MyWorkItemHandler>();
var jobId = await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test"
}, true);
int statusCount = 0;
messageBus.Subscribe<WorkItemStatus>(status => {
Logger.Trace().Message($"Progress: {status.Progress}").Write();
Assert.Equal(jobId, status.WorkItemId);
statusCount++;
});
await job.RunUntilEmptyAsync();
Assert.Equal(11, statusCount);
}
public void WhenCallingRemoveAndIsEmptyShouldNotThrow()
{
var queue = new InMemoryQueue();
Assert.Null(queue.Remove());
}
public async Task CanRunWorkItemWithClassHandler() {
using (var queue = new InMemoryQueue<WorkItemData>()) {
using (var messageBus = new InMemoryMessageBus()) {
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
handlerRegistry.Register<MyWorkItem>(new MyWorkItemHandler(Log));
var jobId = await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test"
}, true);
int statusCount = 0;
messageBus.Subscribe<WorkItemStatus>(status => {
_logger.Trace("Progress: {progress}", status.Progress);
Assert.Equal(jobId, status.WorkItemId);
statusCount++;
});
await job.RunUntilEmptyAsync();
Assert.Equal(11, statusCount);
}
}
}
public async Task CanRunBadWorkItem() {
using (var queue = new InMemoryQueue<WorkItemData>(retries: 2, retryDelay: TimeSpan.FromMilliseconds(500))) {
using (var messageBus = new InMemoryMessageBus()) {
var handlerRegistry = new WorkItemHandlers();
var job = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
handlerRegistry.Register<MyWorkItem>(ctx => {
var jobData = ctx.GetData<MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
throw new Exception();
});
var jobId = await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test"
}, true);
int statusCount = 0;
messageBus.Subscribe<WorkItemStatus>(status => {
_logger.Trace("Progress: {progress}", status.Progress);
Assert.Equal(jobId, status.WorkItemId);
statusCount++;
});
await job.RunUntilEmptyAsync();
Assert.Equal(1, statusCount);
}
}
}
public async Task CanHandleMultipleWorkItemInstances() {
const int workItemCount = 1000;
using (var metrics = new InMemoryMetricsClient(loggerFactory: Log)) {
using (var queue = new InMemoryQueue<WorkItemData>(retries: 0, retryDelay: TimeSpan.Zero, loggerFactory: Log)) {
queue.AttachBehavior(new MetricsQueueBehavior<WorkItemData>(metrics, loggerFactory: Log));
using (var messageBus = new InMemoryMessageBus(Log)) {
var handlerRegistry = new WorkItemHandlers();
var j1 = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
var j2 = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
var j3 = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
int errors = 0;
var jobIds = new ConcurrentDictionary<string, int>();
handlerRegistry.Register<MyWorkItem>(async ctx => {
var jobData = ctx.GetData<MyWorkItem>();
Assert.Equal("Test", jobData.SomeData);
var jobWorkTotal = jobIds.AddOrUpdate(ctx.JobId, 1, (key, value) => value + 1);
if (jobData.Index % 100 == 0)
_logger.Trace("Job {jobId} processing work item #: {jobWorkTotal}", ctx.JobId, jobWorkTotal);
for (int i = 0; i < 10; i++)
await ctx.ReportProgressAsync(10 * i);
if (RandomData.GetBool(1)) {
Interlocked.Increment(ref errors);
throw new Exception("Boom!");
}
});
for (int i = 0; i < workItemCount; i++)
await queue.EnqueueAsync(new MyWorkItem {
SomeData = "Test",
Index = i
}, true);
var completedItems = new List<string>();
object completedItemsLock = new object();
messageBus.Subscribe<WorkItemStatus>(status => {
if (status.Progress == 100)
_logger.Trace("Progress: {progress}", status.Progress);
if (status.Progress < 100)
return;
lock (completedItemsLock)
completedItems.Add(status.WorkItemId);
});
var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10));
var tasks = new List<Task> {
Task.Run(async () => {
await j1.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
}, cancellationTokenSource.Token),
Task.Run(async () => {
await j2.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
}, cancellationTokenSource.Token),
Task.Run(async () => {
await j3.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
}, cancellationTokenSource.Token)
};
try {
await Task.WhenAll(tasks);
await SystemClock.SleepAsync(100);
} catch (TaskCanceledException) {}
_logger.Info("Completed: {completedItems} Errors: {errors}", completedItems.Count, errors);
Assert.Equal(workItemCount, completedItems.Count + errors);
Assert.Equal(3, jobIds.Count);
Assert.Equal(workItemCount, jobIds.Sum(kvp => kvp.Value));
}
}
}
}
public virtual void CanRunWorkItemWithMetrics()
{
var eventRaised = new ManualResetEvent(false);
var metricsClient = new InMemoryMetricsClient();
var behavior = new MetricsQueueBehavior<WorkItemData>(metricsClient, "metric");
var queue = new InMemoryQueue<WorkItemData>(behaviours: new[] { behavior });
queue.Completed += (sender, e) => { eventRaised.Set(); };
var work = new SimpleWorkItem { Id = 1, Data = "Testing" };
queue.Enqueue(work);
var item = queue.Dequeue();
item.Complete();
metricsClient.DisplayStats(_writer);
Assert.True(eventRaised.WaitOne(TimeSpan.FromMinutes(1)));
Assert.Equal(6, metricsClient.Counters.Count);
Assert.Equal(4, metricsClient.Timings.Count);
Assert.Equal(1, metricsClient.Counters["metric.workitemdata.simple.enqueued"]?.CurrentValue);
Assert.Equal(1, metricsClient.Counters["metric.workitemdata.simple.dequeued"]?.CurrentValue);
Assert.Equal(1, metricsClient.Counters["metric.workitemdata.simple.completed"]?.CurrentValue);
Assert.True(0 < metricsClient.Timings["metric.workitemdata.simple.queuetime"]?.Count);
Assert.True(0 < metricsClient.Timings["metric.workitemdata.simple.processtime"]?.Count);
}
public void CanRunQueueJob() {
const int workItemCount = 10000;
var queue = new InMemoryQueue<SampleQueueWorkItem>(0, TimeSpan.Zero);
for (int i = 0; i < workItemCount; i++)
queue.Enqueue(new SampleQueueWorkItem { Created = DateTime.Now, Path = "somepath" + i });
var metrics = new InMemoryMetricsClient();
var job = new SampleQueueJob(queue, metrics);
job.RunUntilEmpty(new CancellationTokenSource(30000).Token);
metrics.DisplayStats(_writer);
Assert.Equal(0, queue.GetQueueStats().Queued);
}
