Example #1
0
        public async Task CanSendMultiple() {
            const int iterations = 100000;
            await StartListeningAsync(iterations);

            var metrics = new InMemoryMetricsClient();

            var sw = Stopwatch.StartNew();
            for (int index = 0; index < iterations; index++) {
                if (index % (iterations / 10) == 0)
                    StopListening();

                await _client.CounterAsync("counter");
                await metrics.CounterAsync("counter");

                if (index % (iterations / 10) == 0)
                    await StartListeningAsync(iterations - index);

                if (index % (iterations / 20) == 0)
                    metrics.DisplayStats(_writer);
            }

            sw.Stop();
            metrics.DisplayStats(_writer);

            // Require at least 10,000 operations/s
            Assert.InRange(sw.ElapsedMilliseconds, 0, (iterations / 10000.0) * 1000);

            await Task.Delay(250);
            var messages = GetMessages();
            int expected = iterations - (iterations / (iterations / 10));
            Assert.InRange(messages.Count, expected - 10, expected + 10);
            foreach (string message in messages)
                Assert.Equal("test.counter:1|c", message);
        }
Example #2
0
        public virtual async Task CanRunMultipleQueueJobs() {
            const int jobCount = 5;
            const int workItemCount = 100;

            Log.SetLogLevel<SampleQueueJob>(LogLevel.Information);
            Log.SetLogLevel<InMemoryMetricsClient>(LogLevel.None);

            using (var metrics = new InMemoryMetricsClient(true, loggerFactory: Log)) {
                var queues = new List<IQueue<SampleQueueWorkItem>>();
                try {
                    for (int i = 0; i < jobCount; i++) {
                        var q = GetSampleWorkItemQueue(retries: 1, retryDelay: TimeSpan.Zero);
                        await q.DeleteQueueAsync();
                        q.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test", Log));
                        queues.Add(q);
                    }
                    _logger.Info("Done setting up queues");

                    var enqueueTask = Run.InParallel(workItemCount, async index => {
                        var queue = queues[RandomData.GetInt(0, jobCount - 1)];
                        await queue.EnqueueAsync(new SampleQueueWorkItem {
                            Created = SystemClock.UtcNow,
                            Path = RandomData.GetString()
                        });
                    });
                    _logger.Info("Done enqueueing");

                    var cancellationTokenSource = new CancellationTokenSource();
                    await Run.InParallel(jobCount, async index => {
                        var queue = queues[index - 1];
                        var job = new SampleQueueJob(queue, metrics, Log);
                        await job.RunUntilEmptyAsync(cancellationTokenSource.Token);
                        cancellationTokenSource.Cancel();
                    });
                    _logger.Info("Done running jobs until empty");

                    await enqueueTask;

                    var queueStats = new List<QueueStats>();
                    for (int i = 0; i < queues.Count; i++) {
                        var stats = await queues[i].GetQueueStatsAsync();
                        _logger.Info("Queue#{i}: Working: {working} Completed: {completed} Abandoned: {abandoned} Error: {errors} Deadletter: {deadletter}", i, stats.Working, stats.Completed, stats.Abandoned, stats.Errors, stats.Deadletter);
                        queueStats.Add(stats);
                    }
                    _logger.Info("Done getting queue stats");

                    await metrics.FlushAsync();
                    _logger.Info("Done flushing metrics");

                    var queueSummary = await metrics.GetQueueStatsAsync("test.samplequeueworkitem");
                    Assert.Equal(queueStats.Sum(s => s.Completed), queueSummary.Completed.Count);
                    Assert.InRange(queueStats.Sum(s => s.Completed), 0, workItemCount);
                } finally {
                    foreach (var q in queues) {
                        await q.DeleteQueueAsync();
                        q.Dispose();
                    }
                }
            }
        }
Example #3
0
        public async Task CanHandleMultipleWorkItemInstances()
        {
            var metrics = new InMemoryMetricsClient();
            var queue = new InMemoryQueue<WorkItemData>(retryDelay: TimeSpan.Zero, retries: 0);
            queue.AttachBehavior(new MetricsQueueBehavior<WorkItemData>(metrics));
            var messageBus = new InMemoryMessageBus();
            var handlerRegistry = new WorkItemHandlers();
            var j1 = new WorkItemJob(queue, messageBus, handlerRegistry);
            var j2 = new WorkItemJob(queue, messageBus, handlerRegistry);
            var j3 = new WorkItemJob(queue, messageBus, handlerRegistry);
            int errors = 0;
            var jobIds = new ConcurrentDictionary<string, int>();

            handlerRegistry.Register<MyWorkItem>(ctx => {
                var jobData = ctx.GetData<MyWorkItem>();
                Assert.Equal("Test", jobData.SomeData);

                jobIds.AddOrUpdate(ctx.JobId, 1, (key, value) => value + 1);
                
                for (int i = 0; i < 10; i++)
                    ctx.ReportProgress(10 * i);

                if (RandomData.GetBool(1))
                {
                    Interlocked.Increment(ref errors);
                    throw new ApplicationException("Boom!");
                }
                
                return TaskHelper.Completed();
            });

            for (int i = 0; i < 100; i++)
                queue.Enqueue(new MyWorkItem { SomeData = "Test", Index = i }, true);
            
            var completedItems = new List<string>();
            object completedItemsLock = new object();
            messageBus.Subscribe<WorkItemStatus>(status =>
            {
                if (status.Progress < 100)
                    return;

                lock (completedItemsLock)
                    completedItems.Add(status.WorkItemId);
            });
            
            var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10));
            var token = cancellationTokenSource.Token;
            var tasks = new List<Task>();
            tasks.AddRange(new[] {
                Task.Run(async () => await j1.RunUntilEmptyAsync(token), token),
                Task.Run(async () => await j2.RunUntilEmptyAsync(token), token),
                Task.Run(async () => await j3.RunUntilEmptyAsync(token), token),
            });

            await Task.WhenAll(tasks);
            Thread.Sleep(10);
            Assert.Equal(100, completedItems.Count + errors);
            Assert.Equal(3, jobIds.Count);
            Assert.Equal(100, jobIds.Sum(kvp => kvp.Value));
        }
        public virtual async Task CanRunQueueJob() {
            const int workItemCount = 100;
            var metrics = new InMemoryMetricsClient();
            var queue = GetSampleWorkItemQueue(retries: 0, retryDelay: TimeSpan.Zero);
            await queue.DeleteQueueAsync();
            queue.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));

            metrics.StartDisplayingStats(TimeSpan.FromSeconds(1), _writer);
            var enqueueTask = Run.InParallel(workItemCount, async index => {
                await queue.EnqueueAsync(new SampleQueueWorkItem {
                    Created = DateTime.Now,
                    Path = "somepath" + index
                });
            });

            var job = new SampleQueueJob(queue, metrics);
            await Task.Delay(10);
            await Task.WhenAll(job.RunUntilEmptyAsync(), enqueueTask);

            metrics.DisplayStats(_writer);

            var stats = await queue.GetQueueStatsAsync();
            Assert.Equal(0, stats.Queued);
            Assert.Equal(workItemCount, stats.Enqueued);
            Assert.Equal(workItemCount, stats.Dequeued);
        }
        public static void RegisterServices(Container container, ILoggerFactory loggerFactory) {
            container.RegisterSingleton<ILoggerFactory>(loggerFactory);
            container.RegisterSingleton(typeof(ILogger<>), typeof(Logger<>));

            ServiceProvider.Current = container;
            container.RegisterSingleton<ISerializer>(() => new JsonNetSerializer());

            var metricsClient = new InMemoryMetricsClient();
            container.RegisterSingleton<IMetricsClient>(metricsClient);
            container.RegisterSingleton<ICacheClient, InMemoryCacheClient>();

            container.RegisterCollection(typeof(IQueueBehavior<ValuesPost>), new[] {
                Lifestyle.Singleton.CreateRegistration(
                    () => new MetricsQueueBehavior<ValuesPost>(metricsClient), container)
            });
            container.RegisterSingleton<IQueue<ValuesPost>>(() => new InMemoryQueue<ValuesPost>(behaviors: container.GetAllInstances<IQueueBehavior<ValuesPost>>()));
            
            container.RegisterCollection(typeof(IQueueBehavior<WorkItemData>), new[] {
                Lifestyle.Singleton.CreateRegistration(
                    () => new MetricsQueueBehavior<WorkItemData>(metricsClient), container)
            });

            var handlers = new WorkItemHandlers();
            handlers.Register<DeleteValueWorkItem, DeleteValueWorkItemHandler>();
            container.RegisterSingleton(handlers);

            container.RegisterSingleton<IQueue<WorkItemData>>(() => new InMemoryQueue<WorkItemData>(behaviors: container.GetAllInstances<IQueueBehavior<WorkItemData>>(), workItemTimeout: TimeSpan.FromHours(1)));
            
            container.RegisterSingleton<IMessageBus, InMemoryMessageBus>();
            container.RegisterSingleton<IMessagePublisher>(container.GetInstance<IMessageBus>);
            container.RegisterSingleton<IMessageSubscriber>(container.GetInstance<IMessageBus>);

            container.RegisterSingleton<ILockProvider, CacheLockProvider>();
            container.RegisterSingleton<IFileStorage>(new InMemoryFileStorage());
        }
Example #6
0
        public void CanRunMultipleQueueJobs()
        {
            const int jobCount = 5;
            const int workItemCount = 1000;
            var metrics = new InMemoryMetricsClient();
            metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(100), _writer);

            var queues = new List<RedisQueue<SampleQueueWorkItem>>();
            for (int i = 0; i < jobCount; i++)
            {
                var q = new RedisQueue<SampleQueueWorkItem>(SharedConnection.GetMuxer(), retries: 3, retryDelay: TimeSpan.FromSeconds(1));
                q.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
                queues.Add(q);
            }

            Task.Run(() =>
            {
                Parallel.For(0, workItemCount, i => {
                    var queue = queues[RandomData.GetInt(0, 4)];
                    queue.Enqueue(new SampleQueueWorkItem { Created = DateTime.Now, Path = RandomData.GetString() });
                });
            });

            Parallel.For(0, jobCount, index =>
            {
                var queue = queues[index];
                var job = new SampleQueueJob(queue, metrics);
                job.RunUntilEmpty();
            });

            metrics.DisplayStats(_writer);
        }
Example #7
0
 public void CanDisplayStatsMultithreaded() {
     var metrics = new InMemoryMetricsClient();
     metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(10), _writer);
     Parallel.For(0, 100, i => {
         metrics.Counter("Test");
         Thread.Sleep(50);
     });
 }
        public async Task CanWaitForCounter() {
            var metrics = new InMemoryMetricsClient();
            metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(50), _writer);
            Task.Run(async () => {
                await Task.Delay(50);
                await metrics.CounterAsync("Test");
                await metrics.CounterAsync("Test");
            });

            var success = await metrics.WaitForCounterAsync("Test", 2, TimeSpan.FromMilliseconds(500));
            Assert.True(success);

            Task.Run(async () => {
                await Task.Delay(50);
                await metrics.CounterAsync("Test");
            });

            success = await metrics.WaitForCounterAsync("Test", timeout: TimeSpan.FromMilliseconds(500));
            Assert.True(success);

            success = await metrics.WaitForCounterAsync("Test", timeout: TimeSpan.FromMilliseconds(100));
            Assert.False(success);

            Task.Run(async () => {
                await Task.Delay(50);
                await metrics.CounterAsync("Test", 2);
            });

            success = await metrics.WaitForCounterAsync("Test", 2, TimeSpan.FromMilliseconds(500));
            Assert.True(success);

            success = await metrics.WaitForCounterAsync("Test", async () => await metrics.CounterAsync("Test"), cancellationToken: TimeSpan.FromMilliseconds(500).ToCancellationToken());
            Assert.True(success);

            Task.Run(async () => {
                await Task.Delay(50);
                await metrics.CounterAsync("Test");
            });

            success = await metrics.WaitForCounterAsync("Test", timeout: TimeSpan.FromMilliseconds(500));
            Assert.True(success);

            metrics.DisplayStats(_writer);
        }
        public virtual async Task CanRunMultipleQueueJobs() {
            const int jobCount = 5;
            const int workItemCount = 100;
            var metrics = new InMemoryMetricsClient();
            metrics.StartDisplayingStats(TimeSpan.FromSeconds(1), _writer);

            var queues = new List<IQueue<SampleQueueWorkItem>>();
            for (int i = 0; i < jobCount; i++) {
                var q = GetSampleWorkItemQueue(retries: 3, retryDelay: TimeSpan.FromSeconds(1));
                await q.DeleteQueueAsync();
                q.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
                queues.Add(q);
            }

            var enqueueTask = Run.InParallel(workItemCount, async index => {
                var queue = queues[RandomData.GetInt(0, jobCount - 1)];
                await queue.EnqueueAsync(new SampleQueueWorkItem {
                    Created = DateTime.Now,
                    Path = RandomData.GetString()
                });
            });

            var cancellationTokenSource = new CancellationTokenSource();
            await Run.InParallel(jobCount, async index => {
                var queue = queues[index - 1];
                var job = new SampleQueueJob(queue, metrics);
                await job.RunUntilEmptyAsync(cancellationTokenSource.Token);
                cancellationTokenSource.Cancel();
            });

            await enqueueTask;

            var queueStats = new List<QueueStats>();
            for (int i = 0; i < queues.Count; i++) {
                var stats = await queues[i].GetQueueStatsAsync();
                Logger.Info().Message($"Queue#{i}: Working: {stats.Working} Completed: {stats.Completed} Abandoned: {stats.Abandoned} Error: {stats.Errors} Deadletter: {stats.Deadletter}").Write();
                queueStats.Add(stats);
            }

            metrics.DisplayStats(_writer);
            Assert.Equal(metrics.GetCount("completed"), queueStats.Sum(s => s.Completed));
            Assert.InRange(queueStats.Sum(s => s.Completed), 0, workItemCount);
         }
Example #10
0
        public void CanRunQueueJob()
        {
            const int workItemCount = 10000;
            var metrics = new InMemoryMetricsClient();
            var queue = new RedisQueue<SampleQueueWorkItem>(SharedConnection.GetMuxer(), null, null, 0, TimeSpan.Zero);
            queue.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));

            metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(100), _writer);
            Task.Factory.StartNew(() => {
                Parallel.For(0, workItemCount, i => {
                    queue.Enqueue(new SampleQueueWorkItem { Created = DateTime.Now, Path = "somepath" + i });
                });
            });

            var job = new SampleQueueJob(queue, metrics);
            job.RunUntilEmpty();
            metrics.DisplayStats(_writer);

            Assert.Equal(0, queue.GetQueueStats().Queued);
        }
Example #11
0
        public async Task CanIncrementCounter() {
            var metrics = new InMemoryMetricsClient();

            await metrics.CounterAsync("c1");
            Assert.Equal(1, metrics.GetCount("c1"));

            await metrics.CounterAsync("c1", 5);
            Assert.Equal(6, metrics.GetCount("c1"));

            var counter = metrics.Counters["c1"];
            Assert.True(counter.Rate > 400);

            await metrics.GaugeAsync("g1", 2.534);
            Assert.Equal(2.534, metrics.GetGaugeValue("g1"));

            await metrics.TimerAsync("t1", 50788);
            var stats = metrics.GetMetricStats();
            Assert.Equal(1, stats.Timings.Count);

            metrics.DisplayStats(_writer);
        }
Example #12
0
        public async void CanWaitForCounter() {
            var metrics = new InMemoryMetricsClient();
            metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(50), _writer);
            Task.Run(() => {
                Thread.Sleep(50);
                metrics.Counter("Test");
                metrics.Counter("Test");
            });
            var success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(500), 2);
            Assert.True(success);

            Task.Run(() => {
                Thread.Sleep(50);
                metrics.Counter("Test");
            });
            success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(500));
            Assert.True(success);

            success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(100));
            Assert.False(success);

            Task.Run(() => {
                Thread.Sleep(50);
                metrics.Counter("Test", 2);
            });
            success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(500), 2);
            Assert.True(success);

            success = await metrics.WaitForCounterAsync("Test", () => metrics.CounterAsync("Test"), TimeSpan.FromMilliseconds(500));
            Assert.True(success);

            Task.Run(() => {
                Thread.Sleep(50);
                metrics.Counter("Test");
            });
            success = metrics.WaitForCounter("Test", TimeSpan.FromMilliseconds(500));
            Assert.True(success);

            metrics.DisplayStats(_writer);
        }
Example #13
0
        public async Task MeasureWorkerThroughput() {
            var queue = GetQueue(retries: 3, workItemTimeout: TimeSpan.FromSeconds(2), retryDelay: TimeSpan.FromSeconds(1));
            if (queue == null)
                return;
            
            using (queue) {
                await queue.DeleteQueueAsync();

                const int workItemCount = 1;
                for (int i = 0; i < workItemCount; i++) {
                    await queue.EnqueueAsync(new SimpleWorkItem {
                        Data = "Hello"
                    });
                }
                Assert.Equal(workItemCount, (await queue.GetQueueStatsAsync()).Queued);

                var countdown = new AsyncCountdownEvent(workItemCount);
                var metrics = new InMemoryMetricsClient();
                await queue.StartWorkingAsync(async workItem => {
                    Assert.Equal("Hello", workItem.Value.Data);
                    await workItem.CompleteAsync();
                    await metrics.CounterAsync("work");
                    countdown.Signal();
                });

                await countdown.WaitAsync(TimeSpan.FromMinutes(1));
                Assert.Equal(0, countdown.CurrentCount);
                _logger.Trace((await metrics.GetCounterStatsAsync("work")).ToString());

                var stats = await queue.GetQueueStatsAsync();
                Assert.Equal(workItemCount, stats.Dequeued);
                Assert.Equal(workItemCount, stats.Completed);
                Assert.Equal(0, stats.Queued);

                _logger.Trace("# Keys: {0}", CountAllKeysAsync());
            }
        }
Example #14
0
        public async Task MeasureThroughput() {
            var queue = GetQueue(retries: 3, workItemTimeout: TimeSpan.FromSeconds(2), retryDelay: TimeSpan.FromSeconds(1));
            if (queue == null)
                return;
            
            using (queue) {
                await queue.DeleteQueueAsync();

                const int workItemCount = 1000;
                for (int i = 0; i < workItemCount; i++) {
                    await queue.EnqueueAsync(new SimpleWorkItem {
                        Data = "Hello"
                    });
                }
                Assert.Equal(workItemCount, (await queue.GetQueueStatsAsync()).Queued);

                var metrics = new InMemoryMetricsClient();
                var workItem = await queue.DequeueAsync(TimeSpan.Zero);
                while (workItem != null) {
                    Assert.Equal("Hello", workItem.Value.Data);
                    await workItem.CompleteAsync();
                    await metrics.CounterAsync("work");

                    workItem = await queue.DequeueAsync(TimeSpan.Zero);
                }
                _logger.Trace((await metrics.GetCounterStatsAsync("work")).ToString());

                var stats = await queue.GetQueueStatsAsync();
                Assert.Equal(workItemCount, stats.Dequeued);
                Assert.Equal(workItemCount, stats.Completed);
                Assert.Equal(0, stats.Queued);

                _logger.Trace("# Keys: {0}", CountAllKeysAsync());
            }
        }
Example #15
0
        public virtual void MeasureThroughput()
        {
            var cacheClient = GetCacheClient();
            if (cacheClient == null)
                return;

            cacheClient.FlushAll();

            const int itemCount = 10000;
            var metrics = new InMemoryMetricsClient();
            for (int i = 0; i < itemCount; i++)
            {
                cacheClient.Set("test", 13422);
                cacheClient.Set("flag", true);
                Assert.Equal(13422, cacheClient.Get<int>("test"));
                Assert.Null(cacheClient.Get<int?>("test2"));
                Assert.True(cacheClient.Get<bool>("flag"));
                metrics.Counter("work");
            }
            metrics.DisplayStats(_writer);
        }
Example #16
0
        public virtual void MeasureSerializerSimpleThroughput()
        {
            var cacheClient = GetCacheClient();
            if (cacheClient == null)
                return;

            cacheClient.FlushAll();

            const int itemCount = 10000;
            var metrics = new InMemoryMetricsClient();
            for (int i = 0; i < itemCount; i++)
            {
                cacheClient.Set("test", new SimpleModel
                {
                    Data1 = "Hello",
                    Data2 = 12
                });
                var model = cacheClient.Get<SimpleModel>("test");
                Assert.NotNull(model);
                Assert.Equal("Hello", model.Data1);
                Assert.Equal(12, model.Data2);
                metrics.Counter("work");
            }
            metrics.DisplayStats(_writer);
        }
Example #17
0
        public virtual void MeasureSerializerComplexThroughput()
        {
            var cacheClient = GetCacheClient();
            if (cacheClient == null)
                return;

            cacheClient.FlushAll();

            const int itemCount = 10000;
            var metrics = new InMemoryMetricsClient();
            for (int i = 0; i < itemCount; i++)
            {
                cacheClient.Set("test", new ComplexModel
                {
                    Data1 = "Hello",
                    Data2 = 12,
                    Data3 = true,
                    Simple = new SimpleModel { Data1 = "hi", Data2 = 13 },
                    Simples = new List<SimpleModel> { new SimpleModel { Data1 = "hey", Data2 = 45 }, new SimpleModel { Data1 = "next", Data2 = 3423 } },
                    DictionarySimples = new Dictionary<string, SimpleModel> { { "sdf", new SimpleModel { Data1 = "Sachin" } } }
                });
                var model = cacheClient.Get<ComplexModel>("test");
                Assert.NotNull(model);
                Assert.Equal("Hello", model.Data1);
                Assert.Equal(12, model.Data2);
                metrics.Counter("work");
            }
            metrics.DisplayStats(_writer);
        }
Example #18
0
        public virtual void CanHandleErrorInWorker()
        {
            var queue = GetQueue(retries: 0);
            if (queue == null)
                return;

            var metrics = new InMemoryMetricsClient();
            queue.AttachBehavior(new MetricsQueueBehavior<SimpleWorkItem>(metrics));

            using (queue) {
                queue.DeleteQueue();

                queue.StartWorking(w => {
                    Debug.WriteLine("WorkAction");
                    Assert.Equal("Hello", w.Value.Data);
                    throw new ApplicationException();
                });

                metrics.DisplayStats(_writer);
                var success = metrics.WaitForCounter("simpleworkitem.hello.abandoned", () => queue.Enqueue(new SimpleWorkItem
                {
                    Data = "Hello"
                }), TimeSpan.FromSeconds(1));
                metrics.DisplayStats(_writer);
                Assert.True(success);
                Assert.Equal(0, queue.GetQueueStats().Completed);
                Assert.Equal(1, queue.GetQueueStats().Errors);
                Assert.Equal(1, queue.GetQueueStats().Deadletter);
            }
        }
Example #19
0
        public void MeasureWorkerThroughput()
        {
            var queue = GetQueue(retries: 3, workItemTimeout: TimeSpan.FromSeconds(2), retryDelay: TimeSpan.FromSeconds(1));
            if (queue == null)
                return;

            FlushAll();

            using (queue) {
                queue.DeleteQueue();

                const int workItemCount = 1000;
                for (int i = 0; i < workItemCount; i++) {
                    queue.Enqueue(new SimpleWorkItem {
                        Data = "Hello"
                    });
                }
                Assert.Equal(workItemCount, queue.GetQueueStats().Queued);

                var countdown = new CountDownLatch(workItemCount);
                var metrics = new InMemoryMetricsClient();
                queue.StartWorking(workItem => {
                    Assert.Equal("Hello", workItem.Value.Data);
                    workItem.Complete();
                    metrics.Counter("work");
                    countdown.Signal();
                });
                countdown.Wait(60 * 1000);
                metrics.DisplayStats(_output);

                var stats = queue.GetQueueStats();
                Assert.Equal(workItemCount, stats.Dequeued);
                Assert.Equal(workItemCount, stats.Completed);
                Assert.Equal(0, stats.Queued);

                Trace.WriteLine(CountAllKeys());
            }
        }
Example #20
0
        public void JobLoopPerf()
        {
            const int iterations = 10000;

            var metrics = new InMemoryMetricsClient();
            var job = new SampleJob(metrics);
            job.RunContinuous(null, iterations);
            metrics.DisplayStats(_writer);
        }
Example #21
0
        public void CanRunQueueJob() {
            const int workItemCount = 10000;
            var queue = new InMemoryQueue<SampleQueueWorkItem>(0, TimeSpan.Zero);

            for (int i = 0; i < workItemCount; i++)
                queue.Enqueue(new SampleQueueWorkItem { Created = DateTime.Now, Path = "somepath" + i });

            var metrics = new InMemoryMetricsClient();
            var job = new SampleQueueJob(queue, metrics);
            job.RunUntilEmpty(new CancellationTokenSource(30000).Token);
            metrics.DisplayStats(_writer);

            Assert.Equal(0, queue.GetQueueStats().Queued);
        }
Example #22
0
        public async Task CanHandleMultipleWorkItemInstances() {
            const int workItemCount = 1000;

            using (var metrics = new InMemoryMetricsClient(loggerFactory: Log)) {
                using (var queue = new InMemoryQueue<WorkItemData>(retries: 0, retryDelay: TimeSpan.Zero, loggerFactory: Log)) {
                    queue.AttachBehavior(new MetricsQueueBehavior<WorkItemData>(metrics, loggerFactory: Log));
                    using (var messageBus = new InMemoryMessageBus(Log)) {
                        var handlerRegistry = new WorkItemHandlers();
                        var j1 = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
                        var j2 = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
                        var j3 = new WorkItemJob(queue, messageBus, handlerRegistry, Log);
                        int errors = 0;

                        var jobIds = new ConcurrentDictionary<string, int>();

                        handlerRegistry.Register<MyWorkItem>(async ctx => {
                            var jobData = ctx.GetData<MyWorkItem>();
                            Assert.Equal("Test", jobData.SomeData);

                            var jobWorkTotal = jobIds.AddOrUpdate(ctx.JobId, 1, (key, value) => value + 1);
                            if (jobData.Index % 100 == 0)
                                _logger.Trace("Job {jobId} processing work item #: {jobWorkTotal}", ctx.JobId, jobWorkTotal);

                            for (int i = 0; i < 10; i++)
                                await ctx.ReportProgressAsync(10 * i);

                            if (RandomData.GetBool(1)) {
                                Interlocked.Increment(ref errors);
                                throw new Exception("Boom!");
                            }
                        });

                        for (int i = 0; i < workItemCount; i++)
                            await queue.EnqueueAsync(new MyWorkItem {
                                SomeData = "Test",
                                Index = i
                            }, true);

                        var completedItems = new List<string>();
                        object completedItemsLock = new object();
                        messageBus.Subscribe<WorkItemStatus>(status => {
                            if (status.Progress == 100)
                                _logger.Trace("Progress: {progress}", status.Progress);

                            if (status.Progress < 100)
                                return;

                            lock (completedItemsLock)
                                completedItems.Add(status.WorkItemId);
                        });
            
                        var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(10));
                        var tasks = new List<Task> {
                            Task.Run(async () => {
                                await j1.RunUntilEmptyAsync(cancellationTokenSource.Token);
                                cancellationTokenSource.Cancel();
                            }, cancellationTokenSource.Token),
                            Task.Run(async () => {
                                await j2.RunUntilEmptyAsync(cancellationTokenSource.Token);
                                cancellationTokenSource.Cancel();
                            }, cancellationTokenSource.Token),
                            Task.Run(async () => {
                                await j3.RunUntilEmptyAsync(cancellationTokenSource.Token);
                                cancellationTokenSource.Cancel();
                            }, cancellationTokenSource.Token)
                        };

                        try {
                            await Task.WhenAll(tasks);
                            await SystemClock.SleepAsync(100);
                        } catch (TaskCanceledException) {}

                        _logger.Info("Completed: {completedItems} Errors: {errors}", completedItems.Count, errors);
            
                        Assert.Equal(workItemCount, completedItems.Count + errors);
                        Assert.Equal(3, jobIds.Count);
                        Assert.Equal(workItemCount, jobIds.Sum(kvp => kvp.Value));
                    }
                }
            }
        }
Example #23
0
        public async Task JobLoopPerf() {
            const int iterations = 10000;

            var metrics = new InMemoryMetricsClient();
            var job = new SampleJob(metrics, Log);
            var sw = Stopwatch.StartNew();
            await job.RunContinuousAsync(null, iterations);
            sw.Stop();
            await metrics.FlushAsync();
            _logger.Trace((await metrics.GetCounterStatsAsync("runs")).ToString());
            _logger.Trace((await metrics.GetCounterStatsAsync("errors")).ToString());
            _logger.Trace((await metrics.GetCounterStatsAsync("failed")).ToString());
            _logger.Trace((await metrics.GetCounterStatsAsync("completed")).ToString());
        }
Example #24
0
        public virtual async Task CanDequeueEfficiently() {
            const int iterations = 100;

            var queue = GetQueue(runQueueMaintenance: false);
            if (queue == null)
                return;
            
            var metrics = new InMemoryMetricsClient();
            queue.AttachBehavior(new MetricsQueueBehavior<SimpleWorkItem>(metrics));

            using (queue) {
                await queue.DeleteQueueAsync();

                Task.Run(async () => {
                    for (int index = 0; index < iterations; index++) {
                        await Task.Delay(RandomData.GetInt(100, 300));
                        await queue.EnqueueAsync(new SimpleWorkItem {
                            Data = "Hello"
                        });
                    }
                    Logger.Trace().Message("Done enqueuing.").Write();
                });

                Logger.Trace().Message("Starting dequeue loop.").Write();
                var sw = Stopwatch.StartNew();
                for (int index = 0; index < iterations; index++) {
                    var item = await queue.DequeueAsync(TimeSpan.FromSeconds(5));
                    Assert.NotNull(item);
                    await item.CompleteAsync();
                }
                sw.Stop();

                metrics.DisplayStats(_writer);

                Assert.InRange(sw.ElapsedMilliseconds, iterations * 100, iterations * 325);
                Assert.InRange(metrics.Timings["simpleworkitem.queuetime"].Average, 0, 25);
            }
        }
Example #25
0
        public virtual async Task CanHandleErrorInWorker() {
            var queue = GetQueue(retries: 0);
            if (queue == null)
                return;

            var metrics = new InMemoryMetricsClient();
            queue.AttachBehavior(new MetricsQueueBehavior<SimpleWorkItem>(metrics));

            using (queue) {
                await queue.DeleteQueueAsync();
				
                queue.StartWorking(w => {
                    Debug.WriteLine("WorkAction");
                    Assert.Equal("Hello", w.Value.Data);
                    throw new ApplicationException();
                });
                
                metrics.DisplayStats(_writer);
                var success = await metrics.WaitForCounterAsync("simpleworkitem.hello.abandoned", async () => await queue.EnqueueAsync(new SimpleWorkItem {
                    Data = "Hello"
                }), cancellationToken: TimeSpan.FromSeconds(1).ToCancellationToken());
                await Task.Delay(10);
                metrics.DisplayStats(_writer);
                Assert.True(success);

                var stats = await queue.GetQueueStatsAsync();
                Assert.Equal(0, stats.Completed);
                Assert.Equal(1, stats.Errors);
                Assert.Equal(1, stats.Deadletter);
            }
        }
Example #26
0
        public void RegisterServices(Container container) {
            // Foundation service provider
            ServiceProvider.Current = container;
            container.RegisterSingle<IDependencyResolver>(() => new SimpleInjectorCoreDependencyResolver(container));

            JsonConvert.DefaultSettings = () => new JsonSerializerSettings {
                DateParseHandling = DateParseHandling.DateTimeOffset
            };

            var contractResolver = new ExceptionlessContractResolver();
            contractResolver.UseDefaultResolverFor(typeof(DataDictionary), typeof(SettingsDictionary), typeof(VersionOne.VersionOneWebHookStack), typeof(VersionOne.VersionOneWebHookEvent));

            var settings = new JsonSerializerSettings {
                MissingMemberHandling = MissingMemberHandling.Ignore,
                DateParseHandling = DateParseHandling.DateTimeOffset,
                ContractResolver = contractResolver
            };
            settings.AddModelConverters();
            
            container.RegisterSingle<IContractResolver>(() => contractResolver);
            container.RegisterSingle<JsonSerializerSettings>(settings);
            container.RegisterSingle<JsonSerializer>(JsonSerializer.Create(settings));
            container.RegisterSingle<ISerializer>(() => new JsonNetSerializer(settings));

            var metricsClient = new InMemoryMetricsClient();
            metricsClient.StartDisplayingStats();
            container.RegisterSingle<IMetricsClient>(metricsClient);

            container.RegisterSingle<IElasticClient>(() => container.GetInstance<ElasticSearchConfiguration>().GetClient(Settings.Current.ElasticSearchConnectionString.Split(',').Select(url => new Uri(url))));
            container.RegisterSingle<ICacheClient, InMemoryCacheClient>();

            container.RegisterSingle<IQueue<EventPost>>(() => new InMemoryQueue<EventPost>(statName: MetricNames.PostsQueueSize, metrics: container.GetInstance<IMetricsClient>()));
            container.RegisterSingle<IQueue<EventUserDescription>>(() => new InMemoryQueue<EventUserDescription>(statName: MetricNames.EventsUserDescriptionQueueSize, metrics: container.GetInstance<IMetricsClient>()));
            container.RegisterSingle<IQueue<EventNotificationWorkItem>>(() => new InMemoryQueue<EventNotificationWorkItem>(statName: MetricNames.EventNotificationQueueSize, metrics: container.GetInstance<IMetricsClient>()));
            container.RegisterSingle<IQueue<WebHookNotification>>(() => new InMemoryQueue<WebHookNotification>(statName: MetricNames.WebHookQueueSize, metrics: container.GetInstance<IMetricsClient>()));
            container.RegisterSingle<IQueue<MailMessage>>(() => new InMemoryQueue<MailMessage>(statName: MetricNames.EmailsQueueSize, metrics: container.GetInstance<IMetricsClient>()));
            container.RegisterSingle<IQueue<StatusMessage>>(() => new InMemoryQueue<StatusMessage>());

            var workItemHandlers = new WorkItemHandlers();
            workItemHandlers.Register<ReindexWorkItem, ReindexWorkItemHandler>();
            container.RegisterSingle<WorkItemHandlers>(workItemHandlers);
            container.RegisterSingle<IQueue<WorkItemData>>(() => new InMemoryQueue<WorkItemData>(statName: MetricNames.WorkItemQueueSize, metrics: container.GetInstance<IMetricsClient>(), workItemTimeout: TimeSpan.FromHours(1)));
            

            container.RegisterSingle<IMessageBus, InMemoryMessageBus>();
            container.RegisterSingle<IMessagePublisher>(container.GetInstance<IMessageBus>);
            container.RegisterSingle<IMessageSubscriber>(container.GetInstance<IMessageBus>);

            if (!String.IsNullOrEmpty(Settings.Current.StorageFolder))
                container.RegisterSingle<IFileStorage>(new FolderFileStorage(Settings.Current.StorageFolder));
            else
                container.RegisterSingle<IFileStorage>(new InMemoryFileStorage());

            container.RegisterSingle<IStackRepository, StackRepository>();
            container.RegisterSingle<IEventRepository, EventRepository>();
            container.RegisterSingle<IOrganizationRepository, OrganizationRepository>();
            container.RegisterSingle<IProjectRepository, ProjectRepository>();
            container.RegisterSingle<IUserRepository, UserRepository>();
            container.RegisterSingle<IWebHookRepository, WebHookRepository>();
            container.RegisterSingle<ITokenRepository, TokenRepository>();
            container.RegisterSingle<IApplicationRepository, ApplicationRepository>();

            container.RegisterSingle<IGeoIPResolver, MindMaxGeoIPResolver>();

            container.RegisterSingle<IValidator<Application>, ApplicationValidator>();
            container.RegisterSingle<IValidator<Organization>, OrganizationValidator>();
            container.RegisterSingle<IValidator<PersistentEvent>, PersistentEventValidator>();
            container.RegisterSingle<IValidator<Project>, ProjectValidator>();
            container.RegisterSingle<IValidator<Stack>, StackValidator>();
            container.RegisterSingle<IValidator<Models.Token>, TokenValidator>();
            container.RegisterSingle<IValidator<UserDescription>, UserDescriptionValidator>();
            container.RegisterSingle<IValidator<User>, UserValidator>();
            container.RegisterSingle<IValidator<WebHook>, WebHookValidator>();

            container.RegisterSingle<IEmailGenerator>(() => new RazorEmailGenerator(@"Mail\Templates"));
            container.RegisterSingle<IMailer, Mailer>();
            if (Settings.Current.WebsiteMode != WebsiteMode.Dev)
                container.RegisterSingle<IMailSender, SmtpMailSender>();
            else
                container.RegisterSingle<IMailSender>(() => new InMemoryMailSender());

            container.Register<ILockProvider, CacheLockProvider>();
            container.Register<StripeEventHandler>();
            container.RegisterSingle<BillingManager>();
            container.RegisterSingle<DataHelper>();
            container.RegisterSingle<EventStats>();
            container.RegisterSingle<EventPipeline>();
            container.RegisterSingle<EventPluginManager>();
            container.RegisterSingle<FormattingPluginManager>();

            container.RegisterSingle<ICoreLastReferenceIdManager, NullCoreLastReferenceIdManager>();
        }
Example #27
0
        public virtual void CanRunWorkItemWithMetrics()
        {
            var eventRaised = new ManualResetEvent(false);

            var metricsClient = new InMemoryMetricsClient();
            var behavior = new MetricsQueueBehavior<WorkItemData>(metricsClient, "metric");
            var queue = new InMemoryQueue<WorkItemData>(behaviours: new[] { behavior });
            queue.Completed += (sender, e) => { eventRaised.Set(); };

            var work = new SimpleWorkItem { Id = 1, Data = "Testing" };

            queue.Enqueue(work);
            var item = queue.Dequeue();
            item.Complete();

            metricsClient.DisplayStats(_writer);

            Assert.True(eventRaised.WaitOne(TimeSpan.FromMinutes(1)));

            Assert.Equal(6, metricsClient.Counters.Count);
            Assert.Equal(4, metricsClient.Timings.Count);

            Assert.Equal(1, metricsClient.Counters["metric.workitemdata.simple.enqueued"]?.CurrentValue);
            Assert.Equal(1, metricsClient.Counters["metric.workitemdata.simple.dequeued"]?.CurrentValue);
            Assert.Equal(1, metricsClient.Counters["metric.workitemdata.simple.completed"]?.CurrentValue);

            Assert.True(0 < metricsClient.Timings["metric.workitemdata.simple.queuetime"]?.Count);
            Assert.True(0 < metricsClient.Timings["metric.workitemdata.simple.processtime"]?.Count);
        }
Example #28
0
        public void MeasureThroughputWithRandomFailures()
        {
            var queue = GetQueue(retries: 3, workItemTimeout: TimeSpan.FromSeconds(2), retryDelay: TimeSpan.Zero);
            if (queue == null)
                return;

            FlushAll();

            using (queue) {
                queue.DeleteQueue();

                const int workItemCount = 1000;
                for (int i = 0; i < workItemCount; i++) {
                    queue.Enqueue(new SimpleWorkItem {
                        Data = "Hello"
                    });
                }
                Assert.Equal(workItemCount, queue.GetQueueStats().Queued);

                var metrics = new InMemoryMetricsClient();
                var workItem = queue.Dequeue(TimeSpan.Zero);
                while (workItem != null) {
                    Assert.Equal("Hello", workItem.Value.Data);
                    if (RandomData.GetBool(10))
                        workItem.Abandon();
                    else
                        workItem.Complete();
                    metrics.Counter("work");

                    workItem = queue.Dequeue(TimeSpan.FromMilliseconds(100));
                }
                metrics.DisplayStats(_output);

                var stats = queue.GetQueueStats();
                Assert.True(stats.Dequeued >= workItemCount);
                Assert.Equal(workItemCount, stats.Completed + stats.Deadletter);
                Assert.Equal(0, stats.Queued);

                Trace.WriteLine(CountAllKeys());
            }
        }
Example #29
0
        public async Task CanDisplayStatsMultithreaded() {
            var metrics = new InMemoryMetricsClient();
            metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(10), _writer);

            await Run.InParallel(100, async i => {
                await metrics.CounterAsync("Test");
                await Task.Delay(50);
            });
        }
Example #30
0
        public virtual async Task CanDequeueEfficiently() {
            const int iterations = 100;

            var queue = GetQueue(runQueueMaintenance: false);
            if (queue == null)
                return;
            
            using (queue) {
                await queue.DeleteQueueAsync();
                await AssertEmptyQueueAsync(queue);

                using (var metrics = new InMemoryMetricsClient()) {
                    queue.AttachBehavior(new MetricsQueueBehavior<SimpleWorkItem>(metrics));

                    Task.Run(async () => {
                                 for (int index = 0; index < iterations; index++) {
                                     await SystemClock.SleepAsync(RandomData.GetInt(10, 30));
                                     await queue.EnqueueAsync(new SimpleWorkItem { Data = "Hello" });
                                 }
                                 _logger.Trace("Done enqueuing.");
                             });

                    _logger.Trace("Starting dequeue loop.");
                    var sw = Stopwatch.StartNew();
                    for (int index = 0; index < iterations; index++) {
                        var item = await queue.DequeueAsync(TimeSpan.FromSeconds(3));
                        Assert.NotNull(item);
                        await item.CompleteAsync();
                    }
                    sw.Stop();

                    Assert.InRange(sw.ElapsedMilliseconds, iterations * 10, iterations * 50);
                    var timing = await metrics.GetTimerStatsAsync("simpleworkitem.queuetime");
                    Assert.InRange(timing.AverageDuration, 0, 25);
                }
            }
        }