public void CanRunMultipleQueueJobs()
{
const int jobCount = 5;
const int workItemCount = 1000;
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(100), _writer);
var queues = new List<RedisQueue<SampleQueueWorkItem>>();
for (int i = 0; i < jobCount; i++)
{
var q = new RedisQueue<SampleQueueWorkItem>(SharedConnection.GetMuxer(), retries: 3, retryDelay: TimeSpan.FromSeconds(1));
q.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
queues.Add(q);
}
Task.Run(() =>
{
Parallel.For(0, workItemCount, i => {
var queue = queues[RandomData.GetInt(0, 4)];
queue.Enqueue(new SampleQueueWorkItem { Created = DateTime.Now, Path = RandomData.GetString() });
});
});
Parallel.For(0, jobCount, index =>
{
var queue = queues[index];
var job = new SampleQueueJob(queue, metrics);
job.RunUntilEmpty();
});
metrics.DisplayStats(_writer);
}
public virtual async Task CanRunQueueJob() {
const int workItemCount = 100;
var metrics = new InMemoryMetricsClient();
var queue = GetSampleWorkItemQueue(retries: 0, retryDelay: TimeSpan.Zero);
await queue.DeleteQueueAsync();
queue.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
metrics.StartDisplayingStats(TimeSpan.FromSeconds(1), _writer);
var enqueueTask = Run.InParallel(workItemCount, async index => {
await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = DateTime.Now,
Path = "somepath" + index
});
});
var job = new SampleQueueJob(queue, metrics);
await Task.Delay(10);
await Task.WhenAll(job.RunUntilEmptyAsync(), enqueueTask);
metrics.DisplayStats(_writer);
var stats = await queue.GetQueueStatsAsync();
Assert.Equal(0, stats.Queued);
Assert.Equal(workItemCount, stats.Enqueued);
Assert.Equal(workItemCount, stats.Dequeued);
}
public void CanDisplayStatsMultithreaded() {
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(10), _writer);
Parallel.For(0, 100, i => {
metrics.Counter("Test");
Thread.Sleep(50);
});
}
public async Task CanWaitForCounter() {
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(50), _writer);
Task.Run(async () => {
await Task.Delay(50);
await metrics.CounterAsync("Test");
await metrics.CounterAsync("Test");
});
var success = await metrics.WaitForCounterAsync("Test", 2, TimeSpan.FromMilliseconds(500));
Assert.True(success);
Task.Run(async () => {
await Task.Delay(50);
await metrics.CounterAsync("Test");
});
success = await metrics.WaitForCounterAsync("Test", timeout: TimeSpan.FromMilliseconds(500));
Assert.True(success);
success = await metrics.WaitForCounterAsync("Test", timeout: TimeSpan.FromMilliseconds(100));
Assert.False(success);
Task.Run(async () => {
await Task.Delay(50);
await metrics.CounterAsync("Test", 2);
});
success = await metrics.WaitForCounterAsync("Test", 2, TimeSpan.FromMilliseconds(500));
Assert.True(success);
success = await metrics.WaitForCounterAsync("Test", async () => await metrics.CounterAsync("Test"), cancellationToken: TimeSpan.FromMilliseconds(500).ToCancellationToken());
Assert.True(success);
Task.Run(async () => {
await Task.Delay(50);
await metrics.CounterAsync("Test");
});
success = await metrics.WaitForCounterAsync("Test", timeout: TimeSpan.FromMilliseconds(500));
Assert.True(success);
metrics.DisplayStats(_writer);
}
public virtual async Task CanRunMultipleQueueJobs() {
const int jobCount = 5;
const int workItemCount = 100;
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromSeconds(1), _writer);
var queues = new List<IQueue<SampleQueueWorkItem>>();
for (int i = 0; i < jobCount; i++) {
var q = GetSampleWorkItemQueue(retries: 3, retryDelay: TimeSpan.FromSeconds(1));
await q.DeleteQueueAsync();
q.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
queues.Add(q);
}
var enqueueTask = Run.InParallel(workItemCount, async index => {
var queue = queues[RandomData.GetInt(0, jobCount - 1)];
await queue.EnqueueAsync(new SampleQueueWorkItem {
Created = DateTime.Now,
Path = RandomData.GetString()
});
});
var cancellationTokenSource = new CancellationTokenSource();
await Run.InParallel(jobCount, async index => {
var queue = queues[index - 1];
var job = new SampleQueueJob(queue, metrics);
await job.RunUntilEmptyAsync(cancellationTokenSource.Token);
cancellationTokenSource.Cancel();
});
await enqueueTask;
var queueStats = new List<QueueStats>();
for (int i = 0; i < queues.Count; i++) {
var stats = await queues[i].GetQueueStatsAsync();
Logger.Info().Message($"Queue#{i}: Working: {stats.Working} Completed: {stats.Completed} Abandoned: {stats.Abandoned} Error: {stats.Errors} Deadletter: {stats.Deadletter}").Write();
queueStats.Add(stats);
}
metrics.DisplayStats(_writer);
Assert.Equal(metrics.GetCount("completed"), queueStats.Sum(s => s.Completed));
Assert.InRange(queueStats.Sum(s => s.Completed), 0, workItemCount);
}
public void CanRunQueueJob()
{
const int workItemCount = 10000;
var metrics = new InMemoryMetricsClient();
var queue = new RedisQueue<SampleQueueWorkItem>(SharedConnection.GetMuxer(), null, null, 0, TimeSpan.Zero);
queue.AttachBehavior(new MetricsQueueBehavior<SampleQueueWorkItem>(metrics, "test"));
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(100), _writer);
Task.Factory.StartNew(() => {
Parallel.For(0, workItemCount, i => {
queue.Enqueue(new SampleQueueWorkItem { Created = DateTime.Now, Path = "somepath" + i });
});
});
var job = new SampleQueueJob(queue, metrics);
job.RunUntilEmpty();
metrics.DisplayStats(_writer);
Assert.Equal(0, queue.GetQueueStats().Queued);
}
public async void CanWaitForCounter() {
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(50), _writer);
Task.Run(() => {
Thread.Sleep(50);
metrics.Counter("Test");
metrics.Counter("Test");
});
var success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(500), 2);
Assert.True(success);
Task.Run(() => {
Thread.Sleep(50);
metrics.Counter("Test");
});
success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(500));
Assert.True(success);
success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(100));
Assert.False(success);
Task.Run(() => {
Thread.Sleep(50);
metrics.Counter("Test", 2);
});
success = await metrics.WaitForCounterAsync("Test", TimeSpan.FromMilliseconds(500), 2);
Assert.True(success);
success = await metrics.WaitForCounterAsync("Test", () => metrics.CounterAsync("Test"), TimeSpan.FromMilliseconds(500));
Assert.True(success);
Task.Run(() => {
Thread.Sleep(50);
metrics.Counter("Test");
});
success = metrics.WaitForCounter("Test", TimeSpan.FromMilliseconds(500));
Assert.True(success);
metrics.DisplayStats(_writer);
}
public async Task CanDisplayStatsMultithreaded() {
var metrics = new InMemoryMetricsClient();
metrics.StartDisplayingStats(TimeSpan.FromMilliseconds(10), _writer);
await Run.InParallel(100, async i => {
await metrics.CounterAsync("Test");
await Task.Delay(50);
});
}
public void RegisterServices(Container container) {
// Foundation service provider
ServiceProvider.Current = container;
container.RegisterSingle<IDependencyResolver>(() => new SimpleInjectorCoreDependencyResolver(container));
JsonConvert.DefaultSettings = () => new JsonSerializerSettings {
DateParseHandling = DateParseHandling.DateTimeOffset
};
var contractResolver = new ExceptionlessContractResolver();
contractResolver.UseDefaultResolverFor(typeof(DataDictionary), typeof(SettingsDictionary), typeof(VersionOne.VersionOneWebHookStack), typeof(VersionOne.VersionOneWebHookEvent));
var settings = new JsonSerializerSettings {
MissingMemberHandling = MissingMemberHandling.Ignore,
DateParseHandling = DateParseHandling.DateTimeOffset,
ContractResolver = contractResolver
};
settings.AddModelConverters();
container.RegisterSingle<IContractResolver>(() => contractResolver);
container.RegisterSingle<JsonSerializerSettings>(settings);
container.RegisterSingle<JsonSerializer>(JsonSerializer.Create(settings));
container.RegisterSingle<ISerializer>(() => new JsonNetSerializer(settings));
var metricsClient = new InMemoryMetricsClient();
metricsClient.StartDisplayingStats();
container.RegisterSingle<IMetricsClient>(metricsClient);
container.RegisterSingle<IElasticClient>(() => container.GetInstance<ElasticSearchConfiguration>().GetClient(Settings.Current.ElasticSearchConnectionString.Split(',').Select(url => new Uri(url))));
container.RegisterSingle<ICacheClient, InMemoryCacheClient>();
container.RegisterSingle<IQueue<EventPost>>(() => new InMemoryQueue<EventPost>(statName: MetricNames.PostsQueueSize, metrics: container.GetInstance<IMetricsClient>()));
container.RegisterSingle<IQueue<EventUserDescription>>(() => new InMemoryQueue<EventUserDescription>(statName: MetricNames.EventsUserDescriptionQueueSize, metrics: container.GetInstance<IMetricsClient>()));
container.RegisterSingle<IQueue<EventNotificationWorkItem>>(() => new InMemoryQueue<EventNotificationWorkItem>(statName: MetricNames.EventNotificationQueueSize, metrics: container.GetInstance<IMetricsClient>()));
container.RegisterSingle<IQueue<WebHookNotification>>(() => new InMemoryQueue<WebHookNotification>(statName: MetricNames.WebHookQueueSize, metrics: container.GetInstance<IMetricsClient>()));
container.RegisterSingle<IQueue<MailMessage>>(() => new InMemoryQueue<MailMessage>(statName: MetricNames.EmailsQueueSize, metrics: container.GetInstance<IMetricsClient>()));
container.RegisterSingle<IQueue<StatusMessage>>(() => new InMemoryQueue<StatusMessage>());
var workItemHandlers = new WorkItemHandlers();
workItemHandlers.Register<ReindexWorkItem, ReindexWorkItemHandler>();
container.RegisterSingle<WorkItemHandlers>(workItemHandlers);
container.RegisterSingle<IQueue<WorkItemData>>(() => new InMemoryQueue<WorkItemData>(statName: MetricNames.WorkItemQueueSize, metrics: container.GetInstance<IMetricsClient>(), workItemTimeout: TimeSpan.FromHours(1)));
container.RegisterSingle<IMessageBus, InMemoryMessageBus>();
container.RegisterSingle<IMessagePublisher>(container.GetInstance<IMessageBus>);
container.RegisterSingle<IMessageSubscriber>(container.GetInstance<IMessageBus>);
if (!String.IsNullOrEmpty(Settings.Current.StorageFolder))
container.RegisterSingle<IFileStorage>(new FolderFileStorage(Settings.Current.StorageFolder));
else
container.RegisterSingle<IFileStorage>(new InMemoryFileStorage());
container.RegisterSingle<IStackRepository, StackRepository>();
container.RegisterSingle<IEventRepository, EventRepository>();
container.RegisterSingle<IOrganizationRepository, OrganizationRepository>();
container.RegisterSingle<IProjectRepository, ProjectRepository>();
container.RegisterSingle<IUserRepository, UserRepository>();
container.RegisterSingle<IWebHookRepository, WebHookRepository>();
container.RegisterSingle<ITokenRepository, TokenRepository>();
container.RegisterSingle<IApplicationRepository, ApplicationRepository>();
container.RegisterSingle<IGeoIPResolver, MindMaxGeoIPResolver>();
container.RegisterSingle<IValidator<Application>, ApplicationValidator>();
container.RegisterSingle<IValidator<Organization>, OrganizationValidator>();
container.RegisterSingle<IValidator<PersistentEvent>, PersistentEventValidator>();
container.RegisterSingle<IValidator<Project>, ProjectValidator>();
container.RegisterSingle<IValidator<Stack>, StackValidator>();
container.RegisterSingle<IValidator<Models.Token>, TokenValidator>();
container.RegisterSingle<IValidator<UserDescription>, UserDescriptionValidator>();
container.RegisterSingle<IValidator<User>, UserValidator>();
container.RegisterSingle<IValidator<WebHook>, WebHookValidator>();
container.RegisterSingle<IEmailGenerator>(() => new RazorEmailGenerator(@"Mail\Templates"));
container.RegisterSingle<IMailer, Mailer>();
if (Settings.Current.WebsiteMode != WebsiteMode.Dev)
container.RegisterSingle<IMailSender, SmtpMailSender>();
else
container.RegisterSingle<IMailSender>(() => new InMemoryMailSender());
container.Register<ILockProvider, CacheLockProvider>();
container.Register<StripeEventHandler>();
container.RegisterSingle<BillingManager>();
container.RegisterSingle<DataHelper>();
container.RegisterSingle<EventStats>();
container.RegisterSingle<EventPipeline>();
container.RegisterSingle<EventPluginManager>();
container.RegisterSingle<FormattingPluginManager>();
container.RegisterSingle<ICoreLastReferenceIdManager, NullCoreLastReferenceIdManager>();
}