private void CreateScheduler()
{
lock (_startup)
{
if (_consumer != null)
{
return;
}
_container = new JobSchedulerContainer(container =>
container.Register(() => _timeFactory, LifeStyles.Singleton));
_scheduler = _container.CreateJobScheduler();
_scheduler.Start();
_consumerContainer = new SchedulerContainer();
_consumerScheduler = _consumerContainer.CreateTaskScheduler();
_taskFactory = _consumerContainer.CreateTaskFactory(_consumerScheduler);
_taskFactory = _consumerContainer.CreateTaskFactory(_consumerScheduler);
_taskFactory.Scheduler.Configuration.MaximumThreads = _configuration.ThreadsMax;
_taskFactory.Scheduler.Configuration.MaxQueueSize = _configuration.QueueMax;
_taskFactory.Scheduler.Configuration.WaitForThreadPoolToFinish =
_configuration.WaitForThreadPoolToFinish;
_taskFactory.Scheduler.Start();
_queueContainer = new QueueContainer <MemoryMessageQueueInit>();
_consumer = _queueContainer.CreateConsumerMethodQueueScheduler(QueueName, Connection,
_taskFactory);
_consumer.Start();
}
}
public void Create_CreateTaskFactory2()
{
using (var test = new SchedulerContainer())
{
Assert.NotNull(test.CreateTaskFactory(test.CreateTaskScheduler()));
}
}
public void Create_CreateTaskFactory2()
{
using (var test = new SchedulerContainer())
{
Assert.NotNull(test.CreateTaskFactory(test.CreateTaskScheduler()));
}
}
/// <summary>
/// 停止所有执行任务，并清空执行状态表
/// </summary>
public void StopAllTask()
{
foreach (var context in AppDomainContextDic.Values)
{
RemoteAction.Invoke(context.Domain, () => { SchedulerContainer.GetContainerInstance().StopSchedule(); });
}
_taskRunLogRespository.ClearTaskRunLog(); // 清空执行状态表
}
public void RunTest <TTransportInit, TMessage>(string queueName,
string connectionString,
bool addInterceptors,
int messageCount,
ILogProvider logProvider,
Func <QueueProducerConfiguration, AdditionalMessageData> generateData,
Action <string, string, QueueProducerConfiguration, long, string, ICreationScope> verify,
bool sendViaBatch,
List <string> routes1,
List <string> routes2,
int runTime,
int timeOut,
int readerCount,
TimeSpan heartBeatTime,
TimeSpan heartBeatMonitorTime,
ICreationScope scope,
string updateTime, bool enableChaos)
where TTransportInit : ITransportInit, new()
where TMessage : class
{
//add data with routes - generate data per route passed in
Parallel.ForEach(routes1, route =>
{
RunTest <TTransportInit, TMessage>(queueName, connectionString, addInterceptors,
messageCount, logProvider, generateData, verify, sendViaBatch, route, scope, false);
});
Parallel.ForEach(routes2, route =>
{
RunTest <TTransportInit, TMessage>(queueName, connectionString, addInterceptors,
messageCount, logProvider, generateData, verify, sendViaBatch, route, scope, false);
});
//run a consumer for each route
using (var schedulerCreator = new SchedulerContainer())
{
var taskScheduler = schedulerCreator.CreateTaskScheduler();
taskScheduler.Configuration.MaximumThreads = routes1.Count + routes2.Count;
taskScheduler.Start();
var taskFactory = schedulerCreator.CreateTaskFactory(taskScheduler);
//spin up and process each route
var running = new List <List <string> > {
routes1, routes2
};
Parallel.ForEach(running, route =>
{
var consumer = new ConsumerAsyncShared <TMessage> {
Factory = taskFactory
};
consumer.RunConsumer <TTransportInit>(queueName, connectionString, addInterceptors,
logProvider, runTime, messageCount, timeOut, readerCount, heartBeatTime, heartBeatMonitorTime, updateTime, enableChaos, route);
});
}
}
public void Create_TaskScheduler()
{
using (var test = new SchedulerContainer())
{
using (test.CreateTaskScheduler())
{
}
}
}
public void PurgeErrorMessages <TTransportInit>(QueueConnection queueConnection,
bool addInterceptors, ILogger logProvider, bool actuallyPurge, ICreationScope scope)
where TTransportInit : ITransportInit, new()
{
using (var trace = SharedSetup.CreateTrace("consumer-error"))
{
using (var metrics = new Metrics.Metrics(queueConnection.Queue))
{
var addInterceptorConsumer = InterceptorAdding.No;
if (addInterceptors)
{
addInterceptorConsumer = InterceptorAdding.ConfigurationOnly;
}
var processedCount = new IncrementWrapper();
using (
var creator = SharedSetup.CreateCreator <TTransportInit>(addInterceptorConsumer, logProvider,
metrics, false, false, scope, trace.Source)
)
{
using (var schedulerCreator =
new SchedulerContainer(
// ReSharper disable once AccessToDisposedClosure
serviceRegister => serviceRegister.Register(() => metrics, LifeStyles.Singleton).RegisterNonScopedSingleton(trace.Source),
options => SharedSetup.SetOptions(options, false)))
{
using (var taskScheduler = schedulerCreator.CreateTaskScheduler())
{
taskScheduler.Start();
var taskFactory = schedulerCreator.CreateTaskFactory(taskScheduler);
using (
var queue =
creator
.CreateConsumerQueueScheduler(
queueConnection, taskFactory))
{
SharedSetup.SetupDefaultConsumerQueueErrorPurge(queue.Configuration, actuallyPurge);
SharedSetup.SetupDefaultErrorRetry(queue.Configuration);
var waitForFinish = new ManualResetEventSlim(false);
waitForFinish.Reset();
//start looking for work
queue.Start <TMessage>((message, notifications) =>
throw new Exception("There should have been no data to process"));
//wait for 30 seconds
waitForFinish.Wait(15000);
}
}
}
}
}
}
}
public void Create_TaskScheduler()
{
using (var test = new SchedulerContainer())
{
using (test.CreateTaskScheduler())
{
}
}
}
public static ITaskFactory CreateFactory(int maxThreads, int maxQueueSize, out SchedulerContainer schedulerCreator)
{
schedulerCreator = new SchedulerContainer();
var taskScheduler = schedulerCreator.CreateTaskScheduler();
taskScheduler.Configuration.MaximumThreads = maxThreads;
taskScheduler.Start();
return(schedulerCreator.CreateTaskFactory(taskScheduler));
}
/// <summary>
/// 开始所有执行的任务，并且结合配置执行状态表数据
/// </summary>
public void StartAllScheduler()
{
GetUseableJobconfigs(); //读取任务配置
foreach (var configs in ConfigDic.Values)
{
foreach (var config in configs)
{
var runlog = new TaskRunLog
{
Pcip = Computer.IpAddress,
Pcmac = Computer.MacAddress,
Pcname = Computer.ComputerName,
Taskid = config.Taskid,
Taskname = config.Taskname,
Taskremark = config.TaskRemark,
Taskserverid = _runnerConfig.TaskServerId,
Taskstatus = 0,
Tasknextruntime = DateTime.Now,
Tasklastruntime = DateTime.Now
};
_taskRunLogRespository.SaveTaskRunLog(runlog);
}
}
#region 根据模块创建新的appdomain,并开始任务
foreach (var config in ConfigDic)
{
var setup = new AppDomainSetup
{
ApplicationName = $"JobLoader_{config.Key}",
ApplicationBase = FileHelper.GetRootPath(),
PrivateBinPath = "ZilLionTask",
CachePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "JobCachePath"),
ShadowCopyFiles = "true"
};
setup.ShadowCopyDirectories = string.Concat(setup.ApplicationBase, ";", setup.PrivateBinPath);
var remoteDomain = AppDomain.CreateDomain($"{config.Key}Domain_{Guid.NewGuid():N}", null, setup);
var context = AppDomainContext.Wrap(remoteDomain);
if (!AppDomainContextDic.ContainsKey(config.Key))
{
AppDomainContextDic.Add(config.Key, context);
}
RemoteAction.Invoke(context.Domain, config.Key, config.Value, _runnerConfig, (k, v, runnerconfig) =>
{
var container = SchedulerContainer.GetContainerInstance();
container.InitScheduler(k, v, runnerconfig);
container.StartScheduler();
});
}
#endregion
}
protected void Application_Start()
{
AreaRegistration.RegisterAllAreas();
FilterConfig.RegisterGlobalFilters(GlobalFilters.Filters);
RouteConfig.RegisterRoutes(RouteTable.Routes);
BundleConfig.RegisterBundles(BundleTable.Bundles);
//Scheduler
SchedulerContainer scheduler = new SchedulerContainer();
scheduler.RunJob();
DevExtremeBundleConfig.RegisterBundles(BundleTable.Bundles);
}
public void Create_Null_Services_Fails()
{
using (var test = new SchedulerContainer(null))
{
Assert.Throws <NullReferenceException>(
delegate
{
// ReSharper disable once AccessToDisposedClosure
test.CreateTaskFactory();
});
}
}
public void Create_Null_Services_Fails()
{
using (var test = new SchedulerContainer(null))
{
Assert.Throws<NullReferenceException>(
delegate
{
// ReSharper disable once AccessToDisposedClosure
test.CreateTaskFactory();
});
}
}
/// <summary>
/// 根据ID立即执行
/// </summary>
/// <param name="config"></param>
public void RunOnceTaskById(TaskConfig config)
{
if (!AppDomainContextDic.ContainsKey(config.TaskModule))
{
return;
}
var context = AppDomainContextDic[config.TaskModule];
RemoteAction.Invoke(context.Domain, config.Taskid,
taskid => { SchedulerContainer.GetContainerInstance().RunOnceTask(taskid); });
var runlog = _taskRunLogRespository.GetTaskRunLogById(config.Taskid);
runlog.Tasklastruntime = DateTime.Now;
_taskRunLogRespository.SaveTaskRunLog(runlog);
}
public void RunTest <TTransportInit, TMessage>(QueueConnection queueConnection,
bool addInterceptors,
int messageCount,
ILogger logProvider,
Func <QueueProducerConfiguration, int, AdditionalMessageData> generateData,
Action <QueueConnection, QueueProducerConfiguration, long, int, ICreationScope> verify,
bool sendViaBatch,
List <int> userValues,
int runTime,
int timeOut,
int readerCount,
TimeSpan heartBeatTime,
TimeSpan heartBeatMonitorTime,
ICreationScope scope,
string updateTime, bool enableChaos,
Action <QueueConsumerConfiguration, int> setQueueOptions)
where TTransportInit : ITransportInit, new()
where TMessage : class
{
//add data with user column
Parallel.ForEach(userValues, userColumn =>
{
RunTest <TTransportInit, TMessage>(queueConnection, addInterceptors,
messageCount, logProvider, generateData, verify, sendViaBatch, userColumn, scope, false);
});
//run a consumer for each data value
using (var schedulerCreator = new SchedulerContainer())
{
var taskScheduler = schedulerCreator.CreateTaskScheduler();
taskScheduler.Configuration.MaximumThreads = userValues.Count * 2;
taskScheduler.Start();
var taskFactory = schedulerCreator.CreateTaskFactory(taskScheduler);
//spin up and process each value
Parallel.ForEach(userValues, userColumn =>
{
var consumer = new ConsumerAsyncShared <TMessage> {
Factory = taskFactory
};
consumer.RunConsumer <TTransportInit>(queueConnection, addInterceptors,
logProvider, runTime, messageCount, timeOut, readerCount, heartBeatTime, heartBeatMonitorTime, updateTime, enableChaos, scope, null, (g) => setQueueOptions(g, userColumn));
});
}
}
/// <summary>
/// 删除指定id任务
/// </summary>
/// <param name="config"></param>
public void DeleteById(TaskConfig config)
{
if (!AppDomainContextDic.ContainsKey(config.TaskModule))
{
return;
}
var context = AppDomainContextDic[config.TaskModule];
RemoteAction.Invoke(context.Domain, config.Taskid,
taskid => { SchedulerContainer.GetContainerInstance().DeleteJob(taskid); });
config.IsDeleted = 1;
_taskConfigRespository.SaveData(config);
var runlog = _taskRunLogRespository.GetTaskRunLogById(config.Taskid);
_taskRunLogRespository.RemoveTaskRunLog(runlog);
}
public void PurgeErrorMessages <TTransportInit>(QueueConnection queueConnection,
bool addInterceptors, ILogger logProvider, bool actuallyPurge, ICreationScope scope)
where TTransportInit : ITransportInit, new()
{
using (var trace = SharedSetup.CreateTrace("consumer-error"))
{
using (var metrics = new Metrics.Metrics(queueConnection.Queue))
{
var addInterceptorConsumer = InterceptorAdding.No;
if (addInterceptors)
{
addInterceptorConsumer = InterceptorAdding.ConfigurationOnly;
}
using (
var creator = SharedSetup.CreateCreator <TTransportInit>(addInterceptorConsumer, logProvider,
metrics, false, false, scope, trace.Source)
)
{
using (var schedulerCreator =
new SchedulerContainer(
// ReSharper disable once AccessToDisposedClosure
serviceRegister => serviceRegister.Register(() => metrics, LifeStyles.Singleton).RegisterNonScopedSingleton(trace.Source)))
{
using (var taskScheduler = schedulerCreator.CreateTaskScheduler())
{
taskScheduler.Start();
var taskFactory = schedulerCreator.CreateTaskFactory(taskScheduler);
using (
var queue =
creator
.CreateConsumerMethodQueueScheduler(
queueConnection, taskFactory))
{
SharedSetup.SetupDefaultConsumerQueueErrorPurge(queue.Configuration, actuallyPurge);
SharedSetup.SetupDefaultErrorRetry(queue.Configuration);
queue.Start();
Thread.Sleep(15000);
}
}
}
}
}
}
}
public void Run(int messageCount, int runtime, int timeOut, int workerCount, int readerCount, int queueSize,
bool useTransactions, int messageType)
{
SchedulerContainer schedulerContainer = null;
if (Factory == null)
{
Factory = CreateFactory(workerCount, queueSize, out schedulerContainer);
}
var queueName = GenerateQueueName.Create();
var logProvider = LoggerShared.Create(queueName, GetType().Name);
using (var queueCreator =
new QueueCreationContainer <SqlServerMessageQueueInit>(
serviceRegister => serviceRegister.Register(() => logProvider, LifeStyles.Singleton)))
{
try
{
using (
var oCreation =
queueCreator.GetQueueCreation <SqlServerMessageQueueCreation>(queueName,
ConnectionInfo.ConnectionString)
)
{
oCreation.Options.EnableDelayedProcessing = true;
oCreation.Options.EnableHeartBeat = !useTransactions;
oCreation.Options.EnableHoldTransactionUntilMessageCommitted = useTransactions;
oCreation.Options.EnableStatus = !useTransactions;
oCreation.Options.EnableStatusTable = true;
var result = oCreation.CreateQueue();
Assert.True(result.Success, result.ErrorMessage);
if (messageType == 1)
{
var producer = new ProducerAsyncShared();
producer.RunTestAsync <SqlServerMessageQueueInit, FakeMessage>(queueName,
ConnectionInfo.ConnectionString, false, messageCount, logProvider, Helpers.GenerateData,
Helpers.Verify, false, oCreation.Scope).Wait(timeOut / 2 * 1000);
var consumer = new ConsumerAsyncShared <FakeMessage> {
Factory = Factory
};
consumer.RunConsumer <SqlServerMessageQueueInit>(queueName, ConnectionInfo.ConnectionString,
false, logProvider,
runtime, messageCount,
timeOut, readerCount, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(12), "second(*%3)");
}
else if (messageType == 2)
{
var producer = new ProducerAsyncShared();
producer.RunTestAsync <SqlServerMessageQueueInit, FakeMessageA>(queueName,
ConnectionInfo.ConnectionString, false, messageCount, logProvider, Helpers.GenerateData,
Helpers.Verify, false, oCreation.Scope).Wait(timeOut / 2 * 1000);
var consumer = new ConsumerAsyncShared <FakeMessageA> {
Factory = Factory
};
consumer.RunConsumer <SqlServerMessageQueueInit>(queueName, ConnectionInfo.ConnectionString,
false, logProvider,
runtime, messageCount,
timeOut, readerCount, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(12), "second(*%3)");
}
else if (messageType == 3)
{
var producer = new ProducerAsyncShared();
producer.RunTestAsync <SqlServerMessageQueueInit, FakeMessageB>(queueName,
ConnectionInfo.ConnectionString, false, messageCount, logProvider, Helpers.GenerateData,
Helpers.Verify, false, oCreation.Scope).Wait(timeOut / 2 * 1000);
var consumer = new ConsumerAsyncShared <FakeMessageB> {
Factory = Factory
};
consumer.RunConsumer <SqlServerMessageQueueInit>(queueName, ConnectionInfo.ConnectionString,
false, logProvider,
runtime, messageCount,
timeOut, readerCount, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(12), "second(*%10)");
}
new VerifyQueueRecordCount(queueName, oCreation.Options).Verify(0, false, false);
}
}
finally
{
schedulerContainer?.Dispose();
using (
var oCreation =
queueCreator.GetQueueCreation <SqlServerMessageQueueCreation>(queueName,
ConnectionInfo.ConnectionString)
)
{
oCreation.RemoveQueue();
}
}
}
}
public void RunConsumer <TTransportInit>(string queueName,
string connectionString,
bool addInterceptors,
int workerCount,
ILogProvider logProvider,
int timeOut,
int readerCount,
int queueSize,
long messageCount,
TimeSpan heartBeatTime,
TimeSpan heartBeatMonitorTime,
string updatetime)
where TTransportInit : ITransportInit, new()
{
using (var metrics = new Metrics.Metrics(queueName))
{
var addInterceptorConsumer = InterceptorAdding.No;
if (addInterceptors)
{
addInterceptorConsumer = InterceptorAdding.ConfigurationOnly;
}
using (
var creator = SharedSetup.CreateCreator <TTransportInit>(addInterceptorConsumer, logProvider, metrics,
true))
{
using (var schedulerCreator = new SchedulerContainer())
{
using (var taskScheduler = schedulerCreator.CreateTaskScheduler())
{
taskScheduler.Configuration.MaximumThreads = workerCount;
taskScheduler.Configuration.MaxQueueSize = queueSize;
taskScheduler.Start();
var taskFactory = schedulerCreator.CreateTaskFactory(taskScheduler);
using (
var queue =
creator
.CreateConsumerMethodQueueScheduler(
queueName, connectionString, taskFactory))
{
SharedSetup.SetupDefaultConsumerQueue(queue.Configuration, readerCount, heartBeatTime,
heartBeatMonitorTime, updatetime, null);
queue.Start();
for (var i = 0; i < timeOut; i++)
{
if (VerifyMetrics.GetPoisonMessageCount(metrics.GetCurrentMetrics()) == messageCount)
{
break;
}
Thread.Sleep(1000);
}
//wait for last error to be saved if needed.
Thread.Sleep(3000);
}
}
VerifyMetrics.VerifyPoisonMessageCount(queueName, metrics.GetCurrentMetrics(), messageCount);
}
}
}
}
public HomeController(JobInfoServices jobInfoService,
SchedulerContainer schedulerContainer)
{
_jobInfoService = jobInfoService;
_schedulerContainer = schedulerContainer;
}
public void RunConsumer <TTransportInit>(string queueName, string connectionString, bool addInterceptors,
ILogProvider logProvider,
int messageCount, int workerCount, int timeOut,
int queueSize, int readerCount,
TimeSpan heartBeatTime, TimeSpan heartBeatMonitorTime,
string updateTime,
string route,
bool enableChaos)
where TTransportInit : ITransportInit, new()
{
if (enableChaos)
{
timeOut *= 2;
}
using (var metrics = new Metrics.Metrics(queueName))
{
var addInterceptorConsumer = InterceptorAdding.No;
if (addInterceptors)
{
addInterceptorConsumer = InterceptorAdding.ConfigurationOnly;
}
var processedCount = new IncrementWrapper();
using (
var creator = SharedSetup.CreateCreator <TTransportInit>(addInterceptorConsumer, logProvider, metrics, false, enableChaos)
)
{
using (var schedulerCreator =
new SchedulerContainer(
// ReSharper disable once AccessToDisposedClosure
serviceRegister => serviceRegister.Register(() => metrics, LifeStyles.Singleton), options => SharedSetup.SetOptions(options, enableChaos)))
{
bool rollBacks;
using (var taskScheduler = schedulerCreator.CreateTaskScheduler())
{
taskScheduler.Configuration.MaximumThreads = workerCount;
taskScheduler.Configuration.MaxQueueSize = queueSize;
taskScheduler.Start();
var taskFactory = schedulerCreator.CreateTaskFactory(taskScheduler);
using (
var queue =
creator
.CreateConsumerQueueScheduler(
queueName, connectionString, taskFactory))
{
rollBacks = queue.Configuration.TransportConfiguration.MessageRollbackSupported;
SharedSetup.SetupDefaultConsumerQueue(queue.Configuration, readerCount, heartBeatTime,
heartBeatMonitorTime, updateTime, route);
SharedSetup.SetupDefaultErrorRetry(queue.Configuration);
var waitForFinish = new ManualResetEventSlim(false);
waitForFinish.Reset();
//start looking for work
queue.Start <TMessage>((message, notifications) =>
{
MessageHandlingShared.HandleFakeMessagesError(processedCount, waitForFinish,
messageCount, message);
});
waitForFinish.Wait(timeOut * 1000);
//wait for last error to be saved if needed.
Thread.Sleep(3000);
}
}
if (rollBacks)
{
VerifyMetrics.VerifyRollBackCount(queueName, metrics.GetCurrentMetrics(), messageCount, 2, 2);
}
}
}
}
}
public static ITaskFactory CreateFactory(int maxThreads, int maxQueueSize, out SchedulerContainer schedulerCreator)
{
schedulerCreator = new SchedulerContainer();
var taskScheduler = schedulerCreator.CreateTaskScheduler();
taskScheduler.Configuration.MaximumThreads = maxThreads;
taskScheduler.Configuration.MaxQueueSize = maxQueueSize;
taskScheduler.Start();
return schedulerCreator.CreateTaskFactory(taskScheduler);
}
static void Main(string[] args)
{
//we are using serilog for sample purposes
var log = new LoggerConfiguration()
.WriteTo.Console()
.MinimumLevel.Debug()
.CreateLogger();
Log.Logger = log;
log.Information("Startup");
log.Information(SharedConfiguration.AllSettings);
//verify that the queue exists
var fileLocation = Path.Combine(Environment.ExpandEnvironmentVariables("%userprofile%"), "Documents");
var queueName = ConfigurationManager.AppSettings.ReadSetting("QueueName");
var connectionString = $"Filename={fileLocation}{ConfigurationManager.AppSettings.ReadSetting("Database")};Connection=shared;";
var queueConnection = new QueueConnection(queueName, connectionString);
using (var createQueueContainer = new QueueCreationContainer <LiteDbMessageQueueInit>(serviceRegister =>
Injectors.AddInjectors(new SerilogAdapter(log), SharedConfiguration.EnableTrace, SharedConfiguration.EnableMetrics, SharedConfiguration.EnableCompression, SharedConfiguration.EnableEncryption, "LiteDbSchedulerConsumer", serviceRegister),
options => Injectors.SetOptions(options, SharedConfiguration.EnableChaos)))
{
using (var createQueue =
createQueueContainer.GetQueueCreation <LiteDbMessageQueueCreation>(queueConnection))
{
if (!createQueue.QueueExists)
{
//the consumer can't do anything if the queue hasn't been created
Log.Error($"Could not find {connectionString}. Verify that you have run the producer, which will create the queue");
return;
}
}
}
using (var schedulerContainer = new SchedulerContainer(serviceRegister =>
Injectors.AddInjectors(new SerilogAdapter(log), SharedConfiguration.EnableTrace, SharedConfiguration.EnableMetrics, SharedConfiguration.EnableCompression, SharedConfiguration.EnableEncryption, "LiteDbSchedulerConsumer", serviceRegister),
options => Injectors.SetOptions(options, SharedConfiguration.EnableChaos)))
{
using (var scheduler = schedulerContainer.CreateTaskScheduler())
{
var factory = schedulerContainer.CreateTaskFactory(scheduler);
factory.Scheduler.Configuration.MaximumThreads = 8; //8 background threads
factory.Scheduler.Configuration.MaxQueueSize = 1; //allow work to be de-queued but held in memory until a thread is free
//note - the same factory can be passed to multiple queue instances - don't dispose the scheduler container until all queues have finished
factory.Scheduler.Start(); //the scheduler must be started before passing it to a queue
using (var queueContainer = new QueueContainer <LiteDbMessageQueueInit>(serviceRegister =>
Injectors.AddInjectors(new SerilogAdapter(log), SharedConfiguration.EnableTrace, SharedConfiguration.EnableMetrics, SharedConfiguration.EnableCompression, SharedConfiguration.EnableEncryption, "LiteDbSchedulerConsumer", serviceRegister),
options => Injectors.SetOptions(options, SharedConfiguration.EnableChaos)))
{
using (var queue = queueContainer.CreateConsumerMethodQueueScheduler(queueConnection, factory))
{
//set some processing options and start looking for work
//in the async model, the worker count is how many threads are querying the queue - the scheduler runs the work
queue.Configuration.Worker.WorkerCount = 1; //lets just run 1 thread that queries the database
queue.Configuration.HeartBeat.UpdateTime = "sec(*%10)"; //set a heartbeat every 10 seconds
queue.Configuration.HeartBeat.MonitorTime =
TimeSpan.FromSeconds(15); //check for dead records every 15 seconds
queue.Configuration.HeartBeat.Time =
TimeSpan.FromSeconds(
35); //records with no heartbeat after 35 seconds are considered dead
//an invalid data exception will be re-tried 3 times, with delays of 3, 6 and then finally 9 seconds
queue.Configuration.TransportConfiguration.RetryDelayBehavior.Add(
typeof(InvalidDataException),
new List <TimeSpan>
{
TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(6), TimeSpan.FromSeconds(9)
});
queue.Configuration.MessageExpiration.Enabled = true;
queue.Configuration.MessageExpiration.MonitorTime =
TimeSpan.FromSeconds(20); //check for expired messages every 20 seconds
queue.Start(); //when running linq statements, there is no message handler, as the producer tells us what to run
Console.WriteLine("Processing messages - press any key to stop");
Console.ReadKey((true));
}
}
}
}
//if jaeger is using udp, sometimes the messages get lost; there doesn't seem to be a flush() call ?
if (SharedConfiguration.EnableTrace)
{
System.Threading.Thread.Sleep(2000);
}
}
public void RunConsumer <TTransportInit>(string queueName, string connectionString, bool addInterceptors,
ILogProvider logProvider,
int messageCount, int workerCount, int timeOut,
int queueSize, int readerCount,
TimeSpan heartBeatTime, TimeSpan heartBeatMonitorTime,
Guid id, string updateTime, bool enableChaos)
where TTransportInit : ITransportInit, new()
{
if (enableChaos)
{
timeOut *= 2;
}
using (var metrics = new Metrics.Metrics(queueName))
{
var addInterceptorConsumer = InterceptorAdding.No;
if (addInterceptors)
{
addInterceptorConsumer = InterceptorAdding.ConfigurationOnly;
}
using (
var creator = SharedSetup.CreateCreator <TTransportInit>(addInterceptorConsumer, logProvider, metrics, false, enableChaos)
)
{
using (var schedulerCreator =
new SchedulerContainer(
// ReSharper disable once AccessToDisposedClosure
serviceRegister => serviceRegister.Register(() => metrics, LifeStyles.Singleton)))
{
bool rollback;
using (var taskScheduler = schedulerCreator.CreateTaskScheduler())
{
taskScheduler.Configuration.MaximumThreads = workerCount;
taskScheduler.Configuration.MaxQueueSize = queueSize;
taskScheduler.Start();
var taskFactory = schedulerCreator.CreateTaskFactory(taskScheduler);
using (
var queue =
creator
.CreateConsumerMethodQueueScheduler(
queueName, connectionString, taskFactory))
{
SharedSetup.SetupDefaultConsumerQueue(queue.Configuration, readerCount, heartBeatTime,
heartBeatMonitorTime, updateTime, null);
SharedSetup.SetupDefaultErrorRetry(queue.Configuration);
rollback = queue.Configuration.TransportConfiguration.MessageRollbackSupported;
queue.Start();
var counter = 0;
while (counter < timeOut)
{
if (MethodIncrementWrapper.Count(id) >= messageCount * 3)
{
break;
}
Thread.Sleep(1000);
counter++;
}
//wait 3 more seconds before starting to shutdown
Thread.Sleep(3000);
}
}
if (rollback)
{
VerifyMetrics.VerifyRollBackCount(queueName, metrics.GetCurrentMetrics(), messageCount, 2, 2);
}
}
}
}
}
public void Run(int messageCount, int runtime, int timeOut, int workerCount, int readerCount, int queueSize,
int messageType)
{
SchedulerContainer schedulerContainer = null;
if (Factory == null)
{
Factory = CreateFactory(workerCount, queueSize, out schedulerContainer);
}
using (var connectionInfo = new IntegrationConnectionInfo())
{
var queueName = GenerateQueueName.Create();
var logProvider = LoggerShared.Create(queueName, GetType().Name);
using (var queueCreator =
new QueueCreationContainer <MemoryMessageQueueInit>(
serviceRegister => serviceRegister.Register(() => logProvider, LifeStyles.Singleton)))
{
try
{
using (
var oCreation =
queueCreator.GetQueueCreation <MessageQueueCreation>(queueName,
connectionInfo.ConnectionString)
)
{
var result = oCreation.CreateQueue();
Assert.True(result.Success, result.ErrorMessage);
if (messageType == 1)
{
var producer = new ProducerAsyncShared();
producer.RunTestAsync <MemoryMessageQueueInit, FakeMessage>(queueName,
connectionInfo.ConnectionString, false, messageCount, logProvider, Helpers.GenerateData,
Helpers.Verify, false, oCreation.Scope, false).Wait(timeOut / 2 * 1000);
var consumer = new ConsumerAsyncShared <FakeMessage> {
Factory = Factory
};
consumer.RunConsumer <MemoryMessageQueueInit>(queueName, connectionInfo.ConnectionString,
false, logProvider,
runtime, messageCount,
timeOut, readerCount, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(35), "second(*%10)", false);
}
else if (messageType == 2)
{
var producer = new ProducerAsyncShared();
producer.RunTestAsync <MemoryMessageQueueInit, FakeMessageA>(queueName,
connectionInfo.ConnectionString, false, messageCount, logProvider, Helpers.GenerateData,
Helpers.Verify, false, oCreation.Scope, false).Wait(timeOut / 2 * 1000);
var consumer = new ConsumerAsyncShared <FakeMessageA> {
Factory = Factory
};
consumer.RunConsumer <MemoryMessageQueueInit>(queueName, connectionInfo.ConnectionString,
false, logProvider,
runtime, messageCount,
timeOut, readerCount, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(35), "second(*%10)", false);
}
else if (messageType == 3)
{
var producer = new ProducerAsyncShared();
producer.RunTestAsync <MemoryMessageQueueInit, FakeMessageB>(queueName,
connectionInfo.ConnectionString, false, messageCount, logProvider, Helpers.GenerateData,
Helpers.Verify, false, oCreation.Scope, false).Wait(timeOut / 2 * 1000);
var consumer = new ConsumerAsyncShared <FakeMessageB> {
Factory = Factory
};
consumer.RunConsumer <MemoryMessageQueueInit>(queueName, connectionInfo.ConnectionString,
false, logProvider,
runtime, messageCount,
timeOut, readerCount,
TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(35), "second(*%10)", false);
}
new VerifyQueueRecordCount().Verify(oCreation.Scope, 0, true);
}
}
finally
{
schedulerContainer?.Dispose();
using (
var oCreation =
queueCreator.GetQueueCreation <MessageQueueCreation>(queueName,
connectionInfo.ConnectionString)
)
{
oCreation.RemoveQueue();
}
}
}
}
}
public void RunConsumer <TTransportInit>(QueueConnection queueConnection,
bool addInterceptors,
int workerCount,
ILogger logProvider,
int timeOut,
int readerCount,
int queueSize,
long messageCount,
TimeSpan heartBeatTime,
TimeSpan heartBeatMonitorTime,
string updateTime,
string route,
bool enableChaos, ICreationScope scope)
where TTransportInit : ITransportInit, new()
{
if (enableChaos)
{
timeOut *= 2;
}
using (var metrics = new Metrics.Metrics(queueConnection.Queue))
{
var addInterceptorConsumer = InterceptorAdding.No;
if (addInterceptors)
{
addInterceptorConsumer = InterceptorAdding.ConfigurationOnly;
}
using (
var creator = SharedSetup.CreateCreator <TTransportInit>(addInterceptorConsumer, logProvider, metrics,
true, enableChaos, scope))
{
using (var schedulerCreator = new SchedulerContainer())
{
using (var taskScheduler = schedulerCreator.CreateTaskScheduler())
{
taskScheduler.Configuration.MaximumThreads = workerCount;
taskScheduler.Configuration.MaxQueueSize = queueSize;
taskScheduler.Start();
var taskFactory = schedulerCreator.CreateTaskFactory(taskScheduler);
using (
var queue =
creator
.CreateConsumerQueueScheduler(
queueConnection, taskFactory))
{
SharedSetup.SetupDefaultConsumerQueue(queue.Configuration, readerCount, heartBeatTime,
heartBeatMonitorTime, updateTime, route);
//start looking for work
queue.Start <TMessage>((message, notifications) =>
{
MessageHandlingShared.HandleFakeMessageNoOp();
});
for (var i = 0; i < timeOut; i++)
{
if (VerifyMetrics.GetPoisonMessageCount(metrics.GetCurrentMetrics()) == messageCount)
{
break;
}
Thread.Sleep(1000);
}
//wait for last error to be saved if needed.
Thread.Sleep(3000);
}
}
VerifyMetrics.VerifyPoisonMessageCount(queueConnection.Queue, metrics.GetCurrentMetrics(), messageCount);
}
}
}
}
static void Main(string[] args)
{
//we are using serilog for sample purposes; any https://github.com/damianh/LibLog provider can be used
var log = new LoggerConfiguration()
.WriteTo.Console()
.MinimumLevel.Debug()
.CreateLogger();
Log.Logger = log;
log.Information("Startup");
log.Information(SharedConfiguration.AllSettings);
var queueName = ConfigurationManager.AppSettings.ReadSetting("QueueName");
var connectionString = ConfigurationManager.AppSettings.ReadSetting("Database");
using (var schedulerContainer = new SchedulerContainer(serviceRegister =>
Injectors.AddInjectors(log, SharedConfiguration.EnableTrace, SharedConfiguration.EnableMetrics,
SharedConfiguration.EnableCompression, SharedConfiguration.EnableEncryption, "RedisConsumerAsync",
serviceRegister), options => Injectors.SetOptions(options, SharedConfiguration.EnableChaos)))
{
using (var scheduler = schedulerContainer.CreateTaskScheduler())
{
var factory = schedulerContainer.CreateTaskFactory(scheduler);
factory.Scheduler.Configuration.MaximumThreads = 8; //8 background threads
factory.Scheduler.Configuration.MaxQueueSize = 1; //allow work to be de-queued but held in memory until a thread is free
//note - the same factory can be passed to multiple queue instances - don't dispose the scheduler container until all queues have finished
factory.Scheduler.Start(); //the scheduler must be started before passing it to a queue
using (var queueContainer = new QueueContainer <RedisQueueInit>(serviceRegister =>
Injectors.AddInjectors(log, SharedConfiguration.EnableTrace, SharedConfiguration.EnableMetrics, SharedConfiguration.EnableCompression, SharedConfiguration.EnableEncryption, "RedisConsumerAsync", serviceRegister)
, options => Injectors.SetOptions(options, SharedConfiguration.EnableChaos)))
{
using (var queue = queueContainer.CreateConsumerQueueScheduler(queueName, connectionString, factory))
{
//set some processing options and start looking for work
//in the async model, the worker count is how many threads are querying the queue - the scheduler runs the work
queue.Configuration.Worker.WorkerCount = 1; //lets just run 1 thread that queries the database
queue.Configuration.HeartBeat.UpdateTime = "sec(*%10)"; //set a heartbeat every 10 seconds
queue.Configuration.HeartBeat.MonitorTime =
TimeSpan.FromSeconds(15); //check for dead records every 15 seconds
queue.Configuration.HeartBeat.Time =
TimeSpan.FromSeconds(
35); //records with no heartbeat after 35 seconds are considered dead
//an invalid data exception will be re-tried 3 times, with delays of 3, 6 and then finally 9 seconds
queue.Configuration.TransportConfiguration.RetryDelayBehavior.Add(
typeof(InvalidDataException),
new List <TimeSpan>
{
TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(6), TimeSpan.FromSeconds(9)
});
queue.Configuration.MessageExpiration.Enabled = true;
queue.Configuration.MessageExpiration.MonitorTime =
TimeSpan.FromSeconds(20); //check for expired messages every 20 seconds
queue.Start <SimpleMessage>(MessageProcessing.HandleMessages);
Console.WriteLine("Processing messages - press any key to stop");
Console.ReadKey((true));
}
}
}
}
//if jaeger is using udp, sometimes the messages get lost; there doesn't seem to be a flush() call ?
if (SharedConfiguration.EnableTrace)
{
System.Threading.Thread.Sleep(2000);
}
}
public void Run(int messageCount, int runtime, int timeOut, int workerCount, int readerCount, int queueSize,
int messageType, ConnectionInfoTypes type)
{
SchedulerContainer schedulerContainer = null;
if (Factory == null)
{
Factory = CreateFactory(workerCount, queueSize, out schedulerContainer);
}
var queueName = GenerateQueueName.Create();
var logProvider = LoggerShared.Create(queueName, GetType().Name);
var connectionString = new ConnectionInfo(type).ConnectionString;
using (var queueCreator =
new QueueCreationContainer <RedisQueueInit>(
serviceRegister => serviceRegister.Register(() => logProvider, LifeStyles.Singleton)))
{
try
{
if (messageType == 1)
{
var producer = new ProducerAsyncShared();
producer.RunTestAsync <RedisQueueInit, FakeMessage>(queueName,
connectionString, false, messageCount, logProvider, Helpers.GenerateData,
Helpers.Verify, false, null).Wait(timeOut * 1000 / 2);
var consumer = new ConsumerAsyncShared <FakeMessage> {
Factory = Factory
};
consumer.RunConsumer <RedisQueueInit>(queueName, connectionString, false,
logProvider,
runtime, messageCount,
timeOut, readerCount, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(12), "second(*%3)");
}
else if (messageType == 2)
{
var producer = new ProducerAsyncShared();
producer.RunTestAsync <RedisQueueInit, FakeMessageA>(queueName,
connectionString, false, messageCount, logProvider, Helpers.GenerateData,
Helpers.Verify, false, null).Wait(timeOut * 1000 / 2);
var consumer = new ConsumerAsyncShared <FakeMessageA> {
Factory = Factory
};
consumer.RunConsumer <RedisQueueInit>(queueName, connectionString, false,
logProvider,
runtime, messageCount,
timeOut, readerCount, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(12), "second(*%3)");
}
else if (messageType == 3)
{
var producer = new ProducerAsyncShared();
producer.RunTestAsync <RedisQueueInit, FakeMessageB>(queueName,
connectionString, false, messageCount, logProvider, Helpers.GenerateData,
Helpers.Verify, false, null).Wait(timeOut * 1000 / 2);
var consumer = new ConsumerAsyncShared <FakeMessageB> {
Factory = Factory
};
consumer.RunConsumer <RedisQueueInit>(queueName, connectionString, false,
logProvider,
runtime, messageCount,
timeOut, readerCount, TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(12), "second(*%3)");
}
using (var count = new VerifyQueueRecordCount(queueName, connectionString))
{
count.Verify(0, false, -1);
}
}
finally
{
schedulerContainer?.Dispose();
using (
var oCreation =
queueCreator.GetQueueCreation <RedisQueueCreation>(queueName,
connectionString)
)
{
oCreation.RemoveQueue();
}
}
}
}
public void RunConsumer <TTransportInit>(QueueConnection queueConnection,
bool addInterceptors,
int workerCount,
ILogger logProvider,
int timeOut,
int readerCount,
int queueSize,
int runTime,
int messageCount,
TimeSpan heartBeatTime,
TimeSpan heartBeatMonitorTime,
Guid id,
string updateTime,
bool enableChaos, ICreationScope scope)
where TTransportInit : ITransportInit, new()
{
if (enableChaos)
{
timeOut *= 2;
}
using (var trace = SharedSetup.CreateTrace("consumer-rollback"))
{
using (var metrics = new Metrics.Metrics(queueConnection.Queue))
{
var addInterceptorConsumer = InterceptorAdding.No;
if (addInterceptors)
{
addInterceptorConsumer = InterceptorAdding.ConfigurationOnly;
}
using (
var creator = SharedSetup.CreateCreator <TTransportInit>(addInterceptorConsumer, logProvider,
metrics, false, enableChaos, scope, trace.Source)
)
{
using (var schedulerCreator = new SchedulerContainer((x) => x.RegisterNonScopedSingleton(trace.Source)))
{
using (var taskScheduler = schedulerCreator.CreateTaskScheduler())
{
taskScheduler.Configuration.MaximumThreads = workerCount;
taskScheduler.Start();
var taskFactory = schedulerCreator.CreateTaskFactory(taskScheduler);
using (
var queue =
creator
.CreateConsumerMethodQueueScheduler(
queueConnection, taskFactory))
{
SharedSetup.SetupDefaultConsumerQueue(queue.Configuration, readerCount,
heartBeatTime,
heartBeatMonitorTime, updateTime, null);
queue.Start();
var counter = 0;
while (counter < timeOut)
{
if (MethodIncrementWrapper.Count(id) >= messageCount)
{
break;
}
Thread.Sleep(1000);
counter++;
}
//wait for queues to commit records
Thread.Sleep(3000);
}
}
Assert.Equal(messageCount, MethodIncrementWrapper.Count(id));
VerifyMetrics.VerifyProcessedCount(queueConnection.Queue, metrics.GetCurrentMetrics(),
messageCount);
VerifyMetrics.VerifyRollBackCount(queueConnection.Queue, metrics.GetCurrentMetrics(),
messageCount, 1, 0);
}
}
}
}
}
public async Task Run <TTransportInit, TTransportCreate>(
QueueConnection queueConnection,
int messageCount, int runtime, int timeOut, int workerCount, int readerCount, int queueSize,
int messageType, bool enableChaos,
Action <TTransportCreate> setOptions,
Func <QueueProducerConfiguration, AdditionalMessageData> generateData,
Action <QueueConnection, QueueProducerConfiguration, long, ICreationScope> verify,
Action <QueueConnection, IBaseTransportOptions, ICreationScope, int, bool, bool> verifyQueueCount)
where TTransportInit : ITransportInit, new()
where TTransportCreate : class, IQueueCreation
{
SchedulerContainer schedulerContainer = null;
if (Factory == null)
{
Factory = CreateFactory(workerCount, queueSize, out schedulerContainer);
}
var logProvider = LoggerShared.Create(queueConnection.Queue, GetType().Name);
using (var queueCreator =
new QueueCreationContainer <TTransportInit>(
serviceRegister => serviceRegister.Register(() => logProvider, LifeStyles.Singleton)))
{
ICreationScope scope = null;
var oCreation = queueCreator.GetQueueCreation <TTransportCreate>(queueConnection);
var time = runtime * 1000 / 2;
try
{
setOptions(oCreation);
var result = oCreation.CreateQueue();
Assert.True(result.Success, result.ErrorMessage);
scope = oCreation.Scope;
if (messageType == 1)
{
var producer = new ProducerAsyncShared();
await producer.RunTestAsync <TTransportInit, FakeMessage>(queueConnection, false,
messageCount, logProvider, generateData,
verify, false, scope, false).ConfigureAwait(false);
Thread.Sleep(time);
var consumer = new ConsumerAsyncShared <FakeMessage> {
Factory = Factory
};
consumer.RunConsumer <TTransportInit>(queueConnection,
false, logProvider,
runtime, messageCount,
timeOut, readerCount, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(35),
"second(*%10)", enableChaos, scope, null);
}
else if (messageType == 2)
{
var producer = new ProducerAsyncShared();
await producer.RunTestAsync <TTransportInit, FakeMessageA>(queueConnection, false,
messageCount, logProvider, generateData,
verify, false, scope, false).ConfigureAwait(false);
Thread.Sleep(time);
var consumer = new ConsumerAsyncShared <FakeMessageA> {
Factory = Factory
};
consumer.RunConsumer <TTransportInit>(queueConnection,
false, logProvider,
runtime, messageCount,
timeOut, readerCount, TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(35),
"second(*%10)", enableChaos, scope, null);
}
else if (messageType == 3)
{
var producer = new ProducerAsyncShared();
await producer.RunTestAsync <TTransportInit, FakeMessageB>(queueConnection, false,
messageCount, logProvider, generateData,
verify, false, oCreation.Scope, false).ConfigureAwait(false);
Thread.Sleep(time);
var consumer = new ConsumerAsyncShared <FakeMessageB> {
Factory = Factory
};
consumer.RunConsumer <TTransportInit>(queueConnection,
false, logProvider,
runtime, messageCount,
timeOut, readerCount,
TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(35), "second(*%10)", enableChaos, scope,
null);
}
verifyQueueCount(queueConnection, oCreation.BaseTransportOptions, scope, 0, false,
false);
}
finally
{
schedulerContainer?.Dispose();
oCreation?.RemoveQueue();
oCreation?.Dispose();
scope?.Dispose();
}
}
}
public void RunConsumer <TTransportInit>(QueueConnection queueConnection,
bool addInterceptors,
int workerCount,
ILogger logProvider,
int timeOut,
int readerCount,
int queueSize,
int runTime,
int messageCount,
TimeSpan heartBeatTime,
TimeSpan heartBeatMonitorTime,
string updateTime,
string route, bool enableChaos, ICreationScope scope)
where TTransportInit : ITransportInit, new()
{
var processedCount = new IncrementWrapper();
var haveIProcessedYouBefore = new ConcurrentDictionary <string, int>();
if (enableChaos)
{
timeOut *= 2;
}
using (var trace = SharedSetup.CreateTrace("consumer-rollback"))
{
using (var metrics = new Metrics.Metrics(queueConnection.Queue))
{
var addInterceptorConsumer = InterceptorAdding.No;
if (addInterceptors)
{
addInterceptorConsumer = InterceptorAdding.ConfigurationOnly;
}
using (
var creator = SharedSetup.CreateCreator <TTransportInit>(addInterceptorConsumer, logProvider,
metrics, false, enableChaos, scope, trace.Source)
)
{
using (var schedulerCreator = new SchedulerContainer((x) => x.RegisterNonScopedSingleton(trace.Source)))
{
using (var taskScheduler = schedulerCreator.CreateTaskScheduler())
{
taskScheduler.Configuration.MaximumThreads = workerCount;
taskScheduler.Start();
var taskFactory = schedulerCreator.CreateTaskFactory(taskScheduler);
using (
var queue =
creator
.CreateConsumerQueueScheduler(
queueConnection, taskFactory))
{
SharedSetup.SetupDefaultConsumerQueue(queue.Configuration, readerCount,
heartBeatTime,
heartBeatMonitorTime, updateTime, route);
var waitForFinish = new ManualResetEventSlim(false);
waitForFinish.Reset();
//start looking for work
queue.Start <TMessage>((message, notifications) =>
{
MessageHandlingShared.HandleFakeMessagesRollback(message, runTime,
processedCount,
messageCount, waitForFinish, haveIProcessedYouBefore);
});
waitForFinish.Wait(timeOut * 1000);
}
}
Assert.Equal(messageCount, processedCount.ProcessedCount);
VerifyMetrics.VerifyProcessedCount(queueConnection.Queue, metrics.GetCurrentMetrics(),
messageCount);
VerifyMetrics.VerifyRollBackCount(queueConnection.Queue, metrics.GetCurrentMetrics(),
messageCount, 1, 0);
}
}
haveIProcessedYouBefore.Clear();
}
}
}
static void Main(string[] args)
{
//we are using serilog for sample purposes
var log = new LoggerConfiguration()
.WriteTo.Console(outputTemplate: "[{Timestamp:HH:mm:ss} {Level:u3}] [{SourceContext}] {Message:lj}{NewLine}{Exception}")
.MinimumLevel.Debug()
.CreateLogger();
Log.Logger = log;
log.Information("Startup");
log.Information(SharedConfiguration.AllSettings);
//verify that the queue exists
var queueName = ConfigurationManager.AppSettings.ReadSetting("QueueName");
var connectionString = ConfigurationManager.AppSettings.ReadSetting("Database");
var queueConnection = new QueueConnection(queueName, connectionString);
using (var createQueueContainer = new QueueCreationContainer <PostgreSqlMessageQueueInit>(serviceRegister =>
Injectors.AddInjectors(Helpers.CreateForSerilog(), SharedConfiguration.EnableTrace, SharedConfiguration.EnableMetrics, SharedConfiguration.EnableCompression, SharedConfiguration.EnableEncryption, "PostgreSqlConsumerAsync", serviceRegister)
, options => Injectors.SetOptions(options, SharedConfiguration.EnableChaos)))
{
using (var createQueue =
createQueueContainer.GetQueueCreation <PostgreSqlMessageQueueCreation>(queueConnection))
{
if (!createQueue.QueueExists)
{
//the consumer can't do anything if the queue hasn't been created
Log.Error($"Could not find {connectionString}. Verify that you have run the producer, which will create the queue");
return;
}
}
}
using (var schedulerContainer = new SchedulerContainer(serviceRegister =>
Injectors.AddInjectors(Helpers.CreateForSerilog(), SharedConfiguration.EnableTrace, SharedConfiguration.EnableMetrics,
SharedConfiguration.EnableCompression, SharedConfiguration.EnableEncryption, "PostgreSqlConsumerAsync",
serviceRegister), options => Injectors.SetOptions(options, SharedConfiguration.EnableChaos)))
{
using (var scheduler = schedulerContainer.CreateTaskScheduler())
{
var factory = schedulerContainer.CreateTaskFactory(scheduler);
factory.Scheduler.Configuration.MaximumThreads = 8; //8 background threads
//note - the same factory can be passed to multiple queue instances - don't dispose the scheduler container until all queues have finished
factory.Scheduler.Start(); //the scheduler must be started before passing it to a queue
using (var queueContainer = new QueueContainer <PostgreSqlMessageQueueInit>(serviceRegister =>
Injectors.AddInjectors(Helpers.CreateForSerilog(), SharedConfiguration.EnableTrace, SharedConfiguration.EnableMetrics, SharedConfiguration.EnableCompression, SharedConfiguration.EnableEncryption, "PostgreSqlConsumerAsync", serviceRegister)
, options => Injectors.SetOptions(options, SharedConfiguration.EnableChaos)))
{
using (var queue = queueContainer.CreateConsumerQueueScheduler(queueConnection, factory))
{
//set some processing options and start looking for work
//in the async model, the worker count is how many threads are querying the queue - the scheduler runs the work
queue.Configuration.Worker.WorkerCount = 1; //lets just run 1 thread that queries the database
queue.Configuration.HeartBeat.UpdateTime = "sec(*%10)"; //set a heartbeat every 10 seconds
queue.Configuration.HeartBeat.MonitorTime =
TimeSpan.FromSeconds(15); //check for dead records every 15 seconds
queue.Configuration.HeartBeat.Time =
TimeSpan.FromSeconds(
35); //records with no heartbeat after 35 seconds are considered dead
//an invalid data exception will be re-tried 3 times, with delays of 3, 6 and then finally 9 seconds
queue.Configuration.TransportConfiguration.RetryDelayBehavior.Add(
typeof(InvalidDataException),
new List <TimeSpan>
{
TimeSpan.FromSeconds(3), TimeSpan.FromSeconds(6), TimeSpan.FromSeconds(9)
});
queue.Configuration.MessageExpiration.Enabled = true;
queue.Configuration.MessageExpiration.MonitorTime =
TimeSpan.FromSeconds(20); //check for expired messages every 20 seconds
queue.Start <SimpleMessage>(MessageProcessing.HandleMessages);
Console.WriteLine("Processing messages - press any key to stop");
Console.ReadKey((true));
}
}
}
}
//if jaeger is using udp, sometimes the messages get lost; there doesn't seem to be a flush() call ?
if (SharedConfiguration.EnableTrace)
{
System.Threading.Thread.Sleep(2000);
}
}
/// <summary>
/// This method returns the default scheduler container.
/// </summary>
/// <returns>The default scheduler.</returns>
protected virtual SchedulerContainer InitialiseSchedulerContainer()
{
var container = new SchedulerContainer(Policy.Scheduler);
return(container);
}
