static void Main(string[] args)
{
var factory = new ConnectionFactory();
factory.HostName = ConfigurationManager.AppSettings["rabbitmqHostName"];
factory.AuthMechanisms = new AuthMechanismFactory[] { new ExternalMechanismFactory() };
factory.Ssl.ServerName = ConfigurationManager.AppSettings["rabbitmqServerName"];
factory.Ssl.CertPath = ConfigurationManager.AppSettings["certificateFilePath"];
factory.Ssl.CertPassphrase = ConfigurationManager.AppSettings["certificatePassphrase"];
factory.Ssl.Enabled = true;
factory.Ssl.Version = SslProtocols.Tls12;
factory.Port = AmqpTcpEndpoint.DefaultAmqpSslPort;
factory.VirtualHost = "/";
using (var connection = factory.CreateConnection())
{
using (var channel = connection.CreateModel())
{
Console.WriteLine(" [*] Publishing messages. To exit press CTRL+C");
int count = 0;
var rand = new Random();
while (true)
{
count++;
WorkProcessor.EnqueueMessage(channel, "Lipsum", new LipsumGeneratorMessage(rand.Next(5)));
Console.WriteLine("Sent message Lipsum " + count);
System.Threading.Thread.Sleep(rand.Next(2000));
}
}
}
}
public void P7_TestWorkProcessorQueueTasks()
{
var wp = new WorkProcessor(2);
Assert.IsTrue(wp.AddTask("A"));
Assert.IsTrue(wp.AddTask("B"));
Assert.IsFalse(wp.AddTask("C"));
}
/// <summary>
/// Starts the pool's crawlers
/// </summary>
public void StartCrawlers()
{
for (int i = 1; i <= numberOfWorkProcessors && i <= jobsInSource; i++)
{
WorkProcessor crw = new WorkProcessor(String.Format("Pool: '{0}' \t Crawler: '{1}'", poolName, i));
crw.StartCrawling(workSource);
crawlers.Add(crw);
}
}
private void Start()
{
if (!isWork)
{
return;
}
workProcessor = WorkProcessor.GetInstance();
testFunction();
}
public void P7_TestTaskDurationMap()
{
var d = WorkProcessor.GetTaskDuration("A");
Assert.AreEqual(1, d);
d = WorkProcessor.GetTaskDuration("Z");
Assert.AreEqual(26, d);
d = WorkProcessor.GetTaskDuration("*");
Assert.AreEqual(-1, d);
}
private void ProcessCmdLine(WorkProcessor processor, LineCmd cmdline)
{
string cmd = cmdline.CmdLine;
Dictionary <string, string> args = null;
int pos = cmdline.CmdLine.IndexOf(' ');
if (pos != -1)
{
cmd = cmdline.CmdLine.Substring(0, pos);
args = cmdline.CmdLine.Substring(pos + 1).Split(' ').ToList()
.ConvertAll(s => { var p = s.IndexOf('='); return(p == -1 ? new { Name = s, Value = "" } : new { Name = s.Substring(0, p), Value = s.Substring(p + 1) }); })
.ToDictionary(s => s.Name, s => s.Value);
}
switch (cmd)
{
case "list":
{
var list = this.pluginManager.GetPlugins();
cmdline.Resp = string.Join("|", list.ToList().ConvertAll(d => d.PluginAssemblyPath).ToArray());
}
break;
case "add":
{
this.pluginManager.AddPlugin(cmdline.Bytes, args["file"]);
}
break;
case "remove":
{
this.pluginManager.RemovePlugin(args["file"]);
}
break;
case "q":
{
cmdline.DisconnectSocket = true;
}
break;
default:
{
}
break;
}
}
public void Start()
{
processor = new WorkProcessor(workProcessor);
serviceBus = ServiceBusFactory.New(sbc =>
{
sbc.ReceiveFrom("rabbitmq://localhost/AsyncWork-PendingWork");
sbc.UseRabbitMq();
sbc.SetPurgeOnStartup(false);
sbc.SetDefaultSerializer(massTransitSerializer);
if (conccurentNumberOfProcessors > 0)
sbc.SetConcurrentConsumerLimit(conccurentNumberOfProcessors);
sbc.Subscribe(subs =>
{
subs.Consumer(typeof(WorkProcessor), x => processor);
});
});
}
public TwoToTwoWorkProcessorThroughputTest()
{
var sequenceBarrier = _ringBuffer.NewBarrier();
_handlers[0] = new ValueAdditionWorkHandler();
_handlers[1] = new ValueAdditionWorkHandler();
_workProcessors[0] = new WorkProcessor <ValueEvent>(_ringBuffer, sequenceBarrier, _handlers[0], new IgnoreExceptionHandler(), _workSequence);
_workProcessors[1] = new WorkProcessor <ValueEvent>(_ringBuffer, sequenceBarrier, _handlers[1], new IgnoreExceptionHandler(), _workSequence);
for (var i = 0; i < _numPublishers; i++)
{
_valuePublishers[i] = new ValuePublisher(_cyclicBarrier, _ringBuffer, _iterations);
}
_ringBuffer.AddGatingSequences(_workProcessors[0].Sequence, _workProcessors[1].Sequence);
}
public void RemoveWorkHandlerLostEventExample()
{
int eventSize = 8;
var countdownEvent = new CountdownEvent(eventSize);
var workSequence = new Sequence();
var disruptor = new Disruptor <StubEvent>(StubEvent.EventFactory, 4);
var ringBuffer = disruptor.Start();
var handler1 = new DynamicHandler(1, countdownEvent);
var processor1 = new WorkProcessor <StubEvent>(ringBuffer, ringBuffer.NewBarrier(), handler1, new FatalExceptionHandler(), workSequence);
var handler2 = new DynamicHandler(2, countdownEvent);
var processor2 = new WorkProcessor <StubEvent>(ringBuffer, ringBuffer.NewBarrier(), handler2, new FatalExceptionHandler(), workSequence);
ringBuffer.AddGatingSequences(processor1.Sequence);
Task.Run(() => processor1.Run());
ringBuffer.AddGatingSequences(processor2.Sequence);
Task.Run(() => processor2.Run());
handler1.AwaitStart();
handler2.AwaitStart();
Thread.Sleep(10);
// processor1 should own an unavailable work sequence
// => this sequence will be dropped by Halt
processor1.Halt();
var producer = new MessageProducer(disruptor, InitData(0, eventSize));
Task.Run(() => producer.Run());
producer.AwaitStart();
handler1.AwaitShutdown();
ringBuffer.RemoveGatingSequence(processor1.Sequence);
// countdownEvent should not reach zero because of the dropped sequence
var await = countdownEvent.Wait(TimeSpan.FromMilliseconds(500));
Assert.IsFalse(await);
}
public TwoPublisherToTwoProcessorWorkProcessorThroughputTest()
: base(TestName, ITERATIONS)
{
sequenceBarrier = ringBuffer.NewBarrier();
workProcessors[0] = new WorkProcessor <ValueEvent>(ringBuffer, sequenceBarrier,
handlers[0], new IgnoreExceptionHandler(),
workSequence);
workProcessors[1] = new WorkProcessor <ValueEvent>(ringBuffer, sequenceBarrier,
handlers[1], new IgnoreExceptionHandler(),
workSequence);
//workerPool = new WorkerPool<ValueEvent>(ringBuffer, sequenceBarrier, new IgnoreExceptionHandler(), handlers);
for (int i = 0; i < NUM_PUBLISHERS; i++)
{
valuePublishers[i] = new ValuePublisher_V3(cyclicBarrier, ringBuffer, ITERATIONS);
}
workProcessors.ToList().ForEach(e => ringBuffer.AddGatingSequences(e.Sequence));
//ringBuffer.AddGatingSequences(/*workerPool.getWorkerSequences()*/workProcessors[0].Sequence, workProcessors[1].Sequence);
}
public TwoPublisherToTwoProcessorWorkProcessorThroughputTest()
: base(TestName, ITERATIONS)
{
sequenceBarrier = ringBuffer.NewBarrier();
workProcessors[0] = new WorkProcessor<ValueEvent>(ringBuffer, sequenceBarrier,
handlers[0], new IgnoreExceptionHandler(),
workSequence);
workProcessors[1] = new WorkProcessor<ValueEvent>(ringBuffer, sequenceBarrier,
handlers[1], new IgnoreExceptionHandler(),
workSequence);
//workerPool = new WorkerPool<ValueEvent>(ringBuffer, sequenceBarrier, new IgnoreExceptionHandler(), handlers);
for (int i = 0; i < NUM_PUBLISHERS; i++)
{
valuePublishers[i] = new ValuePublisher_V3(cyclicBarrier, ringBuffer, ITERATIONS);
}
workProcessors.ToList().ForEach(e => ringBuffer.AddGatingSequences(e.Sequence));
//ringBuffer.AddGatingSequences(/*workerPool.getWorkerSequences()*/workProcessors[0].Sequence, workProcessors[1].Sequence);
}
public void RemoveWorkHandlerLaterTest()
{
var eventSize = 8;
var countdownEvent = new CountdownEvent(eventSize);
var workSequence = new Sequence();
var disruptor = new Disruptor <StubEvent>(StubEvent.EventFactory, 4);
var ringBuffer = disruptor.Start();
var handler1 = new DynamicHandler(1, countdownEvent);
var processor1 = new WorkProcessor <StubEvent>(ringBuffer, ringBuffer.NewBarrier(), handler1, new FatalExceptionHandler(), workSequence);
var handler2 = new DynamicHandler(2, countdownEvent);
var processor2 = new WorkProcessor <StubEvent>(ringBuffer, ringBuffer.NewBarrier(), handler2, new FatalExceptionHandler(), workSequence);
ringBuffer.AddGatingSequences(processor1.Sequence);
Task.Run(() => processor1.Run());
ringBuffer.AddGatingSequences(processor2.Sequence);
Task.Run(() => processor2.Run());
handler1.AwaitStart();
handler2.AwaitStart();
Thread.Sleep(10);
// processor1 should own an unavailable work sequence
// => this sequence should not be dropped by HaltLater
processor1.HaltLater();
var producer = new MessageProducer(disruptor, InitData(0, eventSize));
Task.Run(() => producer.Run());
producer.AwaitStart();
handler1.AwaitShutdown();
ringBuffer.RemoveGatingSequence(processor1.Sequence);
Assert.IsTrue(countdownEvent.Wait(TimeSpan.FromSeconds(3)));
}
public TwoToTwoWorkProcessorThroughputTest()
: base(Test_Disruptor, ITERATIONS)
{
sequenceBarrier = ringBuffer.NewBarrier();
handlers[0] = new ValueAdditionWorkHandler();
handlers[1] = new ValueAdditionWorkHandler();
workProcessors[0] = new WorkProcessor <ValueEvent>(ringBuffer, sequenceBarrier,
handlers[0], new IgnoreExceptionHandler(),
workSequence);
workProcessors[1] = new WorkProcessor <ValueEvent>(ringBuffer, sequenceBarrier,
handlers[1], new IgnoreExceptionHandler(),
workSequence);
for (int i = 0; i < NUM_PUBLISHERS; i++)
{
valuePublishers[i] = new ValuePublisher(cyclicBarrier, ringBuffer, ITERATIONS);
}
ringBuffer.AddGatingSequences(workProcessors[0].Sequence, workProcessors[1].Sequence);
}
/// <summary>
/// Create a worker pool to enable an array of <see cref="IWorkHandler{TEvent}"/>s to consume published sequences.
///
/// This option requires a pre-configured <see cref="RingBuffer{TEvent}"/> which must have <see cref="RingBuffer{TEvent}.AddGatingSequences(ISequence[])"/> called before the work pool is started.
/// </summary>
/// <param name="ringBuffer">of events to be consumed.</param>
/// <param name="barrier">on which the workers will depend.</param>
/// <param name="exceptionHandler">to callback when an error occurs which is not handled by the <see cref="IWorkHandler{TEvent}"/>s.</param>
/// <param name="workHandlers">to distribute the work load across.</param>
public WorkerPool(
RingBuffer <T> ringBuffer,
ISequenceBarrier barrier,
IExceptionHandler <T> exceptionHandler,
params IWorkHandler <T>[] workHandlers)
{
_ringBuffer = ringBuffer;
var numWorkers = workHandlers.Length;
_workProcessors = new WorkProcessor <T> [numWorkers];
for (var i = 0; i < numWorkers; i++)
{
_workProcessors[i] = new WorkProcessor <T>(
ringBuffer,
barrier,
workHandlers[i],
exceptionHandler,
_workSequence);
}
}
public TwoToTwoWorkProcessorThroughputTest()
: base(Test_Disruptor, ITERATIONS)
{
sequenceBarrier = ringBuffer.NewBarrier();
handlers[0] = new ValueAdditionWorkHandler();
handlers[1] = new ValueAdditionWorkHandler();
workProcessors[0] = new WorkProcessor<ValueEvent>(ringBuffer, sequenceBarrier,
handlers[0], new IgnoreExceptionHandler(),
workSequence);
workProcessors[1] = new WorkProcessor<ValueEvent>(ringBuffer, sequenceBarrier,
handlers[1], new IgnoreExceptionHandler(),
workSequence);
for (int i = 0; i < NUM_PUBLISHERS; i++)
{
valuePublishers[i] = new ValuePublisher(cyclicBarrier, ringBuffer, ITERATIONS);
}
ringBuffer.AddGatingSequences(workProcessors[0].Sequence, workProcessors[1].Sequence);
}
public void P7_TestWorkProcessorProcessQueue()
{
var wp = new WorkProcessor(2);
var ready = new List <string> {
"A", "F", "C"
};
wp.LoadWorkQueue(ref ready);
Assert.AreEqual(1, ready.Count);
Assert.AreEqual("C", ready[0]);
List <string> completed;
for (var i = 0; i < 7; i++)
{
completed = wp.ProcessQueue();
if (i == 0)
{
Assert.AreEqual(1, completed.Count);
Assert.AreEqual("A", completed[0]);
}
else if (i == 5)
{
Assert.AreEqual(1, completed.Count);
Assert.AreEqual("F", completed[0]);
}
else if (i > 5)
{
Assert.IsTrue(wp.IsTaskQueueEmpty());
}
else
{
Assert.AreEqual(0, completed.Count);
}
}
}
/// <summary>
/// Create a worker pool to enable an array of <see cref="IWorkHandler{TEvent}"/>s to consume published sequences.
///
/// This option requires a pre-configured <see cref="RingBuffer{TEvent}"/> which must have <see cref="RingBuffer{TEvent}.AddGatingSequences(ISequence[])"/> called before the work pool is started.
/// </summary>
/// <param name="eventFactory">for filling the <see cref="RingBuffer{TEvent}"/>.</param>
/// <param name="exceptionHandler">to callback when an error occurs which is not handled by the <see cref="IWorkHandler{TEvent}"/>s.</param>
/// <param name="workHandlers">to distribute the work load across.</param>
public WorkerPool(
IEventFactory <T> eventFactory,
IExceptionHandler <T> exceptionHandler,
params IWorkHandler <T>[] workHandlers)
{
_ringBuffer = RingBuffer <T> .CreateMultiProducer(eventFactory, 1024);
var barrier = _ringBuffer.NewBarrier();
var numWorkers = workHandlers.Length;
_workProcessors = new WorkProcessor <T> [numWorkers];
for (int i = 0; i < numWorkers; i++)
{
_workProcessors[i] = new WorkProcessor <T>(
_ringBuffer,
barrier,
workHandlers[i],
exceptionHandler,
_workSequence);
}
_ringBuffer.AddGatingSequences(GetWorkerSequences());
}
private static void Main()
{
Paths.SetProcessedDataFolder(@"C:\Users\ceesj\Documents\hogeschool\minor\Code\ProcessedData\");
Paths.SetRawDataFolder(@"C:\Users\ceesj\Documents\hogeschool\minor\Code\Datasources\");
// Paths.SetProcessedDataFolder(@"E:\Hogeschool\Polishift Organization\ProcessedData\");
// Paths.SetRawDataFolder(@"E:\Hogeschool\Polishift Organization\Datasources\");
#region ParseCode
var interestCsvLocation = Paths.RawDataFolder +
@"\Economical & Social\Interest rates (incomplete)\interest_data.csv";
var warCsvLocation = Paths.RawDataFolder + @"\Economical & Social\Wars\war_data.csv";
var tvCsvLocation = Paths.RawDataFolder +
@"\Economical & Social\Households with TV\households_with_television_data.csv";
var populationCsvLocation = Paths.RawDataFolder +
@"\Economical & Social\GDP & Population & GDP Per capita\population_data.csv";
var gdpTotalCsvLocation = Paths.RawDataFolder +
@"\Economical & Social\GDP & Population & GDP Per capita\gdp_data.csv";
var gdpCapitaCsvLocation = Paths.RawDataFolder +
@"\Economical & Social\GDP & Population & GDP Per capita\percapita_gdp_data.csv";
var workCsvLocation = Paths.RawDataFolder + @"\Economical & Social\NMC_5_0.csv";
ICsvModelFactory <WorkModel> modelFactory =
new WorkModelFactory();
var allItemsAsModels = CsvToModel <WorkModel> .ParseAllCsvLinesToModels(
workCsvLocation, modelFactory);
var processor = new WorkProcessor();
processor.SerializeDataToJson(allItemsAsModels);
#endregion
#region Parsing country borders
// var countryBorderDirectory = Paths.ProcessedDataFolder+ @"countryInformation";
// IJsonModelFactory<CountryGeoModel> countryGeoModelFactory = new CountryGeoModelFactory();
// var processor2 = new CountryBordersProcessor();
//
// var allCountryGeoModels =
// JsonToModel<CountryGeoModel>.ParseJsonDirectoryToModels(countryBorderDirectory, countryGeoModelFactory,
// "*.geo.json");
// processor2.SerializeDataToJson(allCountryGeoModels);
//
// var countryBordersRepo = new CountryBordersRepository();
// foreach (var country in countryBordersRepo.GetAll())
// {
// Console.WriteLine("Countrycode: " + country.CountryCode);
// }
#endregion
#region Filtering
/*
* IFilter filter = new YearFilter();
* filter.Filter();
*
* filter = new EuropeFilter();
* filter.Filter();
*/
#endregion
Downloader.DownloadData(Paths.ProcessedDataFolder);
}
private void CreateProcessor(Socket client)
{
var processor = new WorkProcessor(this, client);
processor.Run(ProcessCmdLine);
}
public void execute(WorkProcessor processor)
{
throw new System.NotImplementedException();
}
public EventReleaser(WorkProcessor <T> workProcessor)
{
_workProcessor = workProcessor;
}
