static void Main(string[] args) { var factory = new ConnectionFactory(); factory.HostName = ConfigurationManager.AppSettings["rabbitmqHostName"]; factory.AuthMechanisms = new AuthMechanismFactory[] { new ExternalMechanismFactory() }; factory.Ssl.ServerName = ConfigurationManager.AppSettings["rabbitmqServerName"]; factory.Ssl.CertPath = ConfigurationManager.AppSettings["certificateFilePath"]; factory.Ssl.CertPassphrase = ConfigurationManager.AppSettings["certificatePassphrase"]; factory.Ssl.Enabled = true; factory.Ssl.Version = SslProtocols.Tls12; factory.Port = AmqpTcpEndpoint.DefaultAmqpSslPort; factory.VirtualHost = "/"; using (var connection = factory.CreateConnection()) { using (var channel = connection.CreateModel()) { Console.WriteLine(" [*] Publishing messages. To exit press CTRL+C"); int count = 0; var rand = new Random(); while (true) { count++; WorkProcessor.EnqueueMessage(channel, "Lipsum", new LipsumGeneratorMessage(rand.Next(5))); Console.WriteLine("Sent message Lipsum " + count); System.Threading.Thread.Sleep(rand.Next(2000)); } } } }
public void P7_TestWorkProcessorQueueTasks() { var wp = new WorkProcessor(2); Assert.IsTrue(wp.AddTask("A")); Assert.IsTrue(wp.AddTask("B")); Assert.IsFalse(wp.AddTask("C")); }
/// <summary> /// Starts the pool's crawlers /// </summary> public void StartCrawlers() { for (int i = 1; i <= numberOfWorkProcessors && i <= jobsInSource; i++) { WorkProcessor crw = new WorkProcessor(String.Format("Pool: '{0}' \t Crawler: '{1}'", poolName, i)); crw.StartCrawling(workSource); crawlers.Add(crw); } }
private void Start() { if (!isWork) { return; } workProcessor = WorkProcessor.GetInstance(); testFunction(); }
public void P7_TestTaskDurationMap() { var d = WorkProcessor.GetTaskDuration("A"); Assert.AreEqual(1, d); d = WorkProcessor.GetTaskDuration("Z"); Assert.AreEqual(26, d); d = WorkProcessor.GetTaskDuration("*"); Assert.AreEqual(-1, d); }
private void ProcessCmdLine(WorkProcessor processor, LineCmd cmdline) { string cmd = cmdline.CmdLine; Dictionary <string, string> args = null; int pos = cmdline.CmdLine.IndexOf(' '); if (pos != -1) { cmd = cmdline.CmdLine.Substring(0, pos); args = cmdline.CmdLine.Substring(pos + 1).Split(' ').ToList() .ConvertAll(s => { var p = s.IndexOf('='); return(p == -1 ? new { Name = s, Value = "" } : new { Name = s.Substring(0, p), Value = s.Substring(p + 1) }); }) .ToDictionary(s => s.Name, s => s.Value); } switch (cmd) { case "list": { var list = this.pluginManager.GetPlugins(); cmdline.Resp = string.Join("|", list.ToList().ConvertAll(d => d.PluginAssemblyPath).ToArray()); } break; case "add": { this.pluginManager.AddPlugin(cmdline.Bytes, args["file"]); } break; case "remove": { this.pluginManager.RemovePlugin(args["file"]); } break; case "q": { cmdline.DisconnectSocket = true; } break; default: { } break; } }
public void Start() { processor = new WorkProcessor(workProcessor); serviceBus = ServiceBusFactory.New(sbc => { sbc.ReceiveFrom("rabbitmq://localhost/AsyncWork-PendingWork"); sbc.UseRabbitMq(); sbc.SetPurgeOnStartup(false); sbc.SetDefaultSerializer(massTransitSerializer); if (conccurentNumberOfProcessors > 0) sbc.SetConcurrentConsumerLimit(conccurentNumberOfProcessors); sbc.Subscribe(subs => { subs.Consumer(typeof(WorkProcessor), x => processor); }); }); }
public TwoToTwoWorkProcessorThroughputTest() { var sequenceBarrier = _ringBuffer.NewBarrier(); _handlers[0] = new ValueAdditionWorkHandler(); _handlers[1] = new ValueAdditionWorkHandler(); _workProcessors[0] = new WorkProcessor <ValueEvent>(_ringBuffer, sequenceBarrier, _handlers[0], new IgnoreExceptionHandler(), _workSequence); _workProcessors[1] = new WorkProcessor <ValueEvent>(_ringBuffer, sequenceBarrier, _handlers[1], new IgnoreExceptionHandler(), _workSequence); for (var i = 0; i < _numPublishers; i++) { _valuePublishers[i] = new ValuePublisher(_cyclicBarrier, _ringBuffer, _iterations); } _ringBuffer.AddGatingSequences(_workProcessors[0].Sequence, _workProcessors[1].Sequence); }
public void RemoveWorkHandlerLostEventExample() { int eventSize = 8; var countdownEvent = new CountdownEvent(eventSize); var workSequence = new Sequence(); var disruptor = new Disruptor <StubEvent>(StubEvent.EventFactory, 4); var ringBuffer = disruptor.Start(); var handler1 = new DynamicHandler(1, countdownEvent); var processor1 = new WorkProcessor <StubEvent>(ringBuffer, ringBuffer.NewBarrier(), handler1, new FatalExceptionHandler(), workSequence); var handler2 = new DynamicHandler(2, countdownEvent); var processor2 = new WorkProcessor <StubEvent>(ringBuffer, ringBuffer.NewBarrier(), handler2, new FatalExceptionHandler(), workSequence); ringBuffer.AddGatingSequences(processor1.Sequence); Task.Run(() => processor1.Run()); ringBuffer.AddGatingSequences(processor2.Sequence); Task.Run(() => processor2.Run()); handler1.AwaitStart(); handler2.AwaitStart(); Thread.Sleep(10); // processor1 should own an unavailable work sequence // => this sequence will be dropped by Halt processor1.Halt(); var producer = new MessageProducer(disruptor, InitData(0, eventSize)); Task.Run(() => producer.Run()); producer.AwaitStart(); handler1.AwaitShutdown(); ringBuffer.RemoveGatingSequence(processor1.Sequence); // countdownEvent should not reach zero because of the dropped sequence var await = countdownEvent.Wait(TimeSpan.FromMilliseconds(500)); Assert.IsFalse(await); }
public TwoPublisherToTwoProcessorWorkProcessorThroughputTest() : base(TestName, ITERATIONS) { sequenceBarrier = ringBuffer.NewBarrier(); workProcessors[0] = new WorkProcessor <ValueEvent>(ringBuffer, sequenceBarrier, handlers[0], new IgnoreExceptionHandler(), workSequence); workProcessors[1] = new WorkProcessor <ValueEvent>(ringBuffer, sequenceBarrier, handlers[1], new IgnoreExceptionHandler(), workSequence); //workerPool = new WorkerPool<ValueEvent>(ringBuffer, sequenceBarrier, new IgnoreExceptionHandler(), handlers); for (int i = 0; i < NUM_PUBLISHERS; i++) { valuePublishers[i] = new ValuePublisher_V3(cyclicBarrier, ringBuffer, ITERATIONS); } workProcessors.ToList().ForEach(e => ringBuffer.AddGatingSequences(e.Sequence)); //ringBuffer.AddGatingSequences(/*workerPool.getWorkerSequences()*/workProcessors[0].Sequence, workProcessors[1].Sequence); }
public TwoPublisherToTwoProcessorWorkProcessorThroughputTest() : base(TestName, ITERATIONS) { sequenceBarrier = ringBuffer.NewBarrier(); workProcessors[0] = new WorkProcessor<ValueEvent>(ringBuffer, sequenceBarrier, handlers[0], new IgnoreExceptionHandler(), workSequence); workProcessors[1] = new WorkProcessor<ValueEvent>(ringBuffer, sequenceBarrier, handlers[1], new IgnoreExceptionHandler(), workSequence); //workerPool = new WorkerPool<ValueEvent>(ringBuffer, sequenceBarrier, new IgnoreExceptionHandler(), handlers); for (int i = 0; i < NUM_PUBLISHERS; i++) { valuePublishers[i] = new ValuePublisher_V3(cyclicBarrier, ringBuffer, ITERATIONS); } workProcessors.ToList().ForEach(e => ringBuffer.AddGatingSequences(e.Sequence)); //ringBuffer.AddGatingSequences(/*workerPool.getWorkerSequences()*/workProcessors[0].Sequence, workProcessors[1].Sequence); }
public void RemoveWorkHandlerLaterTest() { var eventSize = 8; var countdownEvent = new CountdownEvent(eventSize); var workSequence = new Sequence(); var disruptor = new Disruptor <StubEvent>(StubEvent.EventFactory, 4); var ringBuffer = disruptor.Start(); var handler1 = new DynamicHandler(1, countdownEvent); var processor1 = new WorkProcessor <StubEvent>(ringBuffer, ringBuffer.NewBarrier(), handler1, new FatalExceptionHandler(), workSequence); var handler2 = new DynamicHandler(2, countdownEvent); var processor2 = new WorkProcessor <StubEvent>(ringBuffer, ringBuffer.NewBarrier(), handler2, new FatalExceptionHandler(), workSequence); ringBuffer.AddGatingSequences(processor1.Sequence); Task.Run(() => processor1.Run()); ringBuffer.AddGatingSequences(processor2.Sequence); Task.Run(() => processor2.Run()); handler1.AwaitStart(); handler2.AwaitStart(); Thread.Sleep(10); // processor1 should own an unavailable work sequence // => this sequence should not be dropped by HaltLater processor1.HaltLater(); var producer = new MessageProducer(disruptor, InitData(0, eventSize)); Task.Run(() => producer.Run()); producer.AwaitStart(); handler1.AwaitShutdown(); ringBuffer.RemoveGatingSequence(processor1.Sequence); Assert.IsTrue(countdownEvent.Wait(TimeSpan.FromSeconds(3))); }
public TwoToTwoWorkProcessorThroughputTest() : base(Test_Disruptor, ITERATIONS) { sequenceBarrier = ringBuffer.NewBarrier(); handlers[0] = new ValueAdditionWorkHandler(); handlers[1] = new ValueAdditionWorkHandler(); workProcessors[0] = new WorkProcessor <ValueEvent>(ringBuffer, sequenceBarrier, handlers[0], new IgnoreExceptionHandler(), workSequence); workProcessors[1] = new WorkProcessor <ValueEvent>(ringBuffer, sequenceBarrier, handlers[1], new IgnoreExceptionHandler(), workSequence); for (int i = 0; i < NUM_PUBLISHERS; i++) { valuePublishers[i] = new ValuePublisher(cyclicBarrier, ringBuffer, ITERATIONS); } ringBuffer.AddGatingSequences(workProcessors[0].Sequence, workProcessors[1].Sequence); }
/// <summary> /// Create a worker pool to enable an array of <see cref="IWorkHandler{TEvent}"/>s to consume published sequences. /// /// This option requires a pre-configured <see cref="RingBuffer{TEvent}"/> which must have <see cref="RingBuffer{TEvent}.AddGatingSequences(ISequence[])"/> called before the work pool is started. /// </summary> /// <param name="ringBuffer">of events to be consumed.</param> /// <param name="barrier">on which the workers will depend.</param> /// <param name="exceptionHandler">to callback when an error occurs which is not handled by the <see cref="IWorkHandler{TEvent}"/>s.</param> /// <param name="workHandlers">to distribute the work load across.</param> public WorkerPool( RingBuffer <T> ringBuffer, ISequenceBarrier barrier, IExceptionHandler <T> exceptionHandler, params IWorkHandler <T>[] workHandlers) { _ringBuffer = ringBuffer; var numWorkers = workHandlers.Length; _workProcessors = new WorkProcessor <T> [numWorkers]; for (var i = 0; i < numWorkers; i++) { _workProcessors[i] = new WorkProcessor <T>( ringBuffer, barrier, workHandlers[i], exceptionHandler, _workSequence); } }
public TwoToTwoWorkProcessorThroughputTest() : base(Test_Disruptor, ITERATIONS) { sequenceBarrier = ringBuffer.NewBarrier(); handlers[0] = new ValueAdditionWorkHandler(); handlers[1] = new ValueAdditionWorkHandler(); workProcessors[0] = new WorkProcessor<ValueEvent>(ringBuffer, sequenceBarrier, handlers[0], new IgnoreExceptionHandler(), workSequence); workProcessors[1] = new WorkProcessor<ValueEvent>(ringBuffer, sequenceBarrier, handlers[1], new IgnoreExceptionHandler(), workSequence); for (int i = 0; i < NUM_PUBLISHERS; i++) { valuePublishers[i] = new ValuePublisher(cyclicBarrier, ringBuffer, ITERATIONS); } ringBuffer.AddGatingSequences(workProcessors[0].Sequence, workProcessors[1].Sequence); }
public void P7_TestWorkProcessorProcessQueue() { var wp = new WorkProcessor(2); var ready = new List <string> { "A", "F", "C" }; wp.LoadWorkQueue(ref ready); Assert.AreEqual(1, ready.Count); Assert.AreEqual("C", ready[0]); List <string> completed; for (var i = 0; i < 7; i++) { completed = wp.ProcessQueue(); if (i == 0) { Assert.AreEqual(1, completed.Count); Assert.AreEqual("A", completed[0]); } else if (i == 5) { Assert.AreEqual(1, completed.Count); Assert.AreEqual("F", completed[0]); } else if (i > 5) { Assert.IsTrue(wp.IsTaskQueueEmpty()); } else { Assert.AreEqual(0, completed.Count); } } }
/// <summary> /// Create a worker pool to enable an array of <see cref="IWorkHandler{TEvent}"/>s to consume published sequences. /// /// This option requires a pre-configured <see cref="RingBuffer{TEvent}"/> which must have <see cref="RingBuffer{TEvent}.AddGatingSequences(ISequence[])"/> called before the work pool is started. /// </summary> /// <param name="eventFactory">for filling the <see cref="RingBuffer{TEvent}"/>.</param> /// <param name="exceptionHandler">to callback when an error occurs which is not handled by the <see cref="IWorkHandler{TEvent}"/>s.</param> /// <param name="workHandlers">to distribute the work load across.</param> public WorkerPool( IEventFactory <T> eventFactory, IExceptionHandler <T> exceptionHandler, params IWorkHandler <T>[] workHandlers) { _ringBuffer = RingBuffer <T> .CreateMultiProducer(eventFactory, 1024); var barrier = _ringBuffer.NewBarrier(); var numWorkers = workHandlers.Length; _workProcessors = new WorkProcessor <T> [numWorkers]; for (int i = 0; i < numWorkers; i++) { _workProcessors[i] = new WorkProcessor <T>( _ringBuffer, barrier, workHandlers[i], exceptionHandler, _workSequence); } _ringBuffer.AddGatingSequences(GetWorkerSequences()); }
private static void Main() { Paths.SetProcessedDataFolder(@"C:\Users\ceesj\Documents\hogeschool\minor\Code\ProcessedData\"); Paths.SetRawDataFolder(@"C:\Users\ceesj\Documents\hogeschool\minor\Code\Datasources\"); // Paths.SetProcessedDataFolder(@"E:\Hogeschool\Polishift Organization\ProcessedData\"); // Paths.SetRawDataFolder(@"E:\Hogeschool\Polishift Organization\Datasources\"); #region ParseCode var interestCsvLocation = Paths.RawDataFolder + @"\Economical & Social\Interest rates (incomplete)\interest_data.csv"; var warCsvLocation = Paths.RawDataFolder + @"\Economical & Social\Wars\war_data.csv"; var tvCsvLocation = Paths.RawDataFolder + @"\Economical & Social\Households with TV\households_with_television_data.csv"; var populationCsvLocation = Paths.RawDataFolder + @"\Economical & Social\GDP & Population & GDP Per capita\population_data.csv"; var gdpTotalCsvLocation = Paths.RawDataFolder + @"\Economical & Social\GDP & Population & GDP Per capita\gdp_data.csv"; var gdpCapitaCsvLocation = Paths.RawDataFolder + @"\Economical & Social\GDP & Population & GDP Per capita\percapita_gdp_data.csv"; var workCsvLocation = Paths.RawDataFolder + @"\Economical & Social\NMC_5_0.csv"; ICsvModelFactory <WorkModel> modelFactory = new WorkModelFactory(); var allItemsAsModels = CsvToModel <WorkModel> .ParseAllCsvLinesToModels( workCsvLocation, modelFactory); var processor = new WorkProcessor(); processor.SerializeDataToJson(allItemsAsModels); #endregion #region Parsing country borders // var countryBorderDirectory = Paths.ProcessedDataFolder+ @"countryInformation"; // IJsonModelFactory<CountryGeoModel> countryGeoModelFactory = new CountryGeoModelFactory(); // var processor2 = new CountryBordersProcessor(); // // var allCountryGeoModels = // JsonToModel<CountryGeoModel>.ParseJsonDirectoryToModels(countryBorderDirectory, countryGeoModelFactory, // "*.geo.json"); // processor2.SerializeDataToJson(allCountryGeoModels); // // var countryBordersRepo = new CountryBordersRepository(); // foreach (var country in countryBordersRepo.GetAll()) // { // Console.WriteLine("Countrycode: " + country.CountryCode); // } #endregion #region Filtering /* * IFilter filter = new YearFilter(); * filter.Filter(); * * filter = new EuropeFilter(); * filter.Filter(); */ #endregion Downloader.DownloadData(Paths.ProcessedDataFolder); }
private void CreateProcessor(Socket client) { var processor = new WorkProcessor(this, client); processor.Run(ProcessCmdLine); }
public void execute(WorkProcessor processor) { throw new System.NotImplementedException(); }
public EventReleaser(WorkProcessor <T> workProcessor) { _workProcessor = workProcessor; }