public void Index <T>(ulong collectionId, IEnumerable <Document> job, IModel <T> model, IndexSession <T> indexSession) { LogInformation($"indexing collection {collectionId}"); var time = Stopwatch.StartNew(); using (var queue = new ProducerConsumerQueue <Document>(document => { foreach (var field in document.Fields) { if (field.Value != null && field.Index) { indexSession.Put(field.DocumentId, field.KeyId, field.Tokens); } } })) { foreach (var document in job) { foreach (var field in document.Fields) { if (field.Value != null && field.Index) { field.Analyze(model); } } queue.Enqueue(document); } } LogInformation($"processed indexing job (collection {collectionId}) in {time.Elapsed}"); }
public void ProducerConsumerSampleTest() { var messageResolverBuilder = new MessageHandlerResolverBuilder(); var eventHandle = new CountdownEvent(5); messageResolverBuilder.AddHandler(typeof(SessionRequestMessage), new SessionRequestMessageHandler( (message, token) => { eventHandle.Signal(); return(Task.CompletedTask); })); var cancellationTokenSource = new CancellationTokenSource(); using (var producerConsumerQueue = new ProducerConsumerQueue(messageResolverBuilder.Build(), cancellationTokenSource.Token)) { var sessionRequestMessage = new SessionRequestMessage( "123", new HandShakeRequest("1", "1"), new VaspInformation("1", "1", "1", null, null, null, null, "")); producerConsumerQueue.Enqueue(sessionRequestMessage); producerConsumerQueue.Enqueue(sessionRequestMessage); producerConsumerQueue.Enqueue(sessionRequestMessage); producerConsumerQueue.Enqueue(sessionRequestMessage); producerConsumerQueue.Enqueue(sessionRequestMessage); eventHandle.Wait(1_000); } }
public void When_disposing_while_items_still_in_queue() { var exceptions = new List <Exception>(); var numbers = new List <int>(); Action <int> work = n => { Thread.Sleep(500.Milliseconds()); numbers.Add(n); }; var pcq = new ProducerConsumerQueue <int>(work, 1); pcq.OnException += (sender, exception) => exceptions.Add(exception); pcq.Add(1); pcq.Add(2); pcq.Add(3); pcq.Dispose(); pcq.Completion.Result.ShouldBeFalse(); numbers.ShouldBe(new[] { 1 }); exceptions.ShouldBeEmpty(); Should.Throw <ObjectDisposedException>(() => pcq.Capacity.ShouldBe(-1)) .Message.ShouldBe("The collection has been disposed.\r\nObject name: 'BlockingCollection'."); Should.Throw <ObjectDisposedException>(() => pcq.PendingCount.ShouldBe((uint)2)) .Message.ShouldBe("The collection has been disposed.\r\nObject name: 'BlockingCollection'."); Should.Throw <ObjectDisposedException>(() => pcq.PendingItems.ShouldBe(new[] { 2, 3 })) .Message.ShouldBe("The collection has been disposed.\r\nObject name: 'BlockingCollection'."); }
public JobQueue( ILogger <JobQueue> logger) { _queue = new ProducerConsumerQueue <AsyncJob>(DispatchJob); _logger = logger; _enqueued = new ConcurrentDictionary <string, AsyncJob>(); }
public TextLog() { InitializeComponent(); _appendQueue = new ProducerConsumerQueue <char>((c) => sizeof(char), 100000); _data = new LargeCollection <char>(65536, (c) => sizeof(char)); _throttle = new Throttle(2.0f); }
public async Task PermiateAll() { var permutes = new HashSet <uint>(); using (var queue = new ProducerConsumerQueue <uint>(uint.MaxValue)) { var permuteTask = new Task(() => { for (long i = uint.MinValue; i < 10000000; i++) { queue.Put(UrlID.PermuteId((uint)i)); if (i % 1000000 == 0) { System.Console.WriteLine("Processed {0}", i); } } queue.FinishedProducing(); }); var checkTask = new Task(() => { while (!queue.HasFinishedProducing) { var toCheck = queue.Take(); Assert.AreEqual(false, permutes.Contains(toCheck), "Value {0} clashed".FormatWith(toCheck)); permutes.Add(toCheck); } }); permuteTask.Start(); checkTask.Start(); await Task.WhenAll(permuteTask, checkTask); } }
/// <summary> /// Connects the adapter. /// </summary> /// <returns></returns> public async Task ConnectAsync() { ThrowIfDisposed(); if (State != ResonanceComponentState.Connected) { Logger.LogInformation("Connecting Adapter..."); try { await OnConnect(); Logger.LogInformation("Adapter Connected."); State = ResonanceComponentState.Connected; if (WriteMode == ResonanceAdapterWriteMode.Queue) { _pushQueue = new ProducerConsumerQueue <byte[]>(); _pushThread = new System.Threading.Thread(PushThreadMethod); _pushThread.IsBackground = true; _pushThread.Name = $"{this} Push Thread"; _pushThread.Start(); } } catch (Exception ex) { Logger.LogError(ex, "Adapter connection error occurred."); throw ex; } } }
public void ThrownExceptionByMultipleWorkersShouldBePublishedCorrectly() { var exceptions = new ConcurrentQueue <Exception>(); Action <MyClass> consumer = x => { Task.Delay(50.Milliseconds()); throw new InvalidDataException("Something went wrong for: " + x.Id); }; var queue = new ProducerConsumerQueue <MyClass>(consumer, 2); queue.OnException += (sender, args) => { exceptions.Enqueue(args.InnerException); }; queue.Add(_getData(0)); queue.Add(_getData(1)); Task.Delay(100.Milliseconds()).Wait(); exceptions.Count.ShouldBe(2); exceptions.ShouldContain(e => e is InvalidDataException && e.Message.Equals("Something went wrong for: 0")); exceptions.ShouldContain(e => e is InvalidDataException && e.Message.Equals("Something went wrong for: 1")); }
public void DisposedQueueShouldCancelConsumersCorrectly() { Exception exceptionThrown = null; var consumed = new ConcurrentBag <int>(); var queue = new ProducerConsumerQueue <int>(i => { Thread.Sleep(50.Milliseconds()); consumed.Add(i); }, 1); queue.OnException += (sender, args) => { using (consumed.Lock(100.Milliseconds())) { exceptionThrown = args; } }; queue.Add(1); queue.Add(2); queue.Add(3); queue.Add(4); queue.Add(5); queue.CompleteAdding(); queue.Completion.Wait(500.Milliseconds()).ShouldBeTrue(); consumed.Count.ShouldBeGreaterThanOrEqualTo(2); Thread.Sleep(1.Seconds()); exceptionThrown.ShouldBeNull(); }
public static void WriteWatSegment( string fileName, string collection, ITextModel model, ILogger logger, string refFileName) { var time = Stopwatch.StartNew(); var collectionId = collection.ToHash(); var storeFieldNames = new HashSet <string> { "title", "description", "url", "filename" }; var indexFieldNames = new HashSet <string> { "title", "description", "url" }; using (var sessionFactory = new SessionFactory(new KeyValueConfiguration("sir.ini"), logger)) using (var writeSession = sessionFactory.CreateWriteSession(collectionId)) using (var indexSession = sessionFactory.CreateIndexSession(collectionId, model)) using (var queue = new ProducerConsumerQueue <IDictionary <string, object> >(1, (document => { sessionFactory.Write(document, writeSession, indexSession, storeFieldNames, indexFieldNames); }))) { foreach (var document in ReadWatFile(fileName, refFileName)) { queue.Enqueue(document); } } logger.LogInformation($"indexed {fileName} in {time.Elapsed}"); }
//! Constructor for asynchronous connections. public MySqlConnection(ProducerConsumerQueue <SqlOperation> queue, MySqlConnectionInfo connectionInfo) { _queue = queue; _connectionInfo = connectionInfo; _connectionFlags = ConnectionFlags.CONNECTION_ASYNC; _worker = new DatabaseWorker(_queue, this); }
private void _conductor_OnIceStateChanged(IceConnectionStates state) { if (!_closeConnection) { if (state == IceConnectionStates.kIceConnectionConnected) { if (!_connectionCompleted) { _connectionCompleted = true; _incomingQueue = new ProducerConsumerQueue <byte[]>(); _receiveThread = new Thread(IncomingQueueThreadMethod); _receiveThread.IsBackground = true; _receiveThread.Start(); _connectionCompletionSource.SetResult(true); } } else if (state == IceConnectionStates.kIceConnectionFailed) { if (_connectionCompleted) { OnFailed(new ResonanceWebRTCConnectionFailedException("Ice candidate connection failed.")); } else { _connectionCompleted = true; _connectionCompletionSource.SetException(new ResonanceWebRTCConnectionFailedException(new ResonanceWebRTCConnectionFailedException("Ice candidate connection failed."))); } } } }
public void StressTest() { queue = new ProducerConsumerQueue(); ThreadStart consumerMethod = new ThreadStart(accumulatingRunner); Thread consumerThread = new Thread(consumerMethod); consumerThread.Start(); Thread.Sleep(500); ArrayList threads = new ArrayList(); for (int i = 0; i < 100; i++) { ThreadStart producerMethod = new ThreadStart(producingThread); threads.Add(new Thread(producerMethod)); } for (int i = 0; i < 100; i++) { ((Thread)threads[i]).Start(); } for (int i = 0; i < 100; i++) { ((Thread)threads[i]).Join(); } consumerThread.Join(); Assert.AreEqual(0, queue.Count); Assert.AreEqual(100000, staticCounter); for (int i = 0; i < 100000; i++) { Assert.AreEqual(i, (int)accumulatedResults[i], "Failed at index " + i); } }
public void DisposedQueueShouldNotAllowAddingNewItems() { var consumed = new ConcurrentBag <MyClass>(); Action <MyClass> consumer = x => { consumed.Add(x); }; var queue = new ProducerConsumerQueue <MyClass>(consumer, 1); ProducerConsumerQueueException thrownException = null; queue.OnException += (sender, args) => { thrownException = args; }; const int WorkItems = 10; for (var i = 0; i < 10; i++) { queue.Add(_getData(i)); } queue.CompleteAdding(); queue.Completion.Wait(5.Seconds()).ShouldBeTrue(); Thread.Sleep(50.Milliseconds()); consumed.Count.ShouldBe(WorkItems); Assert.DoesNotThrow(() => queue.Add(new MyClass())); Thread.Sleep(1.Seconds()); thrownException.ShouldNotBeNull(); thrownException.Message.ShouldBe("Exception occurred when adding item."); }
public void ProducerConsumerQueueOneConsumerTest() { const int initialItemCount = 100; const int producerDelay = 10; const int consumerDelay = producerDelay / 5; var processedItems = new ConcurrentBag <int>(); using (var queue = ProducerConsumerQueue <int> .Start((value, cancellationToken) => { Thread.Sleep(producerDelay); processedItems.Add(value); System.Console.WriteLine(value); }, 1)) { Parallel.ForEach ( Enumerable.Range(0, initialItemCount), new ParallelOptions { MaxDegreeOfParallelism = Environment.ProcessorCount }, value => { // ReSharper disable once AccessToDisposedClosure queue.Enqueue(value); Thread.Sleep(consumerDelay); } ); Thread.Sleep(initialItemCount * consumerDelay); queue.Shutdown(); } Assert.Greater(initialItemCount, processedItems.Count); }
public async Task Complex_1() { var expected = Enumerable.Range(0, 1000).ToArray(); var result = new List <int>(); var @lock = new object(); var counter = 0; var putManyCounter = 0; var takeManyCounter = 0; using (var queue = new ProducerConsumerQueue <int>(10)) { Func <string, Task> produceAction = async(string name) => { while (counter < 1000) { //await Task.Delay(10); var localProduction = new List <int>(); lock (@lock) { var numToProduce = Tools.Maths.RandomNumberGenerator.Next(1, 10); for (var i = 0; i < numToProduce; i++) { if (counter == 1000) { break; } localProduction.Add(counter++); } } await queue.PutManyAsync(localProduction); } }; Func <string, Task> consumeAction = async(string name) => { while (!queue.HasFinishedProducing) { //await Task.Delay(10); var numToConsume = Tools.Maths.RandomNumberGenerator.Next(1, 10); var consumption = await queue.TakeManyAsync(numToConsume); result.AddRange(consumption); } }; Func <Task> produceTask = async() => { await Task.WhenAll(produceAction("Producer 1")); queue.FinishedProducing(); }; Func <Task> consumeTask = async() => { await Task.WhenAll(consumeAction("Consumer 1")); queue.FinishedConsuming(); }; await Task.WhenAll(produceTask(), consumeTask()); Tools.NUnitTool.Print(result); Assert.AreEqual(expected, result); } }
public void When_waiting_for_consumption_to_complete() { var exceptions = new ConcurrentBag <Exception>(); var numbers = new ConcurrentQueue <int>(); Action <int> work = n => { Thread.Sleep(50.Milliseconds()); numbers.Enqueue(n); }; using (var pcq = new ProducerConsumerQueue <int>(work, 1)) { pcq.OnException += (sender, exception) => exceptions.Add(exception); pcq.Add(1); pcq.Add(2); pcq.Add(3); pcq.CompleteAdding(); pcq.Completion.Result.ShouldBeTrue(); numbers.ShouldBe(new[] { 1, 2, 3 }); exceptions.ShouldBeEmpty(); pcq.Capacity.ShouldBe(-1); pcq.PendingCount.ShouldBe((uint)0); pcq.PendingItems.ShouldBeEmpty(); } }
public void SlowConsumerWithFullCapacityShouldReturnFalseWhenTryingToAddNewItemWithinSpecifiedTimeout() { var consumedItems = new List <int>(); var queue = new ProducerConsumerQueue <int>(i => { Thread.Sleep(25); consumedItems.Add(i); }, 1, 1); // Sanity checks consumedItems.ShouldBeEmpty(); queue.WorkerCount.ShouldBe <uint>(1); queue.Capacity.ShouldBe(1); queue.PendingCount.ShouldBe <uint>(0); queue.TryAdd(1, 10.Milliseconds()).ShouldBeTrue(); Thread.Sleep(50); consumedItems.Count.ShouldBe(1); queue.TryAdd(2, 10.Milliseconds()).ShouldBeTrue(); queue.TryAdd(3, 10.Milliseconds()).ShouldBeTrue(); queue.PendingCount.ShouldBe <uint>(1); queue.TryAdd(4, 10.Milliseconds()).ShouldBeFalse(); Thread.Sleep(100); queue.TryAdd(5, 10.Milliseconds()).ShouldBeTrue(); Thread.Sleep(100); consumedItems.Count.ShouldBe(4); }
public Decompressor(int blockSize, string sourceFilename, string destinationFilename, CancellationToken cancellationToken, Action <string, Exception> onException) : base(blockSize, sourceFilename, destinationFilename, Entities.Command.Decompress, cancellationToken, onException) { _decompressingQueue = new ProducerConsumerQueue <ByteBlock>(Environment.ProcessorCount, Decompress); _writingQueue = new ProducerConsumerQueue <ByteBlock>(1, Write); }
public ConcurrentBlockCompressor( ICompressor compressor, string pathToFile, string saveToFile, int blockSize = 1024 ) { _saveToFile = saveToFile; var workerCount = Environment.ProcessorCount * 4; _chunkSize = 100; _compressor = compressor; _fileStreamByBatchWriter = new FileStreamByBatchWriter(saveToFile); _producerConsumerCompressTask = new ProducerConsumerQueue(workerCount); _producerConsumerWriteTask = new ProducerConsumerQueue(); _gzipActionPerformer = new GzipActionPerformer( _producerConsumerCompressTask, _producerConsumerWriteTask, _fileStreamByBatchWriter ); _fileStreamBatchEnumerator = new FileStreamBatchEnumerator(pathToFile, blockSize); }
/// <summary> /// Prevents a default instance of the <see cref="WebRTCAdapter"/> class from being created. /// </summary> private WebRTCAdapter() { _pendingCandidates = new List <RTCIceCandidate>(); _incomingQueue = new ProducerConsumerQueue <byte[]>(); IceServers = new List <WebRTCIceServer>(); ChannelName = "resonance"; }
/// <summary> /// Prevents a default instance of the <see cref="WebRTCAdapter"/> class from being created. /// </summary> private WebRTCAdapterNative() { ConnectionTimeout = TimeSpan.FromSeconds(30); _pendingCandidates = new List <WebRTCIceCandidate>(); _incomingQueue = new ProducerConsumerQueue <byte[]>(); IceServers = new List <WebRTCIceServer>(); }
public void BlockingConsumerWithOneWorkerShouldBlockEverything() { var exceptionThrown = false; var consumed = new ConcurrentBag <MyClass>(); Action <MyClass> consumer = x => { Thread.Sleep(1.Minutes()); consumed.Add(x); }; var queue = new ProducerConsumerQueue <MyClass>(consumer, 1); queue.OnException += (sender, args) => { using (consumed.Lock(100.Milliseconds())) { exceptionThrown = true; } }; queue.PendingCount.ShouldBe <uint>(0); 10.Times(n => queue.Add(_getData(n))); Thread.Sleep(100.Milliseconds()); queue.PendingCount.ShouldBe <uint>(9); queue.Completion.Wait(600.Milliseconds()).ShouldBeFalse(); consumed.Count.ShouldBe(0); exceptionThrown.ShouldBeFalse(); }
public void ShutdownQueueAfterSomeAddsShouldResultInTheProcessOfAllAddedItemsBeforeDisposal() { var consumed = new ConcurrentBag <MyClass>(); Action <MyClass> consumer = x => { consumed.Add(x); }; var queue = new ProducerConsumerQueue <MyClass>(consumer, 1); ProducerConsumerQueueException thrownException = null; queue.OnException += (sender, args) => { thrownException = args; }; 5.Times(n => { queue.Add(_getData(n)); }); queue.CompleteAdding(); queue.Completion.Wait(10.Seconds()).ShouldBeTrue(); Action afterDisposedEnqueues = () => 3.Times(n => { queue.Add(_getData(n)); }); afterDisposedEnqueues.ShouldNotThrow(); Thread.Sleep(100); consumed.Count.ShouldBe(5); Thread.Sleep(1.Seconds()); thrownException.ShouldNotBeNull(); thrownException.Message.ShouldBe("Exception occurred when adding item."); }
public void BlockingConsumerWithTwoWorkersShouldBlockOnlyOneItem() { var exceptionThrown = false; var consumed = new ConcurrentBag <MyClass>(); var counter = 0; Action <MyClass> consumer = x => { if (counter == 0) { counter++; Thread.Sleep(1.Minutes()); } consumed.Add(x); }; var queue = new ProducerConsumerQueue <MyClass>(consumer, 2); queue.OnException += (sender, args) => { using (consumed.Lock(100.Milliseconds())) { exceptionThrown = true; } }; 10.Times(n => queue.Add(_getData(n))); Thread.Sleep(100); exceptionThrown.ShouldBeFalse(); queue.PendingCount.ShouldBe <uint>(0); queue.Completion.Wait(500.Milliseconds()).ShouldBeFalse(); consumed.Count.ShouldBe(9); }
public SlackController(PluginsCollection plugins, IConfigurationProvider config, SessionFactory sessionFactory) { _postMessageUrl = "https://slack.com/api/chat.postMessage"; _queue = new ProducerConsumerQueue <dynamic>(1, callback: Send); _sessionFactory = sessionFactory; _config = config; }
public DatabaseWorker(ProducerConsumerQueue <ISqlOperation> newQueue, MySqlBase <T> mySqlBase) { _queue = newQueue; _mySqlBase = mySqlBase; _cancelationToken = false; _workerThread = new Thread(WorkerThread); _workerThread.Start(); }
public void CreatingWithPositiveBoundedCapacityAndValidConsumerAndValidWorker() { var queue = new ProducerConsumerQueue <int>(i => { /* do nothing; */ }, 2, 1); queue.Capacity.ShouldBe(1); queue.MaximumConcurrencyLevel.ShouldBe <uint>(2); queue.PendingCount.ShouldBe <uint>(0); queue.PendingItems.ShouldBeEmpty(); }
public IconCache() { _images = new Dictionary <string, Image>(); _loadQueue = new ProducerConsumerQueue <Task>(); _loaderThread = new Thread(new ThreadStart(Dequeue)); _loaderThread.IsBackground = true; _loaderThread.Priority = ThreadPriority.BelowNormal; _loaderThread.Start(); }
public void ProducerConsumerStartsTest() { Action <string, CancellationToken> consumerAction = (value, cancellationToken) => { }; var consumerCount = Environment.ProcessorCount; using (var queue = ProducerConsumerQueue <string> .Start(consumerAction, consumerCount)) { Assert.IsNotNull(queue); } }
public void CreatingWithPositiveBoundedCapacityAndValidConsumerButZeroConcurrencyLevel() { Action action = () => { var _ = new ProducerConsumerQueue <int>(i => { /* do nothing; */ }, 0, 1); }; action.ShouldThrow <ArgumentException>() .Message.ShouldBe("maxConcurrencyLevel should be greater than zero."); }
public void CanStopConsumerSide() { queue = new ProducerConsumerQueue(); dequeuedMsg = "This will be null if we break out of dequeue and return a null msg"; ThreadStart consumerMethod = new ThreadStart(runner); Thread consumerThread = new Thread(consumerMethod); consumerThread.Start(); Thread.Sleep(500); consumerThread.Interrupt(); consumerThread.Join(); Thread.Sleep(500); Assert.IsNull(dequeuedMsg); }
public void TestTaskCompletion() { var queue = new ProducerConsumerQueue(); var task = queue.Enqueue(() => { Thread.Sleep(500); }); task.Wait(); Assert.IsFalse(task.IsFaulted); Assert.IsTrue(task.IsCompleted); Assert.IsTrue(task.Status == System.Threading.Tasks.TaskStatus.RanToCompletion); }
public void CanAddElementAsProducerAndRemoveAsConsumer() { queue = new ProducerConsumerQueue(); ThreadStart consumerMethod = new ThreadStart(runner); Thread consumerThread = new Thread(consumerMethod); consumerThread.Start(); Thread.Sleep(500); queue.Enqueue("this is a string"); Thread.Sleep(500); consumerThread.Join(); Assert.AreEqual("this is a string", dequeuedMsg); }
public void TestException() { var queue = new ProducerConsumerQueue(); var task = queue.Enqueue(() => { Thread.Sleep(500); throw new Exception("You failed"); }); try { task.Wait(); } catch (Exception e) { Assert.IsNotNull(e); Assert.IsTrue(e.InnerException.Message == "You failed"); } }
public void Start() { try { contextQueue = new ProducerConsumerQueue <HttpListenerContext>(ProcessRequest, config.NumWorkerThreads); RestLogger.LogInfo("RestServer starting, {0}", config.BaseUrl); listener.Prefixes.Add(config.BaseUrl); IsListening = true; listener.Start(); listenerThread.Start(); } catch(Exception ex) { IsListening = false; LogException("Start", ex); throw; } }