public async void OnDataAvailableShouldCancel()
        {
            var aq = new AsyncCollection<bool>();
            var cancelToken = new CancellationTokenSource();
            Task.Delay(TimeSpan.FromMilliseconds(100)).ContinueWith(t => cancelToken.Cancel());

            await aq.OnDataAvailable(cancelToken.Token);
        }
        public void TryTakeShouldReturnFalseOnEmpty()
        {
            var aq = new AsyncCollection<bool>();

            Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.False, "Task should indicate no data available.");

            bool data;
            Assert.That(aq.TryTake(out data), Is.False, "TryTake should report false on empty collection.");
            Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.False, "Task should indicate no data available.");
        }
        public void OnDataAvailableShouldTriggerWhenDataAdded()
        {
            var aq = new AsyncCollection<bool>();

            Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.False, "Task should indicate no data available.");

            aq.Add(true);

            Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.True, "Task should indicate data available.");
        }
 public async Task OnDataAvailableShouldCancel()//change class Behavers from trowing OperationCanceledException to returning false
 {//do not debug with brack point it harm this test
     var aq = new AsyncCollection<bool>();
     int timeSpen = 100;
     Task waitUntilCancel = aq.OnHasDataAvailable(new CancellationTokenSource(timeSpen).Token);
     await Task.WhenAny(waitUntilCancel, Task.Delay(timeSpen / 2));
     Assert.IsFalse(waitUntilCancel.IsCompleted, "task Should Cancel only when time is up");
     //ToDO FIX
     //   Assert.IsFalse(await waitUntilCancel, "it Should return false when cancel");
 }
        public void DrainShouldBlockWhenDataRemoved()
        {
            var aq = new AsyncCollection<bool>();

            aq.Add(true);
            aq.Add(true);

            Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.True, "Task should indicate data available.");

            var drained = aq.Drain().ToList();
            Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.False, "Task should indicate no data available.");
        }
        public void OnDataAvailableShouldBlockWhenDataRemoved()
        {
            var aq = new AsyncCollection<bool>();

            aq.Add(true);

            Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.True, "Task should indicate data available.");

            bool data;
            aq.TryTake(out data);
            Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.False, "Task should indicate no data available.");
        }
Beispiel #7
0
        public async Task ConsumeLimitOverFilledComplete()
        {               //****************************************
            var MyCollection = new AsyncCollection <int>(10);
            //****************************************

            var ProducerTask = ProducerWithAddComplete(MyCollection, 100, (index) => index);

            var MyTask = ConsumerWithWait(MyCollection, TimeSpan.Zero);

            await ProducerTask;

            _ = await MyTask;
        }
Beispiel #8
0
        public async void TakeAsyncShouldRemoveItemsFromCollection()
        {
            const int expectedCount = 10;

            var collection = new AsyncCollection <int>();

            collection.AddRange(Enumerable.Range(0, expectedCount));

            var data = await collection.TakeAsync(expectedCount, TimeSpan.FromMilliseconds(100), CancellationToken.None);

            Assert.That(data.Count, Is.EqualTo(expectedCount));
            Assert.That(collection.Count, Is.EqualTo(0));
        }
Beispiel #9
0
        public void OnDataAvailableShouldBlockWhenDataRemoved()
        {
            var aq = new AsyncCollection <bool>();

            aq.Add(true);

            Assert.That(aq.OnHasDataAvailable(CancellationToken.None).IsCompleted, Is.True, "Task should indicate data available.");

            bool data;

            aq.TryTake(out data);
            Assert.That(aq.OnHasDataAvailable(CancellationToken.None).IsCompleted, Is.False, "Task should indicate no data available.");
        }
Beispiel #10
0
        public void Add()
        {               //****************************************
            var MyCollection = new AsyncCollection <int>();
            //****************************************

            var MyTask = MyCollection.Add(42);

            //****************************************

            Assert.IsTrue(MyTask.IsCompleted, "Still waiting to add");

            Assert.AreEqual(1, MyCollection.Count, "Count not as expected");
        }
        public void DrainShouldBlockWhenDataRemoved()
        {
            var aq = new AsyncCollection <bool>();

            aq.Add(true);
            aq.Add(true);

            Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.True, "Task should indicate data available.");

            var drained = aq.Drain().ToList();

            Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.False, "Task should indicate no data available.");
        }
        public async Task CancelTake()
        {
            AsyncCollection <int> collection = new AsyncCollection <int>();

            using (CancellationTokenSource cancellationTokenSource = new CancellationTokenSource())
            {
                var t = Task.Run(async() => await collection.TakeAsync(cancellationTokenSource.Token));

                cancellationTokenSource.Cancel();

                await Assert.ThrowsExceptionAsync <OperationCanceledException>(() => t);
            }
        }
Beispiel #13
0
        public void PeekSuccess()
        {               //****************************************
            var MyCollection = new AsyncCollection <int>();

            //****************************************

            _ = MyCollection.TryAdd(42);

            var MyTask = MyCollection.Peek();

            //****************************************

            Assert.IsTrue(MyTask.IsCompleted, "Peek failed unexpectedly");
        }
Beispiel #14
0
        /// <summary>
        /// Initializes a new instance of the ItemProducer class.
        /// </summary>
        /// <param name="queryContext">request context</param>
        /// <param name="querySpecForInit">query spec for initialization</param>
        /// <param name="partitionKeyRange">The partition key range.</param>
        /// <param name="produceAsyncCompleteCallback">The callback to call once you are done fetching.</param>
        /// <param name="equalityComparer">The comparer to use to determine whether the producer has seen a new document.</param>
        /// <param name="initialPageSize">The initial page size.</param>
        /// <param name="initialContinuationToken">The initial continuation token.</param>
        public ItemProducer(
            CosmosQueryContext queryContext,
            SqlQuerySpec querySpecForInit,
            PartitionKeyRange partitionKeyRange,
            ProduceAsyncCompleteDelegate produceAsyncCompleteCallback,
            IEqualityComparer <CosmosElement> equalityComparer,
            long initialPageSize            = 50,
            string initialContinuationToken = null)
        {
            this.bufferedPages = new AsyncCollection <QueryResponse>();

            // We use a binary semaphore to get the behavior of a mutex,
            // since fetching documents from the backend using a continuation token is a critical section.
            this.fetchSemaphore = new SemaphoreSlim(1, 1);
            if (partitionKeyRange == null)
            {
                throw new ArgumentNullException(nameof(partitionKeyRange));
            }

            if (produceAsyncCompleteCallback == null)
            {
                throw new ArgumentNullException(nameof(produceAsyncCompleteCallback));
            }

            if (equalityComparer == null)
            {
                throw new ArgumentNullException(nameof(equalityComparer));
            }

            this.queryContext                 = queryContext;
            this.querySpecForInit             = querySpecForInit;
            this.PartitionKeyRange            = partitionKeyRange;
            this.produceAsyncCompleteCallback = produceAsyncCompleteCallback;
            this.equalityComparer             = equalityComparer;
            this.pageSize = initialPageSize;
            this.currentContinuationToken  = initialContinuationToken;
            this.BackendContinuationToken  = initialContinuationToken;
            this.PreviousContinuationToken = initialContinuationToken;
            if (!string.IsNullOrEmpty(initialContinuationToken))
            {
                this.hasStartedFetching = true;
                this.IsActive           = true;
            }

            this.fetchSchedulingMetrics = new SchedulingStopwatch();
            this.fetchSchedulingMetrics.Ready();
            this.fetchExecutionRangeAccumulator = new FetchExecutionRangeAccumulator();

            this.HasMoreResults = true;
        }
        public ProcDataflowBlock(
            IActionRunner actionRunner,
            int maximumNumberOfActionsInQueue,
            int?maximumDegreeOfParallelism)
        {
            this.actionRunner = actionRunner;
            this.maximumDegreeOfParallelism = maximumDegreeOfParallelism;

            collection = new AsyncCollection <Func <Task> >(new ConcurrentQueue <Func <Task> >(), maximumNumberOfActionsInQueue);

            collectionForReentrantItems = new AsyncCollection <Func <Task> >(new ConcurrentQueue <Func <Task> >());

            nodeId = Guid.NewGuid();
        }
Beispiel #16
0
        public AsyncProcDataflowBlock(
            int maximumNumberOfActionsInQueue,
            int?maximumDegreeOfParallelism,
            CancellationToken cancellationToken = default)
        {
            this.maximumDegreeOfParallelism = maximumDegreeOfParallelism;
            this.cancellationToken          = cancellationToken;

            collection = new AsyncCollection <Func <Task> >(new ConcurrentQueue <Func <Task> >(), maximumNumberOfActionsInQueue);

            collectionForReentrantItems = new AsyncCollection <Func <Task> >(new ConcurrentQueue <Func <Task> >());

            nodeId = Guid.NewGuid();
        }
        public void TakeWithDelay()
        {
            AsyncCollection <int> collection = new AsyncCollection <int>();

            Task.Run(() =>
            {
                Thread.Sleep(100);
                collection.Add(1);
            });

            int item = collection.Take();

            Assert.AreEqual(item, 1);
        }
        public async void TakeAsyncShouldBeAbleToCancel()
        {
            var cancelSource = new CancellationTokenSource();
            var collection   = new AsyncCollection <int>();

            Task.Delay(TimeSpan.FromMilliseconds(100)).ContinueWith(t => cancelSource.Cancel());

            var sw   = Stopwatch.StartNew();
            var data = await collection.TakeAsync(10, TimeSpan.FromMilliseconds(500), cancelSource.Token);

            sw.Stop();

            Assert.That(sw.ElapsedMilliseconds, Is.LessThan(300));
        }
Beispiel #19
0
        public void Clear()
        {
            var bufferSize = _buffer.Length;

            _buffer         = null;
            _offsetSize     = null;
            _emptySegments  = null;
            _filledSegments = null;

            if (bufferSize >= GCThreshold)
            {
                GC.Collect();
            }
        }
        public async void CollectionShouldReportCorrectBufferCount()
        {
            var collection = new AsyncCollection<int>();

            var dataTask = collection.TakeAsync(10, TimeSpan.FromHours(5), CancellationToken.None);

            collection.AddRange(Enumerable.Range(0, 9));
            Assert.That(collection.Count, Is.EqualTo(9));

            collection.Add(1);
            var data = await dataTask;
            Assert.That(data.Count, Is.EqualTo(10));
            Assert.That(collection.Count, Is.EqualTo(0));
        }
Beispiel #21
0
        public async Task CollectionShouldBlockOnMaxBuffer()
        {
            var collection = new AsyncCollection <int>();
            var task       = Task.Factory.StartNew(() => collection.AddRange(Enumerable.Range(0, 10)));
            await TaskTest.WaitFor(() => collection.Count >= 9);

            Assert.That(collection.Count, Is.EqualTo(9), "Buffer should block at 9 items.");
            Assert.That(task.IsCompleted, Is.False, "Task should be blocking on last item.");
            var item = collection.Pop();
            await TaskTest.WaitFor(() => task.IsCompleted);

            Assert.That(task.IsCompleted, Is.True, "Task should complete after room is made in buffer.");
            Assert.That(collection.Count, Is.EqualTo(9), "There should now be 9 items in the buffer.");
        }
Beispiel #22
0
        public async void TakeAsyncShouldBePerformant()
        {
            const int             dataSize   = 1000000;
            AsyncCollection <int> collection = new AsyncCollection <int>();

            collection.AddRange(Enumerable.Range(0, dataSize));
            var sw   = Stopwatch.StartNew();
            var list = await collection.TakeAsync(dataSize, TimeSpan.FromSeconds(1), CancellationToken.None);

            sw.Stop();
            Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
            Assert.That(list.Count, Is.EqualTo(dataSize));
            Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
        }
        public void TakeAsyncFromMultipleThreads()
        {
            AsyncCollection <int> collection = new AsyncCollection <int>();
            int t1Counter = 0;
            int t2Counter = 0;

            var t1 = Task.Run(async() =>
            {
                while (!collection.IsCompleted)
                {
                    try
                    {
                        await collection.TakeAsync();
                        t1Counter++;
                    }
                    catch (InvalidOperationException)
                    {
                    }
                }
            });

            var t2 = Task.Run(async() =>
            {
                while (!collection.IsCompleted)
                {
                    try
                    {
                        await collection.TakeAsync();
                        t2Counter++;
                    }
                    catch (InvalidOperationException)
                    {
                    }
                }
            });

            for (int i = 0; i < 1000; i++)
            {
                collection.Add(i);
            }

            collection.CompleteAdding();

            t1.Wait();
            t2.Wait();

            Assert.IsTrue(t1Counter > 400);
            Assert.IsTrue(t2Counter > 400);
        }
        public async void TakeAsyncShouldWaitXForBatchSizeToCollect()
        {
            const int expectedDelay = 100;
            const int expectedCount = 10;

            var collection = new AsyncCollection <int>();

            collection.AddRange(Enumerable.Range(0, expectedCount));

            var sw   = Stopwatch.StartNew();
            var data = await collection.TakeAsync(expectedCount + 1, TimeSpan.FromMilliseconds(expectedDelay), CancellationToken.None);

            Assert.That(sw.ElapsedMilliseconds, Is.GreaterThanOrEqualTo(expectedDelay));
            Assert.That(data.Count, Is.EqualTo(expectedCount));
        }
Beispiel #25
0
        public async Task TryTakeFromAny([Values(0, 1)] int index)
        {               //****************************************
            var MyCollections = new AsyncCollection <int>[] { new AsyncCollection <int>(), new AsyncCollection <int>() };
            //****************************************

            await MyCollections[index].Add(42);

            var MyResult = AsyncCollection <int> .TryTakeFromAny(MyCollections);

            //****************************************

            Assert.IsTrue(MyResult.HasItem, "Take failed unexpectedly");
            Assert.AreSame(MyCollections[index], MyResult.Source, "Collection was not as expected");
            Assert.AreEqual(42, MyResult.Item, "Item was not as expected");
        }
Beispiel #26
0
        public async Task ConsumePreFilled()
        {               //****************************************
            var MyCollection = new AsyncCollection <int>();
            //****************************************

            var ProducerTask = Producer(MyCollection, 20, (index) => index);

            var MyTask = Consumer(MyCollection);

            await ProducerTask;

            _ = MyCollection.CompleteAdding();

            _ = await MyTask;
        }
Beispiel #27
0
        public async Task AddCompleteAdd()
        {               //****************************************
            var MyCollection = new AsyncCollection <int>();
            //****************************************

            await MyCollection.AddComplete(42);

            Assert.IsFalse(await MyCollection.Add(84));

            //****************************************

            Assert.AreEqual(1, MyCollection.Count, "Count not as expected");

            Assert.IsTrue(MyCollection.IsAddingCompleted, "Not completed");
        }
Beispiel #28
0
        public void CancelsTaskWhenTokenIsCanceled()
        {
            CancellationTokenSource cancelSource = new CancellationTokenSource();
            var task = AsyncCollection <int> .TakeFromAnyAsync(_collections, cancelSource.Token);

            cancelSource.Cancel();
            Func <Task> asyncAct = () => task;

            asyncAct.ShouldThrow <TaskCanceledException>();

            _collections[0].Add(42);
            _collections[1].Add(64);
            _collections[0].Count.Should().Be(1);
            _collections[1].Count.Should().Be(1);
        }
        public async void CollectionShouldReportCorrectBufferCount()
        {
            var collection = new AsyncCollection <int>();

            var dataTask = collection.TakeAsync(10, TimeSpan.FromHours(5), CancellationToken.None);

            collection.AddRange(Enumerable.Range(0, 9));
            Assert.That(collection.Count, Is.EqualTo(9));

            collection.Add(1);
            var data = await dataTask;

            Assert.That(data.Count, Is.EqualTo(10));
            Assert.That(collection.Count, Is.EqualTo(0));
        }
        public void AddAndRemoveShouldBePerformant()
        {
            const int dataSize   = 1000000;
            var       collection = new AsyncCollection <int>();

            var sw           = Stopwatch.StartNew();
            var receivedData = new List <int>();

            Parallel.Invoke(
                () => collection.AddRange(Enumerable.Range(0, dataSize)),
                () => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result);
            sw.Stop();
            Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
            Assert.That(receivedData.Count, Is.EqualTo(dataSize));
            Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
        }
Beispiel #31
0
        private async Task ProducerWithAddComplete <TItem>(AsyncCollection <TItem> collection, int count, Func <int, TItem> producer)
        {
            for (var Index = 0; Index < count; Index++)
            {
                if (Index == count - 1)
                {
                    await collection.AddComplete(producer(Index));
                }
                else
                {
                    await collection.Add(producer(Index));
                }

                await Task.Yield();
            }
        }
Beispiel #32
0
        public SqlConnectionPool(string connectionString, int poolSize)
        {
            PoolSize         = poolSize;
            ConnectionString = connectionString;

            Connections = new AsyncCollection <MySqlConnection>(poolSize);

            for (int i = 0; i < poolSize; i++)
            {
                var connection = new MySqlConnection(connectionString);

                connection.Open();

                Connections.Add(connection);
            }
        }
Beispiel #33
0
        public async Task ConsumeLimit()
        {               //****************************************
            var MyCollection = new AsyncCollection <int>(10);
            //****************************************

            var MyTask = ConsumerWithWait(MyCollection, TimeSpan.Zero);

            foreach (var Item in Enumerable.Range(1, 100))
            {
                await MyCollection.Add(Item);
            }

            _ = MyCollection.CompleteAdding();

            _ = await MyTask;
        }
Beispiel #34
0
        public async Task Consume()
        {               //****************************************
            var MyCollection = new AsyncCollection <int>();
            //****************************************

            var MyTask = Consumer(MyCollection);

            foreach (var Item in Enumerable.Range(1, 10))
            {
                await MyCollection.Add(Item);
            }

            _ = MyCollection.CompleteAdding();

            _ = await MyTask;
        }
        public MainPage()
        {
            this.InitializeComponent();

            Items = new AsyncCollection <Item>(
                async() =>
            {
                return(await Service1.CountAsync());
            },
                async(int skipCount, int takeCount) =>
            {
                Debug.WriteLine(string.Format("skip {0}; take {1}", skipCount, takeCount));
                return(await Service1.GetDataAsync(skipCount, takeCount));
            });
            grid.ItemsSource = Items;
        }
Beispiel #36
0
        public async Task AddAndRemoveShouldBePerformant()
        {
            AsyncCollection <int> collection = new AsyncCollection <int>();

            const int  dataSize     = 1000000;
            List <int> receivedData = null;// new List<int>(dataSize);
            var        sw           = Stopwatch.StartNew();

            var t  = Task.Run(() => collection.AddRange(Enumerable.Range(0, dataSize)));
            var t2 = Task.Run(() => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result);
            await Task.WhenAll(t, t2);

            sw.Stop();
            Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
            Assert.That(receivedData.Count, Is.EqualTo(dataSize));
            Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
        }
Beispiel #37
0
        /// <summary>
        /// Construct a Producer class.
        /// </summary>
        /// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
        /// <param name="maximumAsyncRequests">The maximum async calls allowed before blocking new requests.  -1 indicates unlimited.</param>
        /// <param name="maximumMessageBuffer">The maximum amount of messages to buffer if the async calls are blocking from sending.</param>
        /// <remarks>
        /// The maximumAsyncRequests parameter provides a mechanism for minimizing the amount of async requests in flight at any one time
        /// by blocking the caller requesting the async call.  This affectively puts an upper limit on the amount of times a caller can
        /// call SendMessageAsync before the caller is blocked.
        ///
        /// The MaximumMessageBuffer parameter provides a way to limit the max amount of memory the driver uses should the send pipeline get
        /// overwhelmed and the buffer starts to fill up.  This is an inaccurate limiting memory use as the amount of memory actually used is
        /// dependant on the general message size being buffered.
        ///
        /// A message will start its timeout countdown as soon as it is added to the producer async queue.  If there are a large number of
        /// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
        /// attempted to send to Kafka before a timeout exception is thrown.
        /// </remarks>
        public Producer(IBrokerRouter brokerRouter, int maximumAsyncRequests = MaximumAsyncRequests, int maximumMessageBuffer = MaximumMessageBuffer)
        {
            BrokerRouter           = brokerRouter;
            _maximumAsyncRequests  = maximumAsyncRequests;
            _metadataQueries       = new MetadataQueries(BrokerRouter);
            _asyncCollection       = new AsyncCollection <TopicMessage>();
            _semaphoreMaximumAsync = new SemaphoreSlim(maximumAsyncRequests, maximumAsyncRequests);

            BatchSize      = DefaultBatchSize;
            BatchDelayTime = TimeSpan.FromMilliseconds(DefaultBatchDelayMS);

            _postTask = Task.Run(async() =>
            {
                await BatchSendAsync().ConfigureAwait(false);
                //TODO add log for ending the sending thread.
            });
        }
        public async void TakeAsyncShouldOnlyWaitTimeoutAndReturnWhatItHas()
        {
            const int size = 20;
            var aq = new AsyncCollection<bool>();

            Task.Factory.StartNew(() =>
            {
                //this should take 2000ms to complete
                for (int i = 0; i < size; i++)
                {
                    aq.Add(true);
                    Thread.Sleep(100);
                }
            });

            var result = await aq.TakeAsync(size, TimeSpan.FromMilliseconds(100), CancellationToken.None);

            Assert.That(result.Count, Is.LessThan(size));
        }
Beispiel #39
0
        /// <summary>
        /// Construct socket and open connection to a specified server.
        /// </summary>
        /// <param name="log">Logging facility for verbose messaging of actions.</param>
        /// <param name="endpoint">The IP endpoint to connect to.</param>
        /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
        public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan? maximumReconnectionTimeout = null)
        {
            _log = log;
            _endpoint = endpoint;
            _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);

            _sendTaskQueue = new AsyncCollection<SocketPayloadSendTask>();
            _readTaskQueue = new AsyncCollection<SocketPayloadReadTask>();

            //dedicate a long running task to the read/write operations
            _socketTask = Task.Run(async () => { await DedicatedSocketTask(); });

            _disposeTask = _disposeToken.Token.CreateTask();
            _disposeRegistration = _disposeToken.Token.Register(() =>
            {
                _sendTaskQueue.CompleteAdding();
                _readTaskQueue.CompleteAdding();
            });
        }
Beispiel #40
0
        /// <summary>
        /// Construct socket and open connection to a specified server.
        /// </summary>
        /// <param name="log">Logging facility for verbose messaging of actions.</param>
        /// <param name="endpoint">The IP endpoint to connect to.</param>
        /// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
        public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan? maximumReconnectionTimeout = null)
        {
            _log = log;
            _endpoint = endpoint;
            _maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);

            _sendTaskQueue = new AsyncCollection<SocketPayloadSendTask>();
            _readTaskQueue = new AsyncCollection<SocketPayloadReadTask>();

            //dedicate a long running task to the read/write operations
            _socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None,
                TaskCreationOptions.LongRunning, TaskScheduler.Default);

            _disposeRegistration = _disposeToken.Token.Register(() =>
            {
                _sendTaskQueue.CompleteAdding();
                _readTaskQueue.CompleteAdding();
            });

        }
Beispiel #41
0
        /// <summary>
        /// Construct a Producer class.
        /// </summary>
        /// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
        /// <param name="maximumAsyncRequests">The maximum async calls allowed before blocking new requests.  -1 indicates unlimited.</param>
        /// <param name="maximumMessageBuffer">The maximum amount of messages to buffer if the async calls are blocking from sending.</param>
        /// <remarks>
        /// The maximumAsyncRequests parameter provides a mechanism for minimizing the amount of async requests in flight at any one time
        /// by blocking the caller requesting the async call.  This affectively puts an upper limit on the amount of times a caller can 
        /// call SendMessageAsync before the caller is blocked.
        /// 
        /// The MaximumMessageBuffer parameter provides a way to limit the max amount of memory the driver uses should the send pipeline get
        /// overwhelmed and the buffer starts to fill up.  This is an inaccurate limiting memory use as the amount of memory actually used is 
        /// dependant on the general message size being buffered.
        /// 
        /// A message will start its timeout countdown as soon as it is added to the producer async queue.  If there are a large number of 
        /// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
        /// attempted to send to Kafka before a timeout exception is thrown.
        /// </remarks>
        public Producer(IBrokerRouter brokerRouter, int maximumAsyncRequests = MaximumAsyncRequests, int maximumMessageBuffer = MaximumMessageBuffer)
        {
            BrokerRouter = brokerRouter;
            _maximumAsyncRequests = maximumAsyncRequests;
            _metadataQueries = new MetadataQueries(BrokerRouter);
            _asyncCollection = new AsyncCollection<TopicMessage>();
            _semaphoreMaximumAsync = new SemaphoreSlim(maximumAsyncRequests, maximumAsyncRequests);

            BatchSize = DefaultBatchSize;
            BatchDelayTime = TimeSpan.FromMilliseconds(DefaultBatchDelayMS);

            _postTask = Task.Run(async () =>
            {
                await BatchSendAsync().ConfigureAwait(false);
                //TODO add log for ending the sending thread.
            });

        }
        public async void TakeAsyncShouldBeAbleToCancel()
        {
            var cancelSource = new CancellationTokenSource();
            var collection = new AsyncCollection<int>();

            Task.Delay(TimeSpan.FromMilliseconds(100)).ContinueWith(t => cancelSource.Cancel());

            var sw = Stopwatch.StartNew();
            var data = await collection.TakeAsync(10, TimeSpan.FromMilliseconds(500), cancelSource.Token);
            sw.Stop();

            Assert.That(sw.ElapsedMilliseconds, Is.LessThan(300));
        }
 public void AsyncCollectionConstructs()
 {
     var collection = new AsyncCollection<string>();
     Assert.That(collection, Is.Not.Null);
 }
        public void TakeAsyncShouldPlayNiceWithTPL()
        {
            const int expected = 200;
            const int max = 400;
            var exit = false;
            var collection = new AsyncCollection<int>();

            var dataTask = collection.TakeAsync(expected, TimeSpan.FromSeconds(100), CancellationToken.None);

            dataTask.ContinueWith(x => exit = true);

            Parallel.ForEach(Enumerable.Range(0, max).ToList(),
                   new ParallelOptions { MaxDegreeOfParallelism = 20 },
                   x =>
                   {
                       while (exit == false)
                       {
                           collection.Add(x);
                           Thread.Sleep(100);
                       }
                   });

            Console.WriteLine("Left in collection: {0}", collection.Count);
            Assert.That(dataTask.Result.Count, Is.EqualTo(expected));
        }
 public void CompletedCollectionShouldShowCompletedTrue()
 {
     var collection = new AsyncCollection<int>();
     collection.Add(1);
     Assert.That(collection.IsCompleted, Is.False);
     collection.CompleteAdding();
     Assert.That(collection.IsCompleted, Is.True);
 }
        public async void TakeAsyncShouldReturnAsSoonAsBatchSizeArrived()
        {
            var collection = new AsyncCollection<int>();

            var dataTask = collection.TakeAsync(10, TimeSpan.FromSeconds(5), CancellationToken.None);

            collection.AddRange(Enumerable.Range(0, 10));

            await dataTask;

            Assert.That(collection.Count, Is.EqualTo(0));

        }
        public void AddAndRemoveShouldBePerformant()
        {
            const int dataSize = 1000000;
            var collection = new AsyncCollection<int>();

            var sw = Stopwatch.StartNew();
            var receivedData = new List<int>();
            Parallel.Invoke(
                () => collection.AddRange(Enumerable.Range(0, dataSize)),
                () => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result);
            sw.Stop();
            Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
            Assert.That(receivedData.Count, Is.EqualTo(dataSize));
            Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
        }
 public void CompletedCollectionShouldPreventMoreItemsAdded()
 {
     var collection = new AsyncCollection<int>();
     collection.Add(1);
     collection.CompleteAdding();
     collection.Add(1);
 }
 public async void TakeAsyncShouldBePerformant()
 {
     const int dataSize = 1000000;
     var collection = new AsyncCollection<int>();
     collection.AddRange(Enumerable.Range(0, dataSize));
     var sw = Stopwatch.StartNew();
     var list = await collection.TakeAsync(dataSize, TimeSpan.FromSeconds(1), CancellationToken.None);
     sw.Stop();
     Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
     Assert.That(list.Count, Is.EqualTo(dataSize));
     Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
 }
 public void AddRangeShouldBePerformant()
 {
     var sw = Stopwatch.StartNew();
     var collection = new AsyncCollection<int>();
     collection.AddRange(Enumerable.Range(0, 1000000));
     sw.Stop();
     Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
     Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
 }
        public void TakeAsyncShouldBeThreadSafe()
        {
            const int expected = 10;
            const int max = 100;
            var exit = false;
            var collection = new AsyncCollection<int>();

            var take1 = collection.TakeAsync(expected, TimeSpan.FromSeconds(100), CancellationToken.None);
            var take2 = collection.TakeAsync(expected, TimeSpan.FromSeconds(100), CancellationToken.None);
            var take3 = collection.TakeAsync(expected, TimeSpan.FromSeconds(100), CancellationToken.None);

            take1.ContinueWith(t => Console.WriteLine("Take1 done..."));
            take2.ContinueWith(t => Console.WriteLine("Take2 done..."));
            take3.ContinueWith(t => Console.WriteLine("Take3 done..."));
            Task.WhenAll(take1, take2, take3).ContinueWith(x => exit = true);

            Parallel.ForEach(Enumerable.Range(0, max).ToList(),
                   new ParallelOptions { MaxDegreeOfParallelism = 20 },
                   x =>
                   {
                       while (exit == false)
                       {
                           collection.Add(x);
                           Thread.Sleep(100);
                       }
                   });

            Console.WriteLine("Left in collection: {0}", collection.Count);
            Assert.That(take1.Result.Count, Is.EqualTo(expected));
            Assert.That(take2.Result.Count, Is.EqualTo(expected));
            Assert.That(take3.Result.Count, Is.EqualTo(expected));
        }
        public async void TakeAsyncShouldReturnEmptyListIfNothingFound()
        {
            var aq = new AsyncCollection<bool>();

            var result = await aq.TakeAsync(100, TimeSpan.FromMilliseconds(100), CancellationToken.None).ConfigureAwait(false);

            Assert.That(result, Is.Not.Null);
            Assert.That(result.Count, Is.EqualTo(0));
        }
 public void CollectionShouldBlockOnMaxBuffer()
 {
     var collection = new AsyncCollection<int>();
     var task = Task.Factory.StartNew(() => collection.AddRange(Enumerable.Range(0, 10)));
     TaskTest.WaitFor(() => collection.Count >= 9);
     Assert.That(collection.Count, Is.EqualTo(9), "Buffer should block at 9 items.");
     Assert.That(task.IsCompleted, Is.False, "Task should be blocking on last item.");
     var item = collection.Pop();
     TaskTest.WaitFor(() => task.IsCompleted);
     Assert.That(task.IsCompleted, Is.True, "Task should complete after room is made in buffer.");
     Assert.That(collection.Count, Is.EqualTo(9), "There should now be 9 items in the buffer.");
 }
        /// <summary>
        /// Construct a Producer class.
        /// </summary>
        /// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
        /// <param name="maximumAsyncRequests">The maximum async calls allowed before blocking new requests.  -1 indicates unlimited.</param>
        /// <param name="maximumMessageBuffer">The maximum amount of messages to buffer if the async calls are blocking from sending.</param>
        /// <remarks>
        /// The maximumAsyncRequests parameter provides a mechanism for minimizing the amount of async requests in flight at any one time
        /// by blocking the caller requesting the async call.  This affectively puts an upper limit on the amount of times a caller can
        /// call SendMessageAsync before the caller is blocked.
        ///
        /// The MaximumMessageBuffer parameter provides a way to limit the max amount of memory the driver uses should the send pipeline get
        /// overwhelmed and the buffer starts to fill up.  This is an inaccurate limiting memory use as the amount of memory actually used is
        /// dependant on the general message size being buffered.
        ///
        /// A message will start its timeout countdown as soon as it is added to the producer async queue.  If there are a large number of
        /// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
        /// attempted to send to Kafka before a timeout exception is thrown.
        /// </remarks>
        public Producer(IBrokerRouter brokerRouter, int maximumAsyncRequests = MaximumAsyncRequests, int maximumMessageBuffer = MaximumMessageBuffer)
        {
            BrokerRouter = brokerRouter;
            _protocolGateway = new ProtocolGateway(BrokerRouter);
            _maximumAsyncRequests = maximumAsyncRequests;
            _metadataQueries = new MetadataQueries(BrokerRouter);
            _asyncCollection = new AsyncCollection<TopicMessage>();
            _semaphoreMaximumAsync = new SemaphoreSlim(maximumAsyncRequests, maximumAsyncRequests);

            BatchSize = DefaultBatchSize;
            BatchDelayTime = TimeSpan.FromMilliseconds(DefaultBatchDelayMS);

            _postTask = Task.Run(() =>
            {
                BatchSendAsync();
                BrokerRouter.Log.InfoFormat("ending the sending thread");
            });
        }
        public async void TakeAsyncShouldWaitXForBatchSizeToCollect()
        {
            const int expectedDelay = 100;
            const int expectedCount = 10;

            var collection = new AsyncCollection<int>();
            collection.AddRange(Enumerable.Range(0, expectedCount));

            var sw = Stopwatch.StartNew();
            var data = await collection.TakeAsync(expectedCount + 1, TimeSpan.FromMilliseconds(expectedDelay), CancellationToken.None);

            Assert.That(sw.ElapsedMilliseconds, Is.GreaterThanOrEqualTo(expectedDelay));
            Assert.That(data.Count, Is.EqualTo(expectedCount));
        }
        public async Task AddAndRemoveShouldBePerformant()
        {
            AsyncCollection<int> collection = new AsyncCollection<int>();

            const int dataSize = 1000000;
            List<int> receivedData = null;// new List<int>(dataSize);
            var sw = Stopwatch.StartNew();

            var t = Task.Run(() => collection.AddRange(Enumerable.Range(0, dataSize)));
            var t2 = Task.Run(() => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result);
            await Task.WhenAll(t, t2);
            sw.Stop();
            Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
            Assert.That(receivedData.Count, Is.EqualTo(dataSize));
            Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
        }
        public async void TakeAsyncShouldRemoveItemsFromCollection()
        {
            const int expectedCount = 10;

            var collection = new AsyncCollection<int>();
            collection.AddRange(Enumerable.Range(0, expectedCount));

            var data = await collection.TakeAsync(expectedCount, TimeSpan.FromMilliseconds(100), CancellationToken.None);

            Assert.That(data.Count, Is.EqualTo(expectedCount));
            Assert.That(collection.Count, Is.EqualTo(0));
        }
        public void TestDrainAnApplyNotInInfiniteLoop()
        {
            var aq = new AsyncCollection<bool>();

            aq.Add(true);

            var task = Task.Run(() => aq.DrainAndApply(x => aq.Add(x)));

            Assert.IsTrue(task.Wait(1000));
        }
        public async void TakeAsyncShouldReturnEvenWhileMoreDataArrives()
        {
            var exit = false;
            var collection = new AsyncCollection<int>();

            var sw = Stopwatch.StartNew();
            var dataTask = collection.TakeAsync(10, TimeSpan.FromMilliseconds(5000), CancellationToken.None);


            var highVolumeAdding = Task.Run(() =>
            {
                //high volume of data adds
                while (exit == false)
                {
                    collection.Add(1);
                    Thread.Sleep(5);
                }
            });

            Console.WriteLine("Awaiting data...");
            await dataTask;

            Assert.That(dataTask.Result.Count, Is.EqualTo(10));
            Assert.That(sw.ElapsedMilliseconds, Is.LessThan(5000));
            exit = true;

            Console.WriteLine("Waiting to unwind test...");
            await highVolumeAdding;
        }