public async void OnDataAvailableShouldCancel()
{
var aq = new AsyncCollection<bool>();
var cancelToken = new CancellationTokenSource();
Task.Delay(TimeSpan.FromMilliseconds(100)).ContinueWith(t => cancelToken.Cancel());
await aq.OnDataAvailable(cancelToken.Token);
}
public void TryTakeShouldReturnFalseOnEmpty()
{
var aq = new AsyncCollection<bool>();
Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.False, "Task should indicate no data available.");
bool data;
Assert.That(aq.TryTake(out data), Is.False, "TryTake should report false on empty collection.");
Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.False, "Task should indicate no data available.");
}
public void OnDataAvailableShouldTriggerWhenDataAdded()
{
var aq = new AsyncCollection<bool>();
Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.False, "Task should indicate no data available.");
aq.Add(true);
Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.True, "Task should indicate data available.");
}
public async Task OnDataAvailableShouldCancel()//change class Behavers from trowing OperationCanceledException to returning false
{//do not debug with brack point it harm this test
var aq = new AsyncCollection<bool>();
int timeSpen = 100;
Task waitUntilCancel = aq.OnHasDataAvailable(new CancellationTokenSource(timeSpen).Token);
await Task.WhenAny(waitUntilCancel, Task.Delay(timeSpen / 2));
Assert.IsFalse(waitUntilCancel.IsCompleted, "task Should Cancel only when time is up");
//ToDO FIX
// Assert.IsFalse(await waitUntilCancel, "it Should return false when cancel");
}
public void DrainShouldBlockWhenDataRemoved()
{
var aq = new AsyncCollection<bool>();
aq.Add(true);
aq.Add(true);
Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.True, "Task should indicate data available.");
var drained = aq.Drain().ToList();
Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.False, "Task should indicate no data available.");
}
public void OnDataAvailableShouldBlockWhenDataRemoved()
{
var aq = new AsyncCollection<bool>();
aq.Add(true);
Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.True, "Task should indicate data available.");
bool data;
aq.TryTake(out data);
Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.False, "Task should indicate no data available.");
}
public async Task ConsumeLimitOverFilledComplete()
{ //****************************************
var MyCollection = new AsyncCollection <int>(10);
//****************************************
var ProducerTask = ProducerWithAddComplete(MyCollection, 100, (index) => index);
var MyTask = ConsumerWithWait(MyCollection, TimeSpan.Zero);
await ProducerTask;
_ = await MyTask;
}
public async void TakeAsyncShouldRemoveItemsFromCollection()
{
const int expectedCount = 10;
var collection = new AsyncCollection <int>();
collection.AddRange(Enumerable.Range(0, expectedCount));
var data = await collection.TakeAsync(expectedCount, TimeSpan.FromMilliseconds(100), CancellationToken.None);
Assert.That(data.Count, Is.EqualTo(expectedCount));
Assert.That(collection.Count, Is.EqualTo(0));
}
public void OnDataAvailableShouldBlockWhenDataRemoved()
{
var aq = new AsyncCollection <bool>();
aq.Add(true);
Assert.That(aq.OnHasDataAvailable(CancellationToken.None).IsCompleted, Is.True, "Task should indicate data available.");
bool data;
aq.TryTake(out data);
Assert.That(aq.OnHasDataAvailable(CancellationToken.None).IsCompleted, Is.False, "Task should indicate no data available.");
}
public void Add()
{ //****************************************
var MyCollection = new AsyncCollection <int>();
//****************************************
var MyTask = MyCollection.Add(42);
//****************************************
Assert.IsTrue(MyTask.IsCompleted, "Still waiting to add");
Assert.AreEqual(1, MyCollection.Count, "Count not as expected");
}
public void DrainShouldBlockWhenDataRemoved()
{
var aq = new AsyncCollection <bool>();
aq.Add(true);
aq.Add(true);
Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.True, "Task should indicate data available.");
var drained = aq.Drain().ToList();
Assert.That(aq.OnDataAvailable(CancellationToken.None).IsCompleted, Is.False, "Task should indicate no data available.");
}
public async Task CancelTake()
{
AsyncCollection <int> collection = new AsyncCollection <int>();
using (CancellationTokenSource cancellationTokenSource = new CancellationTokenSource())
{
var t = Task.Run(async() => await collection.TakeAsync(cancellationTokenSource.Token));
cancellationTokenSource.Cancel();
await Assert.ThrowsExceptionAsync <OperationCanceledException>(() => t);
}
}
public void PeekSuccess()
{ //****************************************
var MyCollection = new AsyncCollection <int>();
//****************************************
_ = MyCollection.TryAdd(42);
var MyTask = MyCollection.Peek();
//****************************************
Assert.IsTrue(MyTask.IsCompleted, "Peek failed unexpectedly");
}
/// <summary>
/// Initializes a new instance of the ItemProducer class.
/// </summary>
/// <param name="queryContext">request context</param>
/// <param name="querySpecForInit">query spec for initialization</param>
/// <param name="partitionKeyRange">The partition key range.</param>
/// <param name="produceAsyncCompleteCallback">The callback to call once you are done fetching.</param>
/// <param name="equalityComparer">The comparer to use to determine whether the producer has seen a new document.</param>
/// <param name="initialPageSize">The initial page size.</param>
/// <param name="initialContinuationToken">The initial continuation token.</param>
public ItemProducer(
CosmosQueryContext queryContext,
SqlQuerySpec querySpecForInit,
PartitionKeyRange partitionKeyRange,
ProduceAsyncCompleteDelegate produceAsyncCompleteCallback,
IEqualityComparer <CosmosElement> equalityComparer,
long initialPageSize = 50,
string initialContinuationToken = null)
{
this.bufferedPages = new AsyncCollection <QueryResponse>();
// We use a binary semaphore to get the behavior of a mutex,
// since fetching documents from the backend using a continuation token is a critical section.
this.fetchSemaphore = new SemaphoreSlim(1, 1);
if (partitionKeyRange == null)
{
throw new ArgumentNullException(nameof(partitionKeyRange));
}
if (produceAsyncCompleteCallback == null)
{
throw new ArgumentNullException(nameof(produceAsyncCompleteCallback));
}
if (equalityComparer == null)
{
throw new ArgumentNullException(nameof(equalityComparer));
}
this.queryContext = queryContext;
this.querySpecForInit = querySpecForInit;
this.PartitionKeyRange = partitionKeyRange;
this.produceAsyncCompleteCallback = produceAsyncCompleteCallback;
this.equalityComparer = equalityComparer;
this.pageSize = initialPageSize;
this.currentContinuationToken = initialContinuationToken;
this.BackendContinuationToken = initialContinuationToken;
this.PreviousContinuationToken = initialContinuationToken;
if (!string.IsNullOrEmpty(initialContinuationToken))
{
this.hasStartedFetching = true;
this.IsActive = true;
}
this.fetchSchedulingMetrics = new SchedulingStopwatch();
this.fetchSchedulingMetrics.Ready();
this.fetchExecutionRangeAccumulator = new FetchExecutionRangeAccumulator();
this.HasMoreResults = true;
}
public ProcDataflowBlock(
IActionRunner actionRunner,
int maximumNumberOfActionsInQueue,
int?maximumDegreeOfParallelism)
{
this.actionRunner = actionRunner;
this.maximumDegreeOfParallelism = maximumDegreeOfParallelism;
collection = new AsyncCollection <Func <Task> >(new ConcurrentQueue <Func <Task> >(), maximumNumberOfActionsInQueue);
collectionForReentrantItems = new AsyncCollection <Func <Task> >(new ConcurrentQueue <Func <Task> >());
nodeId = Guid.NewGuid();
}
public AsyncProcDataflowBlock(
int maximumNumberOfActionsInQueue,
int?maximumDegreeOfParallelism,
CancellationToken cancellationToken = default)
{
this.maximumDegreeOfParallelism = maximumDegreeOfParallelism;
this.cancellationToken = cancellationToken;
collection = new AsyncCollection <Func <Task> >(new ConcurrentQueue <Func <Task> >(), maximumNumberOfActionsInQueue);
collectionForReentrantItems = new AsyncCollection <Func <Task> >(new ConcurrentQueue <Func <Task> >());
nodeId = Guid.NewGuid();
}
public void TakeWithDelay()
{
AsyncCollection <int> collection = new AsyncCollection <int>();
Task.Run(() =>
{
Thread.Sleep(100);
collection.Add(1);
});
int item = collection.Take();
Assert.AreEqual(item, 1);
}
public async void TakeAsyncShouldBeAbleToCancel()
{
var cancelSource = new CancellationTokenSource();
var collection = new AsyncCollection <int>();
Task.Delay(TimeSpan.FromMilliseconds(100)).ContinueWith(t => cancelSource.Cancel());
var sw = Stopwatch.StartNew();
var data = await collection.TakeAsync(10, TimeSpan.FromMilliseconds(500), cancelSource.Token);
sw.Stop();
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(300));
}
public void Clear()
{
var bufferSize = _buffer.Length;
_buffer = null;
_offsetSize = null;
_emptySegments = null;
_filledSegments = null;
if (bufferSize >= GCThreshold)
{
GC.Collect();
}
}
public async void CollectionShouldReportCorrectBufferCount()
{
var collection = new AsyncCollection<int>();
var dataTask = collection.TakeAsync(10, TimeSpan.FromHours(5), CancellationToken.None);
collection.AddRange(Enumerable.Range(0, 9));
Assert.That(collection.Count, Is.EqualTo(9));
collection.Add(1);
var data = await dataTask;
Assert.That(data.Count, Is.EqualTo(10));
Assert.That(collection.Count, Is.EqualTo(0));
}
public async Task CollectionShouldBlockOnMaxBuffer()
{
var collection = new AsyncCollection <int>();
var task = Task.Factory.StartNew(() => collection.AddRange(Enumerable.Range(0, 10)));
await TaskTest.WaitFor(() => collection.Count >= 9);
Assert.That(collection.Count, Is.EqualTo(9), "Buffer should block at 9 items.");
Assert.That(task.IsCompleted, Is.False, "Task should be blocking on last item.");
var item = collection.Pop();
await TaskTest.WaitFor(() => task.IsCompleted);
Assert.That(task.IsCompleted, Is.True, "Task should complete after room is made in buffer.");
Assert.That(collection.Count, Is.EqualTo(9), "There should now be 9 items in the buffer.");
}
public async void TakeAsyncShouldBePerformant()
{
const int dataSize = 1000000;
AsyncCollection <int> collection = new AsyncCollection <int>();
collection.AddRange(Enumerable.Range(0, dataSize));
var sw = Stopwatch.StartNew();
var list = await collection.TakeAsync(dataSize, TimeSpan.FromSeconds(1), CancellationToken.None);
sw.Stop();
Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
Assert.That(list.Count, Is.EqualTo(dataSize));
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
}
public void TakeAsyncFromMultipleThreads()
{
AsyncCollection <int> collection = new AsyncCollection <int>();
int t1Counter = 0;
int t2Counter = 0;
var t1 = Task.Run(async() =>
{
while (!collection.IsCompleted)
{
try
{
await collection.TakeAsync();
t1Counter++;
}
catch (InvalidOperationException)
{
}
}
});
var t2 = Task.Run(async() =>
{
while (!collection.IsCompleted)
{
try
{
await collection.TakeAsync();
t2Counter++;
}
catch (InvalidOperationException)
{
}
}
});
for (int i = 0; i < 1000; i++)
{
collection.Add(i);
}
collection.CompleteAdding();
t1.Wait();
t2.Wait();
Assert.IsTrue(t1Counter > 400);
Assert.IsTrue(t2Counter > 400);
}
public async void TakeAsyncShouldWaitXForBatchSizeToCollect()
{
const int expectedDelay = 100;
const int expectedCount = 10;
var collection = new AsyncCollection <int>();
collection.AddRange(Enumerable.Range(0, expectedCount));
var sw = Stopwatch.StartNew();
var data = await collection.TakeAsync(expectedCount + 1, TimeSpan.FromMilliseconds(expectedDelay), CancellationToken.None);
Assert.That(sw.ElapsedMilliseconds, Is.GreaterThanOrEqualTo(expectedDelay));
Assert.That(data.Count, Is.EqualTo(expectedCount));
}
public async Task TryTakeFromAny([Values(0, 1)] int index)
{ //****************************************
var MyCollections = new AsyncCollection <int>[] { new AsyncCollection <int>(), new AsyncCollection <int>() };
//****************************************
await MyCollections[index].Add(42);
var MyResult = AsyncCollection <int> .TryTakeFromAny(MyCollections);
//****************************************
Assert.IsTrue(MyResult.HasItem, "Take failed unexpectedly");
Assert.AreSame(MyCollections[index], MyResult.Source, "Collection was not as expected");
Assert.AreEqual(42, MyResult.Item, "Item was not as expected");
}
public async Task ConsumePreFilled()
{ //****************************************
var MyCollection = new AsyncCollection <int>();
//****************************************
var ProducerTask = Producer(MyCollection, 20, (index) => index);
var MyTask = Consumer(MyCollection);
await ProducerTask;
_ = MyCollection.CompleteAdding();
_ = await MyTask;
}
public async Task AddCompleteAdd()
{ //****************************************
var MyCollection = new AsyncCollection <int>();
//****************************************
await MyCollection.AddComplete(42);
Assert.IsFalse(await MyCollection.Add(84));
//****************************************
Assert.AreEqual(1, MyCollection.Count, "Count not as expected");
Assert.IsTrue(MyCollection.IsAddingCompleted, "Not completed");
}
public void CancelsTaskWhenTokenIsCanceled()
{
CancellationTokenSource cancelSource = new CancellationTokenSource();
var task = AsyncCollection <int> .TakeFromAnyAsync(_collections, cancelSource.Token);
cancelSource.Cancel();
Func <Task> asyncAct = () => task;
asyncAct.ShouldThrow <TaskCanceledException>();
_collections[0].Add(42);
_collections[1].Add(64);
_collections[0].Count.Should().Be(1);
_collections[1].Count.Should().Be(1);
}
public async void CollectionShouldReportCorrectBufferCount()
{
var collection = new AsyncCollection <int>();
var dataTask = collection.TakeAsync(10, TimeSpan.FromHours(5), CancellationToken.None);
collection.AddRange(Enumerable.Range(0, 9));
Assert.That(collection.Count, Is.EqualTo(9));
collection.Add(1);
var data = await dataTask;
Assert.That(data.Count, Is.EqualTo(10));
Assert.That(collection.Count, Is.EqualTo(0));
}
public void AddAndRemoveShouldBePerformant()
{
const int dataSize = 1000000;
var collection = new AsyncCollection <int>();
var sw = Stopwatch.StartNew();
var receivedData = new List <int>();
Parallel.Invoke(
() => collection.AddRange(Enumerable.Range(0, dataSize)),
() => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result);
sw.Stop();
Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
Assert.That(receivedData.Count, Is.EqualTo(dataSize));
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
}
private async Task ProducerWithAddComplete <TItem>(AsyncCollection <TItem> collection, int count, Func <int, TItem> producer)
{
for (var Index = 0; Index < count; Index++)
{
if (Index == count - 1)
{
await collection.AddComplete(producer(Index));
}
else
{
await collection.Add(producer(Index));
}
await Task.Yield();
}
}
public SqlConnectionPool(string connectionString, int poolSize)
{
PoolSize = poolSize;
ConnectionString = connectionString;
Connections = new AsyncCollection <MySqlConnection>(poolSize);
for (int i = 0; i < poolSize; i++)
{
var connection = new MySqlConnection(connectionString);
connection.Open();
Connections.Add(connection);
}
}
public async Task ConsumeLimit()
{ //****************************************
var MyCollection = new AsyncCollection <int>(10);
//****************************************
var MyTask = ConsumerWithWait(MyCollection, TimeSpan.Zero);
foreach (var Item in Enumerable.Range(1, 100))
{
await MyCollection.Add(Item);
}
_ = MyCollection.CompleteAdding();
_ = await MyTask;
}
public async Task Consume()
{ //****************************************
var MyCollection = new AsyncCollection <int>();
//****************************************
var MyTask = Consumer(MyCollection);
foreach (var Item in Enumerable.Range(1, 10))
{
await MyCollection.Add(Item);
}
_ = MyCollection.CompleteAdding();
_ = await MyTask;
}
public MainPage()
{
this.InitializeComponent();
Items = new AsyncCollection <Item>(
async() =>
{
return(await Service1.CountAsync());
},
async(int skipCount, int takeCount) =>
{
Debug.WriteLine(string.Format("skip {0}; take {1}", skipCount, takeCount));
return(await Service1.GetDataAsync(skipCount, takeCount));
});
grid.ItemsSource = Items;
}
public async Task AddAndRemoveShouldBePerformant()
{
AsyncCollection <int> collection = new AsyncCollection <int>();
const int dataSize = 1000000;
List <int> receivedData = null;// new List<int>(dataSize);
var sw = Stopwatch.StartNew();
var t = Task.Run(() => collection.AddRange(Enumerable.Range(0, dataSize)));
var t2 = Task.Run(() => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result);
await Task.WhenAll(t, t2);
sw.Stop();
Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
Assert.That(receivedData.Count, Is.EqualTo(dataSize));
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
}
/// <summary>
/// Construct a Producer class.
/// </summary>
/// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
/// <param name="maximumAsyncRequests">The maximum async calls allowed before blocking new requests. -1 indicates unlimited.</param>
/// <param name="maximumMessageBuffer">The maximum amount of messages to buffer if the async calls are blocking from sending.</param>
/// <remarks>
/// The maximumAsyncRequests parameter provides a mechanism for minimizing the amount of async requests in flight at any one time
/// by blocking the caller requesting the async call. This affectively puts an upper limit on the amount of times a caller can
/// call SendMessageAsync before the caller is blocked.
///
/// The MaximumMessageBuffer parameter provides a way to limit the max amount of memory the driver uses should the send pipeline get
/// overwhelmed and the buffer starts to fill up. This is an inaccurate limiting memory use as the amount of memory actually used is
/// dependant on the general message size being buffered.
///
/// A message will start its timeout countdown as soon as it is added to the producer async queue. If there are a large number of
/// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
/// attempted to send to Kafka before a timeout exception is thrown.
/// </remarks>
public Producer(IBrokerRouter brokerRouter, int maximumAsyncRequests = MaximumAsyncRequests, int maximumMessageBuffer = MaximumMessageBuffer)
{
BrokerRouter = brokerRouter;
_maximumAsyncRequests = maximumAsyncRequests;
_metadataQueries = new MetadataQueries(BrokerRouter);
_asyncCollection = new AsyncCollection <TopicMessage>();
_semaphoreMaximumAsync = new SemaphoreSlim(maximumAsyncRequests, maximumAsyncRequests);
BatchSize = DefaultBatchSize;
BatchDelayTime = TimeSpan.FromMilliseconds(DefaultBatchDelayMS);
_postTask = Task.Run(async() =>
{
await BatchSendAsync().ConfigureAwait(false);
//TODO add log for ending the sending thread.
});
}
public async void TakeAsyncShouldOnlyWaitTimeoutAndReturnWhatItHas()
{
const int size = 20;
var aq = new AsyncCollection<bool>();
Task.Factory.StartNew(() =>
{
//this should take 2000ms to complete
for (int i = 0; i < size; i++)
{
aq.Add(true);
Thread.Sleep(100);
}
});
var result = await aq.TakeAsync(size, TimeSpan.FromMilliseconds(100), CancellationToken.None);
Assert.That(result.Count, Is.LessThan(size));
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan? maximumReconnectionTimeout = null)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
_sendTaskQueue = new AsyncCollection<SocketPayloadSendTask>();
_readTaskQueue = new AsyncCollection<SocketPayloadReadTask>();
//dedicate a long running task to the read/write operations
_socketTask = Task.Run(async () => { await DedicatedSocketTask(); });
_disposeTask = _disposeToken.Token.CreateTask();
_disposeRegistration = _disposeToken.Token.Register(() =>
{
_sendTaskQueue.CompleteAdding();
_readTaskQueue.CompleteAdding();
});
}
/// <summary>
/// Construct socket and open connection to a specified server.
/// </summary>
/// <param name="log">Logging facility for verbose messaging of actions.</param>
/// <param name="endpoint">The IP endpoint to connect to.</param>
/// <param name="maximumReconnectionTimeout">The maximum time to wait when backing off on reconnection attempts.</param>
public KafkaTcpSocket(IKafkaLog log, KafkaEndpoint endpoint, TimeSpan? maximumReconnectionTimeout = null)
{
_log = log;
_endpoint = endpoint;
_maximumReconnectionTimeout = maximumReconnectionTimeout ?? TimeSpan.FromMinutes(MaxReconnectionTimeoutMinutes);
_sendTaskQueue = new AsyncCollection<SocketPayloadSendTask>();
_readTaskQueue = new AsyncCollection<SocketPayloadReadTask>();
//dedicate a long running task to the read/write operations
_socketTask = Task.Factory.StartNew(DedicatedSocketTask, CancellationToken.None,
TaskCreationOptions.LongRunning, TaskScheduler.Default);
_disposeRegistration = _disposeToken.Token.Register(() =>
{
_sendTaskQueue.CompleteAdding();
_readTaskQueue.CompleteAdding();
});
}
/// <summary>
/// Construct a Producer class.
/// </summary>
/// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
/// <param name="maximumAsyncRequests">The maximum async calls allowed before blocking new requests. -1 indicates unlimited.</param>
/// <param name="maximumMessageBuffer">The maximum amount of messages to buffer if the async calls are blocking from sending.</param>
/// <remarks>
/// The maximumAsyncRequests parameter provides a mechanism for minimizing the amount of async requests in flight at any one time
/// by blocking the caller requesting the async call. This affectively puts an upper limit on the amount of times a caller can
/// call SendMessageAsync before the caller is blocked.
///
/// The MaximumMessageBuffer parameter provides a way to limit the max amount of memory the driver uses should the send pipeline get
/// overwhelmed and the buffer starts to fill up. This is an inaccurate limiting memory use as the amount of memory actually used is
/// dependant on the general message size being buffered.
///
/// A message will start its timeout countdown as soon as it is added to the producer async queue. If there are a large number of
/// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
/// attempted to send to Kafka before a timeout exception is thrown.
/// </remarks>
public Producer(IBrokerRouter brokerRouter, int maximumAsyncRequests = MaximumAsyncRequests, int maximumMessageBuffer = MaximumMessageBuffer)
{
BrokerRouter = brokerRouter;
_maximumAsyncRequests = maximumAsyncRequests;
_metadataQueries = new MetadataQueries(BrokerRouter);
_asyncCollection = new AsyncCollection<TopicMessage>();
_semaphoreMaximumAsync = new SemaphoreSlim(maximumAsyncRequests, maximumAsyncRequests);
BatchSize = DefaultBatchSize;
BatchDelayTime = TimeSpan.FromMilliseconds(DefaultBatchDelayMS);
_postTask = Task.Run(async () =>
{
await BatchSendAsync().ConfigureAwait(false);
//TODO add log for ending the sending thread.
});
}
public async void TakeAsyncShouldBeAbleToCancel()
{
var cancelSource = new CancellationTokenSource();
var collection = new AsyncCollection<int>();
Task.Delay(TimeSpan.FromMilliseconds(100)).ContinueWith(t => cancelSource.Cancel());
var sw = Stopwatch.StartNew();
var data = await collection.TakeAsync(10, TimeSpan.FromMilliseconds(500), cancelSource.Token);
sw.Stop();
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(300));
}
public void AsyncCollectionConstructs()
{
var collection = new AsyncCollection<string>();
Assert.That(collection, Is.Not.Null);
}
public void TakeAsyncShouldPlayNiceWithTPL()
{
const int expected = 200;
const int max = 400;
var exit = false;
var collection = new AsyncCollection<int>();
var dataTask = collection.TakeAsync(expected, TimeSpan.FromSeconds(100), CancellationToken.None);
dataTask.ContinueWith(x => exit = true);
Parallel.ForEach(Enumerable.Range(0, max).ToList(),
new ParallelOptions { MaxDegreeOfParallelism = 20 },
x =>
{
while (exit == false)
{
collection.Add(x);
Thread.Sleep(100);
}
});
Console.WriteLine("Left in collection: {0}", collection.Count);
Assert.That(dataTask.Result.Count, Is.EqualTo(expected));
}
public void CompletedCollectionShouldShowCompletedTrue()
{
var collection = new AsyncCollection<int>();
collection.Add(1);
Assert.That(collection.IsCompleted, Is.False);
collection.CompleteAdding();
Assert.That(collection.IsCompleted, Is.True);
}
public async void TakeAsyncShouldReturnAsSoonAsBatchSizeArrived()
{
var collection = new AsyncCollection<int>();
var dataTask = collection.TakeAsync(10, TimeSpan.FromSeconds(5), CancellationToken.None);
collection.AddRange(Enumerable.Range(0, 10));
await dataTask;
Assert.That(collection.Count, Is.EqualTo(0));
}
public void AddAndRemoveShouldBePerformant()
{
const int dataSize = 1000000;
var collection = new AsyncCollection<int>();
var sw = Stopwatch.StartNew();
var receivedData = new List<int>();
Parallel.Invoke(
() => collection.AddRange(Enumerable.Range(0, dataSize)),
() => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result);
sw.Stop();
Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
Assert.That(receivedData.Count, Is.EqualTo(dataSize));
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
}
public void CompletedCollectionShouldPreventMoreItemsAdded()
{
var collection = new AsyncCollection<int>();
collection.Add(1);
collection.CompleteAdding();
collection.Add(1);
}
public async void TakeAsyncShouldBePerformant()
{
const int dataSize = 1000000;
var collection = new AsyncCollection<int>();
collection.AddRange(Enumerable.Range(0, dataSize));
var sw = Stopwatch.StartNew();
var list = await collection.TakeAsync(dataSize, TimeSpan.FromSeconds(1), CancellationToken.None);
sw.Stop();
Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
Assert.That(list.Count, Is.EqualTo(dataSize));
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
}
public void AddRangeShouldBePerformant()
{
var sw = Stopwatch.StartNew();
var collection = new AsyncCollection<int>();
collection.AddRange(Enumerable.Range(0, 1000000));
sw.Stop();
Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
}
public void TakeAsyncShouldBeThreadSafe()
{
const int expected = 10;
const int max = 100;
var exit = false;
var collection = new AsyncCollection<int>();
var take1 = collection.TakeAsync(expected, TimeSpan.FromSeconds(100), CancellationToken.None);
var take2 = collection.TakeAsync(expected, TimeSpan.FromSeconds(100), CancellationToken.None);
var take3 = collection.TakeAsync(expected, TimeSpan.FromSeconds(100), CancellationToken.None);
take1.ContinueWith(t => Console.WriteLine("Take1 done..."));
take2.ContinueWith(t => Console.WriteLine("Take2 done..."));
take3.ContinueWith(t => Console.WriteLine("Take3 done..."));
Task.WhenAll(take1, take2, take3).ContinueWith(x => exit = true);
Parallel.ForEach(Enumerable.Range(0, max).ToList(),
new ParallelOptions { MaxDegreeOfParallelism = 20 },
x =>
{
while (exit == false)
{
collection.Add(x);
Thread.Sleep(100);
}
});
Console.WriteLine("Left in collection: {0}", collection.Count);
Assert.That(take1.Result.Count, Is.EqualTo(expected));
Assert.That(take2.Result.Count, Is.EqualTo(expected));
Assert.That(take3.Result.Count, Is.EqualTo(expected));
}
public async void TakeAsyncShouldReturnEmptyListIfNothingFound()
{
var aq = new AsyncCollection<bool>();
var result = await aq.TakeAsync(100, TimeSpan.FromMilliseconds(100), CancellationToken.None).ConfigureAwait(false);
Assert.That(result, Is.Not.Null);
Assert.That(result.Count, Is.EqualTo(0));
}
public void CollectionShouldBlockOnMaxBuffer()
{
var collection = new AsyncCollection<int>();
var task = Task.Factory.StartNew(() => collection.AddRange(Enumerable.Range(0, 10)));
TaskTest.WaitFor(() => collection.Count >= 9);
Assert.That(collection.Count, Is.EqualTo(9), "Buffer should block at 9 items.");
Assert.That(task.IsCompleted, Is.False, "Task should be blocking on last item.");
var item = collection.Pop();
TaskTest.WaitFor(() => task.IsCompleted);
Assert.That(task.IsCompleted, Is.True, "Task should complete after room is made in buffer.");
Assert.That(collection.Count, Is.EqualTo(9), "There should now be 9 items in the buffer.");
}
/// <summary>
/// Construct a Producer class.
/// </summary>
/// <param name="brokerRouter">The router used to direct produced messages to the correct partition.</param>
/// <param name="maximumAsyncRequests">The maximum async calls allowed before blocking new requests. -1 indicates unlimited.</param>
/// <param name="maximumMessageBuffer">The maximum amount of messages to buffer if the async calls are blocking from sending.</param>
/// <remarks>
/// The maximumAsyncRequests parameter provides a mechanism for minimizing the amount of async requests in flight at any one time
/// by blocking the caller requesting the async call. This affectively puts an upper limit on the amount of times a caller can
/// call SendMessageAsync before the caller is blocked.
///
/// The MaximumMessageBuffer parameter provides a way to limit the max amount of memory the driver uses should the send pipeline get
/// overwhelmed and the buffer starts to fill up. This is an inaccurate limiting memory use as the amount of memory actually used is
/// dependant on the general message size being buffered.
///
/// A message will start its timeout countdown as soon as it is added to the producer async queue. If there are a large number of
/// messages sitting in the async queue then a message may spend its entire timeout cycle waiting in this queue and never getting
/// attempted to send to Kafka before a timeout exception is thrown.
/// </remarks>
public Producer(IBrokerRouter brokerRouter, int maximumAsyncRequests = MaximumAsyncRequests, int maximumMessageBuffer = MaximumMessageBuffer)
{
BrokerRouter = brokerRouter;
_protocolGateway = new ProtocolGateway(BrokerRouter);
_maximumAsyncRequests = maximumAsyncRequests;
_metadataQueries = new MetadataQueries(BrokerRouter);
_asyncCollection = new AsyncCollection<TopicMessage>();
_semaphoreMaximumAsync = new SemaphoreSlim(maximumAsyncRequests, maximumAsyncRequests);
BatchSize = DefaultBatchSize;
BatchDelayTime = TimeSpan.FromMilliseconds(DefaultBatchDelayMS);
_postTask = Task.Run(() =>
{
BatchSendAsync();
BrokerRouter.Log.InfoFormat("ending the sending thread");
});
}
public async void TakeAsyncShouldWaitXForBatchSizeToCollect()
{
const int expectedDelay = 100;
const int expectedCount = 10;
var collection = new AsyncCollection<int>();
collection.AddRange(Enumerable.Range(0, expectedCount));
var sw = Stopwatch.StartNew();
var data = await collection.TakeAsync(expectedCount + 1, TimeSpan.FromMilliseconds(expectedDelay), CancellationToken.None);
Assert.That(sw.ElapsedMilliseconds, Is.GreaterThanOrEqualTo(expectedDelay));
Assert.That(data.Count, Is.EqualTo(expectedCount));
}
public async Task AddAndRemoveShouldBePerformant()
{
AsyncCollection<int> collection = new AsyncCollection<int>();
const int dataSize = 1000000;
List<int> receivedData = null;// new List<int>(dataSize);
var sw = Stopwatch.StartNew();
var t = Task.Run(() => collection.AddRange(Enumerable.Range(0, dataSize)));
var t2 = Task.Run(() => receivedData = collection.TakeAsync(dataSize, TimeSpan.FromSeconds(5), CancellationToken.None).Result);
await Task.WhenAll(t, t2);
sw.Stop();
Console.WriteLine("Performance: {0}", sw.ElapsedMilliseconds);
Assert.That(receivedData.Count, Is.EqualTo(dataSize));
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(200));
}
public async void TakeAsyncShouldRemoveItemsFromCollection()
{
const int expectedCount = 10;
var collection = new AsyncCollection<int>();
collection.AddRange(Enumerable.Range(0, expectedCount));
var data = await collection.TakeAsync(expectedCount, TimeSpan.FromMilliseconds(100), CancellationToken.None);
Assert.That(data.Count, Is.EqualTo(expectedCount));
Assert.That(collection.Count, Is.EqualTo(0));
}
public void TestDrainAnApplyNotInInfiniteLoop()
{
var aq = new AsyncCollection<bool>();
aq.Add(true);
var task = Task.Run(() => aq.DrainAndApply(x => aq.Add(x)));
Assert.IsTrue(task.Wait(1000));
}
public async void TakeAsyncShouldReturnEvenWhileMoreDataArrives()
{
var exit = false;
var collection = new AsyncCollection<int>();
var sw = Stopwatch.StartNew();
var dataTask = collection.TakeAsync(10, TimeSpan.FromMilliseconds(5000), CancellationToken.None);
var highVolumeAdding = Task.Run(() =>
{
//high volume of data adds
while (exit == false)
{
collection.Add(1);
Thread.Sleep(5);
}
});
Console.WriteLine("Awaiting data...");
await dataTask;
Assert.That(dataTask.Result.Count, Is.EqualTo(10));
Assert.That(sw.ElapsedMilliseconds, Is.LessThan(5000));
exit = true;
Console.WriteLine("Waiting to unwind test...");
await highVolumeAdding;
}