public async Task MultithreadingInsertsDontCrash()
{
int insertThreads = 4;
int itemsPerThread = 100;
_queue = new AsyncBatchQueue <int>(11);
List <Task> insertTasks = Enumerable.Range(1, insertThreads)
.Select(
_ => Task.Run(
() =>
{
for (int i = 0; i < itemsPerThread; i++)
{
_queue.Add(42);
}
}))
.ToList();
await Task.WhenAll(insertTasks).ConfigureAwait(true);
_queue.Flush();
int itemsTaken = 0;
while (_queue.Count > 0)
{
itemsTaken += (await _queue.TakeAsync().ConfigureAwait(true)).Count;
}
itemsTaken.Should().Be(insertThreads * itemsPerThread);
}
public async Task MultithreadingInsertsDontCrash()
{
int insertThreads = 4;
int itemsPerThread = 100;
_queue = new AsyncBatchQueue<int>( 11 );
List<Task> insertTasks = Enumerable.Range( 1, insertThreads )
.Select(
_ => Task.Run(
() =>
{
for ( int i = 0; i < itemsPerThread; i++ )
_queue.Add( 42 );
} ) )
.ToList();
await Task.WhenAll( insertTasks );
_queue.Flush();
int itemsTaken = 0;
while ( _queue.Count > 0 )
itemsTaken += ( await _queue.TakeAsync() ).Count;
Assert.AreEqual( insertThreads * itemsPerThread, itemsTaken );
}
public async Task InMemoryQueue_PushAndGet(int batchCount)
{
var secondary = new InMemoryQueue <List <string> >(10);
var q = new AsyncBatchQueue <string>(1000,
new long[] { batchCount },
new Func <string, long>[] { s => 1 }, secondary);
await q.PushSecondaryAsync(Enumerable.Repeat("a", batchCount + 1).ToList());
var output = new List <string>();
var getTask = q.GetNextBatchAsync(output, 10 * 1000).AsTask();
Assert.True(getTask.Wait(1000));
Assert.Equal(batchCount, output.Count);
if (batchCount == 0)
{
return;
}
// second call should return the last item
output.Clear();
await q.GetNextBatchAsync(output, 100);
Assert.Single(output);
}
public async Task GetBatchWhenLimitIsReached(int batchCount)
{
var q = new AsyncBatchQueue <string>(10000,
new long[] { batchCount },
new Func <string, long>[] { s => 1 });
var output = new List <string>();
for (var i = 0; i < batchCount + 1; i++)
{
await q.PushAsync("a");
}
var getTask = q.GetNextBatchAsync(output, 10 * 1000).AsTask();
// first call should returns a full batch
Assert.True(getTask.Wait(1000));
Assert.Equal(batchCount, output.Count);
if (batchCount == 0)
{
return;
}
// second call should return the last item
output.Clear();
await q.GetNextBatchAsync(output, 100);
Assert.Single(output);
}
public async Task InMemoryQueue_ConcurrentRead(int readerCount, int itemCount)
{
var secondary = new InMemoryQueue <List <int> >(1000);
using var cts = new CancellationTokenSource();
using var semaphore = new SemaphoreSlim(0, readerCount);
var results = new List <int>();
var q = new AsyncBatchQueue <int>(10000,
new long[] { 100 },
new Func <int, long>[] { s => 1 }, secondary);
async Task readerTask()
{
var output = new List <int>();
await semaphore.WaitAsync();
// we're trying to test that the readers will 'eventually' read all the items, so we do several pulls here
await q.GetNextBatchAsync(output, 500);
await q.GetNextBatchAsync(output, 500);
await q.GetNextBatchAsync(output, 500);
await Task.Delay(100);
lock (results)
{
results.AddRange(output);
}
};
var readers = new Task[readerCount];
for (var i = 0; i < readerCount; i++)
{
readers[i] = readerTask();
}
for (var i = 0; i < itemCount; i++)
{
if (i % 2 == 0)
{
await q.PushAsync(i);
}
else
{
await q.PushSecondaryAsync(new List <int> {
i
});
}
}
semaphore.Release(readerCount);
await Task.WhenAll(readers);
_output.WriteLine(results.Count.ToString());
_output.WriteLine(q.EstimateSize().ToString());
_output.WriteLine(q.EstimateSecondaryQueueSize().ToString());
Assert.Equal(itemCount, results.Distinct().Count());
}
public async Task InMemoryQueue_PullBothQueues()
{
var secondary = new InMemoryQueue <List <int> >(10);
var q = new AsyncBatchQueue <int>(500,
new long[] { 500 },
new Func <int, long>[] { s => 1 }, secondary);
await q.PushSecondaryAsync(Enumerable.Range(0, 500).ToList());
for (var i = 500; i < 1000; i++)
{
await q.PushAsync(i);
}
// pull 3 times
var output = new List <int>();
await q.GetNextBatchAsync(output, 1000);
await q.GetNextBatchAsync(output, 1000);
await q.GetNextBatchAsync(output, 1000);
Assert.Equal(1000, output.Distinct().Count());
}
public async Task TimerFlushesPendingItems()
{
TimeSpan flushPeriod = TimeSpan.FromMilliseconds( 500 );
_queue = new AsyncBatchQueue<int>( 9999, flushPeriod );
_queue.Add( 42 );
await Task.Delay( flushPeriod + flushPeriod );
var batch = await _queue.TakeAsync();
CollectionAssert.AreEqual( new[] { 42 }, batch.ToList() );
}
public async Task NoRaceBetweenFlushOnAddAndOnDemand()
{
const int attempts = 100 * 1000;
const int batchSize = 5;
_queue = new AsyncBatchQueue <int>(batchSize);
for (int attemptNumber = 0; attemptNumber < attempts; attemptNumber++)
{
AddAllItemsButOne(batchSize);
using (ManualResetEvent trigger = new ManualResetEvent(initialState: false))
{
Task addTask = Task.Run
(
() =>
{
trigger.WaitOne();
_queue.Add(666);
}
);
Task flushTask = Task.Run
(
() =>
{
trigger.WaitOne();
_queue.Flush();
}
);
trigger.Set();
await addTask.ConfigureAwait(true);
await flushTask.ConfigureAwait(true);
IReadOnlyList <int> batch = await _queue.TakeAsync().ConfigureAwait(true);
List <int> allItems = batch.ToList();
// This happens if Flush occurred before Add, which means there's another item from Add left unflushed.
// Gotta flush once more to extract it.
if (batch.Count < batchSize)
{
_queue.Flush();
IReadOnlyList <int> secondBatch = await _queue.TakeAsync().ConfigureAwait(true);
allItems.AddRange(secondBatch);
}
allItems.Count.Should().BeLessOrEqualTo(batchSize, $"Double flush detected at attempt #{attemptNumber}. Items: {String.Join( ", ", allItems )}");
}
}
}
public async Task TimerFlushesPendingItems()
{
TimeSpan flushPeriod = TimeSpan.FromMilliseconds(500);
var timerQueue = new AsyncBatchQueue <int>(9999).WithFlushEvery(flushPeriod);
timerQueue.Add(42);
await Task.Delay(flushPeriod + flushPeriod).ConfigureAwait(true);
var batch = await timerQueue.TakeAsync().ConfigureAwait(true);
batch.Should().BeEqualTo(new[] { 42 });
}
public async Task ManualFlushWorks()
{
int[] array = { 0, 1, 42 };
_queue = new AsyncBatchQueue<int>( 50 );
foreach ( var item in array )
_queue.Add( item );
_queue.Flush();
var batch = await _queue.TakeAsync();
CollectionAssert.AreEqual( array, batch.ToList() );
}
public AWSBufferedSink(string id, string sessionName,
IAppDataFileProvider appDataFileProvider,
ILogger logger,
IMetrics metrics,
IBookmarkManager bookmarkManager,
NetworkStatus networkStatus,
AWSBufferedSinkOptions options)
{
Id = id;
_logger = logger;
_metrics = metrics;
_bookmarkManager = bookmarkManager;
_networkStatus = networkStatus;
_bufferIntervalMs = options.BufferIntervalMs;
_format = options.Format;
_maxSecondaryQueueBatches = options.QueueMaxBatches;
var secondaryQueue = CreateSecondaryQueue(options, sessionName, appDataFileProvider, logger);
_queue = new AsyncBatchQueue <Envelope <T> >(options.QueueSizeItems,
new long[] { options.MaxBatchSize, options.MaxBatchBytes },
new Func <Envelope <T>, long>[] { r => 1, GetRecordSize },
secondaryQueue);
_maxAttempts = options.MaxAttempts;
_jittingFactor = options.JittingFactor;
_backoffFactor = options.BackoffFactor;
_recoveryFactor = options.RecoveryFactor;
_minRateAdjustmentFactor = options.MinRateAdjustmentFactor;
_uploadNetworkPriority = options.UploadNetworkPriority;
if (options.TextDecoration is not null)
{
_textDecorationEvaluator = new TextDecorationEvaluator(options.TextDecoration, ResolveRecordVariables);
}
if (options.TextDecorationEx is not null)
{
_textDecorationEvaluator = new TextDecorationExEvaluator(options.TextDecorationEx, EvaluateVariable, ResolveRecordVariable, logger);
}
if (options.ObjectDecoration is not null)
{
_objectDecorationEvaluator = new ObjectDecorationEvaluator(options.ObjectDecoration, ResolveRecordVariables);
}
if (options.ObjectDecorationEx is not null)
{
_objectDecorationEvaluator = new ObjectDecorationExEvaluator(options.ObjectDecorationEx, EvaluateVariable, ResolveRecordVariable, logger);
}
}
public async Task ManualFlushWorks()
{
int[] array = { 0, 1, 42 };
_queue = new AsyncBatchQueue <int>(50);
foreach (var item in array)
{
_queue.Add(item);
}
_queue.Flush();
var batch = await _queue.TakeAsync().ConfigureAwait(true);
batch.Should().BeEqualTo(array);
}
public async Task FlushesWhenBatchSizeIsReached()
{
int[] array = { 0, 1, 42 };
int index = 0;
_queue = new AsyncBatchQueue<int>( array.Length );
for ( ; index < array.Length - 1; index++ )
_queue.Add( array[ index ] );
var takeTask = _queue.TakeAsync();
Assert.IsFalse( takeTask.IsCompleted );
_queue.Add( array[ index ] );
var batch = await takeTask;
CollectionAssert.AreEqual( array, batch.ToList() );
}
public async Task GetBatchWhenTimerExpires(int batchCount, int remaining)
{
var q = new AsyncBatchQueue <string>(10000,
new long[] { batchCount },
new Func <string, long>[] { s => 1 });
var output = new List <string>();
for (var i = 0; i < batchCount - remaining; i++)
{
await q.PushAsync("a");
}
var getTask = q.GetNextBatchAsync(output, 500).AsTask();
Assert.True(getTask.Wait(5000));
Assert.Equal(batchCount - remaining, output.Count);
}
public async Task PersistentQueue_PushAndGet(int batchCount)
{
var dataDir = Path.Combine(AppContext.BaseDirectory, Guid.NewGuid().ToString());
var queueDir = Path.Combine(dataDir, _queueDirName);
Directory.CreateDirectory(queueDir);
var fileProvider = new ProtectedAppDataFileProvider(dataDir);
try
{
var secondary = new FilePersistentQueue <List <int> >(100, queueDir, new IntegerListSerializer(), fileProvider, NullLogger.Instance);
var q = new AsyncBatchQueue <int>(1000,
new long[] { batchCount },
new Func <int, long>[] { s => 1 }, secondary);
await q.PushSecondaryAsync(Enumerable.Range(0, batchCount + 1).ToList());
var output = new List <int>();
var getTask = q.GetNextBatchAsync(output, 10 * 1000).AsTask();
Assert.True(getTask.Wait(1000));
Assert.Equal(batchCount, output.Count);
if (batchCount == 0)
{
return;
}
// second call should return the last item
output.Clear();
await q.GetNextBatchAsync(output, 100);
Assert.Single(output);
}
finally
{
if (Directory.Exists(dataDir))
{
Directory.Delete(dataDir, true);
}
}
}
public async Task FlushesWhenBatchSizeIsReached()
{
int[] array = { 0, 1, 42 };
int index = 0;
_queue = new AsyncBatchQueue <int>(array.Length);
for ( ; index < array.Length - 1; index++)
{
_queue.Add(array[index]);
}
var takeTask = _queue.TakeAsync();
takeTask.IsCompleted.Should().BeFalse();
_queue.Add(array[index]);
var batch = await takeTask.ConfigureAwait(true);
batch.Should().BeEqualTo(array);
}
public async Task PushToFullQueue_Cancellation()
{
using var cts = new CancellationTokenSource();
var output = new List <string>();
var q = new AsyncBatchQueue <string>(1000,
new long[] { 100 },
new Func <string, long>[] { s => 1 });
for (var i = 0; i < 1000; i++)
{
await q.PushAsync("a");
}
var pushTask = q.PushAsync("b", cts.Token).AsTask();
cts.Cancel();
await Task.Delay(100);
Assert.True(pushTask.IsCompleted);
}
public async Task GetBatch_Cancellation()
{
using var cts = new CancellationTokenSource();
var output = new List <string>();
var q = new AsyncBatchQueue <string>(10000,
new long[] { 100 },
new Func <string, long>[] { s => 1 });
for (var i = 0; i < 99; i++)
{
await q.PushAsync("a");
}
var getTask = q.GetNextBatchAsync(output, int.MaxValue, cts.Token).AsTask();
cts.Cancel();
await Task.Delay(200);
Assert.True(getTask.IsCompleted);
}
public async Task PersistentQueue_PullBothQueues()
{
var dataDir = Path.Combine(AppContext.BaseDirectory, Guid.NewGuid().ToString());
var queueDir = Path.Combine(dataDir, _queueDirName);
Directory.CreateDirectory(queueDir);
var fileProvider = new ProtectedAppDataFileProvider(dataDir);
try
{
var secondary = new FilePersistentQueue <List <int> >(10, queueDir, new IntegerListSerializer(), fileProvider, NullLogger.Instance);
var q = new AsyncBatchQueue <int>(500,
new long[] { 500 },
new Func <int, long>[] { s => 1 }, secondary);
await q.PushSecondaryAsync(Enumerable.Range(0, 500).ToList());
for (var i = 500; i < 1000; i++)
{
await q.PushAsync(i);
}
// pull 3 times
var output = new List <int>();
await q.GetNextBatchAsync(output, 1000);
await q.GetNextBatchAsync(output, 1000);
await q.GetNextBatchAsync(output, 1000);
Assert.Equal(1000, output.Distinct().Count());
}
finally
{
if (Directory.Exists(dataDir))
{
Directory.Delete(dataDir, true);
}
}
}
public async Task LimitIsReached_OrderIsPresevered()
{
var q = new AsyncBatchQueue <int>(10000,
new long[] { 100 },
new Func <int, long>[] { s => 1 });
for (var i = 0; i < 150 + 1; i++)
{
await q.PushAsync(i);
}
var output = new List <int>();
await q.GetNextBatchAsync(output, 1000);
// pull again
await q.GetNextBatchAsync(output, 100);
for (var i = 0; i < 150; i++)
{
Assert.Equal(i, output[i]);
}
}
public async Task PersistentQueue_ConcurrentRead(int readerCount, int itemCount)
{
var dataDir = Path.Combine(AppContext.BaseDirectory, Guid.NewGuid().ToString());
var queueDir = Path.Combine(dataDir, _queueDirName);
Directory.CreateDirectory(queueDir);
var fileProvider = new ProtectedAppDataFileProvider(dataDir);
try
{
var secondary = new FilePersistentQueue <List <int> >(100000, queueDir, new IntegerListSerializer(), fileProvider, NullLogger.Instance);
using var cts = new CancellationTokenSource();
using var semaphore = new SemaphoreSlim(0, readerCount);
var results = new List <int>();
var q = new AsyncBatchQueue <int>(10000,
new long[] { 100 },
new Func <int, long>[] { s => 1 }, secondary);
async Task readerTask()
{
var output = new List <int>();
await semaphore.WaitAsync();
// we're trying to test that the readers will 'eventually' read all the items, so we do several pulls here
await q.GetNextBatchAsync(output, 500);
await q.GetNextBatchAsync(output, 500);
await q.GetNextBatchAsync(output, 500);
await Task.Delay(100);
lock (results)
{
results.AddRange(output);
}
};
var readers = new Task[readerCount];
for (var i = 0; i < readerCount; i++)
{
readers[i] = readerTask();
}
for (var i = 0; i < itemCount; i++)
{
if (i % 2 == 0)
{
await q.PushAsync(i);
}
else
{
await q.PushSecondaryAsync(new List <int> {
i
});
}
}
semaphore.Release(readerCount);
await Task.WhenAll(readers);
_output.WriteLine(results.Count.ToString());
_output.WriteLine(q.EstimateSize().ToString());
_output.WriteLine(q.EstimateSecondaryQueueSize().ToString());
Assert.Equal(itemCount, results.Distinct().Count());
}
finally
{
if (Directory.Exists(dataDir))
{
Directory.Delete(dataDir, true);
}
}
}
public void ThrowsOnIncorrectBatchSize()
{
Action act = () => _queue = new AsyncBatchQueue <int>(0);
act.ShouldThrow <ArgumentOutOfRangeException>();
}
public void ThrowsOnIncorrectBatchSize()
{
_queue = new AsyncBatchQueue<int>( 0 );
}
